479 files changed, 55301 insertions, 25950 deletions

diff --git a/erts/Makefile.in b/erts/Makefile.in
index fabf86db7c..2e63fc469e 100644
--- a/erts/Makefile.in
+++ b/erts/Makefile.in
@@ -1,19 +1,19 @@
 # 
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2006-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2006-2010. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 # 
 include $(ERL_TOP)/make/target.mk
@@ -87,15 +87,20 @@ endif
 # in the same directory...
 local_setup:
 	@cd start_scripts && $(MAKE)
+	@echo `ls $(ERL_TOP)/bin/`
 	@rm -f $(ERL_TOP)/bin/erl $(ERL_TOP)/bin/erlc $(ERL_TOP)/bin/cerl \
 		$(ERL_TOP)/bin/erl.exe $(ERL_TOP)/bin/erlc.exe \
 		$(ERL_TOP)/bin/escript $(ERL_TOP)/bin/escript.exe \
 		$(ERL_TOP)/bin/dialyzer $(ERL_TOP)/bin/dialyzer.exe \
 		$(ERL_TOP)/bin/typer $(ERL_TOP)/bin/typer.exe \
+		$(ERL_TOP)/bin/run_test $(ERL_TOP)/bin/run_test.exe \
+		$(ERL_TOP)/bin/ct_run $(ERL_TOP)/bin/ct_run.exe \
 		$(ERL_TOP)/bin/start*.boot $(ERL_TOP)/bin/start*.script
 	@if [ "X$(TARGET)" = "Xwin32" ]; then \
 		cp $(ERL_TOP)/bin/$(TARGET)/dialyzer.exe $(ERL_TOP)/bin/dialyzer.exe; \
 		cp $(ERL_TOP)/bin/$(TARGET)/typer.exe $(ERL_TOP)/bin/typer.exe; \
+		cp $(ERL_TOP)/bin/$(TARGET)/ct_run.exe $(ERL_TOP)/bin/ct_run.exe; \
+		cp $(ERL_TOP)/bin/$(TARGET)/ct_run.exe $(ERL_TOP)/bin/run_test.exe; \
 		cp $(ERL_TOP)/bin/$(TARGET)/erlc.exe $(ERL_TOP)/bin/erlc.exe; \
 		cp $(ERL_TOP)/bin/$(TARGET)/erl.exe $(ERL_TOP)/bin/erl.exe; \
 		cp $(ERL_TOP)/bin/$(TARGET)/werl.exe $(ERL_TOP)/bin/werl.exe; \
@@ -115,6 +120,8 @@ local_setup:
 	        $(ERL_TOP)/erts/etc/unix/cerl.src > $(ERL_TOP)/bin/cerl; \
 		cp $(ERL_TOP)/bin/$(TARGET)/dialyzer $(ERL_TOP)/bin/dialyzer; \
 		cp $(ERL_TOP)/bin/$(TARGET)/typer $(ERL_TOP)/bin/typer; \
+		cp $(ERL_TOP)/bin/$(TARGET)/ct_run $(ERL_TOP)/bin/ct_run; \
+		ln -s $(ERL_TOP)/bin/ct_run $(ERL_TOP)/bin/run_test; \
 		cp $(ERL_TOP)/bin/$(TARGET)/erlc $(ERL_TOP)/bin/erlc; \
 		cp $(ERL_TOP)/bin/$(TARGET)/escript $(ERL_TOP)/bin/escript; \
 	 	chmod 755 $(ERL_TOP)/bin/erl $(ERL_TOP)/bin/erlc \
diff --git a/erts/aclocal.m4 b/erts/aclocal.m4
index 3d935b7295..b64380e817 100644
--- a/erts/aclocal.m4
+++ b/erts/aclocal.m4
@@ -1,7 +1,7 @@
 dnl
 dnl %CopyrightBegin%
 dnl
-dnl Copyright Ericsson AB 1998-2010. All Rights Reserved.
+dnl Copyright Ericsson AB 1998-2011. All Rights Reserved.
 dnl
 dnl The contents of this file are subject to the Erlang Public License,
 dnl Version 1.1, (the "License"); you may not use this file except in
@@ -386,14 +386,24 @@ AC_DEFUN(LM_SYS_IPV6,
 AC_CACHE_VAL(ac_cv_sys_ipv6_support,
 [ok_so_far=yes
  AC_TRY_COMPILE([#include <sys/types.h>
-#include <netinet/in.h>],
+#ifdef __WIN32__
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#else
+#include <netinet/in.h>
+#endif],
    [struct in6_addr a6; struct sockaddr_in6 s6;], ok_so_far=yes, ok_so_far=no)
 
 if test $ok_so_far = yes; then
   ac_cv_sys_ipv6_support=yes
 else
   AC_TRY_COMPILE([#include <sys/types.h>
-#include <netinet/in.h>],
+#ifdef __WIN32__
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#else
+#include <netinet/in.h>
+#endif],
     [struct in_addr6 a6; struct sockaddr_in6 s6;],
     ac_cv_sys_ipv6_support=in_addr6, ac_cv_sys_ipv6_support=no)
 fi
@@ -512,6 +522,8 @@ dnl
 AC_DEFUN(LM_CHECK_THR_LIB,
 [
 
+NEED_NPTL_PTHREAD_H=no
+
 dnl win32?
 AC_MSG_CHECKING([for native win32 threads])
 if test "X$host_os" = "Xwin32"; then
@@ -519,11 +531,13 @@ if test "X$host_os" = "Xwin32"; then
     THR_DEFS="-DWIN32_THREADS"
     THR_LIBS=
     THR_LIB_NAME=win32_threads
+    THR_LIB_TYPE=win32_threads
 else
     AC_MSG_RESULT(no)
     THR_DEFS=
     THR_LIBS=
     THR_LIB_NAME=
+    THR_LIB_TYPE=posix_unknown
 
 dnl Try to find POSIX threads
 
@@ -584,8 +598,11 @@ dnl On ofs1 the '-pthread' switch should be used
 		    AC_MSG_WARN([result yes guessed because of cross compilation])
 		fi
 		if test $nptl = yes; then
+		    THR_LIB_TYPE=posix_nptl
 		    need_nptl_incldir=no
-		    AC_CHECK_HEADER(nptl/pthread.h, need_nptl_incldir=yes)
+		    AC_CHECK_HEADER(nptl/pthread.h,
+				    [need_nptl_incldir=yes
+				     NEED_NPTL_PTHREAD_H=yes])
 		    if test $need_nptl_incldir = yes; then
 			# Ahh...
 			nptl_path="$C_INCLUDE_PATH:$CPATH"
@@ -649,6 +666,19 @@ fi
 
 ])
 
+AC_DEFUN(ERL_INTERNAL_LIBS,
+[
+
+ERTS_INTERNAL_X_LIBS=
+
+AC_CHECK_LIB(kstat, kstat_open,
+[AC_DEFINE(HAVE_KSTAT, 1, [Define if you have kstat])
+ERTS_INTERNAL_X_LIBS="$ERTS_INTERNAL_X_LIBS -lkstat"])
+
+AC_SUBST(ERTS_INTERNAL_X_LIBS)
+
+])
+
 dnl ----------------------------------------------------------------------
 dnl
 dnl ERL_FIND_ETHR_LIB
@@ -672,10 +702,14 @@ AC_DEFUN(ERL_FIND_ETHR_LIB,
 [
 
 LM_CHECK_THR_LIB
+ERL_INTERNAL_LIBS
 
+ethr_have_native_atomics=no
+ethr_have_native_spinlock=no
 ETHR_THR_LIB_BASE="$THR_LIB_NAME"
+ETHR_THR_LIB_BASE_TYPE="$THR_LIB_TYPE"
 ETHR_DEFS="$THR_DEFS"
-ETHR_X_LIBS="$THR_LIBS"
+ETHR_X_LIBS="$THR_LIBS $ERTS_INTERNAL_X_LIBS"
 ETHR_LIBS=
 ETHR_LIB_NAME=
 
@@ -687,6 +721,7 @@ ethr_lib_name=ethread
 case "$THR_LIB_NAME" in
 
     win32_threads)
+	ETHR_THR_LIB_BASE_DIR=win
 	# * _WIN32_WINNT >= 0x0400 is needed for
 	#   TryEnterCriticalSection
 	# * _WIN32_WINNT >= 0x0403 is needed for
@@ -712,10 +747,128 @@ case "$THR_LIB_NAME" in
         if test $found_win32_winnt = no; then
 	    AC_MSG_ERROR([-D_WIN32_WINNT missing in CPPFLAGS])
         fi
+
 	AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads])
+
+	have_ilckd=no
+	AC_MSG_CHECKING([for _InterlockedCompareExchange64()])
+	AC_TRY_LINK([
+			#define WIN32_LEAN_AND_MEAN
+			#include <windows.h>
+		    ],
+		    [
+			volatile __int64 *var;
+			_InterlockedCompareExchange64(var, (__int64) 1, (__int64) 0);
+			return 0;
+		    ],
+		    have_ilckd=yes)
+	AC_MSG_RESULT([$have_ilckd])
+	test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()])
+
+	AC_CHECK_SIZEOF(void *)
+	case "$ac_cv_sizeof_void_p-$have_ilckd" in
+	    8-no)
+		ethr_have_native_atomics=no
+	  	ethr_have_native_spinlock=no;;
+	    *)
+		ethr_have_native_atomics=yes
+	  	ethr_have_native_spinlock=yes;;
+	esac
+
+	have_ilckd=no
+	AC_MSG_CHECKING([for _InterlockedDecrement64()])
+	AC_TRY_LINK([
+			#define WIN32_LEAN_AND_MEAN
+			#include <windows.h>
+		    ],
+		    [
+			volatile __int64 *var;
+			_InterlockedDecrement64(var);
+			return 0;
+		    ],
+		    have_ilckd=yes)
+	AC_MSG_RESULT([$have_ilckd])
+	test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()])
+
+	have_ilckd=no
+	AC_MSG_CHECKING([for _InterlockedIncrement64()])
+	AC_TRY_LINK([
+			#define WIN32_LEAN_AND_MEAN
+			#include <windows.h>
+		    ],
+		    [
+			volatile __int64 *var;
+			_InterlockedIncrement64(var);
+			return 0;
+		    ],
+		    have_ilckd=yes)
+	AC_MSG_RESULT([$have_ilckd])
+	test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()])
+
+	have_ilckd=no
+	AC_MSG_CHECKING([for _InterlockedExchangeAdd64()])
+	AC_TRY_LINK([
+			#define WIN32_LEAN_AND_MEAN
+			#include <windows.h>
+		    ],
+		    [
+			volatile __int64 *var;
+			_InterlockedExchangeAdd64(var, (__int64) 1);
+			return 0;
+		    ],
+		    have_ilckd=yes)
+	AC_MSG_RESULT([$have_ilckd])
+	test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()])
+
+	have_ilckd=no
+	AC_MSG_CHECKING([for _InterlockedExchange64()])
+	AC_TRY_LINK([
+			#define WIN32_LEAN_AND_MEAN
+			#include <windows.h>
+		    ],
+		    [
+			volatile __int64 *var;
+			_InterlockedExchange64(var, (__int64) 1);
+			return 0;
+		    ],
+		    have_ilckd=yes)
+	AC_MSG_RESULT([$have_ilckd])
+	test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()])
+
+	have_ilckd=no
+	AC_MSG_CHECKING([for _InterlockedAnd64()])
+	AC_TRY_LINK([
+			#define WIN32_LEAN_AND_MEAN
+			#include <windows.h>
+		    ],
+		    [
+			volatile __int64 *var;
+			_InterlockedAnd64(var, (__int64) 1);
+			return 0;
+		    ],
+		    have_ilckd=yes)
+	AC_MSG_RESULT([$have_ilckd])
+	test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()])
+
+	have_ilckd=no
+	AC_MSG_CHECKING([for _InterlockedOr64()])
+	AC_TRY_LINK([
+			#define WIN32_LEAN_AND_MEAN
+			#include <windows.h>
+		    ],
+		    [
+			volatile __int64 *var;
+			_InterlockedOr64(var, (__int64) 1);
+			return 0;
+		    ],
+		    have_ilckd=yes)
+	AC_MSG_RESULT([$have_ilckd])
+	test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()])
+
 	;;
 
     pthread)
+	ETHR_THR_LIB_BASE_DIR=pthread
     	AC_DEFINE(ETHR_PTHREADS, 1, [Define if you have pthreads])
 	case $host_os in
 	    openbsd*)
@@ -771,9 +924,7 @@ case "$THR_LIB_NAME" in
 		if test $usable_sigaltstack = no; then
 		    ETHR_DEFS="$ETHR_DEFS -DETHR_UNUSABLE_SIGALTSTACK"
 		fi
-
-		AC_DEFINE(ETHR_INIT_MUTEX_IN_CHILD_AT_FORK, 1, \
-[Define if mutexes should be reinitialized (instead of unlocked) in child at fork.]) ;;
+		;;
 	    *) ;;
 	esac
 
@@ -799,6 +950,15 @@ case "$THR_LIB_NAME" in
 			AC_DEFINE(ETHR_HAVE_MIT_PTHREAD_H, 1, \
 [Define if the pthread.h header file is in pthread/mit directory.]))
 
+	if test $NEED_NPTL_PTHREAD_H = yes; then
+	    AC_DEFINE(ETHR_NEED_NPTL_PTHREAD_H, 1, \
+[Define if you need the <nptl/pthread.h> header file.])
+	fi
+
+	AC_CHECK_HEADER(sched.h, \
+			AC_DEFINE(ETHR_HAVE_SCHED_H, 1, \
+[Define if you have the <sched.h> header file.]))
+
 	AC_CHECK_HEADER(sys/time.h, \
 			AC_DEFINE(ETHR_HAVE_SYS_TIME_H, 1, \
 [Define if you have the <sys/time.h> header file.]))
@@ -814,39 +974,212 @@ case "$THR_LIB_NAME" in
 	dnl Check for functions
 	dnl
 
-	AC_CHECK_FUNC(pthread_atfork, \
-			AC_DEFINE(ETHR_HAVE_PTHREAD_ATFORK, 1, \
-[Define if you have the pthread_atfork function.]))
-	AC_CHECK_FUNC(pthread_mutexattr_settype, \
-			AC_DEFINE(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE, 1, \
-[Define if you have the pthread_mutexattr_settype function.]))
-	AC_CHECK_FUNC(pthread_mutexattr_setkind_np, \
-			AC_DEFINE(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP, 1, \
-[Define if you have the pthread_mutexattr_setkind_np function.]))
 	AC_CHECK_FUNC(pthread_spin_lock, \
-			AC_DEFINE(ETHR_HAVE_PTHREAD_SPIN_LOCK, 1, \
-[Define if you have the pthread_spin_lock function.]))
-	test "$force_linux_pthread_rwlocks" = "yes" || {
-	    force_linux_pthread_rwlocks=no
-	}
-	case "$force_linux_pthread_rwlocks-$host_os" in
-		no-linux*) # Writers may get starved
-			# TODO: write a test that tests the implementation
-			;;
-		*)
-			AC_CHECK_FUNC(pthread_rwlock_init, \
-				AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCK_INIT, 1, \
-[Define if you have a pthread_rwlock implementation that can be used.]))
-			;;
-	esac
+			[ethr_have_native_spinlock=yes \
+			 AC_DEFINE(ETHR_HAVE_PTHREAD_SPIN_LOCK, 1, \
+[Define if you have the pthread_spin_lock function.])])
+
+	have_sched_yield=no
+	have_librt_sched_yield=no
+	AC_CHECK_FUNC(sched_yield, [have_sched_yield=yes])
+	if test $have_sched_yield = no; then
+	    AC_CHECK_LIB(rt, sched_yield,
+			 [have_librt_sched_yield=yes
+			  ETHR_X_LIBS="$ETHR_X_LIBS -lrt"])
+	fi
+	if test $have_sched_yield = yes || test $have_librt_sched_yield = yes; then
+	    AC_DEFINE(ETHR_HAVE_SCHED_YIELD, 1, [Define if you have the sched_yield() function.])
+	    AC_MSG_CHECKING([whether sched_yield() returns an int])
+	    sched_yield_ret_int=no
+	    AC_TRY_COMPILE([
+				#ifdef ETHR_HAVE_SCHED_H
+				#include <sched.h>
+				#endif
+			   ],
+			   [int sched_yield();],
+			   [sched_yield_ret_int=yes])
+	    AC_MSG_RESULT([$sched_yield_ret_int])
+	    if test $sched_yield_ret_int = yes; then
+		AC_DEFINE(ETHR_SCHED_YIELD_RET_INT, 1, [Define if sched_yield() returns an int.])
+	    fi
+	fi
+
+	have_pthread_yield=no
+	AC_CHECK_FUNC(pthread_yield, [have_pthread_yield=yes])
+	if test $have_pthread_yield = yes; then
+	    AC_DEFINE(ETHR_HAVE_PTHREAD_YIELD, 1, [Define if you have the pthread_yield() function.])
+	    AC_MSG_CHECKING([whether pthread_yield() returns an int])
+	    pthread_yield_ret_int=no
+	    AC_TRY_COMPILE([
+				#if defined(ETHR_NEED_NPTL_PTHREAD_H)
+				#include <nptl/pthread.h>
+				#elif defined(ETHR_HAVE_MIT_PTHREAD_H)
+				#include <pthread/mit/pthread.h>
+				#elif defined(ETHR_HAVE_PTHREAD_H)
+				#include <pthread.h>
+				#endif
+			   ],
+			   [int pthread_yield();],
+			   [pthread_yield_ret_int=yes])
+	    AC_MSG_RESULT([$pthread_yield_ret_int])
+	    if test $pthread_yield_ret_int = yes; then
+		AC_DEFINE(ETHR_PTHREAD_YIELD_RET_INT, 1, [Define if pthread_yield() returns an int.])
+	    fi
+	fi
+
+	have_pthread_rwlock_init=no
+	AC_CHECK_FUNC(pthread_rwlock_init, [have_pthread_rwlock_init=yes])
+	if test $have_pthread_rwlock_init = yes; then
+
+	    ethr_have_pthread_rwlockattr_setkind_np=no
+	    AC_CHECK_FUNC(pthread_rwlockattr_setkind_np,
+			  [ethr_have_pthread_rwlockattr_setkind_np=yes])
+
+	    if test $ethr_have_pthread_rwlockattr_setkind_np = yes; then
+		AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP, 1, \
+[Define if you have the pthread_rwlockattr_setkind_np() function.])
+
+		AC_MSG_CHECKING([for PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP])
+		ethr_pthread_rwlock_writer_nonrecursive_initializer_np=no
+		AC_TRY_LINK([
+				#if defined(ETHR_NEED_NPTL_PTHREAD_H)
+				#include <nptl/pthread.h>
+				#elif defined(ETHR_HAVE_MIT_PTHREAD_H)
+				#include <pthread/mit/pthread.h>
+				#elif defined(ETHR_HAVE_PTHREAD_H)
+				#include <pthread.h>
+				#endif
+			    ],
+			    [
+				pthread_rwlockattr_t *attr;
+				return pthread_rwlockattr_setkind_np(attr,
+				    PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
+			    ],
+			    [ethr_pthread_rwlock_writer_nonrecursive_initializer_np=yes])
+		AC_MSG_RESULT([$ethr_pthread_rwlock_writer_nonrecursive_initializer_np])
+		if test $ethr_pthread_rwlock_writer_nonrecursive_initializer_np = yes; then
+		    AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP, 1, \
+[Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock attribute.])
+		fi
+	    fi
+	fi
+
+	if test "$force_pthread_rwlocks" = "yes"; then
+
+	    AC_DEFINE(ETHR_FORCE_PTHREAD_RWLOCK, 1, \
+[Define if you want to force usage of pthread rwlocks])
+
+	    if test $have_pthread_rwlock_init = yes; then
+		AC_MSG_WARN([Forced usage of pthread rwlocks. Note that this implementation may suffer from starvation issues.])
+	    else
+		AC_MSG_ERROR([User forced usage of pthread rwlock, but no such implementation was found])
+	    fi
+	fi
+
 	AC_CHECK_FUNC(pthread_attr_setguardsize, \
 			AC_DEFINE(ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE, 1, \
 [Define if you have the pthread_attr_setguardsize function.]))
 
+	linux_futex=no
+	AC_MSG_CHECKING([for Linux futexes])
+	AC_TRY_LINK([
+			#include <sys/syscall.h>
+			#include <unistd.h>
+			#include <linux/futex.h>
+			#include <sys/time.h>
+		    ],
+		    [
+			int i = 1;
+			syscall(__NR_futex, (void *) &i, FUTEX_WAKE, 1,
+				(void*)0,(void*)0, 0);
+			syscall(__NR_futex, (void *) &i, FUTEX_WAIT, 0,
+				(void*)0,(void*)0, 0);
+			return 0;
+		    ],
+		    linux_futex=yes)
+	AC_MSG_RESULT([$linux_futex])
+	test $linux_futex = yes && AC_DEFINE(ETHR_HAVE_LINUX_FUTEX, 1, [Define if you have a linux futex implementation.])
+
+	AC_MSG_CHECKING([for GCC atomic operations])
+	ethr_have_gcc_atomic_ops=no
+	AC_TRY_LINK([],
+		    [
+			long res;
+			volatile long val;
+			res = __sync_val_compare_and_swap(&val, (long) 1, (long) 0);
+			res = __sync_add_and_fetch(&val, (long) 1);
+			res = __sync_sub_and_fetch(&val, (long) 1);
+			res = __sync_fetch_and_and(&val, (long) 1);
+			res = __sync_fetch_and_or(&val, (long) 1);
+		    ],
+		    [ethr_have_native_atomics=yes
+		     ethr_have_gcc_atomic_ops=yes])
+	AC_MSG_RESULT([$ethr_have_gcc_atomic_ops])
+	test $ethr_have_gcc_atomic_ops = yes && AC_DEFINE(ETHR_HAVE_GCC_ATOMIC_OPS, 1, [Define if you have gcc atomic operations])
+
+	case "$host_cpu" in
+	  sun4u | sparc64 | sun4v)
+		ethr_have_native_atomics=yes;; 
+	  i86pc | i*86 | x86_64 | amd64)
+		ethr_have_native_atomics=yes;;
+	  macppc | ppc | "Power Macintosh")
+		ethr_have_native_atomics=yes;;
+	  tile)
+		ethr_have_native_atomics=yes;;
+	  *)
+		;;
+	esac
+
+	AC_MSG_CHECKING([for a usable libatomic_ops implementation])
+	case "x$with_libatomic_ops" in
+	    xno | xyes | x)
+		libatomic_ops_include=
+		;;
+	    *)
+		if test -d "${with_libatomic_ops}/include"; then
+		    libatomic_ops_include="-I$with_libatomic_ops/include"
+		    CPPFLAGS="$CPPFLAGS $libatomic_ops_include"
+		else
+		    AC_MSG_ERROR([libatomic_ops include directory $with_libatomic_ops/include not found])
+		fi;;
+	esac
+	ethr_have_libatomic_ops=no
+	AC_TRY_LINK([#include "atomic_ops.h"],
+		    [
+			volatile AO_t x;
+			AO_t y;
+			int z;
+
+			AO_nop_full();
+			AO_store(&x, (AO_t) 0);
+			z = AO_load(&x);
+			z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1);
+		    ],
+		    [ethr_have_native_atomics=yes
+		     ethr_have_libatomic_ops=yes])
+	AC_MSG_RESULT([$ethr_have_libatomic_ops])
+	if test $ethr_have_libatomic_ops = yes; then
+	    AC_CHECK_SIZEOF(AO_t, ,
+			    [
+				#include <stdio.h>
+				#include "atomic_ops.h"
+			    ])
+	    AC_DEFINE_UNQUOTED(ETHR_SIZEOF_AO_T, $ac_cv_sizeof_AO_t, [Define to the size of AO_t if libatomic_ops is used])
+
+	    AC_DEFINE(ETHR_HAVE_LIBATOMIC_OPS, 1, [Define if you have libatomic_ops atomic operations])
+	    if test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then
+		AC_DEFINE(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS, 1, [Define if you prefer libatomic_ops native ethread implementations])
+	    fi
+	    ETHR_DEFS="$ETHR_DEFS $libatomic_ops_include"
+	elif test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then
+	    AC_MSG_ERROR([No usable libatomic_ops implementation found])
+	fi
+
 	dnl Restore LIBS
 	LIBS=$saved_libs
 	dnl restore CPPFLAGS
 	CPPFLAGS=$saved_cppflags
+
 	;;
     *)
 	;;
@@ -862,17 +1195,111 @@ fi
 
 if test "x$ETHR_THR_LIB_BASE" != "x"; then
 	ETHR_DEFS="-DUSE_THREADS $ETHR_DEFS"
-	ETHR_LIBS="-l$ethr_lib_name $ETHR_X_LIBS"
+	ETHR_LIBS="-l$ethr_lib_name -lerts_internal_r $ETHR_X_LIBS"
 	ETHR_LIB_NAME=$ethr_lib_name
 fi
 
 AC_CHECK_SIZEOF(void *)
 AC_DEFINE_UNQUOTED(ETHR_SIZEOF_PTR, $ac_cv_sizeof_void_p, [Define to the size of pointers])
 
-if test "X$disable_native_ethr_impls" = "Xyes"; then
-	AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations])
+AC_CHECK_SIZEOF(int)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF_INT, $ac_cv_sizeof_int, [Define to the size of int])
+AC_CHECK_SIZEOF(long)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG, $ac_cv_sizeof_long, [Define to the size of long])
+AC_CHECK_SIZEOF(long long)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG_LONG, $ac_cv_sizeof_long_long, [Define to the size of long long])
+AC_CHECK_SIZEOF(__int64)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT64, $ac_cv_sizeof___int64, [Define to the size of __int64])
+
+
+case X$erl_xcomp_bigendian in
+    X) ;;
+    Xyes|Xno) ac_cv_c_bigendian=$erl_xcomp_bigendian;;
+    *) AC_MSG_ERROR([Bad erl_xcomp_bigendian value: $erl_xcomp_bigendian]);;
+esac
+
+AC_C_BIGENDIAN
+
+if test "$ac_cv_c_bigendian" = "yes"; then
+    AC_DEFINE(ETHR_BIGENDIAN, 1, [Define if bigendian])
 fi
 
+AC_ARG_ENABLE(native-ethr-impls,
+	      AS_HELP_STRING([--disable-native-ethr-impls],
+                             [disable native ethread implementations]),
+[ case "$enableval" in
+    no) disable_native_ethr_impls=yes ;;
+    *)  disable_native_ethr_impls=no ;;
+  esac ], disable_native_ethr_impls=no)
+
+test "X$disable_native_ethr_impls" = "Xyes" &&
+  AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations])
+
+AC_ARG_ENABLE(prefer-gcc-native-ethr-impls,
+	      AS_HELP_STRING([--enable-prefer-gcc-native-ethr-impls],
+			     [prefer gcc native ethread implementations]),
+[ case "$enableval" in
+    yes) enable_prefer_gcc_native_ethr_impls=yes ;;
+    *)  enable_prefer_gcc_native_ethr_impls=no ;;
+  esac ], enable_prefer_gcc_native_ethr_impls=no)
+
+test $enable_prefer_gcc_native_ethr_impls = yes &&
+  AC_DEFINE(ETHR_PREFER_GCC_NATIVE_IMPLS, 1, [Define if you prefer gcc native ethread implementations])
+
+AC_ARG_WITH(libatomic_ops,
+	    AS_HELP_STRING([--with-libatomic_ops=PATH],
+			   [specify and prefer usage of libatomic_ops in the ethread library]))
+
+AC_ARG_ENABLE(ethread-pre-pentium4-compatibility,
+	      AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility],
+			     [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]),
+[
+  case "$enable_ethread_pre_pentium4_compatibility" in
+    yes|no) ;;
+    *) enable_ethread_pre_pentium4_compatibility=check;;
+  esac
+],
+[enable_ethread_pre_pentium4_compatibility=check])
+
+test "$cross_compiling" != "yes" || enable_ethread_pre_pentium4_compatibility=no
+
+case "$enable_ethread_pre_pentium4_compatibility-$host_cpu" in
+  check-i86pc | check-i*86)
+    AC_MSG_CHECKING([whether pre pentium 4 compatibility should forced])
+    AC_RUN_IFELSE([
+#if defined(__GNUC__)
+#  if defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS)
+#    define CHECK_LIBATOMIC_OPS__
+#  else
+#    define CHECK_GCC_ASM__
+#  endif
+#elif defined(ETHR_HAVE_LIBATOMIC_OPS)
+#  define CHECK_LIBATOMIC_OPS__
+#endif
+#if defined(CHECK_LIBATOMIC_OPS__)
+#include "atomic_ops.h"
+#endif
+int main(void)
+{
+#if defined(CHECK_GCC_ASM__)
+    __asm__ __volatile__("mfence" : : : "memory");
+#elif defined(CHECK_LIBATOMIC_OPS__)
+    AO_nop_full();
+#endif
+    return 0;
+}
+	],
+	[enable_ethread_pre_pentium4_compatibility=no],
+	[enable_ethread_pre_pentium4_compatibility=yes],
+	[enable_ethread_pre_pentium4_compatibility=no])
+    AC_MSG_RESULT([$enable_ethread_pre_pentium4_compatibility]);;
+  *)
+    ;;
+esac
+
+test $enable_ethread_pre_pentium4_compatibility = yes &&
+  AC_DEFINE(ETHR_PRE_PENTIUM4_COMPAT, 1, [Define if you want compatibilty with x86 processors before pentium4.])
+
 AC_DEFINE(ETHR_HAVE_ETHREAD_DEFINES, 1, \
 [Define if you have all ethread defines])
 
@@ -881,6 +1308,7 @@ AC_SUBST(ETHR_LIBS)
 AC_SUBST(ETHR_LIB_NAME)
 AC_SUBST(ETHR_DEFS)
 AC_SUBST(ETHR_THR_LIB_BASE)
+AC_SUBST(ETHR_THR_LIB_BASE_DIR)
 
 ])
 
diff --git a/erts/autoconf/configure.vxworks b/erts/autoconf/configure.vxworks
index 70d7bdbaf2..23a93faa31 100755
--- a/erts/autoconf/configure.vxworks
+++ b/erts/autoconf/configure.vxworks
@@ -2,7 +2,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -101,7 +101,6 @@ epmd_dir=${ERL_TOP}/erts/epmd/src
 os_mon_dir=${ERL_TOP}/lib/os_mon/c_src
 orber_dir=${ERL_TOP}/lib/orber/c_src
 ic_dir=${ERL_TOP}/lib/ic/c_src
-asn1_dir=${ERL_TOP}/lib/asn1/c_src
 internal_tools_dir=${ERL_TOP}
 libdir=${ERL_TOP}/lib
 tsdir=$libdir/test_server/src
@@ -122,7 +121,6 @@ CONFIG_FILES="${ERL_TOP}/erts/emulator/$host/Makefile
 	    $os_mon_dir/$host/Makefile
 	    $zlibdir/$host/Makefile
 	    $ic_dir/$host/Makefile
-	    $asn1_dir/$host/Makefile
 	    $runtime_tools_dir/$host/Makefile
 	    $tools_dir/$host/Makefile
 	    $orber_dir/$host/Makefile"
diff --git a/erts/autoconf/vxworks/sed.general b/erts/autoconf/vxworks/sed.general
index f725a6f9ca..88697b788d 100644
--- a/erts/autoconf/vxworks/sed.general
+++ b/erts/autoconf/vxworks/sed.general
@@ -1,7 +1,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -55,12 +55,16 @@ s|@ETHR_LIBS@||
 s|@ETHR_LIB_NAME@||
 s|@ETHR_DEFS@||
 s|@ETHR_THR_LIB_BASE@||
+s|@ETHR_THR_LIB_BASE_DIR@||
 s|@EMU_THR_DEFS@||
 s|@EMU_THR_LIBS@||
 s|@EMU_THR_LIB_NAME@|ethread|
 s|@ERTS_ENABLE_KERNEL_POLL@|no|
+s|@ERTS_INTERNAL_X_LIBS@||
 s|@cc_root@|/clearcase/otp/|
 # Define VxWorks even though cross-compiling. 
+s|@CROSS_COMPILING|yes|
+
 s|@HCFLAGS@|-DVXWORKS|
 s|@HCLIBS@||
 s|@ENABLE_ALLOC_TYPE_VARS@||
diff --git a/erts/autoconf/win32.config.cache b/erts/autoconf/win32.config.cache.static
index 31dfe510cd..d25b1df9d9 100755
--- a/erts/autoconf/win32.config.cache
+++ b/erts/autoconf/win32.config.cache.static
@@ -61,7 +61,6 @@ ac_cv_func_fork=${ac_cv_func_fork=no}
 ac_cv_func_fork_works=${ac_cv_func_fork_works=no}
 ac_cv_func_fpsetmask=${ac_cv_func_fpsetmask=no}
 ac_cv_func_fstat=${ac_cv_func_fstat=yes}
-ac_cv_func_getaddrinfo=${ac_cv_func_getaddrinfo=no}
 ac_cv_func_gethostbyaddr=${ac_cv_func_gethostbyaddr=no}
 ac_cv_func_gethostbyaddr_r=${ac_cv_func_gethostbyaddr_r=no}
 ac_cv_func_gethostbyname=${ac_cv_func_gethostbyname=no}
@@ -71,7 +70,6 @@ ac_cv_func_gethostname=${ac_cv_func_gethostname=no}
 ac_cv_func_gethrtime=${ac_cv_func_gethrtime=no}
 ac_cv_func_getipnodebyaddr=${ac_cv_func_getipnodebyaddr=no}
 ac_cv_func_getipnodebyname=${ac_cv_func_getipnodebyname=no}
-ac_cv_func_getnameinfo=${ac_cv_func_getnameinfo=no}
 ac_cv_func_getpagesize=${ac_cv_func_getpagesize=no}
 ac_cv_func_gettimeofday=${ac_cv_func_gettimeofday=no}
 ac_cv_func_gmtime_r=${ac_cv_func_gmtime_r=no}
@@ -212,7 +210,6 @@ ac_cv_sizeof_void_p=${ac_cv_sizeof_void_p=4}
 ac_cv_struct_exception=${ac_cv_struct_exception=no}
 ac_cv_struct_sockaddr_sa_len=${ac_cv_struct_sockaddr_sa_len=no}
 ac_cv_struct_tm=${ac_cv_struct_tm=time.h}
-ac_cv_sys_ipv6_support=${ac_cv_sys_ipv6_support=no}
 ac_cv_sys_multicast_support=${ac_cv_sys_multicast_support=no}
 ac_cv_type_char=${ac_cv_type_char=yes}
 ac_cv_type_int=${ac_cv_type_int=yes}
diff --git a/erts/configure.in b/erts/configure.in
index 5fa1245b13..fac07f8b6a 100644
--- a/erts/configure.in
+++ b/erts/configure.in
@@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. -*-m4-*-
 
 dnl %CopyrightBegin%
 dnl
-dnl Copyright Ericsson AB 1997-2010. All Rights Reserved.
+dnl Copyright Ericsson AB 1997-2011. All Rights Reserved.
 dnl
 dnl The contents of this file are subject to the Erlang Public License,
 dnl Version 1.1, (the "License"); you may not use this file except in
@@ -61,6 +61,9 @@ if test x"${ERL_TOP}/erts" != x"$srcdir"; then
 fi
 erl_top=${ERL_TOP}
 
+# Remove old configuration information
+/bin/rm -f "$ERL_TOP/erts/CONF_INFO"
+
 # echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 # echo X
 # echo "X srcdir  = $srcdir"
@@ -87,6 +90,13 @@ else
     host_os=$host
 fi
 
+if test "$cross_compiling" = "yes"; then
+    CROSS_COMPILING=yes
+else
+    CROSS_COMPILING=no
+fi
+AC_SUBST(CROSS_COMPILING)
+
 ERL_XCOMP_SYSROOT_INIT
 
 AC_ISC_POSIX
@@ -100,7 +110,8 @@ ENABLE_ALLOC_TYPE_VARS=
 AC_SUBST(ENABLE_ALLOC_TYPE_VARS)
 
 AC_ARG_ENABLE(bootstrap-only,
-[  --enable-bootstrap-only enable bootstrap only configuration],
+AS_HELP_STRING([--enable-bootstrap-only],
+               [enable bootstrap only configuration]),
 [ if test "X$enableval" = "Xyes"; then
 	# Disable stuff not necessary in a bootstrap only system in order
 	# to speed up things by reducing the amount of stuff needing to be
@@ -116,38 +127,46 @@ AC_ARG_ENABLE(bootstrap-only,
 ])
 
 AC_ARG_ENABLE(threads,
-[  --enable-threads        enable async thread support
-  --disable-threads       disable async thread support],
+AS_HELP_STRING([--enable-threads], [enable async thread support])
+AS_HELP_STRING([--disable-threads], [disable async thread support]),
 [ case "$enableval" in
     no) enable_threads=no ;;
     *)  enable_threads=yes ;;
   esac ], enable_threads=unknown)
 
+AC_ARG_ENABLE(halfword-emulator,
+AS_HELP_STRING([--enable-halfword-emulator],
+               [enable halfword emulator (only for 64bit builds)]),
+[ case "$enableval" in
+    no) enable_halfword_emualtor=no ;;
+    *)  enable_halfword_emulator=yes ;;
+  esac ], enable_halfword_emulator=unknown)
+
 AC_ARG_ENABLE(smp-support,
-[  --enable-smp-support    enable smp support
-  --disable-smp-support   disable smp support],
+AS_HELP_STRING([--enable-smp-support], [enable smp support])
+AS_HELP_STRING([--disable-smp-support], [disable smp support]),
 [ case "$enableval" in
     no) enable_smp_support=no ;;
     *)  enable_smp_support=yes ;;
   esac ], enable_smp_support=unknown)
 
 AC_ARG_WITH(termcap,
-[  --with-termcap          use termcap (default)
-  --without-termcap       do not use any termcap libraries (ncurses,curses,termcap,termlib)],
+AS_HELP_STRING([--with-termcap], [use termcap (default)])
+AS_HELP_STRING([--without-termcap],
+               [do not use any termcap libraries (ncurses,curses,termcap,termlib)]),
 [],
 [with_termcap=yes])
 
 
 AC_ARG_ENABLE(hybrid-heap,
-[  --enable-hybrid-heap    enable hybrid heap
-  --disable-hybrid-heap   disable hybrid heap],
+AS_HELP_STRING([--enable-hybrid-heap], [enable hybrid heap]),
 [ case "$enableval" in
     no) enable_hybrid_heap=no ;;
     *)  enable_hybrid_heap=yes ;;
   esac ], enable_hybrid_heap=unknown)
 
 AC_ARG_ENABLE(lock-checking,
-[  --enable-lock-checking  enable lock checking],
+AS_HELP_STRING([--enable-lock-checking], [enable lock checking]),
 [ case "$enableval" in
     no) enable_lock_check=no ;;
     *)  enable_lock_check=yes ;;
@@ -156,16 +175,15 @@ AC_ARG_ENABLE(lock-checking,
   enable_lock_check=no)
 
 AC_ARG_ENABLE(lock-counter,
-[  --enable-lock-counter    enable lock counters
-  --disable-lock-counter   disable lock counters],
+AS_HELP_STRING([--enable-lock-counter], [enable lock counters]),
 [ case "$enableval" in
     no) enable_lock_count=no ;;
     *)  enable_lock_count=yes ;;
   esac ], enable_lock_count=no)
 
 AC_ARG_ENABLE(kernel-poll,
-[  --enable-kernel-poll    enable kernel poll support
-  --disable-kernel-poll   disable kernel poll support],
+AS_HELP_STRING([--enable-kernel-poll], [enable kernel poll support])
+AS_HELP_STRING([--disable-kernel-poll], [disable kernel poll support]),
 [ case "$enableval" in
     no) enable_kernel_poll=no ;;
     *)  enable_kernel_poll=yes ;;
@@ -173,28 +191,27 @@ AC_ARG_ENABLE(kernel-poll,
 
 
 AC_ARG_ENABLE(sctp,
-[  --enable-sctp           enable sctp support
-  --disable-sctp          disable sctp support],
+AS_HELP_STRING([--enable-sctp], [enable sctp support])
+AS_HELP_STRING([--disable-sctp], [disable sctp support]),
 [ case "$enableval" in
     no) enable_sctp=no ;;
     *)  enable_sctp=yes ;;
   esac ], enable_sctp=unknown)
 
 AC_ARG_ENABLE(hipe,
-[  --enable-hipe           enable hipe support
-  --disable-hipe          disable hipe support])
+AS_HELP_STRING([--enable-hipe], [enable hipe support])
+AS_HELP_STRING([--disable-hipe], [disable hipe support]))
 
 AC_ARG_ENABLE(native-libs,
-[  --enable-native-libs    compile Erlang libraries to native code])
+AS_HELP_STRING([--enable-native-libs],
+               [compile Erlang libraries to native code]))
 
 AC_ARG_ENABLE(tsp,
-[  --enable-tsp            compile tsp app])
-
-AC_ARG_ENABLE(elib-malloc,
-[  --enable-elib-malloc    use elib_malloc instead of normal malloc])
+AS_HELP_STRING([--enable-tsp], [compile tsp app]))
 
 AC_ARG_ENABLE(fp-exceptions,
-[  --enable-fp-exceptions    Use hardware floating point exceptions (default if hipe enabled)],
+AS_HELP_STRING([--enable-fp-exceptions],
+               [use hardware floating point exceptions (default if hipe enabled)]),
 [ case "$enableval" in
     no) enable_fp_exceptions=no ;;
     *)  enable_fp_exceptions=yes ;;
@@ -202,7 +219,8 @@ AC_ARG_ENABLE(fp-exceptions,
 ],enable_fp_exceptions=auto)
 
 AC_ARG_ENABLE(darwin-universal,
-[  --enable-darwin-universal    build universal binaries on darwin i386],
+AS_HELP_STRING([--enable-darwin-universal],
+               [build universal binaries on darwin i386]),
 [ case "$enableval" in
     no) enable_darwin_universal=no ;;
     *)  enable_darwin_univeral=yes ;;
@@ -211,7 +229,7 @@ AC_ARG_ENABLE(darwin-universal,
 
 
 AC_ARG_ENABLE(darwin-64bit,
-[  --enable-darwin-64bit    build 64bit binaries on darwin],
+AS_HELP_STRING([--enable-darwin-64bit], [build 64bit binaries on darwin]),
 [ case "$enableval" in
     no) enable_darwin_64bit=no ;;
     *)  enable_darwin_64bit=yes ;;
@@ -219,7 +237,8 @@ AC_ARG_ENABLE(darwin-64bit,
 ],enable_darwin_64bit=no)
 
 AC_ARG_ENABLE(m64-build,
-[  --enable-m64-build    build 64bit binaries using the -m64 flag to (g)cc],
+AS_HELP_STRING([--enable-m64-build],
+               [build 64bit binaries using the -m64 flag to (g)cc]),
 [ case "$enableval" in
     no) enable_m64_build=no ;;
     *)  enable_m64_build=yes ;;
@@ -227,7 +246,8 @@ AC_ARG_ENABLE(m64-build,
 ],enable_m64_build=no)
 
 AC_ARG_ENABLE(m32-build,
-[  --enable-m32-build    build 32bit binaries using the -m32 flag to (g)cc],
+AS_HELP_STRING([--enable-m32-build],
+               [build 32bit binaries using the -m32 flag to (g)cc]),
 [ case "$enableval" in
     no) enable_m32_build=no ;;
     *)
@@ -240,7 +260,7 @@ AC_ARG_ENABLE(m32-build,
 ],enable_m32_build=no)
 
 AC_ARG_ENABLE(fixalloc,
-[  --disable-fixalloc      disable the use of fix_alloc])
+AS_HELP_STRING([--disable-fixalloc], [disable the use of fix_alloc]))
 if test x${enable_fixalloc} = xno ; then
   AC_DEFINE(NO_FIX_ALLOC,[],
 	    [Define if you don't want the fix allocator in Erlang])
@@ -248,8 +268,9 @@ fi
 
 AC_SUBST(PERFCTR_PATH)
 AC_ARG_WITH(perfctr,
-[  --with-perfctr=PATH     specify location of perfctr include and lib
-  --without-perfctr       don't use perfctr (default)])
+AS_HELP_STRING([--with-perfctr=PATH],
+               [specify location of perfctr include and lib])
+AS_HELP_STRING([--without-perfctr], [don't use perfctr (default)]))
 
 if test "x$with_perfctr" = "xno" -o "x$with_perfctr" = "x" ; then
     PERFCTR_PATH=
@@ -263,26 +284,13 @@ else
 fi
 
 AC_ARG_ENABLE(clock-gettime,
-[  --enable-clock-gettime  Use clock-gettime for time correction],
+AS_HELP_STRING([--enable-clock-gettime],
+               [use clock-gettime for time correction]),
 [ case "$enableval" in
     no) clock_gettime_correction=no ;;
     *) clock_gettime_correction=yes ;;
   esac ], clock_gettime_correction=unknown)
 
-AC_ARG_ENABLE(native-ethr-impls,
-[  --enable-native-ethr-impls enable native ethread implementations
-  --disable-native-ethr-impls disable native ethread implementations],
-[ case "$enableval" in
-    no) disable_native_ethr_impls=yes ;;
-    *)  disable_native_ethr_impls=no ;;
-  esac ], disable_native_ethr_impls=no)
-
-dnl Defined in libraries/megaco/configure.in but we need it here
-dnl also in order to show it to the "top user"
-
-AC_ARG_ENABLE(megaco_flex_scanner_lineno,
-[  --disable-megaco-flex-scanner-lineno disable megaco flex scanner lineno])
-
 dnl Magic test for clearcase.
 OTP_RELEASE=
 if test "${ERLANG_COMMERCIAL_BUILD}" != ""; then
@@ -572,6 +580,11 @@ AC_SUBST(WFLAGS)
 AC_SUBST(CFLAG_RUNTIME_LIBRARY_PATH)
 
 AC_CHECK_SIZEOF(void *) # Needed for ARCH and smp checks below
+if test "x$ac_cv_sizeof_void_p" = x8; then
+  AC_SUBST(EXTERNAL_WORD_SIZE, 64)
+else
+  AC_SUBST(EXTERNAL_WORD_SIZE, 32)
+fi
 
 dnl
 dnl Figure out operating system and cpu architecture
@@ -749,6 +762,26 @@ esac
 	
 AC_SUBST(LIBCARBON)
 
+dnl Check if we should/can build a halfword emulator
+
+AC_MSG_CHECKING(if we are building a halfword emulator (32bit heap on 64bit machine))
+if test "$enable_halfword_emulator" = "yes"; then
+	if test "$ARCH" = "amd64"; then
+		AC_DEFINE(HALFWORD_HEAP_EMULATOR, [1],
+			  [Define if building a halfword-heap 64bit emulator])
+		ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS halfword"
+		AC_MSG_RESULT([yes])
+	else
+		AC_MSG_ERROR(no; halfword emulator not supported on this architecture)
+	fi
+else
+	AC_MSG_RESULT([no])
+fi
+
+
+
+
+
 dnl some tests below will call this if we haven't already - and autoconf
 dnl can't handle those tests being done conditionally at runtime
 AC_PROG_CPP
@@ -794,13 +827,14 @@ fi
 AC_CHECK_PROGS(XSLTPROC, xsltproc)
 if test -z "$XSLTPROC"; then
   echo "xsltproc" >> doc/CONF_INFO
-  AC_MSG_WARN([No 'xsltproc' command found: the documentation can not be built])
+  AC_MSG_WARN([No 'xsltproc' command found: the documentation cannot be built])
 fi
 
 AC_CHECK_PROGS(FOP, fop)
 if test -z "$FOP"; then
+  FOP="$ERL_TOP/make/fakefop"
   echo "fop" >> doc/CONF_INFO
-  AC_MSG_WARN([No 'fop' command found: the documentation can not be built])
+  AC_MSG_WARN([No 'fop' command found: going to generate placeholder PDF files])
 fi
 
 dnl
@@ -1021,11 +1055,48 @@ if test $ERTS_BUILD_SMP_EMU = yes; then
 	fi
 
 	AC_DEFINE(ERTS_HAVE_SMP_EMU, 1, [Define if the smp emulator is built])
+
+	case "$ethr_have_native_atomics-$ethr_have_native_spinlock" in
+	  yes-*)
+		;;
+
+	  no-yes)
+
+		test -f "$ERL_TOP/erts/CONF_INFO" ||
+		    echo "" > "$ERL_TOP/erts/CONF_INFO"
+		cat >> $ERL_TOP/erts/CONF_INFO <<EOF
+
+                 No native atomic implementation available.
+                 Fallbacks implemented using spinlocks will be
+                 used. Note that the performance of the SMP
+                 runtime system will suffer due to this.
+
+EOF
+		;;
+
+	  no-no)
+
+		test -f "$ERL_TOP/erts/CONF_INFO" ||
+		    echo "" > "$ERL_TOP/erts/CONF_INFO"
+		cat >> "$ERL_TOP/erts/CONF_INFO" <<EOF
+
+                 No native atomic implementation, nor no native
+                 spinlock implementation available. Fallbacks
+                 implemented using mutexes will be used. Note
+                 that the performance of the SMP runtime system
+                 will suffer much due to this.
+
+EOF
+		;;
+
+	esac
+
 	enable_threads=force
 fi
 
 AC_SUBST(ERTS_BUILD_SMP_EMU)
 
+AC_CHECK_FUNCS([posix_fadvise])
 
 
 #
@@ -1120,17 +1191,15 @@ else
 		enable_child_waiter_thread=yes
 		;;
 	    linux*)
-		AC_DEFINE(USE_RECURSIVE_MALLOC_MUTEX,[1],
-			[Define if malloc should use a recursive mutex])
 		AC_MSG_CHECKING([whether dlopen() needs to be called before first call to dlerror()])
-		if test "x$ETHR_THR_LIB_BASE_NAME" != "xnptl"; then
+		if test "x$ETHR_THR_LIB_BASE_TYPE" != "xposix_nptl"; then
 		    AC_DEFINE(ERTS_NEED_DLOPEN_BEFORE_DLERROR,[1],
 			[Define if dlopen() needs to be called before first call to dlerror()])
 			AC_MSG_RESULT(yes)
 		else
 			AC_MSG_RESULT(no)
 		fi
-		if test "x$ETHR_THR_LIB_BASE_NAME" != "xnptl"; then
+		if test "x$ETHR_THR_LIB_BASE_TYPE" != "xposix_nptl"; then
 		    # Child waiter thread cannot be enabled
 		    disable_child_waiter_thread=yes
 		    enable_child_waiter_thread=no
@@ -1192,13 +1261,7 @@ fi
 
 AC_SUBST(EMU_LOCK_CHECKING)
 
-ERTS_INTERNAL_X_LIBS=
-
-AC_CHECK_LIB(kstat, kstat_open,
-[AC_DEFINE(HAVE_KSTAT, 1, [Define if you have kstat])
-ERTS_INTERNAL_X_LIBS="$ERTS_INTERNAL_X_LIBS -lkstat"])
-
-AC_SUBST(ERTS_INTERNAL_X_LIBS)
+ERL_INTERNAL_LIBS
 
 dnl THR_LIBS and THR_DEFS are only used by odbc
 THR_LIBS=$ETHR_X_LIBS
@@ -1243,8 +1306,7 @@ dnl zlib
 dnl -------------
 
 AC_ARG_ENABLE(shared-zlib,
-[  --enable-shared-zlib        enable using shared zlib library
-  --disable-shared-zlib       disable shared zlib, compile own zlib source (default)],
+AS_HELP_STRING([--enable-shared-zlib], [enable using shared zlib library]),
 [ case "$enableval" in
     no) enable_shared_zlib=no ;;
     *)  enable_shared_zlib=yes ;;
@@ -1423,7 +1485,7 @@ AC_CHECK_HEADERS(fcntl.h limits.h unistd.h syslog.h dlfcn.h ieeefp.h \
                  sys/ioctl.h sys/time.h sys/uio.h \
                  sys/socket.h sys/sockio.h sys/socketio.h \
                  net/errno.h malloc.h mach-o/dyld.h arpa/nameser.h \
-		 pty.h util.h utmp.h langinfo.h poll.h)
+		 pty.h util.h utmp.h langinfo.h poll.h sdkddkver.h)
 
 AC_CHECK_HEADER(sys/resource.h,
 	[AC_DEFINE(HAVE_SYS_RESOURCE_H, 1,
@@ -1623,18 +1685,62 @@ LIBS="$LIBS $EMU_THR_X_LIBS"
 
 dnl Check if we have these, in which case we'll try to build
 dnl inet_gethost with ipv6 support.
-AC_CHECK_FUNC(getaddrinfo, have_getaddrinfo=yes, have_getaddrinfo=no)
+AC_CHECK_HEADERS(windows.h)
+AC_CHECK_HEADERS(winsock2.h)
+AC_CHECK_HEADERS(ws2tcpip.h,[],[],[
+#ifdef HAVE_WINSOCK2_H
+#include <winsock2.h>
+#endif
+#ifdef HAVE_WINDOWS_H
+#include <windows.h>
+#endif
+])
+dnl AC_CHECK_FUNC(getaddrinfo, have_getaddrinfo=yes, have_getaddrinfo=no)
+AC_MSG_CHECKING(for getaddrinfo)
+AC_TRY_LINK([
+#include <stdlib.h>
+#include <string.h>
+#ifdef HAVE_WINSOCK2_H
+#include <winsock2.h>
+#endif
+#ifdef HAVE_WINDOWS_H
+#include <windows.h>
+#endif
+#ifdef HAVE_WS2TCPIP_H
+#include <ws2tcpip.h>
+#endif
+#ifndef __WIN32__
+#include <sys/socket.h>
+#include <netdb.h>
+#endif
+], 
+[
+getaddrinfo("","",NULL,NULL);
+],have_getaddrinfo=yes, have_getaddrinfo=no)
 if test $have_getaddrinfo = yes; then
+        AC_MSG_RESULT([yes])
         AC_MSG_CHECKING([whether getaddrinfo accepts enough flags])
-        AC_TRY_RUN([
+        AC_TRY_COMPILE([
 #include <stdlib.h>
 #include <string.h>
+#ifdef HAVE_WINSOCK2_H
+#include <winsock2.h>
+#endif
+#ifdef HAVE_WINDOWS_H
+#include <windows.h>
+#endif
+#ifdef HAVE_WS2TCPIP_H
+#include <ws2tcpip.h>
+#endif
+#ifndef __WIN32__
 #include <sys/socket.h>
 #include <netdb.h>
-int main(int argc, char **argv) {
+#endif
+],
+[
     struct addrinfo hints, *ai;
     memset(&hints, 0, sizeof(hints));
-    hints.ai_flags = (AI_CANONNAME|AI_V4MAPPED|AI_ADDRCONFIG);
+    hints.ai_flags = AI_CANONNAME;
     hints.ai_socktype = SOCK_STREAM;
     hints.ai_family = AF_INET6;
     if (getaddrinfo("::", NULL, &hints, &ai) == 0) {
@@ -1643,32 +1749,57 @@ int main(int argc, char **argv) {
     } else {
 	exit(1);
     }
-}
-	],, have_getaddrinfo=no,
-	[
-	case X$erl_xcomp_getaddrinfo in
-	    X) have_getaddrinfo=cross;;
-	    Xyes|Xno) have_getaddrinfo=$erl_xcomp_getaddrinfo;;
-	    *) AC_MSG_ERROR([Bad erl_xcomp_getaddrinfo value: $erl_xcomp_getaddrinfo]);;
-	esac
-	])
+],, have_getaddrinfo=no)
 	AC_MSG_RESULT($have_getaddrinfo)
 	case $have_getaddrinfo in
 	    yes)
 		AC_DEFINE(HAVE_GETADDRINFO, [1], 
 		          [Define to 1 if you have a good `getaddrinfo' function.]);;
-	    cross)
-		AC_MSG_WARN([result no guessed because of cross compilation]);;
 	    *) ;;
 	esac
+else
+    AC_MSG_RESULT([no])
+fi
+AC_MSG_CHECKING(for getnameinfo)
+AC_TRY_LINK([
+#include <stdlib.h>
+#include <string.h>
+#ifdef HAVE_WINSOCK2_H
+#include <winsock2.h>
+#endif
+#ifdef HAVE_WINDOWS_H
+#include <windows.h>
+#endif
+#ifdef HAVE_WS2TCPIP_H
+#include <ws2tcpip.h>
+#endif
+#ifndef __WIN32__
+#include <sys/socket.h>
+#include <netdb.h>
+#endif
+], 
+[
+getnameinfo(NULL,0,NULL,0,NULL,0,0);
+],have_getnameinfo=yes, have_getnameinfo=no)
+if test $have_getnameinfo = yes; then
+        AC_MSG_RESULT([yes])
+	AC_DEFINE(HAVE_GETNAMEINFO, [1], 
+		  [Define to 1 if you have a good `getnameinfo' function.])
+else
+	AC_MSG_RESULT([no])	  
 fi
-AC_CHECK_FUNCS([getnameinfo getipnodebyname getipnodebyaddr gethostbyname2])
+
+
+AC_CHECK_FUNCS([getipnodebyname getipnodebyaddr gethostbyname2])
 
 AC_CHECK_FUNCS([ieee_handler fpsetmask finite isnan isinf res_gethostbyname dlopen \
 		pread pwrite writev memmove strerror strerror_r strncasecmp \
-		gethrtime localtime_r gmtime_r mremap memcpy mallopt \
+		gethrtime localtime_r gmtime_r inet_pton mmap mremap memcpy mallopt \
 		sbrk _sbrk __sbrk brk _brk __brk \
 		flockfile fstat strlcpy strlcat setsid posix2time setlocale nl_langinfo poll])
+
+AC_CHECK_DECLS([posix2time],,,[#include <time.h>])
+
 if test "X$host" = "Xwin32"; then
 	ac_cv_func_setvbuf_reversed=yes
 fi
@@ -1698,7 +1829,6 @@ if test $disable_vfork = true; then
 fi
 
 AC_FUNC_VPRINTF
-AC_FUNC_MMAP
 
 dnl The AC_DEFINEs are necessary for autoheader to work. :-(
 dnl for gzio
@@ -1730,6 +1860,15 @@ fi
 dnl Need by run_erl.
 AC_CHECK_FUNCS([openpty])
 
+dnl fdatasync syscall (Unix only)
+AC_CHECK_FUNCS([fdatasync])
+
+dnl Find which C libraries are required to use fdatasync
+AC_SEARCH_LIBS(fdatasync, [rt])
+
+AC_CHECK_HEADERS(net/if_dl.h ifaddrs.h netpacket/packet.h)
+AC_CHECK_FUNCS([getifaddrs])
+
 dnl ----------------------------------------------------------------------
 dnl Checks for features/quirks in the system that affects Erlang.
 dnl ----------------------------------------------------------------------
@@ -1791,6 +1930,27 @@ if test $processor_bind_functionality = yes; then
 	AC_DEFINE(HAVE_PROCESSOR_BIND, 1, [Define if you have processor_bind functionality])
 fi
 
+AC_MSG_CHECKING([for cpuset_getaffinity/cpuset_setaffinity])
+AC_TRY_COMPILE([
+#include <sys/param.h>
+#include <sys/cpuset.h>
+],
+[
+	int res;
+	cpuset_t cpuset;
+	CPU_ZERO(&cpuset);
+	CPU_SET(1, &cpuset);
+	res = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(cpuset_t), &cpuset);
+	res = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(cpuset_t), &cpuset);
+	res = CPU_ISSET(1, &cpuset);
+	CPU_CLR(1, &cpuset);
+],
+		cpuset_xetaffinity=yes,
+		cpuset_xetaffinity=no)
+AC_MSG_RESULT([$cpuset_xetaffinity])
+if test $cpuset_xetaffinity = yes; then
+	AC_DEFINE(HAVE_CPUSET_xETAFFINITY, 1, [Define if you have cpuset_getaffinity/cpuset_setaffinity])
+fi
 
 AC_CACHE_CHECK([for 'end' symbol],
 		erts_cv_have_end_symbol,
@@ -2240,9 +2400,15 @@ if test "$cross_compiling" != "yes" && test X${enable_hipe} != Xno; then
   if test -z "$M4"; then
 	enable_hipe=no
    	AC_MSG_NOTICE([HiPE disabled as no valid m4 is found in PATH])
+  elif test "$enable_halfword_emulator" = "yes"; then
+       if test X${enable_hipe} = Xyes; then
+              AC_MSG_ERROR([HiPE can not be combined with halfword emulator (yet)])
+       else		    
+   	      AC_MSG_NOTICE([HiPE auto-disabled on halfword emulator])
+       fi	
   else
   	case "$ARCH-$OPSYS" in
-	    x86-linux|amd64-linux|x86-darwin*|amd64-darwin*|ppc-linux|ppc-darwin|arm-linux|amd64-freebsd|x86-freebsd|x86-sol2|amd64-sol2|ultrasparc-linux)
+	    x86-linux|amd64-linux|x86-darwin*|amd64-darwin*|ppc-linux|ppc64-linux|ppc-darwin|arm-linux|amd64-freebsd|x86-freebsd|x86-sol2|amd64-sol2|ultrasparc-linux)
       		enable_hipe=yes
       		;;	
   	esac
@@ -2615,7 +2781,7 @@ static __inline__ int check_fpe(double f)
  * Implement SIGFPE handler based on CPU/OS combination
  */
 
-#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__i386__) || defined(__x86_64__))) || (defined(__OpenBSD__) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__))
+#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__i386__) || defined(__x86_64__))) || ((defined(__OpenBSD__) || defined(__NetBSD__)) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__))
 
 #if defined(__linux__) && defined(__i386__)
 #if !defined(X86_FXSR_MAGIC)
@@ -2729,6 +2895,11 @@ static void fpe_sig_action(int sig, siginfo_t *si, void *puc)
     struct fxsave64 *fxsave = uc->sc_fpstate;
     fxsave->fx_mxcsr = 0x1F80;
     fxsave->fx_fsw &= ~0xFF;
+#elif defined(__NetBSD__) && defined(__x86_64__)
+    mcontext_t *mc = &uc->uc_mcontext;
+    struct fxsave64 *fxsave = (struct fxsave64 *)&mc->__fpregs;
+    fxsave->fx_mxcsr = 0x1F80;
+    fxsave->fx_fsw &= ~0xFF;
 #elif defined(__sun__) && defined(__x86_64__)
     mcontext_t *mc = &uc->uc_mcontext;
     struct fpchip_state *fpstate = &mc->fpregs.fp_reg_set.fpchip_state;
@@ -2883,7 +3054,7 @@ case $enable_hybrid_heap-$host_os in
     no-*)
 	AC_MSG_RESULT([no; disabled by user])
 	ERTS_BUILD_HYBRID_EMU=no;;
-    *-win32|*-vxworks|*-ose) # vxworks and ose have their own "configure scripts"...
+    *-win32|*-vxworks) # vxworks have their own "configure scripts"...
 	AC_MSG_RESULT([no; default on this platform])
 	ERTS_BUILD_HYBRID_EMU=no;;
     *)
@@ -2946,17 +3117,6 @@ if test "x$HIPE_ENABLED" = "xyes" ; then
 fi
 
 #
-# Check if we should use elib_malloc.
-#
-
-if test X${enable_elib_malloc} = Xyes; then
- AC_DEFINE(ENABLE_ELIB_MALLOC,[],[Define to enable use of elib_malloc (a malloc() replacement)])
- AC_DEFINE(ELIB_HEAP_SBRK,[],[Elib sbrk])
- AC_DEFINE(ELIB_ALLOC_IS_CLIB,[],[Use elib malloc as clib])
- AC_DEFINE(ELIB_SORTED_BLOCKS,[],[Define to enable the use of sorted blocks when using elib_malloc])
-fi
-
-#
 # Check for working poll().
 #
 AC_MSG_CHECKING([for working poll()])
@@ -3364,6 +3524,8 @@ dnl use "PATH/include" and "PATH/lib".
 AC_SUBST(SSL_INCLUDE)
 AC_SUBST(SSL_ROOT)
 AC_SUBST(SSL_LIBDIR)
+AC_SUBST(SSL_CRYPTO_LIBNAME)
+AC_SUBST(SSL_SSL_LIBNAME)
 AC_SUBST(SSL_CC_RUNTIME_LIBRARY_PATH)
 AC_SUBST(SSL_LD_RUNTIME_LIBRARY_PATH)
 AC_SUBST(SSL_DED_LD_RUNTIME_LIBRARY_PATH)
@@ -3372,14 +3534,16 @@ AC_SUBST(SSL_LINK_WITH_KERBEROS)
 AC_SUBST(STATIC_KERBEROS_LIBS)
 AC_SUBST(SSL_LINK_WITH_ZLIB)
 AC_SUBST(STATIC_ZLIB_LIBS)
-AC_SUBST(OPENSSL_CMD)
 
 std_ssl_locations="/usr/local /usr/sfw /opt/local /usr /usr/pkg /usr/local/openssl /usr/lib/openssl /usr/openssl /usr/local/ssl /usr/lib/ssl /usr/ssl"
 
 AC_ARG_WITH(ssl-zlib,
-[  --with-ssl-zlib=PATH    specify location of ZLib to be used by OpenSSL
-  --with-ssl-zlib         link SSL with  Zlib (default if found)
-  --without-ssl-zlib      don't link SSL with ZLib])
+AS_HELP_STRING([--with-ssl-zlib=PATH],
+               [specify location of ZLib to be used by OpenSSL])
+AS_HELP_STRING([--with-ssl-zlib],
+               [link SSL with  Zlib (default if found)])
+AS_HELP_STRING([--without-ssl-zlib],
+               [don't link SSL with ZLib]))
 
 
 if  test "x$with_ssl_zlib" = "xno"; then
@@ -3448,13 +3612,13 @@ fi
 		
 			
 AC_ARG_WITH(ssl,
-[  --with-ssl=PATH         specify location of OpenSSL include and lib
-  --with-ssl              use SSL (default)
-  --without-ssl           don't use SSL])
+AS_HELP_STRING([--with-ssl=PATH], [specify location of OpenSSL include and lib])
+AS_HELP_STRING([--with-ssl], [use SSL (default)])
+AS_HELP_STRING([--without-ssl], [don't use SSL]))
 
 AC_ARG_ENABLE(dynamic-ssl-lib,
-[  --enable-dynamic-ssl-lib        enable using dynamic openssl libraries
-  --disable-dynamic-ssl-lib       disable using dynamic openssl libraries],
+AS_HELP_STRING([--disable-dynamic-ssl-lib],
+               [disable using dynamic openssl libraries]),
 [ case "$enableval" in
     no) enable_dynamic_ssl=no ;;
     *)  enable_dynamic_ssl=yes ;;
@@ -3505,6 +3669,12 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in
 		urp="uninstall/openssl_is1/inno setup: app path"
 		rp="$wrp$urp"
 		if regtool -q get "$rp" > /dev/null; then
+		   true
+		else
+		   urp="uninstall/openssl (32-bit)_is1/inno setup: app path"
+		   rp="$wrp$urp"
+		fi	
+		if regtool -q get "$rp" > /dev/null; then
 			ssl_install_dir=`regtool -q get "$rp"`
 			# Try hard to get rid of spaces...
 			if cygpath -d "$ssl_install_dir" > /dev/null 2>&1; then
@@ -3519,19 +3689,41 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in
     CRYPTO_APP=crypto
     SSH_APP=ssh
 
-    AC_MSG_CHECKING(for OpenSSL >= 0.9.7 in standard locations) 
+    SSL_CRYPTO_LIBNAME=crypto
+    SSL_SSL_LIBNAME=ssl
+
+    AC_MSG_CHECKING(for OpenSSL >= 0.9.7 in standard locations)
     for rdir in $extra_dir /cygdrive/c/OpenSSL $std_ssl_locations; do
 	dir="$erl_xcomp_sysroot$rdir"
 	if test -f "$erl_xcomp_isysroot$rdir/include/openssl/opensslv.h"; then
 		is_real_ssl=yes
 		SSL_ROOT="$dir"
 		if test "x$MIXED_CYGWIN" = "xyes" ; then
-			if test -f "$dir/lib/VC/ssleay32.lib" || \
-			   test -f "$dir/lib/VC/openssl.lib"; then 
+			if test -f "$dir/lib/VC/libeay32.lib"; then
 				SSL_RUNTIME_LIBDIR="$rdir/lib/VC"
 				SSL_LIBDIR="$dir/lib/VC"
-			elif test -f "$dir/lib/ssleay32.lib" || \
-			     test -f "$dir/lib/openssl.lib"; then
+				SSL_CRYPTO_LIBNAME=libeay32
+				SSL_SSL_LIBNAME=ssleay32
+			elif test -f "$dir/lib/VC/openssl.lib"; then 
+				SSL_RUNTIME_LIBDIR="$rdir/lib/VC"
+				SSL_LIBDIR="$dir/lib/VC"
+			elif test -f $dir/lib/VC/libeay32MD.lib; then
+		            SSL_CRYPTO_LIBNAME=libeay32MD
+		            SSL_SSL_LIBNAME=ssleay32MD
+			    if test "x$enable_dynamic_ssl" = "xno" && \
+	                       test -f  $dir/lib/VC/static/libeay32MD.lib; then
+				  SSL_RUNTIME_LIBDIR="$rdir/lib/VC/static"
+				  SSL_LIBDIR="$dir/lib/VC/static"
+		            else
+			          SSL_RUNTIME_LIBDIR="$rdir/lib/VC"
+				  SSL_LIBDIR="$dir/lib/VC"
+                            fi 
+			elif test -f "$dir/lib/libeay32.lib"; then
+				SSL_RUNTIME_LIBDIR="$rdir/lib"
+				SSL_LIBDIR="$dir/lib"
+				SSL_CRYPTO_LIBNAME=libeay32
+				SSL_CRYPTO_LIBNAME=ssleay32
+			elif test -f "$dir/lib/openssl.lib"; then
 				SSL_RUNTIME_LIBDIR="$rdir/lib"
 				SSL_LIBDIR="$dir/lib"
 			else
@@ -3564,21 +3756,6 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in
 			SSL_DYNAMIC_ONLY=yes
 		fi
 		SSL_BINDIR="$rdir/bin"
-dnl 		Should one use EXEEXT or ac_exeext?
-		if test -f "$erl_xcomp_sysroot$SSL_BINDIR/openssl$EXEEXT"; then
-		    if test "$cross_compiling" = "yes"; then
-			dnl Cannot test it; hope it is working...
-			OPENSSL_CMD="$SSL_BINDIR/openssl"
-		    else
-			if "$SSL_BINDIR/openssl" version > /dev/null 2>&1; then
-				OPENSSL_CMD="$SSL_BINDIR/openssl"
-			else
-				is_real_ssl=no
-			fi
-		    fi
-		else
-			is_real_ssl=no
-		fi
 		if test "x$is_real_ssl" = "xyes" ; then
 			SSL_INCLUDE="-I$dir/include"
 			old_CPPFLAGS=$CPPFLAGS
@@ -3642,7 +3819,6 @@ dnl 		Should one use EXEEXT or ac_exeext?
 		  SSL_RUNTIME_LIB="/usr/lib"
 		  SSL_LIB="$erl_xcomp_sysroot/usr/lib"
 		  SSL_BINDIR="/usr/sbin"
-		  OPENSSL_CMD="$SSL_BINDIR/openssl"
 		  dnl OpenBSD requires us to link with -L and -l
 		  SSL_DYNAMIC_ONLY="yes" 
 		fi
@@ -3702,20 +3878,37 @@ dnl		so it is - be adoptable
     fi
     ;;
   *)
-    if test "$cross_compiling" = "yes"; then
-	case "$with_ssl" in
-	    "$erl_xcomp_sysroot"*) ;;
-	    *) AC_MSG_ERROR([Invalid path to option --with-ssl=PATH (not a subdirectory to cross system root)]);;
-	esac
-    fi
-
     # Option given with PATH to package
     if test ! -d "$with_ssl" ; then
        AC_MSG_ERROR(Invalid path to option --with-ssl=PATH)
     fi
     SSL_ROOT="$with_ssl"
+    SSL_CRYPTO_LIBNAME=crypto
+    SSL_SSL_LIBNAME=ssl
     if test "x$MIXED_CYGWIN" = "xyes" && test -d "$with_ssl/lib/VC"; then
-	SSL_LIBDIR="$with_ssl/lib/VC"
+	if test -f "$with_ssl/lib/VC/libeay32.lib"; then
+	    SSL_LIBDIR="$with_ssl/lib/VC"
+	    SSL_CRYPTO_LIBNAME=libeay32
+	    SSL_SSL_LIBNAME=ssleay32
+	elif test -f "$with_ssl/lib/VC/openssl.lib"; then 
+	    SSL_LIBDIR="$with_ssl/lib/VC"
+	elif test -f $with_ssl/lib/VC/libeay32MD.lib; then
+	    SSL_CRYPTO_LIBNAME=libeay32MD
+            SSL_SSL_LIBNAME=ssleay32MD
+	    if test "x$enable_dynamic_ssl" = "xno" && \
+	       test -f  $with_ssl/lib/VC/static/libeay32MD.lib; then
+		SSL_LIBDIR="$with_ssl/lib/VC/static"
+	    else
+		SSL_LIBDIR="$with_ssl/lib/VC"
+            fi 
+	elif test -f "$with_ssl/lib/libeay32.lib"; then
+	    SSL_LIBDIR="$with_ssl/lib"
+	    SSL_CRYPTO_LIBNAME=libeay32
+	    SSL_CRYPTO_LIBNAME=ssleay32
+	else
+	    # This probably wont work, but that's what the user said, so...
+	    SSL_LIBDIR="$with_ssl/lib"
+	fi
     elif test "x$ac_cv_sizeof_void_p" = "x8"; then
 	if test -f "$with_ssl/lib64/libcrypto.a"; then
 		SSL_LIBDIR="$with_ssl/lib64"
@@ -3735,7 +3928,6 @@ dnl		so it is - be adoptable
 	SSL_DYNAMIC_ONLY=yes
     fi
     SSL_INCLUDE="-I$with_ssl/include"
-    OPENSSL_CMD="$with_ssl/bin/openssl"
     SSL_APP=ssl
     CRYPTO_APP=crypto
     SSH_APP=ssh
@@ -3941,9 +4133,9 @@ esac
 
 			
 AC_ARG_WITH(javac,
-[  --with-javac=JAVAC      specify Java compiler to use
-  --with-javac            use a Java compiler if found (default)
-  --without-javac         don't use any Java compiler])
+AS_HELP_STRING([--with-javac=JAVAC], [specify Java compiler to use])
+AS_HELP_STRING([--with-javac], [use a Java compiler if found (default)])
+AS_HELP_STRING([--without-javac], [don't use any Java compiler]))
 
 dnl
 dnl Then there are a number of apps which needs a java compiler...
@@ -4132,7 +4324,6 @@ dnl
   ../lib/ic/c_src/$host/Makefile:../lib/ic/c_src/Makefile.in
   ../lib/os_mon/c_src/$host/Makefile:../lib/os_mon/c_src/Makefile.in
   ../lib/ssl/c_src/$host/Makefile:../lib/ssl/c_src/Makefile.in
-  ../lib/ssl/examples/certs/$host/Makefile:../lib/ssl/examples/certs/Makefile.in
   ../lib/crypto/c_src/$host/Makefile:../lib/crypto/c_src/Makefile.in
   ../lib/orber/c_src/$host/Makefile:../lib/orber/c_src/Makefile.in
   ../lib/runtime_tools/c_src/$host/Makefile:../lib/runtime_tools/c_src/Makefile.in
diff --git a/erts/doc/src/Makefile b/erts/doc/src/Makefile
index 3dfefa2001..6578923fe1 100644
--- a/erts/doc/src/Makefile
+++ b/erts/doc/src/Makefile
@@ -1,19 +1,19 @@
 # 
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 1997-2010. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 # 
 include $(ERL_TOP)/make/target.mk
@@ -45,6 +45,7 @@ XML_REF1_FILES = epmd.xml \
 
 XML_REF3_FILES = \
 	driver_entry.xml \
+	erl_nif.xml \
 	erl_set_memory_block.xml \
 	erl_driver.xml \
 	erl_prim_loader.xml \
diff --git a/erts/doc/src/alt_dist.xml b/erts/doc/src/alt_dist.xml
index a929aec97f..36d83a685b 100644
--- a/erts/doc/src/alt_dist.xml
+++ b/erts/doc/src/alt_dist.xml
@@ -434,7 +434,7 @@
 (13)     Word received;            /* Bytes received */
 (14)     struct uds_data *partner; /* The partner in an accept/listen pair */
 (15)     struct uds_data *next;    /* Next structure in list */
-(16)     /* The input buffer and it's data */
+(16)     /* The input buffer and its data */
 (17)     int buffer_size;          /* The allocated size of the input buffer */
 (18)     int buffer_pos;           /* Current position in input buffer */
 (19)     int header_pos;           /* Where the current header is in the 
@@ -825,7 +825,7 @@
         I/O vector itself. One can use this to allocate the binaries
         for the queue "manually" in the driver, but we'll just fill
         the binary array with NULL values (line 7) , which will make
-        the runtime system allocate it's own buffers when we call
+        the runtime system allocate its own buffers when we call
         <c><![CDATA[driver_enqv]]></c> (line 37).</p>
       <p></p>
       <p>The routine builds an I/O vector containing the header bytes
@@ -942,7 +942,7 @@
          between invocations of Erlang nodes with the same name.</item>
       </list>
       <p>The control interface gets a buffer to return its value in,
-        but is free to allocate it's own buffer is the provided one is
+        but is free to allocate its own buffer is the provided one is
         to small. Here is the code for <c><![CDATA[uds_control]]></c>:</p>
       <code type="none"><![CDATA[
 ( 1) static int uds_control(ErlDrvData handle, unsigned int command, 
@@ -1042,7 +1042,7 @@
       <c><![CDATA[net_kernel:start/1]]></c> function, which is useful as it starts
       the distribution on a running system, where tracing/debugging
       can be performed. The <c><![CDATA[net_kernel:start/1]]></c> routine takes a
-      list as it's single argument. The lists first element should be
+      list as its single argument. The lists first element should be
       the node name (without the "@hostname") as an atom, and the second (and
       last) element should be one of the atoms <c><![CDATA[shortnames]]></c> or 
       <c><![CDATA[longnames]]></c>. In the example case <c><![CDATA[shortnames]]></c> is
diff --git a/erts/doc/src/driver.xml b/erts/doc/src/driver.xml
index 12c79aee90..9f246c4a6c 100644
--- a/erts/doc/src/driver.xml
+++ b/erts/doc/src/driver.xml
@@ -4,7 +4,7 @@
 <chapter>
   <header>
     <copyright>
-      <year>2001</year><year>2010</year>
+      <year>2001</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -63,7 +63,8 @@
     <p>This is a simple driver for accessing a postgres
       database using the libpq C client library. Postgres
       is used because it's free and open source. For information
-      on postgres, refer to the website www.postgres.org.</p>
+      on postgres, refer to the website
+      <url href="http://www.postgres.org">www.postgres.org</url>.</p>
     <p>The driver is synchronous, it uses the synchronous calls of
       the client library. This is only for simplicity, and is
       generally not good, since it will
@@ -196,18 +197,21 @@ static ErlDrvData start(ErlDrvPort port, char *command)
     <p>We call disconnect to log out from the database.
       (This should have been done from Erlang, but just in case.)</p>
     <code type="none"><![CDATA[
- static int do_disconnect(our_data_t* data, ei_x_buff* x);
+static int do_disconnect(our_data_t* data, ei_x_buff* x);
 
 static void stop(ErlDrvData drv_data)
 {
-    do_disconnect((our_data_t*)drv_data, NULL);
+    our_data_t* data = (our_data_t*)drv_data;
+
+    do_disconnect(data, NULL);
+    driver_free(data);
 }
     ]]></code>
     <p>We use the binary format only to return data to the emulator;
       input data is a string paramater for <c><![CDATA[connect]]></c> and
       <c><![CDATA[select]]></c>. The returned data consists of Erlang terms.</p>
     <p>The functions <c><![CDATA[get_s]]></c> and <c><![CDATA[ei_x_to_new_binary]]></c> are
-      utilities that is used to make the code shorter. <c><![CDATA[get_s]]></c>
+      utilities that are used to make the code shorter. <c><![CDATA[get_s]]></c>
       duplicates the string and zero-terminates it, since the
       postgres client library wants that. <c><![CDATA[ei_x_to_new_binary]]></c>
       takes an <c><![CDATA[ei_x_buff]]></c> buffer and allocates a binary and
@@ -241,7 +245,7 @@ static int control(ErlDrvData drv_data, unsigned int command, char *buf,
     return r;
 }
     ]]></code>
-    <p>In <c><![CDATA[do_connect]]></c> is where we log in to the database. If the connection
+    <p><c><![CDATA[do_connect]]></c> is where we log in to the database. If the connection
       was successful we store the connection handle in our driver
       data, and return ok. Otherwise, we return the error message
       from postgres, and store <c><![CDATA[NULL]]></c> in the driver data.</p>
@@ -261,7 +265,7 @@ static int do_connect(const char *s, our_data_t* data, ei_x_buff* x)
 }
     ]]></code>
     <p>If we are connected (if the connection handle is not <c><![CDATA[NULL]]></c>),
-      we log out from the database. We need to check if a we should
+      we log out from the database. We need to check if we should
       encode an ok, since we might get here from the <c><![CDATA[stop]]></c>
       function, which doesn't return data to the emulator.</p>
     <code type="none"><![CDATA[
@@ -276,7 +280,7 @@ static int do_disconnect(our_data_t* data, ei_x_buff* x)
     return 0;
 }
     ]]></code>
-    <p>We execute a query and encodes the result. Encoding is done
+    <p>We execute a query and encode the result. Encoding is done
       in another C module, <c><![CDATA[pg_encode.c]]></c> which is also provided
       as sample code.</p>
     <code type="none"><![CDATA[
@@ -288,7 +292,7 @@ static int do_select(const char* s, our_data_t* data, ei_x_buff* x)
     return 0;
 }
     ]]></code>
-    <p>Here we simply checks the result from postgres, and
+    <p>Here we simply check the result from postgres, and
       if it's data we encode it as lists of lists with
       column data. Everything from postgres is C strings,
       so we just use <c><![CDATA[ei_x_encode_string]]></c> to send
@@ -389,7 +393,7 @@ disconnect(Port) ->
 select(Port, Query) ->
     binary_to_term(port_control(Port, ?DRV_SELECT, Query)).
     ]]></code>
-    <p>The api is simple: <c><![CDATA[connect/1]]></c> loads the driver, opens it
+    <p>The API is simple: <c><![CDATA[connect/1]]></c> loads the driver, opens it
       and logs on to the database, returning the Erlang port
       if successful, <c><![CDATA[select/2]]></c> sends a query to the driver,
       and returns the result, <c><![CDATA[disconnect/1]]></c> closes the
@@ -413,8 +417,8 @@ select(Port, Query) ->
     <title>Sample asynchronous driver</title>
     <p>Sometimes database queries can take long time to
       complete, in our <c><![CDATA[pg_sync]]></c> driver, the emulator
-      halts while the driver is doing it's job. This is
-      often not acceptable, since no other Erlang processes
+      halts while the driver is doing its job. This is
+      often not acceptable, since no other Erlang process
       gets a chance to do anything. To improve on our
       postgres driver, we reimplement it using the asynchronous
       calls in LibPQ.</p>
@@ -469,7 +473,7 @@ typedef struct our_data_t {
       whether the driver is waiting for a connection or waiting
       for the result of a query. (This is needed since the entry
       <c><![CDATA[ready_io]]></c> will be called both when connecting and
-      when there is query result.)</p>
+      when there is a query result.)</p>
     <code type="none"><![CDATA[
 static int do_connect(const char *s, our_data_t* data)
 {
@@ -568,7 +572,7 @@ static void ready_io(ErlDrvData drv_data, ErlDrvEvent event)
       connection is successful, or error if it's not. If the
       connection is not yet established, we simply return; <c><![CDATA[ready_io]]></c>
       will be called again.</p>
-    <p>If we have result from a connect, indicated that we have data in
+    <p>If we have a result from a connect, indicated by having data in
       the <c><![CDATA[x]]></c> buffer, we no longer need to select on
       output (<c><![CDATA[ready_output]]></c>), so we remove this by calling
       <c><![CDATA[driver_select]]></c>.</p>
@@ -627,9 +631,9 @@ return_port_data(Port) ->
       message queue. The function <c><![CDATA[return_port_data]]></c> above
       receives data from the port. Since the data is in
       binary format, we use <c><![CDATA[binary_to_term/1]]></c> to convert
-      it to Erlang term. Note that the driver is opened in
-      binary mode, <c><![CDATA[open_port/2]]></c> is called with the option
-      <c><![CDATA[[binary]]]></c>. This means that data sent from the driver
+      it to an Erlang term. Note that the driver is opened in
+      binary mode (<c><![CDATA[open_port/2]]></c> is called with the option
+      <c><![CDATA[[binary]]]></c>). This means that data sent from the driver
       to the emulator is sent as binaries. Without the <c><![CDATA[binary]]></c>
       option, they would have been lists of integers.</p>
   </section>
@@ -643,15 +647,15 @@ return_port_data(Port) ->
       of a list of integers. For large lists (more than 100000
       elements), this will take some time, so we will perform this
       as an asynchronous task.</p>
-    <p>The asynchronous api for drivers are quite complicated. First
+    <p>The asynchronous API for drivers is quite complicated. First
       of all, the work must be prepared. In our example we do this
       in <c><![CDATA[output]]></c>. We could have used <c><![CDATA[control]]></c> just as well,
       but we want some variation in our examples. In our driver, we allocate
-      a structure that contains all needed for the asynchronous task
+      a structure that contains anything that's needed for the asynchronous task
       to do the work. This is done in the main emulator thread.
       Then the asynchronous function is called from a driver thread,
-      separate from the main emulator thread. Note that the driver-
-      functions are not reentrant, so they shouldn't be used.
+      separate from the main emulator thread. Note that the driver-functions
+      are not reentrant, so they shouldn't be used.
       Finally, after the function is completed, the driver callback
       <c><![CDATA[ready_async]]></c> is called from the main emulator thread,
       this is where we return the result to Erlang. (We can't
@@ -689,7 +693,7 @@ static ErlDrvEntry next_perm_driver_entry = {
       be sent later from the <c><![CDATA[ready_async]]></c> call-back.</p>
     <p>The <c><![CDATA[async_data]]></c> will be passed to the <c><![CDATA[do_perm]]></c> function.
       We do not use a <c><![CDATA[async_free]]></c> function (the last argument to
-      <c><![CDATA[driver_async]]></c>, it's only used if the task is cancelled
+      <c><![CDATA[driver_async]]></c>), it's only used if the task is cancelled
       programmatically.</p>
     <code type="none"><![CDATA[
 struct our_async_data {
@@ -740,7 +744,7 @@ static void ready_async(ErlDrvData drv_data, ErlDrvThreadData async_data)
     ErlDrvPort port = reinterpret_cast<ErlDrvPort>(drv_data);
     our_async_data* d = reinterpret_cast<our_async_data*>(async_data);
     int n = d->data.size(), result_n = n*2 + 3;
-    ErlDrvTermData* result = new ErlDrvTermData[result_n], * rp = result;
+    ErlDrvTermData *result = new ErlDrvTermData[result_n], *rp = result;
     for (vector<int>::iterator i = d->data.begin();
          i != d->data.end(); ++i) {
         *rp++ = ERL_DRV_INT;
diff --git a/erts/doc/src/driver_entry.xml b/erts/doc/src/driver_entry.xml
index e71b48bd92..8bdd154cb9 100644
--- a/erts/doc/src/driver_entry.xml
+++ b/erts/doc/src/driver_entry.xml
@@ -4,7 +4,7 @@
 <cref>
   <header>
     <copyright>
-      <year>2001</year><year>2010</year>
+      <year>2001</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -36,7 +36,7 @@
   <description>
     <p>As of erts version 5.5.3 the driver interface has been extended
       (see <seealso marker="driver_entry#extended_marker">extended marker</seealso>).
-      The extended interface introduce
+      The extended interface introduces
       <seealso marker="erl_driver#version_management">version management</seealso>,
       the possibility to pass capability flags
       (see <seealso marker="driver_entry#driver_flags">driver flags</seealso>)
@@ -45,21 +45,21 @@
     <note>
       <p>Old drivers (compiled with an <c>erl_driver.h</c> from an
         earlier erts version than 5.5.3) have to be recompiled
-        (but does not have to use the extended interface).</p>
+        (but do not have to use the extended interface).</p>
     </note>
     <p>The <c>driver_entry</c> structure is a C struct that all erlang
-      drivers defines. It contains entry points for the erlang driver
+      drivers define. It contains entry points for the erlang driver
       that are called by the erlang emulator when erlang code accesses
       the driver.</p>
     <p>      
       <marker id="emulator"></marker>
       The <seealso marker="driver_entry">erl_driver</seealso> driver
-      API functions needs a port handle
+      API functions need a port handle
       that identifies the driver instance (and the port in the
       emulator). This is only passed to the <c>start</c> function, but
       not to the other functions. The <c>start</c> function returns a
       driver-defined handle that is passed to the other functions. A
-      common practice is to have the <c>start</c> function allocating
+      common practice is to have the <c>start</c> function allocate
       some application-defined structure and stash the <c>port</c>
       handle in it, to use it later with the driver API functions.</p>
     <p>The driver call-back functions are called synchronously from the
@@ -121,7 +121,7 @@ typedef struct erl_drv_entry {
                                    the port */
     void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); 
                                 /* called when we have input from one of 
-                                   the driver's handles) */
+                                   the driver's handles */
     void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event);  
                                 /* called when output is possible to one of 
                                    the driver's handles */
@@ -133,7 +133,7 @@ typedef struct erl_drv_entry {
     int (*control)(ErlDrvData drv_data, unsigned int command, char *buf, 
                    int len, char **rbuf, int rlen); 
                                 /* "ioctl" for drivers - invoked by 
-                                   port_control/3) */
+                                   port_control/3 */
     void (*timeout)(ErlDrvData drv_data);        /* Handling of timeout in driver */
     void (*outputv)(ErlDrvData drv_data, ErlIOVec *ev);
                                 /* called when we have output from erlang
@@ -146,7 +146,7 @@ typedef struct erl_drv_entry {
                                    before 'stop' can be called */
     int (*call)(ErlDrvData drv_data, unsigned int command, char *buf, 
                    int len, char **rbuf, int rlen, unsigned int *flags); 
-                                /* Works mostly like 'control', a syncronous
+                                /* Works mostly like 'control', a synchronous
                                    call into the driver. */
     void (*event)(ErlDrvData drv_data, ErlDrvEvent event,
                   ErlDrvEventData event_data);
@@ -172,7 +172,7 @@ typedef struct erl_drv_entry {
           added to the driver list.) The driver should return 0, or if
           the driver can't initialize, -1.</p>
       </item>
-      <tag><marker id="start"/>int (*start)(ErlDrvPort port, char* command)</tag>
+      <tag><marker id="start"/>ErlDrvData (*start)(ErlDrvPort port, char* command)</tag>
       <item>
         <p>This is called when the driver is instantiated, when
           <c>open_port/2</c> is called. The driver should return a
@@ -188,7 +188,9 @@ typedef struct erl_drv_entry {
         <p>This is called when the port is closed, with
           <c>port_close/1</c> or <c>Port ! {self(), close}</c>. Note
           that terminating the port owner process also closes the
-          port.</p>
+          port. If <c>drv_data</c> is a pointer to memory allocated in
+	  <c>start</c>, then <c>stop</c> is the place to deallocate that
+	  memory.</p>
       </item>
       <tag><marker id="output"/>void (*output)(ErlDrvData drv_data, char *buf, int len)</tag>
       <item>
@@ -217,6 +219,10 @@ typedef struct erl_drv_entry {
           completes, write to the pipe (use <c>SetEvent</c> on
           Windows), this will make the emulator call
           <c>ready_input</c> or <c>ready_output</c>.</p>
+          <p>Spurious events may happen. That is, calls to <c>ready_input</c>
+          or <c>ready_output</c> even though no real events are signaled. In
+          reality it should be rare (and OS dependant), but a robust driver
+          must nevertheless be able to handle such cases.</p>
       </item>
       <tag><marker id="driver_name"/>char *driver_name</tag>
       <item>
@@ -233,7 +239,7 @@ typedef struct erl_drv_entry {
       </item>
       <tag>void *handle</tag>
       <item>
-        <p>This field is reserved for the emulators internal use. The
+        <p>This field is reserved for the emulator's internal use. The
           emulator will modify this field; therefore, it is important
           that the <c>driver_entry</c> isn't declared <c>const</c>.</p> 
       </item>
@@ -397,7 +403,7 @@ typedef struct erl_drv_entry {
       <tag>void *handle2</tag>
       <item>
         <p>
-          This field is reserved for the emulators internal use. The
+          This field is reserved for the emulator's internal use. The
           emulator will modify this field; therefore, it is important
           that the <c>driver_entry</c> isn't declared <c>const</c>.
         </p>
diff --git a/erts/doc/src/epmd.xml b/erts/doc/src/epmd.xml
index 796ab3820b..411e627c85 100644
--- a/erts/doc/src/epmd.xml
+++ b/erts/doc/src/epmd.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>1996</year><year>2009</year>
+      <year>1996</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -31,9 +31,23 @@
     <rev>A</rev>
     <file>epmd.xml</file>
   </header>
+
   <com>epmd</com>
-  <comsummary>Erlang Port Mapper Daemon </comsummary>
+  <comsummary>
+  <p>Erlang Port Mapper Daemon</p>
+  <taglist>
+  <tag><c><![CDATA[epmd [-d|-debug] [DbgExtra...] [-port No] [-daemon] [-relaxed_command_check]]]></c></tag>
+  <item>
+  <p>Starts the port mapper daemon</p>
+  </item>
+  <tag><c><![CDATA[epmd [-d|-debug] [-port No] [-names|-kill|-stop Name]]]></c></tag>
+  <item>
+  <p>Communicates with a running port mapper daemon</p>
+  </item>
+  </taglist>
+  </comsummary>
   <description>
+
     <p>This daemon acts as a name server on all hosts  involved  in
       distributed  Erlang computations. When an Erlang node 
       starts, the node has a name and it obtains an address from the host 
@@ -46,51 +60,193 @@
       The job of the <c><![CDATA[epmd]]></c> daemon is to keep track of which
       node name listens on which address. Hence, <c><![CDATA[epmd]]></c> map
       symbolic node names to machine addresses.</p>
-    <p>The daemon is started automatically by the Erlang start-up script.</p>
-    <p>The program <c><![CDATA[epmd]]></c> can also be used for a variety of other 
-      purposes, for example checking the DNS (Domain Name System) 
-      configuration of a host.</p>
+
+      <p>The TCP/IP <c>epmd</c> daemon actually only keeps track of
+      the <c>Name</c> (first) part of an Erlang node name, the <c>Host</c>
+      part (whatever is after the <c><![CDATA[@]]></c> is implicit in the
+      node name where the <c>epmd</c> daemon was actually contacted,
+      as is the IP address where the Erlang node can be
+      reached. Consistent and correct TCP naming services are
+      therefore required for an Erlang network to function
+      correctly.</p>
+
+    <taglist>
+    <tag>Starting the port mapper daemon</tag>
+    <item>
+
+        <p>The daemon is started automatically by the <c>erl</c>
+        command if the node is to be distributed and there is no
+        running instance present. If automatically launched,
+        environment variables has to be used to alter the behavior of
+        the daemon. See the <seealso
+        marker="#environment_variables">Environment
+        variables</seealso> section below.</p>
+
+        <p>If the -daemon argument is not given, the
+        <c><![CDATA[epmd]]></c> runs as a normal program with the
+	controlling terminal of the shell in which it is
+	started. Normally, it should run as a daemon.</p>
+
+	<p>Regular start-up options are described in the
+	<seealso marker="#daemon_flags">Regular options</seealso>
+	section below.</p>
+
+	<p>The <c>DbgExtra</c> options are described in the
+	<seealso marker="#debug_flags">DbgExtra options</seealso>
+        section below.</p>
+
+     </item>
+     <tag>Communicating with a running port mapper daemon</tag>
+     <item>
+
+	<p>Communicating with the running epmd daemon by means of the
+	<c>epmd</c> program is done primarily for debugging
+	purposes.</p>
+
+	<p>The different queries are described in the <seealso
+        marker="#interactive_flags">Interactive options</seealso>
+        section below.</p>
+
+     </item>
+     </taglist>
   </description>
-  <funcs>
-    <func>
-      <name>epmd [-daemon] </name>
-      <fsummary>Start a name server as a daemon</fsummary>
-      <desc>
-        <p>Starts a name server as a daemon. If it has no argument, the
-          <c><![CDATA[epmd]]></c> runs as a normal program with the controlling terminal 
-          of the shell in which it is started. Normally, it should run as a 
-          daemon.</p>
-      </desc>
-    </func>
-    <func>
-      <name>epmd -names</name>
-      <fsummary>Request the names of the registered Erlang nodes on  this host</fsummary>
-      <desc>
-        <p>Requests the names of the local Erlang nodes <c><![CDATA[epmd]]></c> has 
-          registered.</p>
-      </desc>
-    </func>
-    <func>
-      <name>epmd -kill</name>
-      <fsummary>Kill the <c><![CDATA[epmd]]></c>process</fsummary>
-      <desc>
-        <p>Kills the <c><![CDATA[epmd]]></c> process.</p>
-      </desc>
-    </func>
-    <func>
-      <name>epmd -help</name>
-      <fsummary>List options</fsummary>
-      <desc>
-        <p>Write short info about the usage including some debugging
-          options not listed here.</p>
-      </desc>
-    </func>
-  </funcs>
+  <section>
+    <marker id="daemon_flags"></marker>
+    <title>Regular options</title>
+
+    <p>These options are available when starting the actual name server. The name server is normally started automatically by the <c>erl</c> command (if not already available), but it can also be started at i.e. system start-up.</p>
+    <taglist>
+    <tag><c><![CDATA[-address List]]></c></tag>
+    <item>
+    <p>Let this instance of <c>epmd</c> listen only on the
+      comma-separated list of IP addresses and on the loopback address
+      (which is implicitely added to the list if it has not been
+      specified). This can also be set using the
+      <c><![CDATA[ERL_EPMD_ADDRESS]]></c> environment variable, see the
+      section <seealso marker="#environment_variables">Environment
+      variables</seealso> below.</p>
+    </item>
+    <tag><c><![CDATA[-port No]]></c></tag>
+    <item>
+      <p>Let this instance of epmd listen to another TCP port than
+      default 4369. This can also be set using the
+      <c><![CDATA[ERL_EPMD_PORT]]></c> environment variable, see the
+      section <seealso marker="#environment_variables">Environment
+      variables</seealso> below</p>
+    </item>
+    <tag><c><![CDATA[-d | -debug]]></c></tag>
+    <item>
+
+      <p>Enable debug output. The more <c>-d</c> flags given, the more
+      debug output you will get (to a certain limit). This option is
+      most useful when the epmd daemon is not started as a daemon.</p>
+    </item>
+    <tag><c><![CDATA[-daemon]]></c></tag>
+    <item>
+      <p>Start epmd detached from the controlling terminal. Logging will end up in syslog when available and correctly configured. If the epmd daemon is started at boot, this option should definitely be used. It is also used when the <c>erl</c> command automatically starts <c>epmd</c>.</p>
+    </item>
+    <tag><c><![CDATA[-relaxed_command_check]]></c></tag>
+    <item>
+      <p>Start the epmd program with relaxed command checking (mostly for backward compatibility). This affects the following:</p>
+      <list type="bulleted">
+          <item>
+            <p>With relaxed command checking, the <c>epmd</c> daemon can be killed from the localhost with i.e. <c>epmd -kill</c> even if there are active nodes registered. Normally only daemons with an empty node database can be killed with the <c>epmd -kill</c> command.</p>
+	  </item>
+	  <item>
+	    <p>The <c>epmd -stop</c> command (and the corresponding messages to epmd, as can be given using <c>erl_interface/ei</c>) is normally always ignored, as it opens up for strange situation when two nodes of the same name can be alive at the same time. A node unregisters itself by just closing the connection to epmd, why the <c>stop</c> command was only intended for use in debugging situations.</p>
+	    <p>With relaxed command checking enabled, you can forcibly unregister live nodes.</p>
+	  </item>
+        </list>
+	<p>Relaxed command checking can also be enabled by setting the environment variable <c>ERL_EPMD_RELAXED_COMMAND_CHECK</c> prior to starting <c>epmd</c>.</p>
+	<p>Only use relaxed command checking on systems with very limited interactive usage.</p>
+    </item>
+  </taglist>
+  </section>
 
   <section>
+    <marker id="debug_flags"></marker>
+    <title>DbgExtra options</title>
+    <p>These options are purely for debugging and testing epmd clients, they should not be used in normal operation.</p>
+
+    <taglist>
+    <tag><c><![CDATA[-packet_timeout Seconds]]></c></tag>
+    <item>
+    <p>Set the number of seconds a connection can be
+    inactive before epmd times out and closes the
+    connection (default 60).</p>
+    </item>
+    <tag><c><![CDATA[-delay_accept Seconds]]></c></tag>
+    <item>
+    <p>To simulate a busy server you can insert a delay between epmd
+    gets notified about that a new connection is requested and
+    when the connections gets accepted.</p>
+    </item>
+    <tag><c><![CDATA[-delay_write Seconds]]></c></tag>
+    <item>
+    <p>Also a simulation of a busy server. Inserts
+    a delay before a reply is sent.</p>
+    </item>
+    </taglist>
+  </section>
+  <section>
+    <marker id="interactive_flags"></marker>
+    <title>Interactive options</title>
+    <p>These options make <c>epmd</c> run as an interactive command displaying the results of sending queries ta an already running instance of <c>epmd</c>. The epmd contacted is always on the local node, but the <c>-port</c> option can be used to select between instances if several are running using different port on the host.</p>
+    <taglist>
+    <tag><c><![CDATA[-port No]]></c></tag>
+    <item>
+      <p>Contacts the <c>epmd</c> listening on the given TCP port number
+      (default 4369). This can also be set using the
+      <c><![CDATA[ERL_EPMD_PORT]]></c> environment variable, see the
+      section <seealso marker="#environment_variables">Environment
+      variables</seealso> below</p>
+    </item>
+    <tag><c><![CDATA[-names]]></c></tag>
+    <item>
+      <p>List names registered with the currently running epmd</p>
+    </item>
+    <tag><c><![CDATA[-kill]]></c></tag>
+    <item>
+      <p>Kill the currently running <c>epmd</c>.</p>
+
+      <p>Killing the running <c>epmd</c> is only allowed if <c>epmd
+        -names</c> show an empty database or
+        <c>-relaxed_command_check</c> was given when the running
+        instance of <c>epmd</c> was started. Note that
+        <c>-relaxed_command_check</c> is given when starting the
+        daemon that is to accept killing when it has live nodes
+        registered. When running epmd interactively,
+        <c>-relaxed_command_check</c> has no effect. A daemon that is
+        started without relaxed command checking has to be killed
+        using i.e. signals or some other OS specific method if it has
+        active clients registered.</p>
+    </item>
+    <tag><c><![CDATA[-stop Name]]></c></tag>
+    <item>
+       <p>Forcibly unregister a live node from <c>epmd</c>'s database</p>
+
+       <p>This command can only be used when contacting <c>epmd</c>
+       instances started with the <c>-relaxed_command_check</c>
+       flag. Note that relaxed command checking has to be enabled for
+       the <c>epmd</c> daemon contacted, When running epmd
+       interactively,
+        <c>-relaxed_command_check</c> has no effect.</p>
+    </item>
+  </taglist>
+  </section>
+  <section>
     <marker id="environment_variables"></marker>
     <title>Environment variables</title>
     <taglist>
+      <tag><c><![CDATA[ERL_EPMD_ADDRESS]]></c></tag>
+      <item>
+        <p>This environment variable may be set to a comma-separated
+          list of IP addresses, in which case the <c>epmd</c> daemon
+          will listen only on the specified address(es) and on the
+          loopback address (which is implicitely added to the list if it
+          has not been specified). The default behaviour is to listen on
+          all available IP addresses.</p>
+      </item>
       <tag><c><![CDATA[ERL_EPMD_PORT]]></c></tag>
       <item>
         <p>This environment variable can contain the port number epmd will use.
@@ -99,6 +255,15 @@
           independent clusters of nodes, to co-exist on the same host.
           All nodes in a cluster must use the same epmd port number.</p>
       </item>
+      <tag><c><![CDATA[ERL_EPMD_RELAXED_COMMAND_CHECK]]></c></tag>
+      <item>
+        <p>If set prior to start, the <c>epmd</c> daemon will behave
+        as if the <c>-relaxed_command_check</c> option was given at
+        start-up. If consequently setting this option before starting
+        the Erlang virtual machine, the automatically started
+        <c>epmd</c> will accept the <c>-kill</c> and <c>-stop</c>
+        commands without restrictions.</p>
+      </item>
     </taglist>
   </section>
 
@@ -116,5 +281,29 @@
       silently be ignored.
       </p>
   </section>
+  <section>
+  <title>Access restrictions</title>
+  <p>The <c>epmd</c> daemon accepts messages from both localhost and
+  remote hosts. However, only the query commands are answered (and
+  acted upon) if the query comes from a remote host. It is always an
+  error to try to register a nodename if the client is not a process
+  located on the same host as the <c>epmd</c> instance is running on,
+  why such requests are considered hostile and the connection is
+  immediately closed.</p>
+
+  <p>The queries accepted from remote nodes are:</p>
+      <list type="bulleted">
+          <item>
+	  <p>Port queries - i.e. on which port does the node with a given
+	  name listen</p>
+	  </item>
+	  <item>
+	  <p>Name listing - i.e. give a list of all names registered on
+	  the host</p>
+	  </item>
+     </list>
+  <p>To restrict access further, firewall software has to be used.</p>
+  </section>
+
 </comref>
 
diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml
index bb741c7836..514ee5ffaf 100644
--- a/erts/doc/src/erl.xml
+++ b/erts/doc/src/erl.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>1996</year><year>2010</year>
+      <year>1996</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -41,6 +41,26 @@
       to scroll back to text which has scrolled off the screen.
       The <c><![CDATA[erl]]></c> program must be used, however, in pipelines or if
       you want to redirect standard input or output.</p>
+      <note><p>As of ERTS version 5.8 (OTP-R14A) the runtime system will by
+               default bind schedulers to logical processors using the
+	       <c>default_bind</c> bind type if the amount of schedulers are
+	       at least equal to the amount of logical processors configured,
+	       binding of schedulers is supported, and a CPU topology is
+	       available at startup.
+	    </p><p>
+	       If the Erlang runtime system is the only operating system
+	       process that binds threads to logical processors, this
+	       improves the performance of the runtime system. However,
+	       if other operating system processes (as for example
+	       another Erlang runtime system) also bind threads to
+	       logical processors, there might be a performance penalty
+	       instead. If this is the case you, are are advised to
+	       unbind the schedulers using the
+	       <seealso marker="#+sbt">+sbtu</seealso> command line argument,
+	       or by invoking
+	       <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type,
+	       unbound)</seealso>.</p>
+      </note>
   </description>
   <funcs>
     <func>
@@ -211,7 +231,8 @@
       <tag><c><![CDATA[-detached]]></c></tag>
       <item>
         <p>Starts the Erlang runtime system detached from the system
-          console. Useful for running daemons and backgrounds processes.</p>
+          console. Useful for running daemons and backgrounds processes. Implies
+          <c><![CDATA[-noinput]]></c>.</p>
       </item>
       <tag><c><![CDATA[-emu_args]]></c></tag>
       <item>
@@ -521,6 +542,28 @@
 	<p>Calling <c>erlang:halt/1</c> with a string argument will still
 	produce a crash dump.</p>
       </item>
+      <tag><c><![CDATA[+e Number]]></c></tag>
+      <item>
+        <p>Set max number of ETS tables.</p>
+      </item>
+      <tag><c><![CDATA[+ec]]></c></tag>
+      <item>
+        <p>Force the <c>compressed</c> option on all ETS tables.
+              Only intended for test and evaluation.</p>
+      </item>
+      <tag><c><![CDATA[+fnl]]></c></tag>
+      <item>
+        <p>The VM works with file names as if they are encoded using the ISO-latin-1 encoding, disallowing Unicode characters with codepoints beyond 255. This is default on operating systems that have transparent file naming, i.e. all Unixes except MacOSX.</p>
+      </item>
+      <tag><c><![CDATA[+fnu]]></c></tag>
+      <item>
+        <p>The VM works with file names as if they are encoded using UTF-8 (or some other system specific Unicode encoding). This is the default on operating systems that enforce Unicode encoding, i.e. Windows and MacOSX.</p>
+	<p>By enabling Unicode file name translation on systems where this is not default, you open up to the possibility that some file names can not be interpreted by the VM and therefore will be returned to the program as raw binaries. The option is therefore considered experimental.</p>
+      </item>
+      <tag><c><![CDATA[+fna]]></c></tag>
+      <item>
+        <p>Selection between <c>+fnl</c> and <c>+fnu</c> is done based on the current locale settings in the OS, meaning that if you have set your terminal for UTF-8 encoding, the filesystem is expected to use the same encoding for filenames (use with care).</p>
+      </item>
       <tag><c><![CDATA[+hms Size]]></c></tag>
       <item>
         <p>Sets the default heap size of processes to the size
@@ -583,6 +626,24 @@
       <item>
         <p>Force ets memory block to be moved on realloc.</p>
       </item>
+      <tag><marker id="+rg"><c><![CDATA[+rg ReaderGroupsLimit]]></c></marker></tag>
+      <item>
+        <p>Limits the amount of reader groups used by read/write locks
+	   optimized for read operations in the Erlang runtime system. By
+	   default the reader groups limit equals 8.</p>
+	<p>When the amount of schedulers is less than or equal to the reader
+	   groups limit, each scheduler has its own reader group. When the
+	   amount of schedulers is larger than the reader groups limit,
+	   schedulers share reader groups. Shared reader groups degrades
+	   read lock and read unlock performance while a large amount of
+	   reader groups degrades write lock performance, so the limit is a
+	   tradeoff between performance for read operations and performance
+	   for write operations. Each reader group currently consumes 64 byte
+	   in each read/write lock. Also note that a runtime system using
+	   shared reader groups benefits from <seealso marker="#+sbt">binding
+	   schedulers to logical processors</seealso>, since the reader groups
+	   are distributed better between schedulers.</p>
+      </item>
       <tag><marker id="+S"><c><![CDATA[+S Schedulers:SchedulerOnline]]></c></marker></tag>
       <item>
         <p>Sets the amount of scheduler threads to create and scheduler
@@ -647,8 +708,8 @@
 	      <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, default_bind)</seealso>.
 	      </p></item>
 	    </taglist>
-	    <p>Binding of schedulers are currently only supported on newer
-	       Linux and Solaris systems.</p>
+	    <p>Binding of schedulers is currently only supported on newer
+	       Linux, Solaris, FreeBSD, and Windows systems.</p>
 	    <p>If no CPU topology is available when the <c>+sbt</c> flag
 	       is processed and <c>BindType</c> is any other type than
 	       <c>u</c>, the runtime system will fail to start. CPU
@@ -657,6 +718,22 @@
 	       that the <c>+sct</c> flag may have to be passed before the
 	       <c>+sbt</c> flag on the command line (in case no CPU topology
 	       has been automatically detected).</p>
+            <p>The runtime system will by default bind schedulers to logical
+	       processors using the <c>default_bind</c> bind type if the amount
+	       of schedulers are at least equal to the amount of logical
+	       processors configured, binding of schedulers is supported,
+	       and a CPU topology is available at startup.
+	    </p>
+	    <p><em>NOTE:</em> If the Erlang runtime system is the only operating
+	       system process that binds threads to logical processors, this
+	       improves the performance of the runtime system. However, if other
+	       operating system processes (as for example another Erlang runtime
+	       system) also bind threads to logical processors, there might be a
+	       performance penalty instead. If this is the case you, are advised
+	       to unbind the schedulers using the <c>+sbtu</c> command line
+	       argument, or by invoking
+	       <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type,
+	       unbound)</seealso>.</p>
 	    <p>For more information, see
 	    <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, SchedulerBindType)</seealso>.
 	    </p>
@@ -777,14 +854,28 @@
 	    <p>For more information, see
 	    <seealso marker="erlang#system_flag_cpu_topology">erlang:system_flag(cpu_topology, CpuTopology)</seealso>.</p>
           </item>
+	  <tag><marker id="+swt"><c>+swt very_low|low|medium|high|very_high</c></marker></tag>
+	  <item>
+             <p>Set scheduler wakeup threshold. Default is <c>medium</c>.
+	        The threshold determines when to wake up sleeping schedulers
+		when more work than can be handled by currently awake schedulers
+		exist. A low threshold will cause earlier wakeups, and a high
+		threshold will cause later wakeups. Early wakeups will
+		distribute work over multiple schedulers faster, but work will
+		more easily bounce between schedulers.
+	     </p>
+             <p><em>NOTE:</em> This flag may be removed or changed at any time
+	        without prior notice.
+	     </p>
+	  </item>
+	  <tag><marker id="sched_thread_stack_size"><c><![CDATA[+sss size]]></c></marker></tag>
+	  <item>
+	    <p>Suggested stack size, in kilowords, for scheduler threads.
+            Valid range is 4-8192 kilowords. The default stack size
+            is OS dependent.</p>
+	  </item>
         </taglist>
       </item>
-      <tag><marker id="sched_thread_stack_size"><c><![CDATA[+sss size]]></c></marker></tag>
-      <item>
-        <p>Suggested stack size, in kilowords, for scheduler threads.
-          Valid range is 4-8192 kilowords. The default stack size
-          is OS dependent.</p>
-      </item>
       <tag><marker id="+t"><c><![CDATA[+t size]]></c></marker></tag>
       <item>
         <p>Set the maximum number of atoms the VM can handle. Default is 1048576.</p>
@@ -838,6 +929,25 @@
           <seealso marker="kernel:error_logger#warning_map/0">error_logger(3)</seealso>
           for further information.</p>
       </item>
+      <tag><c><![CDATA[+zFlag Value]]></c></tag>
+      <item>
+        <p>Miscellaneous flags.</p>
+        <taglist>
+          <tag><marker id="+zdbbl"><c>+zdbbl size</c></marker></tag>
+          <item>
+            <p>Set the distribution buffer busy limit
+	    (<seealso marker="erlang#system_info_dist_buf_busy_limit">dist_buf_busy_limit</seealso>)
+	    in kilobytes. Valid range is 1-2097151. Default is 1024.</p>
+            <p>A larger buffer limit will allow processes to buffer
+            more outgoing messages over the distribution. When the
+            buffer limit has been reached, sending processes will be
+            suspended until the buffer size has shrunk. The buffer
+            limit is per distribution channel. A higher limit will
+            give lower latency and higher throughput at the expense
+            of higher memory usage.</p>
+          </item>
+        </taglist>
+      </item>
     </taglist>
   </section>
 
@@ -894,6 +1004,15 @@
 	  add to the code path.
   	  See <seealso marker="kernel:code">code(3)</seealso>.</p>
       </item>
+      <tag><c><![CDATA[ERL_EPMD_ADDRESS]]></c></tag>
+      <item>
+        <p>This environment variable may be set to a comma-separated
+          list of IP addresses, in which case the
+          <seealso marker="epmd">epmd</seealso> daemon
+          will listen only on the specified address(es) and on the
+          loopback address (which is implicitely added to the list if it
+          has not been specified).</p>
+      </item>
       <tag><c><![CDATA[ERL_EPMD_PORT]]></c></tag>
       <item>
         <p>This environment variable can contain the port number to use when
@@ -906,6 +1025,49 @@
   </section>
 
   <section>
+    <marker id="configuration"></marker>
+    <title>Configuration</title>
+    <p>The standard Erlang/OTP system can be re-configured to change the default
+       behavior on start-up.</p>
+    <taglist>
+       <tag>The .erlang Start-up File</tag>
+       <item>
+         <p>When Erlang/OTP is started, the system searches for a file named .erlang
+            in the directory where Erlang/OTP is started. If not found, the user's home
+            directory is searched for an .erlang file.</p>
+         <p>If an .erlang file is found, it is assumed to contain valid Erlang expressions.
+            These expressions are evaluated as if they were input to the shell.</p>
+         <p>A typical .erlang file contains a set of search paths, for example:</p>
+    <code type="none"><![CDATA[
+        io:format("executing user profile in HOME/.erlang\n",[]).
+        code:add_path("/home/calvin/test/ebin").
+        code:add_path("/home/hobbes/bigappl-1.2/ebin").
+        io:format(".erlang rc finished\n",[]).
+    ]]></code>
+       </item>
+       <tag>user_default and shell_default</tag>
+       <item>
+         <p>Functions in the shell which are not prefixed by a module name are assumed
+           to be functional objects (Funs), built-in functions (BIFs), or belong to the
+           module user_default or shell_default.</p>
+         <p>To include private shell commands, define them in a module user_default and
+           add the following argument as the first line in the .erlang file.</p>
+    <code type="none"><![CDATA[
+         code:load_abs("..../user_default").
+    ]]></code>
+       </item>
+       <tag>erl</tag>
+       <item>
+         <p>If the contents of .erlang are changed and a private version of
+           user_default is defined, it is possible to customize the Erlang/OTP environment.
+           More powerful changes can be made by supplying command line arguments in the
+           start-up script erl. Refer to erl(1) and <seealso marker="init">init(3)</seealso>
+           for further information.</p>
+       </item>
+    </taglist>
+  </section>
+
+   <section>
     <title>SEE ALSO</title>
     <p><seealso marker="init">init(3)</seealso>,
       <seealso marker="erl_prim_loader">erl_prim_loader(3)</seealso>,
diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml
index 5978af178a..6c725fc82d 100644
--- a/erts/doc/src/erl_dist_protocol.xml
+++ b/erts/doc/src/erl_dist_protocol.xml
@@ -5,7 +5,7 @@
   <header>
     <copyright>
       <year>2007</year>
-      <year>2007</year>
+      <year>2011</year>
       <holder>Ericsson AB, All Rights Reserved</holder>
     </copyright>
     <legalnotice>
@@ -206,7 +206,7 @@ By default EPMD listens on port 4369.
       <section>
 	<title>Unregister a node from the EPMD</title>
 	<p>
-	  A node unregister itself from the EPMD by simply closing the
+	  A node unregisters itself from the EPMD by simply closing the
 	  TCP connection towards EPMD established when the node was registered.
 	</p>
       </section>
diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml
index 5061230a33..2fb03954b6 100644
--- a/erts/doc/src/erl_driver.xml
+++ b/erts/doc/src/erl_driver.xml
@@ -4,7 +4,7 @@
 <cref>
   <header>
     <copyright>
-      <year>2001</year><year>2010</year>
+      <year>2001</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -56,16 +56,16 @@
       instance is connected to an Erlang port. Every port has a port
       owner process. Communication with the port is normally done
       through the port owner process.</p>
-    <p>Most of the functions takes the <c>port</c> handle as an
+    <p>Most of the functions take the <c>port</c> handle as an
       argument. This identifies the driver instance. Note that this
       port handle must be stored by the driver, it is not given when
       the driver is called from the emulator (see
       <seealso marker="driver_entry#emulator">driver_entry</seealso>).</p>
-    <p>Some of the functions takes a parameter of type
+    <p>Some of the functions take a parameter of type
       <c>ErlDrvBinary</c>, a driver binary. It should be both
-      allocated and freed by the caller. Using a binary directly avoid
+      allocated and freed by the caller. Using a binary directly avoids
       one extra copying of data.</p>
-    <p>Many of the output functions has a "header buffer", with
+    <p>Many of the output functions have a "header buffer", with
       <c>hbuf</c> and <c>hlen</c> parameters. This buffer is sent as a
       list before the binary (or list, depending on port mode) that is
       sent. This is convenient when matching on messages received from
@@ -92,7 +92,7 @@
       with SMP support without being rewritten if driver
       level locking is used.</p>
     <note>
-      <p>It is assumed that drivers does not access other drivers. If
+      <p>It is assumed that drivers do not access other drivers. If
         drivers should access each other they have to provide their own
         mechanism for thread safe synchronization. Such "inter driver
         communication" is strongly discouraged.</p>
@@ -113,12 +113,12 @@
         call-backs may be made from different threads.</p>
     </note>
     <p>Most functions in this API are <em>not</em> thread-safe, i.e.,
-      they may <em>not</em> be called from an arbitrary thread. Function
+      they may <em>not</em> be called from an arbitrary thread. Functions
       that are not documented as thread-safe may only be called from
       driver call-backs or function calls descending from a driver
       call-back call. Note that driver call-backs may be called from
       different threads. This, however, is not a problem for any
-      functions in this API, since the emulator have control over
+      function in this API, since the emulator has control over
       these threads.</p>
     <note>
       <p>Functions not explicitly documented as thread-safe are
@@ -155,10 +155,10 @@
           more information.</p>
       </item>
       <tag>Output functions</tag>
-      <item>With the output functions, the driver sends data back
+      <item>With the output functions, the driver sends data back to
        the emulator. They will be received as messages by the port owner
        process, see <c>open_port/2</c>. The vector function and the
-       function taking a driver binary is faster, because that avoid
+       function taking a driver binary are faster, because they avoid
        copying the data buffer. There is also a fast way of sending
        terms from the driver, without going through the binary term
        format.</item>
@@ -193,14 +193,14 @@
 	 use functionality from the POSIX thread API or the Windows
 	 native thread API.
       </p>
-      <p>The Erlang driver thread API only return error codes when it is
+      <p>The Erlang driver thread API only returns error codes when it is
          reasonable to recover from an error condition. If it isn't reasonable
 	 to recover from an error condition, the whole runtime system is
 	 terminated. For example, if a create mutex operation fails, an error
 	 code is returned, but if a lock operation on a mutex fails, the
 	 whole runtime system is terminated.
       </p>
-      <p>Note that there exist no "condition variable wait with timeout" in
+      <p>Note that there exists no "condition variable wait with timeout" in
          the Erlang driver thread API. This is due to issues with
 	 <c>pthread_cond_timedwait()</c>. When the system clock suddenly
 	 is changed, it isn't always guaranteed that you will wake up from
@@ -241,7 +241,7 @@
 	 to give you better error reports.
       </p>
       </item>
-      <tag>Adding / remove drivers</tag>
+      <tag>Adding / removing drivers</tag>
       <item>A driver can add and later remove drivers.</item>
       <tag>Monitoring processes</tag>
       <item>A driver can monitor a process that does not own a port.</item>
@@ -262,7 +262,7 @@
           could, under rare circumstances, mean that drivers have to
           be slightly modified. If so, this will of course be documented.
           <c>ERL_DRV_EXTENDED_MINOR_VERSION</c> will be incremented when
-          new features are added. The runtime system use the minor version
+          new features are added. The runtime system uses the minor version
           of the driver to determine what features to use.
           The runtime system will refuse to load a driver if the major
           versions differ, or if the major versions are equal and the
@@ -273,7 +273,7 @@
           It can, however, not make sure that it isn't incompatible. Therefore,
           when loading a driver that doesn't use the extended driver
           interface, there is a risk that it will be loaded also when
-          the driver is incompatible. When the driver use the extended driver
+          the driver is incompatible. When the driver uses the extended driver
           interface, the emulator can verify that it isn't of an incompatible
           driver version. You are therefore advised to use the extended driver
           interface.</p>
@@ -309,7 +309,7 @@ typedef struct ErlDrvSysInfo {
         <seealso marker="#driver_system_info">driver_system_info()</seealso>
         will write the system information when passed a reference to
         a <c>ErlDrvSysInfo</c> structure. A description of the
-        fields in the structure follow:
+        fields in the structure follows:
       </p>
       <taglist>
         <tag><c>driver_major_version</c></tag>
@@ -347,14 +347,6 @@ typedef struct ErlDrvSysInfo {
         <item>A value <c>!= 0</c> if the runtime system has SMP support;
            otherwise, <c>0</c>.
         </item>
-        <tag><c>thread_support</c></tag>
-        <item>A value <c>!= 0</c> if the runtime system has thread support;
-           otherwise, <c>0</c>.
-        </item>
-        <tag><c>smp_support</c></tag>
-        <item>A value <c>!= 0</c> if the runtime system has SMP support;
-           otherwise, <c>0</c>.
-        </item>
         <tag><c>async_threads</c></tag>
         <item>The number of async threads in the async thread pool used
            by <seealso marker="#driver_async">driver_async()</seealso>
@@ -401,8 +393,8 @@ typedef struct ErlDrvBinary {
             <seealso marker="#driver_binary_dec_refc">driver_binary_dec_refc()</seealso>.</p>
         </note>
         <p>Some driver calls, such as <c>driver_enq_binary</c>,
-          increments the driver reference count, and others, such as
-          <c>driver_deq</c> decrements it.</p>
+          increment the driver reference count, and others, such as
+          <c>driver_deq</c> decrement it.</p>
         <p>Using a driver binary instead of a normal buffer, is often
           faster, since the emulator doesn't need to copy the data,
           only the pointer is used.</p>
@@ -415,7 +407,7 @@ typedef struct ErlDrvBinary {
           <c>driver_outputv</c> calls, and in the queue. Also the
           driver call-back <seealso marker="driver_entry#outputv">outputv</seealso> uses driver
           binaries.</p>
-        <p>If the driver of some reason or another, wants to keep a
+        <p>If the driver for some reason or another, wants to keep a
           driver binary around, in a static variable for instance, the
           reference count should be incremented,
           and the binary can later be freed in the <seealso marker="driver_entry#stop">stop</seealso> call-back, with
@@ -423,7 +415,7 @@ typedef struct ErlDrvBinary {
         <p>Note that since a driver binary is shared by the driver and
           the emulator, a binary received from the emulator or sent to
           the emulator, must not be changed by the driver.</p>
-        <p>From erts version 5.5 (OTP release R11B), orig_bytes is 
+        <p>Since erts version 5.5 (OTP release R11B), orig_bytes is
           guaranteed to be properly aligned for storage of an array of
           doubles (usually 8-byte aligned).</p>
       </item>
@@ -447,7 +439,7 @@ typedef struct ErlIOVec {
   int vsize;
   int size;
   SysIOVec* iov;
-  >ErlDrvBinary** binv;
+  ErlDrvBinary** binv;
 } ErlIOVec;
 </code>
         <p>The I/O vector used by the emulator and drivers, is a list
@@ -495,17 +487,17 @@ typedef struct ErlIOVec {
           Currently, the only port specific data that the emulator
           associates with the port data lock is the driver queue.</p>
         <p>Normally a driver instance does not have a port data lock. If
-          the driver instance want to use a port data lock, it has to
+          the driver instance wants to use a port data lock, it has to
           create the port data lock by calling
           <seealso marker="#driver_pdl_create">driver_pdl_create()</seealso>.
           <em>NOTE</em>: Once the port data lock has been created, every
-          access to data associated with the port data lock have to be done
+          access to data associated with the port data lock has to be done
           while having the port data lock locked. The port data lock is
           locked, and unlocked, respectively, by use of
           <seealso marker="#driver_pdl_lock">driver_pdl_lock()</seealso>, and
           <seealso marker="#driver_pdl_unlock">driver_pdl_unlock()</seealso>.</p>
         <p>A port data lock is reference counted, and when the reference
-          count reach zero, it will be destroyed. The emulator will at
+          count reaches zero, it will be destroyed. The emulator will at
           least increment the reference count once when the lock is
           created and decrement it once when the port associated with
           the lock terminates. The emulator will also increment the
@@ -545,7 +537,7 @@ typedef struct ErlIOVec {
 	</p>
         <taglist>
           <tag>suggested_stack_size</tag>
-          <item>A suggestion, in kilo-words, on how large stack to use. A value less
+          <item>A suggestion, in kilo-words, on how large a stack to use. A value less
 	        than zero means default size.
 	  </item>
         </taglist>
@@ -648,7 +640,7 @@ typedef struct ErlIOVec {
           opened.</p>
         <p>The data is queued in the port owner process' message
           queue. Note that this does not yield to the emulator. (Since
-          the driver and the emulator runs in the same thread.)</p>
+          the driver and the emulator run in the same thread.)</p>
         <p>The parameter <c>buf</c> points to the data to send, and
           <c>len</c> is the number of bytes.</p>
         <p>The return value for all output functions is 0. (Unless the
@@ -749,7 +741,7 @@ typedef struct ErlIOVec {
           function <seealso marker="driver_entry#emulator">timeout</seealso> is called.</p>
         <p>Note that there is only one timer on each driver instance;
           setting a new timer will replace an older one.</p>
-        <p>Return value i 0 (-1 only when the <c>timeout</c> driver
+        <p>Return value is 0 (-1 only when the <c>timeout</c> driver
           function is <c>NULL</c>).</p>
       </desc>
     </func>
@@ -781,7 +773,7 @@ typedef struct ErlIOVec {
         <marker id="driver_get_now"></marker>
         <p>This function reads a timestamp into the memory pointed to by 
           the parameter <c>now</c>. See the description of <seealso marker="#ErlDrvNowData">ErlDrvNowData</seealso> for
-          specification of it's fields.  </p>
+          specification of its fields.  </p>
         <p>The return value is 0 unless the <c>now</c> pointer is not
           valid, in which case it is &lt; 0.  </p>
       </desc>
@@ -799,20 +791,20 @@ typedef struct ErlIOVec {
           event object must be a socket or pipe (or other object that
           <c>select</c>/<c>poll</c> can use).
           On windows, the Win32 API function <c>WaitForMultipleObjects</c>
-          is used. This places other restriction on the event object.
+          is used. This places other restrictions on the event object.
           Refer to the Win32 SDK documentation.</p>
         <p>The <c>on</c> parameter should be <c>1</c> for setting events
           and <c>0</c> for clearing them.</p>
-        <p>The <c>mode</c> argument is bitwise-or combination of
+        <p>The <c>mode</c> argument is a bitwise-or combination of
           <c>ERL_DRV_READ</c>, <c>ERL_DRV_WRITE</c> and <c>ERL_DRV_USE</c>.
-          The first two specifies whether to wait for read events and/or write
+          The first two specify whether to wait for read events and/or write
           events. A fired read event will call
           <seealso marker="driver_entry#ready_input">ready_input</seealso>
           while a fired write event will call
           <seealso marker="driver_entry#ready_output">ready_output</seealso>.
           </p>
     <note>
-      <p>Some OS (Windows) does not differ between read and write events.
+      <p>Some OS (Windows) do not differentiate between read and write events.
          The call-back for a fired event then only depends on the value of <c>mode</c>.</p>
     </note>
         <p><c>ERL_DRV_USE</c> specifies if we are using the event object or if we want to close it.
@@ -834,9 +826,9 @@ typedef struct ErlIOVec {
          as before. But it is recommended to update them to use <c>ERL_DRV_USE</c> and
          <c>stop_select</c> to make sure that event objects are closed in a safe way.</p>
     </note>           
-        <p>The return value is 0 (Failure, -1, only if the
+        <p>The return value is 0 (failure, -1, only if the
           <c>ready_input</c>/<c>ready_output</c> is
-          <c>NULL</c>.</p>
+          <c>NULL</c>).</p>
       </desc>
     </func>
     <func>
@@ -1076,7 +1068,7 @@ typedef struct ErlIOVec {
           array of <c>SysIOVec</c>s. It also returns the number of
           elements in <c>vlen</c>. This is the only way to get data
           out of the queue.</p>
-        <p>Nothing is remove from the queue by this function, that must be done
+        <p>Nothing is removed from the queue by this function, that must be done
           with <c>driver_deq</c>.</p>
         <p>The returned array is suitable to use with the Unix system
           call <c>writev</c>.</p>
@@ -1209,7 +1201,7 @@ typedef struct ErlIOVec {
       <fsummary>Stop monitoring a process from a driver</fsummary>
       <desc>
         <marker id="driver_demonitor_process"></marker>
-        <p>This function cancels an monitor created earlier. </p>
+        <p>This function cancels a monitor created earlier. </p>
         <p>The function returns 0 if a monitor was removed and &gt; 0
           if the monitor did no longer exist.</p>
       </desc>
@@ -1326,7 +1318,7 @@ typedef struct ErlIOVec {
         <p>This function signals to erlang that the driver has
           encountered an EOF and should be closed, unless the port was
           opened with the <c>eof</c> option, in that case eof is sent
-          to the port. Otherwise, the port is close and an
+          to the port. Otherwise, the port is closed and an
           <c>'EXIT'</c> message is sent to the port owner process.</p>
         <p>The return value is 0.</p>
       </desc>
@@ -1349,8 +1341,8 @@ typedef struct ErlIOVec {
           (<c>driver_failure</c>).</p>
         <p>The driver should fail only when in severe error situations,
           when the driver cannot possibly keep open, for instance
-          buffer allocation gets out of memory. Normal errors is more
-          appropriate to handle with sending error codes with
+          buffer allocation gets out of memory. For normal errors
+          it is more appropriate to send error codes with
           <c>driver_output</c>.</p>
         <p>The return value is 0.</p>
       </desc>
@@ -1371,7 +1363,7 @@ typedef struct ErlIOVec {
         <p>This function returns the process id of the process that
 	  made the current call to the driver. The process id can be
 	  used with <c>driver_send_term</c> to send back data to the
-	  caller. <c>driver_caller()</c> only return valid data
+	  caller. <c>driver_caller()</c> only returns valid data
 	  when currently executing in one of the following driver
 	  callbacks:</p>
 	  <taglist>
@@ -1409,7 +1401,7 @@ typedef struct ErlIOVec {
           tuple, the elements are given first, and then the tuple
           term, with a count. Likewise for lists.</p>
         <p>A tuple must be specified with the number of elements. (The
-          elements precedes the <c>ERL_DRV_TUPLE</c> term.)</p>
+          elements precede the <c>ERL_DRV_TUPLE</c> term.)</p>
         <p>A list must be specified with the number of elements,
           including the tail, which is the last term preceding
           <c>ERL_DRV_LIST</c>.</p>
@@ -1518,7 +1510,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
     };
     driver_output_term(port, spec, sizeof(spec) / sizeof(spec[0]));
         ]]></code>
-	<p>If you want to pass a binary and doesn't already have the content
+	<p>If you want to pass a binary and don't already have the content
 	of the binary in an <c>ErlDrvBinary</c>, you can benefit from using
 	<c>ERL_DRV_BUF2BINARY</c> instead of creating an <c>ErlDrvBinary</c>
 	via <c>driver_alloc_binary()</c> and then pass the binary via
@@ -1565,7 +1557,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
           <em>other</em> processes than the port owner process. The
           <c>receiver</c> parameter specifies the process to receive
           the data.</p>
-        <p>The parameters <c>term</c> and <c>n</c> does the same thing
+        <p>The parameters <c>term</c> and <c>n</c> do the same thing
           as in <seealso marker="#driver_output_term">driver_output_term</seealso>.</p>
         <p>This function is only thread-safe when the emulator with SMP
           support is used.</p>
@@ -1660,7 +1652,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
       <desc>
         <marker id="driver_lock_driver"></marker>
         <p>This function locks the driver used by the port <c>port</c>
-          in memory for the rest of the emulator process
+          in memory for the rest of the emulator process'
           lifetime. After this call, the driver behaves as one of Erlang's 
           statically linked in drivers.</p>
       </desc>
@@ -1904,7 +1896,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
 		 corresponding to one of the involved thread identifiers
 		 has terminated since the thread identifier was saved,
 		 the result of <c>erl_drv_equal_tids()</c> might not give
-		 expected result.
+		 the expected result.
 	</p></note>
         <p>This function is thread-safe.</p>
       </desc>
diff --git a/erts/doc/src/erl_ext_dist.xml b/erts/doc/src/erl_ext_dist.xml
index c2d58d1ef1..fd2da2cfe3 100644
--- a/erts/doc/src/erl_ext_dist.xml
+++ b/erts/doc/src/erl_ext_dist.xml
@@ -5,7 +5,7 @@
   <header>
     <copyright>
       <year>2007</year>
-      <year>2007</year>
+      <year>2011</year>
       <holder>Ericsson AB, All Rights Reserved</holder>
     </copyright>
     <legalnotice>
diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml
index 0dd46a951a..cdce4ec0b8 100644
--- a/erts/doc/src/erl_nif.xml
+++ b/erts/doc/src/erl_nif.xml
@@ -4,7 +4,7 @@
 <cref>
   <header>
     <copyright>
-      <year>2001</year><year>2009</year>
+      <year>2001</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -34,18 +34,29 @@
   <lib>erl_nif</lib>
   <libsummary>API functions for an Erlang NIF library</libsummary>
   <description>
-     <warning><p>The NIF concept was introduced in R13B03 as an
-	EXPERIMENTAL feature. The interfaces may be changed in any way
-	in coming releases. The plan is however to lift the experimental label and
-  maintain interface backward compatibility from R14B.</p>
-  <p>Incompatible changes in <em>R13B04</em>:</p>
+     <note><p>The NIF concept is officially supported from R14B. NIF source code
+     written for earlier experimental versions might need adaption to run on R14B.</p>
+    <p>No incompatible changes between <em>R14B</em> and R14A.</p>
+    <p>Incompatible changes between <em>R14A</em> and R13B04:</p>
+  <list>
+    <item>Environment argument removed for <c>enif_alloc</c>,
+    <c>enif_realloc</c>, <c>enif_free</c>, <c>enif_alloc_binary</c>,
+    <c>enif_realloc_binary</c>, <c>enif_release_binary</c>,
+    <c>enif_alloc_resource</c>, <c>enif_release_resource</c>,
+    <c>enif_is_identical</c> and <c>enif_compare</c>.</item>
+    <item>Character encoding argument added to <c>enif_get_atom</c>
+    and <c>enif_make_existing_atom</c>.</item>
+    <item>Module argument added to <c>enif_open_resource_type</c>
+      while changing name spaces of resource types from global to module local.</item>
+  </list>
+  <p>Incompatible changes between <em>R13B04</em> and R13B03:</p>
   <list>
     <item>The function prototypes of the NIFs have  changed to expect <c>argc</c> and <c>argv</c>
 	  arguments. The arity of a NIF is by that no longer limited to 3.</item>
     <item><c>enif_get_data</c> renamed as <c>enif_priv_data</c>.</item>
     <item><c>enif_make_string</c> got a third argument for character encoding.</item>
   </list>
-  </warning>
+  </note>
 
     <p>A NIF library contains native implementation of some functions
     of an Erlang module. The native implemented functions (NIFs) are
@@ -109,9 +120,9 @@ ok
      the new directive <seealso
      marker="doc/reference_manual:code_loading#on_load">on_load</seealso> to automatically
      load the NIF library when the module is loaded.</p>
-     <note><p>A NIF must be exported or used locally by the module (or both).
-     An unused local stub function will be optimized away by the compiler
-      causing loading of the NIF library to fail.</p>
+     <note><p>A NIF does not have to be exported, it can be local to the module.
+     Note however that unused local stub functions will be optimized
+     away by the compiler causing loading of the NIF library to fail.</p>
     </note>
      <p>A loaded NIF library is tied to the Erlang module code version
      that loaded it. If the module is upgraded with a new version, the
@@ -122,7 +133,7 @@ ok
      will be shared as well. To avoid unintentionally shared static
      data, each Erlang module code can keep its own private data. This
      private data can be set when the NIF library is loaded and
-     then retrieved by calling <seealso marker="#enif_priv_data">enif_priv_data()</seealso>.</p>
+     then retrieved by calling <seealso marker="#enif_priv_data">enif_priv_data</seealso>.</p>
     <p>There is no way to explicitly unload a NIF library. A library will be
      automatically unloaded when the module code that it belongs to is purged
      by the code server. A NIF library will also be unloaded if it is replaced
@@ -137,14 +148,20 @@ ok
     <taglist>
       <tag>Read and write Erlang terms</tag>
       <item><p>Any Erlang terms can be passed to a NIF as function arguments and
-      be returned as function return values. The terms are of C-type <c>ERL_NIF_TERM</c>
+      be returned as function return values. The terms are of C-type
+      <seealso marker="#ERL_NIF_TERM">ERL_NIF_TERM</seealso>
       and can only be read or written using API functions. Most functions to read
       the content of a term are prefixed <c>enif_get_</c> and usually return
       true (or false) if the term was of the expected type (or not).    
       The functions to write terms are all prefixed <c>enif_make_</c> and usually
       return the created <c>ERL_NIF_TERM</c>. There are also some functions
       to query terms, like <c>enif_is_atom</c>, <c>enif_is_identical</c> and
-      <c>enif_compare</c>.</p></item>
+      <c>enif_compare</c>.</p>
+      <p>All terms of type <c>ERL_NIF_TERM</c> belong to an environment of type
+      <seealso marker="#ErlNifEnv">ErlNifEnv</seealso>. The lifetime of a term is
+      controlled by the lifetime of its environment object. All API functions that read
+      or write terms has the environment, that the term belongs to, as the first
+      function argument.</p></item>
       <tag>Binaries</tag>
       <item><p>Terms of type binary are accessed with the help of the struct type
       <seealso marker="#ErlNifBinary">ErlNifBinary</seealso>    
@@ -161,8 +178,10 @@ ok
       <seealso marker="#enif_release_binary">enif_release_binary</seealso>
       or made read-only by transferring it to an Erlang term with
       <seealso marker="#enif_make_binary">enif_make_binary</seealso>.
-      But it does not have do happen in the same NIF call. Read-only binaries
-      does not have to be released.</p>
+      But it does not have to happen in the same NIF call. Read-only binaries
+      do not have to be released.</p>
+      <p><seealso marker="#enif_make_new_binary">enif_make_new_binary</seealso>
+      can be used as a shortcut to allocate and return a binary in the same NIF call.</p>
       <p>Binaries are sequences of whole bytes. Bitstrings with an arbitrary
       bit length have no support yet.</p>
        </item>
@@ -170,43 +189,34 @@ ok
       <item><p>The use of resource objects is a way to return pointers to
       native data structures from a NIF in a safe way. A resource object is
       just a block of memory allocated with
-      <seealso marker="#enif_alloc_resource">enif_alloc_resource()</seealso>.
+      <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.
       A handle ("safe pointer") to this memory block can then be returned to Erlang by the use of
-      <seealso marker="#enif_make_resource">enif_make_resource()</seealso>.
+      <seealso marker="#enif_make_resource">enif_make_resource</seealso>.
       The term returned by <c>enif_make_resource</c>
-      is totally opaque in nature. It can be stored and passed between processses
-      on the same node, but the only real end usage is to pass it back as argument to a NIF.
-      The NIF can then do <seealso marker="#enif_get_resource">enif_get_resource()</seealso>
+      is totally opaque in nature. It can be stored and passed between processes
+      on the same node, but the only real end usage is to pass it back as an argument to a NIF.
+      The NIF can then call <seealso marker="#enif_get_resource">enif_get_resource</seealso>
       and get back a pointer to the memory block that is guaranteed to still be
       valid. A resource object will not be deallocated until the last handle term
       has been garbage collected by the VM and the resource has been
-      released with <seealso marker="#enif_release_resource">enif_release_resource()</seealso>
+      released with <seealso marker="#enif_release_resource">enif_release_resource</seealso>
       (not necessarily in that order).</p>      
       <p>All resource objects are created as instances of some <em>resource type</em>.
       This makes resources from different modules to be distinguishable. 
       A resource type is created by calling
-      <seealso marker="#enif_open_resource_type">enif_open_resource_type()</seealso>
+      <seealso marker="#enif_open_resource_type">enif_open_resource_type</seealso>
       when a library is loaded. Objects of that resource type can then later be allocated
       and <c>enif_get_resource</c> verifies that the resource is of the expected type.
       A resource type can have a user supplied destructor function that is
       automatically called when resources of that type are released (by either
       the garbage collector or <c>enif_release_resource</c>). Resource types
-      are uniquely identified by a supplied name string.</p>
-      <p>Resource types support upgrade in runtime by allowing a loaded NIF
-      library to takeover an already existing resource type and thereby
-      "inherit" all existing objects of that type. The destructor of the new
-      library will thereafter be called for the inherited objects and the
-      library with the old destructor function can be safely unloaded. Existing
-      resource objects, of a module that is upgraded, must either be deleted
-      or taken over by the new NIF library. The unloading of a library will be
-      postponed as long as it exists resource objects with a destructor
-      function in the library.
-      </p>
-      <p>Here is a template example of how to create and return a resource object.</p>
+      are uniquely identified by a supplied name string and the name of the
+      implementing module.</p>
+      <marker id="enif_resource_example"/><p>Here is a template example of how to create and return a resource object.</p>
       <p/>
       <code type="none">
     ERL_NIF_TERM term;
-    MyStruct* ptr = enif_alloc_resource(env, my_resource_type, sizeof(MyStruct));
+    MyStruct* ptr = enif_alloc_resource(my_resource_type, sizeof(MyStruct));
 
     /* initialize struct ... */
 
@@ -216,21 +226,46 @@ ok
         /* store 'ptr' in static variable, private data or other resource object */
     }
     else {
-        enif_release_resource(env, obj);
+        enif_release_resource(obj);
         /* resource now only owned by "Erlang" */
     }
     return term;
-}
-</code>
-
+    </code>
+      <p>Note that once <c>enif_make_resource</c> creates the term to
+      return to Erlang, the code can choose to either keep its own
+      native pointer to the allocated struct and release it later, or
+      release it immediately and rely solely on the garbage collector
+      to eventually deallocate the resource object when it collects
+      the term.</p> 
+      <p>Another usage of resource objects is to create binary terms with
+      user defined memory management.
+      <seealso marker="#enif_make_resource_binary">enif_make_resource_binary</seealso>
+      will create a binary term that is connected to a resource object. The
+      destructor of the resource will be called when the binary is garbage
+      collected, at which time the binary data can be released. An example of
+      this can be a binary term consisting of data from a <c>mmap</c>'ed file.
+      The destructor can then do <c>munmap</c> to release the memory
+      region.</p>
+      <p>Resource types support upgrade in runtime by allowing a loaded NIF
+      library to takeover an already existing resource type and thereby
+      "inherit" all existing objects of that type. The destructor of the new
+      library will thereafter be called for the inherited objects and the
+      library with the old destructor function can be safely unloaded. Existing
+      resource objects, of a module that is upgraded, must either be deleted
+      or taken over by the new NIF library. The unloading of a library will be
+      postponed as long as there exist resource objects with a destructor
+      function in the library.
+      </p>
       </item>
       <tag>Threads and concurrency</tag>
       <item><p>A NIF is thread-safe without any explicit synchronization as
       long as it acts as a pure function and only reads the supplied
       arguments. As soon as you write towards a shared state either through
       static variables or <seealso marker="#enif_priv_data">enif_priv_data</seealso>
-      you need to supply your own explicit synchronization. Resource objects
-      will also require synchronization if you treat them as mutable.</p>
+      you need to supply your own explicit synchronization. This includes terms
+      in process independent environments that are shared between threads.
+      Resource objects will also require synchronization if you treat them as
+      mutable.</p>
       <p>The library initialization callbacks <c>load</c>, <c>reload</c> and
       <c>upgrade</c> are all thread-safe even for shared state data.</p>
       <p>Avoid doing lengthy work in NIF calls as that may degrade the
@@ -262,8 +297,8 @@ ok
        <item><p><c>load</c> is called when the NIF library is loaded
         and there is no previously loaded library for this module.</p>
         <p><c>*priv_data</c> can be set to point to some private data
-           that the library needs in able to keep a state between NIF 
-           calls. <c>enif_priv_data()</c> will return this pointer.
+           that the library needs in order to keep a state between NIF
+           calls. <c>enif_priv_data</c> will return this pointer.
            <c>*priv_data</c> will be initialized to NULL when <c>load</c> is
            called.</p> 
         <p><c>load_info</c> is the second argument to <seealso
@@ -315,19 +350,37 @@ ok
        <item>
         <p>Variables of type <c>ERL_NIF_TERM</c> can refer to any Erlang term.
         This is an opaque type and values of it can only by used either as
-        arguments to API functions or as return values from NIFs. A variable of
-        type <c>ERL_NIF_TERM</c> is only valid until the NIF call, where it was
-        obtained, returns.</p>
+        arguments to API functions or as return values from NIFs. All
+        <c>ERL_NIF_TERM</c>'s belong to an environment
+        (<seealso marker="#ErlNifEnv">ErlNifEnv</seealso>). A term can not be
+        destructed individually, it is valid until its environment is destructed.</p>
       </item>
      <tag><marker id="ErlNifEnv"/>ErlNifEnv</tag>
       <item>
-        <p><c>ErlNifEnv</c> contains information about the context in
-          which a NIF call is made. This pointer should not be
-          dereferenced in any way, but only passed on to API
-          functions. An <c>ErlNifEnv</c> pointer is only valid until
-          the function, where is what supplied as argument,
-          returns. There is thus useless and dangerous to store <c>ErlNifEnv</c>
-          pointers in between NIF calls.</p>
+        <p><c>ErlNifEnv</c> represents an environment that can host Erlang terms.
+        All terms in an environment are valid as long as the environment is valid.
+        <c>ErlNifEnv</c> is an opaque type and pointers to it can only be passed
+        on to API functions. There are two types of environments; process
+        bound and process independent.</p>
+        <p>A <em>process bound environment</em> is passed as the first argument to all NIFs.
+        All function arguments passed to a NIF will belong to that environment.
+        The return value from a NIF must also be a term belonging to the same
+        environment.
+        In addition a process bound environment contains transient information
+        about the calling Erlang process. The environment is only valid in the 
+        thread where it was supplied as argument until the NIF returns. It is
+        thus useless and dangerous to store pointers to process bound
+        environments between NIF calls. </p>
+        <p>A <em>process independent environment</em> is created by calling
+        <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>. It can be
+        used to store terms between NIF calls and to send terms with
+        <seealso marker="#enif_send">enif_send</seealso>. A process
+        independent environment with all its terms is valid until you explicitly
+        invalidates it with <seealso marker="#enif_free_env">enif_free_env</seealso>
+        or <c>enif_send</c>.</p>
+        <p>All elements of a list/tuple must belong to the same environment as the
+        list/tuple itself. Terms can be copied between environments with
+        <seealso marker="#enif_make_copy">enif_make_copy</seealso>.</p>
       </item>
     <tag><marker id="ErlNifFunc"/>ErlNifFunc</tag>
      <item>
@@ -361,7 +414,18 @@ typedef struct {
         <p><c>ErlNifBinary</c> contains transient information about an
           inspected binary term. <c>data</c> is a pointer to a buffer
           of <c>size</c> bytes with the raw content of the binary.</p>
+          <p>Note that <c>ErlNifBinary</c> is a semi-opaque type and you are
+          only allowed to read fields <c>size</c> and <c>data</c>.</p>
       </item>
+      <tag><marker id="ErlNifPid"/>ErlNifPid</tag>
+       <item>
+          <p><c>ErlNifPid</c> is a process identifier (pid). In contrast to
+          pid terms (instances of <c>ERL_NIF_TERM</c>), <c>ErlNifPid</c>'s are self
+          contained and not bound to any
+          <seealso marker="#ErlNifEnv">environment</seealso>. <c>ErlNifPid</c>
+          is an opaque type.</p>
+        </item>
+
       <tag><marker id="ErlNifResourceType"/>ErlNifResourceType</tag>
        <item>
           <p>Each instance of <c>ErlNifResourceType</c> represent a class of
@@ -386,9 +450,9 @@ typedef enum {
     ERL_NIF_LATIN1
 }ErlNifCharEncoding;
 </code>
-              <p>The character encoding used in strings. The only supported
-              encoding is currently <c>ERL_NIF_LATIN1</c> for iso-latin-1
-              (8-bit ascii).</p>
+              <p>The character encoding used in strings and atoms. The only
+              supported encoding is currently <c>ERL_NIF_LATIN1</c> for
+              iso-latin-1 (8-bit ascii).</p>
             </item>
             <tag><marker id="ErlNifSysInfo"/>ErlNifSysInfo</tag>
              <item>
@@ -396,33 +460,53 @@ typedef enum {
                 to return information about the runtime system. Contains currently
                 the exact same content as <seealso marker="erl_driver#ErlDrvSysInfo">ErlDrvSysInfo</seealso>.</p>
               </item>
+              <tag><marker id="ErlNifSInt64"/>ErlNifSInt64</tag>
+               <item><p>A native signed 64-bit integer type.</p></item>
+              <tag><marker id="ErlNifUInt64"/>ErlNifUInt64</tag>
+               <item><p>A native unsigned 64-bit integer type.</p></item>
 
     </taglist>
   </section>
 
   <funcs>
-    <func><name><ret>void*</ret><nametext>enif_alloc(ErlNifEnv* env, size_t size)</nametext></name>
+    <func><name><ret>void*</ret><nametext>enif_alloc(size_t size)</nametext></name>
       <fsummary>Allocate dynamic memory.</fsummary>
       <desc><p>Allocate memory of <c>size</c> bytes. Return NULL if allocation failed.</p></desc>
     </func>
-    <func><name><ret>int</ret><nametext>enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin)</nametext></name>
+    <func><name><ret>int</ret><nametext>enif_alloc_binary(size_t size, ErlNifBinary* bin)</nametext></name>
       <fsummary>Create a new binary.</fsummary>
-      <desc><p>Allocate a new binary of size of <c>size</c>
+      <desc><p>Allocate a new binary of size <c>size</c>
       bytes. Initialize the structure pointed to by <c>bin</c> to
       refer to the allocated binary. The binary must either be released by 
-      <seealso marker="#enif_release_binary">enif_release_binary()</seealso> 
+      <seealso marker="#enif_release_binary">enif_release_binary</seealso> 
       or ownership transferred to an Erlang term with 
-      <seealso marker="#enif_make_binary">enif_make_binary()</seealso>. 
+      <seealso marker="#enif_make_binary">enif_make_binary</seealso>. 
       An allocated (and owned) <c>ErlNifBinary</c> can be kept between NIF
       calls.</p>
-        <p>Return false if allocation failed.</p>
+        <p>Return true on success or false if allocation failed.</p>
+      </desc>
+    </func>
+    <func><name><ret>ErlNifEnv*</ret><nametext>enif_alloc_env()</nametext></name>
+      <fsummary>Create a new environment</fsummary>
+      <desc><p>Allocate a new process independent environment. The environment can
+      be used to hold terms that is not bound to any process. Such terms can
+      later be copied to a process environment with
+      <seealso marker="#enif_make_copy">enif_make_copy</seealso>
+      or be sent to a process as a message with <seealso marker="#enif_send">enif_send</seealso>.</p>
+      <p>Return pointer to the new environment.</p>
       </desc>
     </func>
-    <func><name><ret>void*</ret><nametext>enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned size)</nametext></name>
+    <func><name><ret>void*</ret><nametext>enif_alloc_resource(ErlNifResourceType* type, unsigned size)</nametext></name>
       <fsummary>Allocate a memory managed resource object</fsummary>
       <desc><p>Allocate a memory managed resource object of type <c>type</c> and size <c>size</c> bytes.</p></desc>
     </func>
-    <func><name><ret>int</ret><nametext>enif_compare(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name>
+    <func><name><ret>void</ret><nametext>enif_clear_env(ErlNifEnv* env)</nametext></name>
+      <fsummary>Clear an environment for reuse.</fsummary>
+      <desc><p>Free all terms in an environment and clear it for reuse. The environment must
+      have been allocated with <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>.
+      </p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_compare(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name>
       <fsummary>Compare two terms</fsummary>
       <desc><p>Return an integer less than, equal to, or greater than
       zero if <c>lhs</c> is found, respectively, to be less than,
@@ -432,77 +516,104 @@ typedef enum {
     </func>
     <func><name><ret>void</ret><nametext>enif_cond_broadcast(ErlNifCond *cnd)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_broadcast">erl_drv_cond_broadcast()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_broadcast">erl_drv_cond_broadcast</seealso>.
           </p></desc>
     </func>
     <func><name><ret>ErlNifCond*</ret><nametext>enif_cond_create(char *name)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_create">erl_drv_cond_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_create">erl_drv_cond_create</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_cond_destroy(ErlNifCond *cnd)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_destroy">erl_drv_cond_destroy()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_destroy">erl_drv_cond_destroy</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_cond_signal(ErlNifCond *cnd)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_signal">erl_drv_cond_signal()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_signal">erl_drv_cond_signal</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_cond_wait(ErlNifCond *cnd, ErlNifMutex *mtx)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_wait">erl_drv_cond_wait()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_wait">erl_drv_cond_wait</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_equal_tids(ErlNifTid tid1, ErlNifTid tid2)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_equal_tids">erl_drv_equal_tids()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_equal_tids">erl_drv_equal_tids</seealso>.
           </p></desc>
     </func>
-    <func><name><ret>void</ret><nametext>enif_free(ErlNifEnv* env, void* ptr)</nametext></name>
+    <func><name><ret>void</ret><nametext>enif_free(void* ptr)</nametext></name>
       <fsummary>Free dynamic memory</fsummary>
       <desc><p>Free memory allocated by <c>enif_alloc</c>.</p></desc>
     </func>
-    <func><name><ret>int</ret><nametext>enif_get_atom(ErlNifEnv* env, 
-                  ERL_NIF_TERM term, char* buf, unsigned size)
-                             </nametext></name>
+    <func><name><ret>void</ret><nametext>enif_free_env(ErlNifEnv* env)</nametext></name>
+      <fsummary>Free an environment allocated with enif_alloc_env</fsummary>
+      <desc><p>Free an environment allocated with <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>.
+      All terms created in the environment will be freed as well.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_get_atom(ErlNifEnv* env, ERL_NIF_TERM term, char* buf, unsigned size, ErlNifCharEncoding encode)</nametext></name>
       <fsummary>Get the text representation of an atom term</fsummary>
       <desc><p>Write a null-terminated string, in the buffer pointed to by
               <c>buf</c> of size <c>size</c>, consisting of the string 
-              representation of the atom <c>term</c>. Return the number of bytes
-              written (including terminating null character) or 0 if
+              representation of the atom <c>term</c> with encoding
+              <seealso marker="#ErlNifCharEncoding">encode</seealso>. Return
+              the number of bytes written (including terminating null character) or 0 if
               <c>term</c> is not an atom with maximum length of 
               <c>size-1</c>.</p></desc>
     </func>
+    <func><name><ret>int</ret><nametext>enif_get_atom_length(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len, ErlNifCharEncoding encode)</nametext></name>
+      <fsummary>Get the length of atom <c>term</c>.</fsummary>
+      <desc><p>Set <c>*len</c> to the length (number of bytes excluding
+      terminating null character) of the atom <c>term</c> with encoding
+      <c>encode</c>. Return true on success or false if <c>term</c> is not an
+      atom.</p></desc>
+    </func>
     <func><name><ret>int</ret><nametext>enif_get_double(ErlNifEnv* env, ERL_NIF_TERM term, double* dp)</nametext></name>
       <fsummary>Read a floating-point number term.</fsummary>
       <desc><p>Set <c>*dp</c> to the floating point value of
-      <c>term</c> or return false if <c>term</c> is not a float.</p></desc>
+      <c>term</c>. Return true on success or false if <c>term</c> is not a float.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_int(ErlNifEnv* env, ERL_NIF_TERM term, int* ip)</nametext></name>
-      <fsummary>Read an integer term.</fsummary>
+      <fsummary>Read an integer term</fsummary>
+      <desc><p>Set <c>*ip</c> to the integer value of
+      <c>term</c>. Return true on success or false if <c>term</c> is not an
+      integer or is outside the bounds of type <c>int</c>.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_get_int64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifSInt64* ip)</nametext></name>
+      <fsummary>Read a 64-bit integer term</fsummary>
       <desc><p>Set <c>*ip</c> to the integer value of
-      <c>term</c> or return false if <c>term</c> is not an integer or is
-      outside the bounds of type <c>int</c></p></desc>
+      <c>term</c>. Return true on success or false if <c>term</c> is not an
+      integer or is outside the bounds of a signed 64-bit integer.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid)</nametext></name>
+    <fsummary>Read an local pid term</fsummary>
+    <desc><p>If <c>term</c> is the pid of a node local process, initialize the
+    pid variable <c>*pid</c> from it and return true. Otherwise return false.
+    No check if the process is alive is done.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_list_cell(ErlNifEnv* env, ERL_NIF_TERM list, ERL_NIF_TERM* head, ERL_NIF_TERM* tail)</nametext></name>
       <fsummary>Get head and tail from a list</fsummary>
       <desc><p>Set <c>*head</c> and <c>*tail</c> from
-      <c>list</c> or return false if <c>list</c> is not a non-empty
-      list.</p></desc>
+      <c>list</c> and return true, or return false if <c>list</c> is not a
+      non-empty list.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_get_list_length(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len)</nametext></name>
+      <fsummary>Get the length of list <c>term</c>.</fsummary>
+      <desc><p>Set <c>*len</c> to the length of list <c>term</c> and return true,
+      or return false if <c>term</c> is not a list.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_long(ErlNifEnv* env, ERL_NIF_TERM term, long int* ip)</nametext></name>
       <fsummary>Read an long integer term.</fsummary>
-      <desc><p>Set <c>*ip</c> to the long integer value of
-      <c>term</c> or return false if <c>term</c> is not an integer or is
+      <desc><p>Set <c>*ip</c> to the long integer value of <c>term</c> and
+      return true, or return false if <c>term</c> is not an integer or is
       outside the bounds of type <c>long int</c>.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp)</nametext></name>
       <fsummary>Get the pointer to a resource object</fsummary>
-      <desc><p>Set <c>*objp</c> to point to the resource object referred to by <c>term</c>.
-      The pointer is valid until the calling NIF returns and should not be released.</p>
-      <p>Return false if <c>term</c> is not a handle to a resource object
+      <desc><p>Set <c>*objp</c> to point to the resource object referred to by <c>term</c>.</p>
+      <p>Return true on success or false if <c>term</c> is not a handle to a resource object
       of type <c>type</c>.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_string(ErlNifEnv* env, 
@@ -527,27 +638,32 @@ typedef enum {
       <c>*arity</c> to the number of elements. Note that the array
       is read-only and <c>(*array)[N-1]</c> will be the Nth element of
       the tuple. <c>*array</c> is undefined if the arity of the tuple
-      is zero.</p><p>Return false if <c>term</c> is not a
+      is zero.</p><p>Return true on success or false if <c>term</c> is not a
       tuple.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_uint(ErlNifEnv* env, ERL_NIF_TERM term, unsigned int* ip)</nametext></name>
       <fsummary>Read an unsigned integer term.</fsummary>
-      <desc><p>Set <c>*ip</c> to the unsigned integer value of
-      <c>term</c> or return false if <c>term</c> is not an unsigned integer or is
-      outside the bounds of type <c>unsigned int</c></p></desc>
+      <desc><p>Set <c>*ip</c> to the unsigned integer value of <c>term</c> and
+      return true, or return false if <c>term</c> is not an unsigned integer or
+      is outside the bounds of type <c>unsigned int</c>.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_get_uint64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifUInt64* ip)</nametext></name>
+      <fsummary>Read an unsigned 64-bit integer term.</fsummary>
+      <desc><p>Set <c>*ip</c> to the unsigned integer value of <c>term</c> and
+      return true, or return false if <c>term</c> is not an unsigned integer or
+      is outside the bounds of an unsigned 64-bit integer.</p></desc>
     </func>
-
     <func><name><ret>int</ret><nametext>enif_get_ulong(ErlNifEnv* env, ERL_NIF_TERM term, unsigned long* ip)</nametext></name>
       <fsummary>Read an unsigned integer term.</fsummary>
-      <desc><p>Set <c>*ip</c> to the unsigned long integer value of
-      <c>term</c> or return false if <c>term</c> is not an unsigned integer or is
-      outside the bounds of type <c>unsigned long</c></p></desc>
+      <desc><p>Set <c>*ip</c> to the unsigned long integer value of <c>term</c>
+      and return true, or return false if <c>term</c> is not an unsigned integer or is
+      outside the bounds of type <c>unsigned long</c>.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_inspect_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, ErlNifBinary* bin)</nametext></name>
       <fsummary>Inspect the content of a binary</fsummary>
       <desc><p>Initialize the structure pointed to by <c>bin</c> with 
               information about the binary term
-      <c>bin_term</c>. Return false if <c>bin_term</c> is not a binary.</p></desc> 
+      <c>bin_term</c>. Return true on success or false if <c>bin_term</c> is not a binary.</p></desc> 
     </func>
     <func><name><ret>int</ret><nametext>enif_inspect_iolist_as_binary(ErlNifEnv* 
                                 env, ERL_NIF_TERM term, ErlNifBinary* bin)
@@ -556,7 +672,7 @@ typedef enum {
       <desc><p>Initialize the structure pointed to by <c>bin</c> with one
       continuous buffer with the same byte content as <c>iolist</c>. As with 
       inspect_binary, the data pointed to by <c>bin</c> is transient and does 
-      not need to be released. Return false if <c>iolist</c> is not an
+      not need to be released. Return true on success or false if <c>iolist</c> is not an
       iolist.</p>
       </desc> 
     </func>
@@ -572,11 +688,15 @@ typedef enum {
       <fsummary>Determine if a term is an empty list</fsummary>
       <desc><p>Return true if <c>term</c> is an empty list.</p></desc>
     </func>
+    <marker id="enif_is_exception"/><func><name><ret>int</ret><nametext>enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
+      <fsummary>Determine if a term is an exception</fsummary>
+      <desc><p>Return true if <c>term</c> is an exception.</p></desc>
+    </func>
     <func><name><ret>int</ret><nametext>enif_is_fun(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
       <fsummary>Determine if a term is a fun</fsummary>
       <desc><p>Return true if <c>term</c> is a fun.</p></desc>
     </func>
-    <func><name><ret>int</ret><nametext>enif_is_identical(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name>
+    <func><name><ret>int</ret><nametext>enif_is_identical(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name>
       <fsummary>Erlang operator =:=</fsummary>
       <desc><p>Return true if the two terms are identical. Corresponds to the
       Erlang operators <c>=:=</c> and
@@ -590,19 +710,46 @@ typedef enum {
       <fsummary>Determine if a term is a port</fsummary>
       <desc><p>Return true if <c>term</c> is a port.</p></desc>
     </func>
-
     <func><name><ret>int</ret><nametext>enif_is_ref(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
       <fsummary>Determine if a term is a reference</fsummary>
       <desc><p>Return true if <c>term</c> is a reference.</p></desc>
     </func>
+    <func><name><ret>int</ret><nametext>enif_is_tuple(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
+      <fsummary>Determine if a term is a tuple</fsummary>
+      <desc><p>Return true if <c>term</c> is a tuple.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
+      <fsummary>Determine if a term is a list</fsummary>
+      <desc><p>Return true if <c>term</c> is a list.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_keep_resource(void* obj)</nametext></name>
+      <fsummary>Add a reference to a resource object</fsummary>
+      <desc><p>Add a reference to resource object <c>obj</c> obtained from
+      <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.
+      Each call to <c>enif_keep_resource</c> for an object must be balanced by
+      a call to <seealso marker="#enif_release_resource">enif_release_resource</seealso>
+      before the object will be destructed.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_atom(ErlNifEnv* env, const char* name)</nametext></name>
       <fsummary>Create an atom term</fsummary>
-      <desc><p>Create an atom term from the C-string <c>name</c>. Unlike other terms, atom
-      terms may be saved and used between NIF calls.</p></desc>
+      <desc><p>Create an atom term from the null-terminated C-string <c>name</c>
+      with iso-latin-1 encoding.</p></desc>
+    </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len)</nametext></name>
+      <fsummary>Create an atom term</fsummary>
+      <desc><p>Create an atom term from the string <c>name</c> with length <c>len</c>.
+      Null-characters are treated as any other characters.</p></desc>
     </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_badarg(ErlNifEnv* env)</nametext></name>
       <fsummary>Make a badarg exception.</fsummary>
-      <desc><p>Make a badarg exception to be returned from a NIF.</p></desc>
+      <desc><p>Make a badarg exception to be returned from a NIF, and set
+      an associated exception reason in <c>env</c>. If
+      <c>enif_make_badarg</c> is called, the term it returns <em>must</em>
+      be returned from the function that called it. No other return value
+      is allowed. Also, the term returned from <c>enif_make_badarg</c> may
+      be passed only to
+      <seealso marker="#enif_is_exception">enif_is_exception</seealso> and
+      not to any other NIF API function.</p></desc>
     </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)</nametext></name>
       <fsummary>Make a binary term.</fsummary>
@@ -611,21 +758,40 @@ typedef enum {
               <c>bin</c> should be considered read-only for the rest of the NIF
               call and then as released.</p></desc>
     </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)</nametext></name>
+      <fsummary>Make a copy of a term.</fsummary>
+      <desc><p>Make a copy of term <c>src_term</c>. The copy will be created in
+      environment <c>dst_env</c>. The source term may be located in any
+      environment.</p></desc>        
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_double(ErlNifEnv* env, double d)</nametext></name>
-      <fsummary>Create an floating-point term</fsummary>
-      <desc><p>Create an floating-point term from a <c>double</c>.</p></desc>
+      <fsummary>Create a floating-point term</fsummary>
+      <desc><p>Create a floating-point term from a <c>double</c>.</p></desc>
     </func>
-    <func><name><ret>int</ret><nametext>enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom)</nametext></name>
+    <func><name><ret>int</ret><nametext>enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding encode)</nametext></name>
       <fsummary>Create an existing atom term</fsummary>
       <desc><p>Try to create the term of an already existing atom from
-      the C-string <c>name</c>. If the atom already exist store the
-      term in <c>*atom</c> and return true, otherwise return
-      false.</p></desc>
+      the null-terminated C-string <c>name</c> with encoding
+      <seealso marker="#ErlNifCharEncoding">encode</seealso>. If the atom
+      already exists store the term in <c>*atom</c> and return true, otherwise
+      return false.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding encoding)</nametext></name>
+      <fsummary>Create an existing atom term</fsummary>
+      <desc><p>Try to create the term of an already existing atom from the
+      string <c>name</c> with length <c>len</c> and encoding
+      <seealso marker="#ErlNifCharEncoding">encode</seealso>. Null-characters
+      are treated as any other characters. If the atom already exists store the term
+      in <c>*atom</c> and return true, otherwise return false.</p></desc>
     </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_int(ErlNifEnv* env, int i)</nametext></name>
       <fsummary>Create an integer term</fsummary>
       <desc><p>Create an integer term.</p></desc>
     </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_int64(ErlNifEnv* env, ErlNifSInt64 i)</nametext></name>
+      <fsummary>Create an integer term</fsummary>
+      <desc><p>Create an integer term from a signed 64-bit integer.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list(ErlNifEnv* env, unsigned cnt, ...)</nametext></name>
       <fsummary>Create a list term.</fsummary>
       <desc><p>Create an ordinary list term of length <c>cnt</c>. Expects
@@ -644,7 +810,7 @@ typedef enum {
       <fsummary>Create a list term.</fsummary>
       <desc><p>Create an ordinary list term with length indicated by the
       function name. Prefer these functions (macros) over the variadic
-      <c>enif_make_list</c>  to get compile time error if the number of
+      <c>enif_make_list</c>  to get a compile time error if the number of
       arguments does not match.</p></desc>
     </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list_cell(ErlNifEnv* env, ERL_NIF_TERM head, ERL_NIF_TERM tail)</nametext></name>
@@ -660,6 +826,20 @@ typedef enum {
       <fsummary>Create an integer term from a long int</fsummary>
       <desc><p>Create an integer term from a <c>long int</c>.</p></desc>
     </func>
+    <func><name><ret>unsigned char*</ret><nametext>enif_make_new_binary(ErlNifEnv* env, size_t size, ERL_NIF_TERM* termp)</nametext></name>
+      <fsummary>Allocate and create a new binary term</fsummary>
+      <desc><p>Allocate a binary of size <c>size</c> bytes and create an owning
+      term. The binary data is mutable until the calling NIF returns. This is a
+      quick way to create a new binary without having to use
+      <seealso marker="#ErlNifBinary">ErlNifBinary</seealso>. The drawbacks are
+      that the binary can not be kept between NIF calls and it can not be
+      reallocated.</p><p>Return a pointer to the raw binary data and set
+      <c>*termp</c> to the binary term.</p></desc>
+    </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_pid(ErlNifEnv* env, const ErlNifPid* pid)</nametext></name>
+      <fsummary>Make a pid term</fsummary>
+      <desc><p>Make a pid term from <c>*pid</c>.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_ref(ErlNifEnv* env)</nametext></name>
       <fsummary>Create a reference.</fsummary>
       <desc><p>Create a reference like <seealso marker="erlang#make_ref-0">erlang:make_ref/0</seealso>.</p></desc>
@@ -668,20 +848,50 @@ typedef enum {
       <fsummary>Create an opaque handle to a resource object</fsummary>
       <desc><p>Create an opaque handle to a memory managed resource object
       obtained by <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.
-      No ownership transfer is done, the resource object still needs to be released by
-      <seealso marker="#enif_release_resource">enif_release_resource</seealso>.</p>
-      <p>Note that the only defined behaviour when using of a resource term in
+      No ownership transfer is done, as the resource object still needs to be released by
+      <seealso marker="#enif_release_resource">enif_release_resource</seealso>,
+      but note that the call to <c>enif_release_resource</c> can occur
+      immediately after obtaining the term from <c>enif_make_resource</c>,
+      in which case the resource object will be deallocated when the
+      term is garbage collected. See the
+      <seealso marker="#enif_resource_example">example of creating and
+      returning a resource object</seealso> for more details.</p> 
+      <p>Note that the only defined behaviour of using a resource term in
       an Erlang program is to store it and send it between processes on the
       same node. Other operations such as matching or <c>term_to_binary</c>
       will have unpredictable (but harmless) results.</p></desc>
     </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_resource_binary(ErlNifEnv* env, void* obj, const void* data, size_t size)</nametext></name>
+      <fsummary>Create a custom binary term</fsummary>
+      <desc><p>Create a binary term that is memory managed by a resource object
+      <c>obj</c> obtained by <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.
+      The returned binary term will consist of <c>size</c> bytes pointed to
+      by <c>data</c>. This raw binary data must be kept readable and unchanged
+      until the destructor of the resource is called. The binary data may be
+      stored external to the resource object in which case it is the responsibility
+      of the destructor to release the data.</p>
+      <p>Several binary terms may be managed by the same resource object. The
+      destructor will not be called until the last binary is garbage collected.
+      This can be useful as a way to return different parts of a larger binary
+      buffer.</p>
+      <p>As with <seealso marker="#enif_make_resource">enif_make_resource</seealso>,
+      no ownership transfer is done. The resource still needs to be released with
+      <seealso marker="#enif_release_resource">enif_release_resource</seealso>.</p>
+      </desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_string(ErlNifEnv* env, const char* string, ErlNifCharEncoding encoding)</nametext></name>
       <fsummary>Create a string.</fsummary>
       <desc><p>Create a list containing the characters of the
       null-terminated string <c>string</c> with encoding <seealso marker="#ErlNifCharEncoding">encoding</seealso>.</p></desc>
     </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_string_len(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding encoding)</nametext></name>
+      <fsummary>Create a string.</fsummary>
+      <desc><p>Create a list containing the characters of the string <c>string</c> with
+      length <c>len</c> and encoding <seealso marker="#ErlNifCharEncoding">encoding</seealso>.
+      Null-characters are treated as any other characters.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_sub_binary(ErlNifEnv* 
-      env, ERL_NIF_TERM bin_term, unsigned pos, unsigned size)</nametext></name>
+      env, ERL_NIF_TERM bin_term, size_t pos, size_t size)</nametext></name>
       <fsummary>Make a subbinary term.</fsummary>
       <desc><p>Make a subbinary of binary <c>bin_term</c>, starting at 
               zero-based position <c>pos</c> with a length of <c>size</c> bytes. 
@@ -707,7 +917,7 @@ typedef enum {
       <fsummary>Create a tuple term.</fsummary>
       <desc><p>Create a tuple term with length indicated by the
       function name. Prefer these functions (macros) over the variadic
-      <c>enif_make_tuple</c>  to get compile time error if the number of
+      <c>enif_make_tuple</c>  to get a compile time error if the number of
       arguments does not match.</p></desc>
     </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_tuple_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt)</nametext></name>
@@ -719,36 +929,41 @@ typedef enum {
       <fsummary>Create an unsigned integer term</fsummary>
       <desc><p>Create an integer term from an <c>unsigned int</c>.</p></desc>
     </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i)</nametext></name>
+      <fsummary>Create an unsigned integer term</fsummary>
+      <desc><p>Create an integer term from an unsigned 64-bit integer.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_ulong(ErlNifEnv* env, unsigned long i)</nametext></name>
       <fsummary>Create an integer term from an unsigned long int</fsummary>
       <desc><p>Create an integer term from an <c>unsigned long int</c>.</p></desc>
     </func>
     <func><name><ret>ErlNifMutex*</ret><nametext>enif_mutex_create(char *name)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_create">erl_drv_mutex_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_create">erl_drv_mutex_create</seealso>.
     </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_mutex_destroy(ErlNifMutex *mtx)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_destroy">erl_drv_mutex_destroy()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_destroy">erl_drv_mutex_destroy</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_mutex_lock(ErlNifMutex *mtx)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_lock">erl_drv_mutex_lock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_lock">erl_drv_mutex_lock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_mutex_trylock(ErlNifMutex *mtx)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_trylock">erl_drv_mutex_trylock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_trylock">erl_drv_mutex_trylock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_mutex_unlock(ErlNifMutex *mtx)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_unlock">erl_drv_mutex_unlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_unlock">erl_drv_mutex_unlock</seealso>.
           </p></desc>
     </func>
-    <func><name><ret>ErlNifResourceType*</ret><nametext>enif_open_resource_type(ErlNifEnv* env, const char* name,
+    <func><name><ret>ErlNifResourceType*</ret><nametext>enif_open_resource_type(ErlNifEnv* env,
+                             const char* module_str, const char* name,
                              ErlNifResourceDtor* dtor, ErlNifResourceFlags flags, ErlNifResourceFlags* tried)</nametext></name>
       <fsummary>Create or takeover a resource type</fsummary>
       <desc><p>Create or takeover a resource type identified by the string
@@ -762,10 +977,10 @@ typedef enum {
                The supplied destructor <c>dtor</c> will be called both for existing instances
                as well as new instances not yet created by the calling NIF library.</item>            
       </taglist>
-      <p>The two flag values can be combined with bitwise-or. To avoid unintentionally
-      name clashes a good practice is to include the module name as part of the
-      type <c>name</c>. The <c>dtor</c> may  be <c>NULL</c> in case no destructor
-      is needed.</p>
+      <p>The two flag values can be combined with bitwise-or. The name of the
+      resource type is local to the calling module. Argument <c>module_str</c>
+      is not (yet) used and must be NULL. The <c>dtor</c> may  be <c>NULL</c>
+      in case no destructor is needed.</p>
       <p>On success, return a pointer to the resource type and <c>*tried</c>
       will be set to either <c>ERL_NIF_RT_CREATE</c> or
       <c>ERL_NIF_RT_TAKEOVER</c> to indicate what was actually done.
@@ -782,129 +997,163 @@ typedef enum {
       <c>reload</c> or <c>upgrade</c>.</p>
       <p>Was previously named <c>enif_get_data</c>.</p></desc>
     </func>
-    <func><name><ret>void</ret><nametext>enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size)</nametext></name>
+    <func><name><ret>int</ret><nametext>enif_realloc_binary(ErlNifBinary* bin, size_t size)</nametext></name>
       <fsummary>Change the size of a binary.</fsummary>
       <desc><p>Change the size of a binary <c>bin</c>. The source binary
       may be read-only, in which case it will be left untouched and
-      a mutable copy is allocated and assigned to <c>*bin</c>.</p></desc>
+      a mutable copy is allocated and assigned to <c>*bin</c>. Return true on success,
+      false if memory allocation failed.</p></desc>
     </func>
-    <func><name><ret>void</ret><nametext>enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin)</nametext></name>
+    <func><name><ret>void</ret><nametext>enif_release_binary(ErlNifBinary* bin)</nametext></name>
       <fsummary>Release a binary.</fsummary>
-      <desc><p>Release a binary obtained 
-              from <c>enif_alloc_binary</c>.</p></desc>
+      <desc><p>Release a binary obtained from <c>enif_alloc_binary</c>.</p></desc>
     </func>
-    <func><name><ret>void</ret><nametext>enif_release_resource(ErlNifEnv* env, void* obj)</nametext></name>
+    <func><name><ret>void</ret><nametext>enif_release_resource(void* obj)</nametext></name>
       <fsummary>Release a resource object.</fsummary>
-      <desc><p>Release a resource objects obtained from <c>enif_alloc_resource</c>.
-      The object may still be alive if it is referred to by Erlang terms. Each call to
-      <c>enif_release_resource</c> must correspond to a previous call to <c>enif_alloc_resource</c>.
-      References made by <c>enif_make_resource</c> can only be released by the garbage collector.</p></desc>
+      <desc><p>Remove a reference to resource object <c>obj</c>obtained from
+      <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.
+      The resource object will be destructed when the last reference is removed.
+      Each call to <c>enif_release_resource</c> must correspond to a previous
+      call to <c>enif_alloc_resource</c> or
+      <seealso marker="#enif_keep_resource">enif_keep_resource</seealso>.
+      References made by <seealso marker="#enif_make_resource">enif_make_resource</seealso>
+      can only be removed by the garbage collector.</p></desc>
     </func>
     <func><name><ret>ErlNifRWLock*</ret><nametext>enif_rwlock_create(char *name)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_create">erl_drv_rwlock_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_create">erl_drv_rwlock_create</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_rwlock_destroy(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_destroy">erl_drv_rwlock_destroy()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_destroy">erl_drv_rwlock_destroy</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_rwlock_rlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rlock">erl_drv_rwlock_rlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rlock">erl_drv_rwlock_rlock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_rwlock_runlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_runlock">erl_drv_rwlock_runlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_runlock">erl_drv_rwlock_runlock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_rwlock_rwlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwlock">erl_drv_rwlock_rwlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwlock">erl_drv_rwlock_rwlock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_rwlock_rwunlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwunlock">erl_drv_rwlock_rwunlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwunlock">erl_drv_rwlock_rwunlock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_rwlock_tryrlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrlock">erl_drv_rwlock_tryrlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrlock">erl_drv_rwlock_tryrlock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_rwlock_tryrwlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrwlock">erl_drv_rwlock_tryrwlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrwlock">erl_drv_rwlock_tryrwlock</seealso>.
           </p></desc>
     </func>
-    <func><name><ret>unsigned</ret><nametext>enif_sizeof_resource(ErlNifEnv* env, void* obj)</nametext></name>
+    <func><name><ret>ErlNifPid*</ret><nametext>enif_self(ErlNifEnv* caller_env, ErlNifPid* pid)</nametext></name>
+      <fsummary>Get the pid of the calling process.</fsummary>
+      <desc><p>Initialize the pid variable <c>*pid</c> to represent the
+      calling process. Return <c>pid</c>.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_send(ErlNifEnv* env, ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)</nametext></name>
+      <fsummary>Send a message to a process.</fsummary>
+      <desc><p>Send a message to a process.</p>
+      <taglist>
+        <tag><c>env</c></tag>
+              <item>The environment of the calling process. Must be NULL if and
+              only if calling from a created thread.</item>
+        <tag><c>*to_pid</c></tag>
+              <item>The pid of the receiving process. The pid should refer to a process on the local node.</item>
+        <tag><c>msg_env</c></tag>
+              <item>The environment of the message term. Must be a process
+              independent environment allocated with
+               <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>.</item>
+        <tag><c>msg</c></tag>
+              <item>The message term to send.</item>
+      </taglist>
+      <p>Return true on success, or false if <c>*to_pid</c> does not refer to an alive local process.</p>
+      <p>The message environment <c>msg_env</c> with all its terms (including
+      <c>msg</c>) will be invalidated by a successful call to <c>enif_send</c>. The environment
+      should either be freed with <seealso marker="#enif_free_env">enif_free_env</seealso>
+      of cleared for reuse with <seealso marker="#enif_clear_env">enif_clear_env</seealso>.</p>
+      <p>This function is only thread-safe when the emulator with SMP support is used.
+      It can only be used in a non-SMP emulator from a NIF-calling thread.</p>
+      </desc>
+    </func>
+    <func><name><ret>unsigned</ret><nametext>enif_sizeof_resource(void* obj)</nametext></name>
       <fsummary>Get the byte size of a resource object</fsummary>
       <desc><p>Get the byte size of a resource object <c>obj</c> obtained by
-      <c>enif_alloc_resource</c>.</p></desc>
+      <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.</p></desc>
     </func>
     <func>
       <name><ret>void</ret><nametext>enif_system_info(ErlNifSysInfo *sys_info_ptr, size_t size)</nametext></name>
       <fsummary>Get information about the Erlang runtime system</fsummary>
-      <desc><p>Same as <seealso marker="erl_driver#driver_system_info">driver_system_info()</seealso>.
+      <desc><p>Same as <seealso marker="erl_driver#driver_system_info">driver_system_info</seealso>.
             </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_thread_create(char *name,ErlNifTid *tid,void * (*func)(void *),void *args,ErlNifThreadOpts *opts)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_create">erl_drv_thread_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_create">erl_drv_thread_create</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_thread_exit(void *resp)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_exit">erl_drv_thread_exit()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_exit">erl_drv_thread_exit</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_thread_join(ErlNifTid, void **respp)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_join">erl_drv_thread_join ()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_join">erl_drv_thread_join </seealso>.
           </p></desc>
     </func>
     <func><name><ret>ErlNifThreadOpts*</ret><nametext>enif_thread_opts_create(char *name)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_create">erl_drv_thread_opts_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_create">erl_drv_thread_opts_create</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_thread_opts_destroy(ErlNifThreadOpts *opts)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_destroy">erl_drv_thread_opts_destroy()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_destroy">erl_drv_thread_opts_destroy</seealso>.
           </p></desc>
     </func>
     <func><name><ret>ErlNifTid</ret><nametext>enif_thread_self(void)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_self">erl_drv_thread_self()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_self">erl_drv_thread_self</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_tsd_key_create(char *name, ErlNifTSDKey *key)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_create">erl_drv_tsd_key_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_create">erl_drv_tsd_key_create</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_tsd_key_destroy(ErlNifTSDKey key)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_destroy">erl_drv_tsd_key_destroy()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_destroy">erl_drv_tsd_key_destroy</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void*</ret><nametext>enif_tsd_get(ErlNifTSDKey key)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_get">erl_drv_tsd_get()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_get">erl_drv_tsd_get</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_tsd_set(ErlNifTSDKey key, void *data)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_set">erl_drv_tsd_set()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_set">erl_drv_tsd_set</seealso>.
           </p></desc>
     </func>
   </funcs>
   <section>
     <title>SEE ALSO</title>
-    <p><seealso marker="erlang#load_nif-2">load_nif(3)</seealso></p>
+    <p><seealso marker="erlang#load_nif-2">erlang:load_nif/2</seealso></p>
   </section>
 </cref>
 
diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml
index 46f8df4683..f98e15cb52 100644
--- a/erts/doc/src/erlang.xml
+++ b/erts/doc/src/erlang.xml
@@ -4,7 +4,7 @@
 <erlref>
   <header>
     <copyright>
-      <year>1996</year><year>2010</year>
+      <year>1996</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -80,7 +80,7 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
-      <name>adler32(Data) -> int()</name>
+      <name>erlang:adler32(Data) -> int()</name>
       <fsummary>Compute adler32 checksum</fsummary>
       <type>
         <v>Data = iodata()</v>
@@ -90,7 +90,7 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
-      <name>adler32(OldAdler, Data) -> int()</name>
+      <name>erlang:adler32(OldAdler, Data) -> int()</name>
       <fsummary>Compute adler32 checksum</fsummary>
       <type>
         <v>OldAdler = int()</v>
@@ -102,17 +102,17 @@ iolist() = [char() | binary() | iolist()]
 	<c>Data</c>.</p>
 	<p>The following code:</p>
 	<code>
-	X = adler32(Data1),
-	Y = adler32(X,Data2).
+	X = erlang:adler32(Data1),
+	Y = erlang:adler32(X,Data2).
 	</code>
 	<p>- would assign the same value to <c>Y</c> as this would:</p>
 	<code>
-	Y = adler32([Data1,Data2]).
+	Y = erlang:adler32([Data1,Data2]).
 	</code>
       </desc>
     </func>
     <func>
-      <name>adler32_combine(FirstAdler, SecondAdler, SecondSize) -> int()</name>
+      <name>erlang:adler32_combine(FirstAdler, SecondAdler, SecondSize) -> int()</name>
       <fsummary>Combine two adler32 checksums</fsummary>
       <type>
         <v>FirstAdler = SecondAdler = int()</v>
@@ -124,14 +124,14 @@ iolist() = [char() | binary() | iolist()]
 	the second checksum to be known.</p>
 	<p>The following code:</p>
 	<code>
-	Y = adler32(Data1),
-	Z = adler32(Y,Data2).
+	Y = erlang:adler32(Data1),
+	Z = erlang:adler32(Y,Data2).
 	</code>
 	<p>- would assign the same value to <c>Z</c> as this would:</p>
 	<code>
-	X = adler32(Data1),
-	Y = adler32(Data2),
-	Z = adler32_combine(X,Y,iolist_size(Data2)).
+	X = erlang:adler32(Data1),
+	Y = erlang:adler32(Data2),
+	Z = erlang:adler32_combine(X,Y,iolist_size(Data2)).
 	</code>
       </desc>
     </func>
@@ -147,7 +147,7 @@ iolist() = [char() | binary() | iolist()]
           <c>Tuple1</c>, and contains the elements in <c>Tuple1</c>
           followed by <c>Term</c> as the last element. Semantically
           equivalent to
-          <c>list_to_tuple(tuple_to_list(Tuple ++ [Term])</c>, but much
+          <c>list_to_tuple(tuple_to_list(Tuple) ++ [Term])</c>, but much
           faster.</p>
         <pre>
 > <input>erlang:append_element({one, two}, three).</input>
@@ -253,6 +253,54 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
+      <name>binary_part(Subject, PosLen) -> binary()</name>
+      <fsummary>Extracts a part of a binary</fsummary>
+      <type>
+        <v>Subject = binary()</v>
+	<v>PosLen = {Start,Length}</v>
+	<v>Start = int()</v>
+	<v>Length = int()</v>
+      </type>
+      <desc>
+      <p>Extracts the part of the binary described by <c>PosLen</c>.</p>
+
+      <p>Negative length can be used to extract bytes at the end of a binary:</p>
+
+<code>
+1> Bin = &lt;&lt;1,2,3,4,5,6,7,8,9,10&gt;&gt;.
+2> binary_part(Bin,{byte_size(Bin), -5)).
+&lt;&lt;6,7,8,9,10&gt;&gt;
+</code>
+
+      <p>If <c>PosLen</c> in any way references outside the binary, a <c>badarg</c> exception is raised.</p>
+
+      <p><c>Start</c> is zero-based, i.e:</p>
+<code>
+1> Bin = &lt;&lt;1,2,3&gt;&gt;
+2> binary_part(Bin,{0,2}).
+&lt;&lt;1,2&gt;&gt;
+</code>
+
+      <p>See the STDLIB module <c>binary</c> for details about the <c>PosLen</c> semantics.</p>
+
+      <p>Allowed in guard tests.</p>
+      </desc>
+    </func>
+    <func>
+      <name>binary_part(Subject, Start, Length) -> binary()</name>
+      <fsummary>Extracts a part of a binary</fsummary>
+      <type>
+        <v>Subject = binary()</v>
+	<v>Start = int()</v>
+	<v>Length = int()</v>
+      </type>
+      <desc>
+      <p>The same as <c>binary_part(Subject, {Pos, Len})</c>.</p>
+
+      <p>Allowed in guard tests.</p>
+      </desc>
+    </func>
+    <func>
       <name>binary_to_atom(Binary, Encoding) -> atom()</name>
       <fsummary>Convert from text representation to an atom</fsummary>
       <type>
@@ -318,6 +366,11 @@ iolist() = [char() | binary() | iolist()]
           corresponding to the bytes from position <c>Start</c> to
           position <c>Stop</c> in <c>Binary</c>. Positions in the
           binary are numbered starting from 1.</p>
+
+	  <note><p>This function's indexing style of using one-based indices for
+	  binaries is deprecated. New code should use the functions in
+	  the STDLIB module <c>binary</c> instead. They consequently
+	  use the same (zero-based) style of indexing.</p></note>
       </desc>
     </func>
     <func>
@@ -354,7 +407,7 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
-      <name>erlang:binary_to_term(Binary, Opts) -> term()</name>
+      <name>binary_to_term(Binary, Opts) -> term()</name>
       <fsummary>Decode an Erlang external term format binary</fsummary>
       <type>
         <v>Opts = [safe]</v>
@@ -446,7 +499,7 @@ iolist() = [char() | binary() | iolist()]
       <name>erlang:cancel_timer(TimerRef) -> Time | false</name>
       <fsummary>Cancel a timer</fsummary>
       <type>
-        <v>TimerRef = ref()</v>
+        <v>TimerRef = reference()</v>
         <v>Time = int()</v>
       </type>
       <desc>
@@ -500,7 +553,7 @@ false</pre>
       </desc>
     </func>
     <func>
-      <name>crc32(Data) -> int()</name>
+      <name>erlang:crc32(Data) -> int()</name>
       <fsummary>Compute crc32 (IEEE 802.3) checksum</fsummary>
       <type>
         <v>Data = iodata()</v>
@@ -510,7 +563,7 @@ false</pre>
       </desc>
     </func>
     <func>
-      <name>crc32(OldCrc, Data) -> int()</name>
+      <name>erlang:crc32(OldCrc, Data) -> int()</name>
       <fsummary>Compute crc32 (IEEE 802.3) checksum</fsummary>
       <type>
         <v>OldCrc = int()</v>
@@ -522,17 +575,17 @@ false</pre>
 	<c>Data</c>.</p>
 	<p>The following code:</p>
 	<code>
-	X = crc32(Data1),
-	Y = crc32(X,Data2).
+	X = erlang:crc32(Data1),
+	Y = erlang:crc32(X,Data2).
 	</code>
 	<p>- would assign the same value to <c>Y</c> as this would:</p>
 	<code>
-	Y = crc32([Data1,Data2]).
+	Y = erlang:crc32([Data1,Data2]).
 	</code>
       </desc>
     </func>
     <func>
-      <name>crc32_combine(FirstCrc, SecondCrc, SecondSize) -> int()</name>
+      <name>erlang:crc32_combine(FirstCrc, SecondCrc, SecondSize) -> int()</name>
       <fsummary>Combine two crc32 (IEEE 802.3) checksums</fsummary>
       <type>
         <v>FirstCrc = SecondCrc = int()</v>
@@ -544,14 +597,14 @@ false</pre>
 	the second checksum to be known.</p>
 	<p>The following code:</p>
 	<code>
-	Y = crc32(Data1),
-	Z = crc32(Y,Data2).
+	Y = erlang:crc32(Data1),
+	Z = erlang:crc32(Y,Data2).
 	</code>
 	<p>- would assign the same value to <c>Z</c> as this would:</p>
 	<code>
-	X = crc32(Data1),
-	Y = crc32(Data2),
-	Z = crc32_combine(X,Y,iolist_size(Data2)).
+	X = erlang:crc32(Data1),
+	Y = erlang:crc32(Data2),
+	Z = erlang:crc32_combine(X,Y,iolist_size(Data2)).
 	</code>
       </desc>
     </func>
@@ -571,7 +624,7 @@ false</pre>
       </desc>
     </func>
     <func>
-      <name>decode_packet(Type,Bin,Options) -> {ok,Packet,Rest} | {more,Length} | {error,Reason}</name>
+      <name>erlang:decode_packet(Type,Bin,Options) -> {ok,Packet,Rest} | {more,Length} | {error,Reason}</name>
       <fsummary>Extracts a protocol packet from a binary</fsummary>
       <type>
         <v>Bin = binary()</v>
@@ -707,18 +760,18 @@ false</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:demonitor(MonitorRef) -> true</name>
+      <name>demonitor(MonitorRef) -> true</name>
       <fsummary>Stop monitoring</fsummary>
       <type>
-        <v>MonitorRef = ref()</v>
+        <v>MonitorRef = reference()</v>
       </type>
       <desc>
         <p>If <c>MonitorRef</c> is a reference which the calling process
           obtained by calling
-          <seealso marker="#monitor/2">erlang:monitor/2</seealso>,
+          <seealso marker="#monitor/2">monitor/2</seealso>,
           this monitoring is turned off. If the monitoring is already
           turned off, nothing happens.</p>
-        <p>Once <c>erlang:demonitor(MonitorRef)</c> has returned it is
+        <p>Once <c>demonitor(MonitorRef)</c> has returned it is
           guaranteed that no <c>{'DOWN', MonitorRef, _, _, _}</c> message
           due to the monitor will be placed in the callers message queue
           in the future. A <c>{'DOWN', MonitorRef, _, _, _}</c> message
@@ -726,10 +779,10 @@ false</pre>
           the call, though. Therefore, in most cases, it is advisable
           to remove such a <c>'DOWN'</c> message from the message queue
           after monitoring has been stopped. 
-          <seealso marker="#demonitor/2">erlang:demonitor(MonitorRef, [flush])</seealso> can be used instead of
-          <c>erlang:demonitor(MonitorRef)</c> if this cleanup is wanted.</p>
+          <seealso marker="#demonitor/2">demonitor(MonitorRef, [flush])</seealso> can be used instead of
+          <c>demonitor(MonitorRef)</c> if this cleanup is wanted.</p>
         <note>
-          <p>Prior to OTP release R11B (erts version 5.5) <c>erlang:demonitor/1</c>
+          <p>Prior to OTP release R11B (erts version 5.5) <c>demonitor/1</c>
             behaved completely asynchronous, i.e., the monitor was active
             until the "demonitor signal" reached the monitored entity. This
             had one undesirable effect, though. You could never know when
@@ -747,10 +800,10 @@ false</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:demonitor(MonitorRef, OptionList) -> true|false</name>
+      <name>demonitor(MonitorRef, OptionList) -> true|false</name>
       <fsummary>Stop monitoring</fsummary>
       <type>
-        <v>MonitorRef = ref()</v>
+        <v>MonitorRef = reference()</v>
         <v>OptionList = [Option]</v>
         <v>Option = flush</v>
         <v>Option = info</v>
@@ -759,8 +812,8 @@ false</pre>
         <p>The returned value is <c>true</c> unless <c>info</c> is part
 	   of <c>OptionList</c>.
 	</p>
-        <p><c>erlang:demonitor(MonitorRef, [])</c> is equivalent to
-          <seealso marker="#demonitor/1">erlang:demonitor(MonitorRef)</seealso>.</p>
+        <p><c>demonitor(MonitorRef, [])</c> is equivalent to
+          <seealso marker="#demonitor/1">demonitor(MonitorRef)</seealso>.</p>
         <p>Currently the following <c>Option</c>s are valid:</p>
         <taglist>
           <tag><c>flush</c></tag>
@@ -768,11 +821,11 @@ false</pre>
             <p>Remove (one) <c>{_, MonitorRef, _, _, _}</c> message,
               if there is one, from the callers message queue after
               monitoring has been stopped.</p>
-            <p>Calling <c>erlang:demonitor(MonitorRef, [flush])</c>
+            <p>Calling <c>demonitor(MonitorRef, [flush])</c>
               is equivalent to the following, but more efficient:</p>
             <code type="none">
 
-    erlang:demonitor(MonitorRef),
+    demonitor(MonitorRef),
     receive
         {_, MonitorRef, _, _, _} ->
             true
@@ -810,7 +863,7 @@ false</pre>
         </note>
         <p>Failure: <c>badarg</c> if <c>OptionList</c> is not a list, or
           if <c>Option</c> is not a valid option, or the same failure as for
-          <seealso marker="#demonitor/1">erlang:demonitor/1</seealso></p>
+          <seealso marker="#demonitor/1">demonitor/1</seealso></p>
       </desc>
     </func>
     <func>
@@ -891,7 +944,7 @@ b</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:error(Reason)</name>
+      <name>error(Reason)</name>
       <fsummary>Stop execution with a given reason</fsummary>
       <type>
         <v>Reason = term()</v>
@@ -904,7 +957,7 @@ b</pre>
           function first). Since evaluating this function causes
           the process to terminate, it has no return value.</p>
         <pre>
-> <input>catch erlang:error(foobar).</input>
+> <input>catch error(foobar).</input>
 {'EXIT',{foobar,[{erl_eval,do_apply,5},
                  {erl_eval,expr,5},
                  {shell,exprs,6},
@@ -913,7 +966,7 @@ b</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:error(Reason, Args)</name>
+      <name>error(Reason, Args)</name>
       <fsummary>Stop execution with a given reason</fsummary>
       <type>
         <v>Reason = term()</v>
@@ -1430,7 +1483,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:integer_to_list(Integer, Base) -> string()</name>
+      <name>integer_to_list(Integer, Base) -> string()</name>
       <fsummary>Text representation of an integer</fsummary>
       <type>
         <v>Integer = int()</v>
@@ -1440,7 +1493,7 @@ os_prompt%</pre>
         <p>Returns a string which corresponds to the text
           representation of <c>Integer</c> in base <c>Base</c>.</p>
         <pre>
-> <input>erlang:integer_to_list(1023, 16).</input>
+> <input>integer_to_list(1023, 16).</input>
 "3FF"</pre>
       </desc>
     </func>
@@ -1879,7 +1932,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:list_to_integer(String, Base) -> int()</name>
+      <name>list_to_integer(String, Base) -> int()</name>
       <fsummary>Convert from text representation to an integer</fsummary>
       <type>
         <v>String = string()</v>
@@ -1889,7 +1942,7 @@ os_prompt%</pre>
         <p>Returns an integer whose text representation in base
           <c>Base</c> is <c>String</c>.</p>
         <pre>
-> <input>erlang:list_to_integer("3FF", 16).</input>
+> <input>list_to_integer("3FF", 16).</input>
 1023</pre>
         <p>Failure: <c>badarg</c> if <c>String</c> contains a bad
           representation of an integer.</p>
@@ -1981,16 +2034,18 @@ os_prompt%</pre>
         <v>Text = string()</v>
       </type>
       <desc>
-        <warning>
-          <p>This BIF is still an experimental feature. The interface
-          may be changed in any way in future releases.</p><p>In
-          R13B03 the return value on failure was
+        <note>
+          <p>In releases older than OTP R14B, NIFs were an
+          experimental feature. Versions of OTP older than R14B might
+          have different and possibly incompatible NIF semantics and
+          interfaces. For example, in R13B03 the return value on
+          failure was
           <c>{error,Reason,Text}</c>.</p>
-        </warning>
+        </note>
         <p>Loads and links a dynamic library containing native
         implemented functions (NIFs) for a module. <c>Path</c> is a
         file path to the sharable object/dynamic library file minus
-        the OS-dependant file extension (.so for Unix and .ddl for
+        the OS-dependent file extension (.so for Unix and .dll for
         Windows). See <seealso marker="erl_nif">erl_nif</seealso>
         on how to implement a NIF library.</p> 
         <p><c>LoadInfo</c> can be any term. It will be passed on to
@@ -2107,7 +2162,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>make_ref() -> ref()</name>
+      <name>make_ref() -> reference()</name>
       <fsummary>Return an almost unique reference</fsummary>
       <desc>
         <p>Returns an almost unique reference.</p>
@@ -2156,7 +2211,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:max(Term1, Term2) -> Maximum</name>
+      <name>max(Term1, Term2) -> Maximum</name>
       <fsummary>Return the largest of two term</fsummary>
       <type>
         <v>Term1 = Term2 = Maximum = term()</v>
@@ -2301,6 +2356,14 @@ os_prompt%</pre>
               <seealso marker="tools:instrument">instrument(3)</seealso>
               and/or <seealso marker="erts:erl">erl(1)</seealso>.</p>
           </item>
+          <tag><c>low</c></tag>
+          <item>
+            <p>Only on 64-bit halfword emulator.</p>
+            <p>The total amount of memory allocated in low memory areas
+	      that are restricted to less than 4 Gb even though
+	      the system may have more physical memory.</p>
+	    <p>May be removed in future releases of halfword emulator.</p>
+          </item>
         </taglist>
         <note>
           <p>The <c>system</c> value is not complete. Some allocated
@@ -2405,7 +2468,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:min(Term1, Term2) -> Minimum</name>
+      <name>min(Term1, Term2) -> Minimum</name>
       <fsummary>Return the smallest of two term</fsummary>
       <type>
         <v>Term1 = Term2 = Minimum = term()</v>
@@ -2433,7 +2496,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:monitor(Type, Item) -> MonitorRef</name>
+      <name>monitor(Type, Item) -> MonitorRef</name>
       <fsummary>Start monitoring</fsummary>
       <type>
         <v>Type = process</v>
@@ -2469,7 +2532,7 @@ os_prompt%</pre>
         <note>
           <p>When a process is monitored by registered name, the process
             that has the registered name at the time when
-            <c>erlang:monitor/2</c> is called will be monitored.
+            <c>monitor/2</c> is called will be monitored.
             The monitor will not be effected, if the registered name is
             unregistered.</p>
         </note>
@@ -2503,20 +2566,20 @@ os_prompt%</pre>
           </item>
         </taglist>
         <note>
-          <p>If/when <c>erlang:monitor/2</c> is extended (e.g. to
+          <p>If/when <c>monitor/2</c> is extended (e.g. to
             handle other item types than <c>process</c>), other
             possible values for <c>Object</c>, and <c>Info</c> in the
             <c>'DOWN'</c> message will be introduced.</p>
         </note>
         <p>The monitoring is turned off either when the <c>'DOWN'</c>
           message is sent, or when
-          <seealso marker="#demonitor/1">erlang:demonitor/1</seealso>
+          <seealso marker="#demonitor/1">demonitor/1</seealso>
           is called.</p>
         <p>If an attempt is made to monitor a process on an older node
           (where remote process monitoring is not implemented or one
           where remote process monitoring by registered name is not
           implemented), the call fails with <c>badarg</c>.</p>
-        <p>Making several calls to <c>erlang:monitor/2</c> for the same
+        <p>Making several calls to <c>monitor/2</c> for the same
           <c>Item</c> is not an error; it results in as many, completely
           independent, monitorings.</p>
         <note>
@@ -2591,6 +2654,37 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
+      <name>erlang:nif_error(Reason)</name>
+      <fsummary>Stop execution with a given reason</fsummary>
+      <type>
+        <v>Reason = term()</v>
+      </type>
+      <desc>
+        <p>Works exactly like
+	 <seealso marker="#error/1">erlang:error/1</seealso>,
+	 but Dialyzer thinks that this BIF will return an arbitrary term.
+	 When used in a stub function for a NIF to generate an
+	 exception when the NIF library is not loaded, Dialyzer
+	 will not generate false warnings.</p>
+      </desc>
+    </func>
+    <func>
+      <name>erlang:nif_error(Reason, Args)</name>
+      <fsummary>Stop execution with a given reason</fsummary>
+      <type>
+        <v>Reason = term()</v>
+        <v>Args = [term()]</v>
+      </type>
+      <desc>
+        <p>Works exactly like
+	 <seealso marker="#error/2">erlang:error/2</seealso>,
+	 but Dialyzer thinks that this BIF will return an arbitrary term.
+	 When used in a stub function for a NIF to generate an
+	 exception when the NIF library is not loaded, Dialyzer
+	 will not generate false warnings.</p>
+      </desc>
+    </func>
+    <func>
       <name>node() -> Node</name>
       <fsummary>Name of the local node</fsummary>
       <type>
@@ -2606,7 +2700,7 @@ os_prompt%</pre>
       <name>node(Arg) -> Node</name>
       <fsummary>At which node is a pid, port or reference located</fsummary>
       <type>
-        <v>Arg = pid() | port() | ref()</v>
+        <v>Arg = pid() | port() | reference()</v>
         <v>Node = node()</v>
       </type>
       <desc>
@@ -2683,8 +2777,10 @@ os_prompt%</pre>
           Otherwise, some other point in time is chosen. It is also
           guaranteed that subsequent calls to this BIF returns
           continuously increasing values. Hence, the return value from
-          <c>now()</c> can be used to generate unique time-stamps. It
-          can only be used to check the local time of day if
+          <c>now()</c> can be used to generate unique time-stamps,
+          and if it is called in a tight loop on a fast machine
+          the time of the node can become skewed.</p>
+        <p>It can only be used to check the local time of day if
           the time-zone info of the underlying operating system is
           properly configured.</p>
       </desc>
@@ -2693,14 +2789,17 @@ os_prompt%</pre>
       <name>open_port(PortName, PortSettings) -> port()</name>
       <fsummary>Open a port</fsummary>
       <type>
-        <v>PortName = {spawn, Command} | {spawn_driver, Command} | {spawn_executable, Command} | {fd, In, Out}</v>
+        <v>PortName = {spawn, Command} | {spawn_driver, Command} | {spawn_executable, FileName} | {fd, In, Out}</v>
         <v>&nbsp;Command = string()</v>
+	<v>&nbsp;FileName = [ FileNameChar ] | binary()</v>
+	<v>&nbsp;FileNameChar = int() (1..255 or any Unicode codepoint, see description)</v> 
         <v>&nbsp;In = Out = int()</v>
         <v>PortSettings = [Opt]</v>
-        <v>&nbsp;Opt = {packet, N} | stream | {line, L} | {cd, Dir} | {env, Env} | {args, [ string() ]} | {arg0, string()} | exit_status | use_stdio | nouse_stdio | stderr_to_stdout | in | out | binary | eof</v>
+        <v>&nbsp;Opt = {packet, N} | stream | {line, L} | {cd, Dir} | {env, Env} | {args, [ ArgString ]} | {arg0, ArgString} | exit_status | use_stdio | nouse_stdio | stderr_to_stdout | in | out | binary | eof</v>
         <v>&nbsp;&nbsp;N = 1 | 2 | 4</v>
         <v>&nbsp;&nbsp;L = int()</v>
         <v>&nbsp;&nbsp;Dir = string()</v>
+	<v>&nbsp;&nbsp;ArgString = [ FileNameChar ] | binary()</v>
         <v>&nbsp;&nbsp;Env = [{Name, Val}]</v>
         <v>&nbsp;&nbsp;&nbsp;Name = string()</v>
         <v>&nbsp;&nbsp;&nbsp;Val = string() | false</v>
@@ -2749,7 +2848,7 @@ os_prompt%</pre>
           <item>
 
 	      <p>Works like <c>{spawn, Command}</c>, but only runs
-	      external executables. The <c>Command</c> in it's whole
+	      external executables. The <c>Command</c> in its whole
 	      is used as the name of the executable, including any
 	      spaces. If arguments are to be passed, the
 	      <c>args</c> and <c>arg0</c> <c>PortSettings</c> can be used.</p>
@@ -2763,7 +2862,26 @@ os_prompt%</pre>
 	      executed, the appropriate command interpreter will
 	      implicitly be invoked, but there will still be no
 	      command argument expansion or implicit PATH search.</p>
-	      
+
+	      <p>The name of the executable as well as the arguments
+	      given in <c>args</c> and <c>arg0</c> is subject to
+	      Unicode file name translation if the system is running
+	      in Unicode file name mode. To avoid
+	      translation or force i.e. UTF-8, supply the executable
+	      and/or arguments as a binary in the correct
+	      encoding. See the <seealso
+	      marker="kernel:file">file</seealso> module, the
+	      <seealso marker="kernel:file#native_name_encoding/0">
+	      file:native_name_encoding/0</seealso> function and the 
+	      <seealso marker="stdlib:unicode_usage">stdlib users guide
+	      </seealso> for details.</p>
+
+	      <note>The characters in the name (if given as a list)
+	      can only be &gt; 255 if the Erlang VM is started in
+	      Unicode file name translation mode, otherwise the name
+	      of the executable is limited to the ISO-latin-1
+	      character set.</note>
+  
 	      <p>If the <c>Command</c> cannot be run, an error
 	      exception, with the posix error code as the reason, is
 	      raised. The error reason may differ between operating
@@ -2866,6 +2984,21 @@ os_prompt%</pre>
 	    should not be given in this list. The proper executable name will
 	    automatically be used as argv[0] where applicable.</p>
 
+	    <p>When the Erlang VM is running in Unicode file name
+	    mode, the arguments can contain any Unicode characters and
+	    will be translated into whatever is appropriate on the
+	    underlying OS, which means UTF-8 for all platforms except
+	    Windows, which has other (more transparent) ways of
+	    dealing with Unicode arguments to programs. To avoid
+	    Unicode translation of arguments, they can be supplied as
+	    binaries in whatever encoding is deemed appropriate.</p>
+
+	      <note>The characters in the arguments (if given as a
+	      list of characters) can only be &gt; 255 if the Erlang
+	      VM is started in Unicode file name mode,
+	      otherwise the arguments are limited to the
+	      ISO-latin-1 character set.</note>
+
 	    <p>If one, for any reason, wants to explicitly set the
 	    program name in the argument vector, the <c>arg0</c>
 	    option can be used.</p>
@@ -2881,6 +3014,9 @@ os_prompt%</pre>
 	    responds to this is highly system dependent and no specific 
 	    effect is guaranteed.</p>
 
+	    <p>The unicode file name translation rules of the
+	    <c>args</c> option apply to this option as well.</p>
+
 	  </item> 
 
           <tag><c>exit_status</c></tag>
@@ -2926,7 +3062,7 @@ os_prompt%</pre>
 	    The standard input and standard output handles of the port program
 	    will, if this option is supplied, be opened with the flag
 	    FILE_FLAG_OVERLAPPED, so that the port program can (and has to) do
-	    overlapped I/O on it's standard handles. This is not normally
+	    overlapped I/O on its standard handles. This is not normally
 	    the case for simple port programs, but an option of value for the
 	    experienced Windows programmer. <em>On all other platforms, this 
 	    option is silently discarded</em>.</p> 
@@ -3147,7 +3283,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:port_command(Port, Data, OptionList) -> true|false</name>
+      <name>port_command(Port, Data, OptionList) -> true|false</name>
       <fsummary>Send data to a port</fsummary>
       <type>
         <v>Port = port() | atom()</v>
@@ -3183,10 +3319,6 @@ os_prompt%</pre>
         <note>
           <p>More options may be added in the future.</p>
         </note>
-        <note>
-          <p><c>erlang:port_command/3</c> is currently not auto imported, but
-	     it is planned to be auto imported in OTP R14.</p>
-        </note>
         <p>Failures:</p>
         <taglist>
           <tag><c>badarg</c></tag>
@@ -3815,11 +3947,11 @@ os_prompt%</pre>
           <tag><c>{monitored_by, Pids}</c></tag>
           <item>
             <p>A list of pids that are monitoring the process (with
-              <c>erlang:monitor/2</c>).</p>
+              <c>monitor/2</c>).</p>
           </item>
           <tag><c>{monitors, Monitors}</c></tag>
           <item>
-            <p>A list of monitors (started by <c>erlang:monitor/2</c>)
+            <p>A list of monitors (started by <c>monitor/2</c>)
               that are active for the process. For a local process
               monitor or a remote process monitor by pid, the list item
               is <c>{process, Pid}</c>, and for a remote process
@@ -3856,7 +3988,8 @@ os_prompt%</pre>
           <tag><c>{status, Status}</c></tag>
           <item>
             <p><c>Status</c> is the status of the process. <c>Status</c>
-              is <c>waiting</c> (waiting for a message), <c>running</c>,
+              is <c>exiting</c>, <c>garbage_collecting</c>,
+              <c>waiting</c> (for a message), <c>running</c>,
               <c>runnable</c> (ready to run, but another process is
               running), or <c>suspended</c> (suspended on a "busy" port
               or by the <c>erlang:suspend_process/[1,2]</c> BIF).</p>
@@ -4008,7 +4141,7 @@ os_prompt%</pre>
           terminate, it has no return value - unless the arguments are
           invalid, in which case the function <em>returns the error reason</em>, that is <c>badarg</c>. If you want to be
           really sure not to return you can call
-          <c>erlang:error(erlang:raise(Class, Reason, Stacktrace))</c>
+          <c>error(erlang:raise(Class, Reason, Stacktrace))</c>
           and hope to distinguish exceptions later.</p>
       </desc>
     </func>
@@ -4016,7 +4149,7 @@ os_prompt%</pre>
       <name>erlang:read_timer(TimerRef) -> int() | false</name>
       <fsummary>Number of milliseconds remaining for a timer</fsummary>
       <type>
-        <v>TimerRef = ref()</v>
+        <v>TimerRef = reference()</v>
       </type>
       <desc>
         <p><c>TimerRef</c> is a timer reference returned by
@@ -4039,7 +4172,7 @@ os_prompt%</pre>
       <name>erlang:ref_to_list(Ref) -> string()</name>
       <fsummary>Text representation of a reference</fsummary>
       <type>
-        <v>Ref = ref()</v>
+        <v>Ref = reference()</v>
       </type>
       <desc>
         <p>Returns a string which corresponds to the text
@@ -4218,7 +4351,7 @@ true</pre>
         <v>Dest = pid() | RegName </v>
         <v>&nbsp;LocalPid = pid() (of a process, alive or dead, on the local node)</v>
         <v>Msg = term()</v>
-        <v>TimerRef = ref()</v>
+        <v>TimerRef = reference()</v>
       </type>
       <desc>
         <p>Starts a timer which will send the message <c>Msg</c>
@@ -4431,7 +4564,7 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>spawn(Node, Module, Function, ArgumentList) -> pid()</name>
+      <name>spawn(Node, Module, Function, Args) -> pid()</name>
       <fsummary>Create a new process with a function as entry point on a given node</fsummary>
       <type>
         <v>Node = node()</v>
@@ -4609,7 +4742,7 @@ true</pre>
           <tag><c>monitor</c></tag>
           <item>
             <p>Monitor the new process (just like
-              <seealso marker="#monitor/2">erlang:monitor/2</seealso> does).</p>
+              <seealso marker="#monitor/2">monitor/2</seealso> does).</p>
           </item>
           <tag><c>{priority, Level}</c></tag>
           <item>
@@ -4747,7 +4880,7 @@ true</pre>
         <v>&nbsp;LocalPid = pid() (of a process, alive or dead, on the local node)</v>
         <v>&nbsp;RegName = atom()</v>
         <v>Msg = term()</v>
-        <v>TimerRef = ref()</v>
+        <v>TimerRef = reference()</v>
       </type>
       <desc>
         <p>Starts a timer which will send the message
@@ -5093,9 +5226,9 @@ true</pre>
 	       schedulers actually have bound as requested, call
 	       <seealso marker="#system_info_scheduler_bindings">erlang:system_info(scheduler_bindings)</seealso>.
 	    </p>
-            <p>Schedulers can currently only be bound on newer Linux
-	       and Solaris systems, but more systems will be supported
-	       in the future.
+            <p>Schedulers can currently only be bound on newer Linux,
+	       Solaris, FreeBSD, and Windows systems, but more systems will be
+	       supported in the future.
 	    </p>
             <p>In order for the runtime system to be able to bind schedulers,
 	       the CPU topology needs to be known. If the runtime system fails
@@ -5103,10 +5236,21 @@ true</pre>
 	       For more information on how to define the CPU topology, see 
 	       <seealso marker="#system_flag_cpu_topology">erlang:system_flag(cpu_topology, CpuTopology)</seealso>.
 	    </p>
-	    <p><em>NOTE:</em> If other programs on the system have bound
-	       to processors, e.g. another Erlang runtime system, you
-	       may loose performance when binding schedulers. Therefore,
-	       schedulers are by default not bound.</p>
+            <p>The runtime system will by default bind schedulers to logical
+	       processors using the <c>default_bind</c> bind type if the amount
+	       of schedulers are at least equal to the amount of logical
+	       processors configured, binding of schedulers is supported,
+	       and a CPU topology is available at startup.
+	    </p>
+	    <p><em>NOTE:</em> If the Erlang runtime system is the only
+	       operating system process that binds threads to logical processors,
+	       this improves the performance of the runtime system. However,
+	       if other operating system processes (as for example another Erlang
+	       runtime system) also bind threads to logical processors, there
+	       might be a performance penalty instead. If this is the case you,
+	       are are advised to unbind the schedulers using the
+	       <seealso marker="erl#+sbt">+sbtu</seealso> command line argument,
+	       or <c>erlang:system_flag(scheduler_bind_type, unbound)</c>.</p>
 	    <p>Schedulers can be bound in different ways. The <c>How</c>
 	       argument determines how schedulers are bound. <c>How</c> can
 	       currently be one of:</p>
@@ -5271,7 +5415,7 @@ true</pre>
             <p>Returns <c>{Allocator, Version, Features, Settings}.</c></p>
             <p>Types:</p>
             <list type="bulleted">
-              <item><c>Allocator = undefined | elib_malloc | glibc</c></item>
+              <item><c>Allocator = undefined | glibc</c></item>
               <item><c>Version = [int()]</c></item>
               <item><c>Features = [atom()]</c></item>
               <item><c>Settings = [{Subsystem, [{Parameter, Value}]}]</c></item>
@@ -5286,7 +5430,7 @@ true</pre>
                   implementation used. If <c>Allocator</c> equals
                   <c>undefined</c>, the <c>malloc()</c> implementation
                   used could not be identified. Currently
-                  <c>elib_malloc</c> and <c>glibc</c> can be identified.</p>
+                  <c>glibc</c> can be identified.</p>
               </item>
               <item>
                 <p><c>Version</c> is a list of integers (but not a
@@ -5363,6 +5507,16 @@ true</pre>
 	    <seealso marker="#system_info_allocator_tuple">erlang:system_info({allocator, Alloc})</seealso>.
 	    </p>
           </item>
+          <tag><c>build_type</c></tag>
+          <item>
+            <p>Returns an atom describing the build type of the runtime
+	       system. This is normally the atom <c>opt</c> for optimized.
+               Other possible return values are <c>debug</c>, <c>purify</c>,
+	       <c>quantify</c>, <c>purecov</c>, <c>gcov</c>, <c>valgrind</c>,
+	       <c>gprof</c>, and <c>lcnt</c>. Possible return values
+	       may be added and/or removed at any time without prior notice.
+	     </p>
+          </item>
           <tag><c>c_compiler_used</c></tag>
           <item>
             <p>Returns a two-tuple describing the C compiler used when
@@ -5440,7 +5594,7 @@ true</pre>
 	       <c>CpuTopology</c> type to change.
 	    </p>
           </item>
-          <tag><c>{cpu_topology, defined}</c></tag>
+          <tag><marker id="system_info_cpu_topology_defined"><c>{cpu_topology, defined}</c></marker></tag>
           <item>
             <p>Returns the user defined <c>CpuTopology</c>. For more
 	       information see the documentation of
@@ -5450,12 +5604,14 @@ true</pre>
 	       argument.
 	    </p>
           </item>
-          <tag><c>{cpu_topology, detected}</c></tag>
+          <tag><marker id="system_info_cpu_topology_detected"><c>{cpu_topology, detected}</c></marker></tag>
           <item>
             <p>Returns the automatically detected <c>CpuTopology</c>. The
 	       emulator currently only detects the CPU topology on some newer
-	       linux and solaris systems. For more information see the 
-               documentation of the
+	       Linux, Solaris, FreeBSD, and Windows systems. On Windows system with
+	       more than 32 logical processors the CPU topology is not detected.
+	    </p>
+            <p>For more information see the documentation of the
 	       <seealso marker="#system_info_cpu_topology">cpu_topology</seealso>
 	       argument.
 	    </p>
@@ -5513,52 +5669,16 @@ true</pre>
           </item>
           <tag><c>elib_malloc</c></tag>
           <item>
-            <p>If the emulator uses the <c>elib_malloc</c> memory
-              allocator, a list of two-element tuples containing status
-              information is returned; otherwise, <c>false</c> is
-              returned. The list currently contains the following
-              two-element tuples (all sizes are presented in bytes):</p>
-            <taglist>
-              <tag><c>{heap_size, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the current heap size.</p>
-              </item>
-              <tag><c>{max_alloced_size, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the maximum amount of memory
-                  allocated on the heap since the emulator started.</p>
-              </item>
-              <tag><c>{alloced_size, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the current amount of memory
-                  allocated on the heap.</p>
-              </item>
-              <tag><c>{free_size, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the current amount of free
-                  memory on the heap.</p>
-              </item>
-              <tag><c>{no_alloced_blocks, No}</c></tag>
-              <item>
-                <p>Where <c>No</c> is the current number of allocated
-                  blocks on the heap.</p>
-              </item>
-              <tag><c>{no_free_blocks, No}</c></tag>
-              <item>
-                <p>Where <c>No</c> is the current number of free blocks
-                  on the heap.</p>
-              </item>
-              <tag><c>{smallest_alloced_block, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the size of the smallest
-                  allocated block on the heap.</p>
-              </item>
-              <tag><c>{largest_free_block, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the size of the largest free
-                  block on the heap.</p>
-              </item>
-            </taglist>
+            <p>This option will be removed in a future release.
+	      The return value will always be <c>false</c> since
+	      the elib_malloc allocator has been removed.</p>
+          </item>
+          <tag><marker id="system_info_dist_buf_busy_limit"><c>dist_buf_busy_limit</c></marker></tag>
+          <item>
+            <p>Returns the value of the distribution buffer busy limit
+	    in bytes. This limit can be set on startup by passing the
+	    <seealso marker="erl#+zdbbl">+zdbbl</seealso> command line
+	    flag to <c>erl</c>.</p>
           </item>
           <tag><c>fullsweep_after</c></tag>
           <item>
@@ -5634,11 +5754,34 @@ true</pre>
               information see the <seealso marker="erts:crash_dump">"How to interpret the Erlang crash dumps"</seealso> chapter
               in the ERTS User's Guide.</p>
           </item>
-	  <tag><c>logical_processors</c></tag>
+	  <tag><marker id="logical_processors"><c>logical_processors</c></marker></tag>
+          <item>
+            <p>Returns the detected number of logical processors configured
+	       on the system. The return value is either an integer, or
+	       the atom <c>unknown</c> if the emulator wasn't able to
+	       detect logical processors configured.
+	    </p>
+          </item>
+	  <tag><marker id="logical_processors_available"><c>logical_processors_available</c></marker></tag>
           <item>
-            <p>Returns the number of logical processors detected on the
-	       system as an integer or the atom <c>unknown</c> if the
-	       emulator wasn't able to detect any.
+            <p>Returns the detected number of logical processors available to
+	       the Erlang runtime system. The return value is either an
+	       integer, or the atom <c>unknown</c> if the emulator wasn't
+	       able to detect logical processors available. The number
+	       of logical processors available is less than or equal to
+	       the number of <seealso marker="#logical_processors_online">logical
+	       processors online</seealso>.
+	    </p>
+          </item>
+	  <tag><marker id="logical_processors_online"><c>logical_processors_online</c></marker></tag>
+          <item>
+            <p>Returns the detected number of logical processors online on
+	       the system. The return value is either an integer,
+	       or the atom <c>unknown</c> if the emulator wasn't able to
+	       detect logical processors online. The number of logical
+	       processors online is less than or equal to the number of
+	       <seealso marker="#logical_processors">logical processors
+	       configured</seealso>.
 	    </p>
           </item>
           <tag><c>machine</c></tag>
@@ -5843,6 +5986,26 @@ true</pre>
               <c>get_tcw</c> in "Match Specifications in Erlang",
               <seealso marker="erts:match_spec#get_tcw">ERTS User's Guide</seealso>.</p>
           </item>
+          <tag><marker id="update_cpu_info"><c>update_cpu_info</c></marker></tag>
+          <item>
+            <p>The runtime system rereads the CPU information available and
+	       updates its internally stored information about the
+	       <seealso marker="#system_info_cpu_topology_detected">detected CPU
+	       topology</seealso> and the amount of logical processors
+	       <seealso marker="#logical_processors">configured</seealso>,
+	       <seealso marker="#logical_processors_online">online</seealso>, and
+	       <seealso marker="#logical_processors_available">available</seealso>.
+	       If the CPU information has changed since the last time it was read,
+	       the atom <c>changed</c> is returned; otherwise, the atom
+	       <c>unchanged</c> is returned. If the CPU information has changed
+	       you probably want to
+	       <seealso marker="#system_flag_schedulers_online">adjust the amount
+	       of schedulers online</seealso>. You typically want to have as
+	       many schedulers online as
+	       <seealso marker="#logical_processors_available">logical processors
+	       available</seealso>.
+	    </p>
+          </item>
           <tag><marker id="system_info_version"><c>version</c></marker></tag>
           <item>
             <p>Returns a string containing the version number of the
@@ -5850,9 +6013,23 @@ true</pre>
           </item>
           <tag><c>wordsize</c></tag>
           <item>
-            <p>Returns the word size in bytes as an integer, i.e. on a
-              32-bit architecture 4 is returned, and on a 64-bit
-              architecture 8 is returned.</p>
+            <p>Same as <c>{wordsize, internal}</c></p>
+          </item>
+          <tag><c>{wordsize, internal}</c></tag>
+          <item>
+            <p>Returns the size of Erlang term words in bytes as an
+              integer, i.e. on a 32-bit architecture 4 is returned,
+              and on a pure 64-bit architecture 8 is returned. On a
+              halfword 64-bit emulator, 4 is returned, as the Erlang
+              terms are stored using a virtual wordsize of half the
+              systems wordsize.</p>
+          </item>
+          <tag><c>{wordsize, external}</c></tag>
+          <item>
+	    <p>Returns the true wordsize of the emulator, i.e. the size
+	       of a pointer, in bytes as an integer. On a pure 32-bit
+	       architecture 4 is returned, on both a halfword and pure
+	       64-bit architecture, 8 is returned.</p>
           </item>
         </taglist>
         <note>
@@ -6645,6 +6822,17 @@ true</pre>
               See also
               <seealso marker="#trace_pattern/3">erlang:trace_pattern/3</seealso>.</p>
           </item>
+          <tag><c>call_time</c></tag>
+          <item>
+	      <p>Return the call time values for this function or
+              <c>true</c> for the pseudo function <c>on_load</c> if call
+	      time tracing is active. Returns <c>false</c> otherwise.
+	      The call time values returned, <c>[{Pid, Count, S, Us}]</c>,
+	      is a list of each process that has executed the function and its specific counters.
+              See also
+              <seealso marker="#trace_pattern/3">erlang:trace_pattern/3</seealso>.</p>
+          </item>
+
           <tag><c>all</c></tag>
           <item>
             <p>Return a list containing the <c>{Item, Value}</c> tuples
@@ -6747,13 +6935,13 @@ true</pre>
           </item>
           <tag><c>restart</c></tag>
           <item>
-            <p>For the <c>FlagList</c> option <c>call_count</c>:
+	      <p>For the <c>FlagList</c> option <c>call_count</c> and <c>call_time</c>:
               restart the existing counters. The behaviour is undefined
               for other <c>FlagList</c> options.</p>
           </item>
           <tag><c>pause</c></tag>
           <item>
-            <p>For the <c>FlagList</c> option <c>call_count</c>: pause
+	      <p>For the <c>FlagList</c> option <c>call_count</c> and <c>call_time</c>: pause
               the existing counters. The behaviour is undefined for
               other <c>FlagList</c> options.</p>
           </item>
@@ -6808,6 +6996,23 @@ true</pre>
             <p>The counter value can be read with
               <seealso marker="#trace_info/2">erlang:trace_info/2</seealso>.</p>
           </item>
+          <tag><c>call_time</c></tag>
+          <item>
+            <p>Starts (<c>MatchSpec == true</c>) or stops
+              (<c>MatchSpec == false</c>) call time tracing for all
+              types of function calls. For every function a counter is
+	      incremented when the function is called. Time spent in the function
+	      is accumulated in two other counters, seconds and micro-seconds.
+	      The counters are stored for each call traced process.</p>
+            <p>If call time tracing is started while already running,
+              the count and time is restarted from zero. Running counters can be
+              paused with <c>MatchSpec == pause</c>. Paused and running
+              counters can be restarted from zero with
+              <c>MatchSpec == restart</c>.</p>
+            <p>The counter value can be read with
+              <seealso marker="#trace_info/2">erlang:trace_info/2</seealso>.</p>
+          </item>
+
         </taglist>
         <p>The <c>global</c> and <c>local</c> options are mutually
           exclusive and <c>global</c> is the default (if no options are
@@ -6815,7 +7020,7 @@ true</pre>
           perform a kind of local tracing, and can also not be combined
           with <c>global</c>. A function can be either globally or
           locally traced. If global tracing is specified for a
-          specified set of functions; local, meta and call count
+          specified set of functions; local, meta, call time and call count
           tracing for the matching set of local functions will be
           disabled, and vice versa.</p>
         <p>When disabling trace, the option must match the type of trace
diff --git a/erts/doc/src/erlc.xml b/erts/doc/src/erlc.xml
index 1e8960c22c..ebf76a2afe 100644
--- a/erts/doc/src/erlc.xml
+++ b/erts/doc/src/erlc.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>1997</year><year>2010</year>
+      <year>1997</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -141,6 +141,50 @@
           for compiling native code, which needs to be compiled with the same
           run-time system that it should be run on.</p>
       </item>
+      <tag>-M</tag>
+      <item>
+        <p>Produces a Makefile rule to track headers dependencies. The
+          rule is sent to stdout. No object file is produced.
+        </p>
+      </item>
+      <tag>-MF <em>Makefile</em></tag>
+      <item>
+        <p>Like the <c><![CDATA[-M]]></c> option above, except that the
+          Makefile is written to <em>Makefile</em>. No object
+          file is produced.
+        </p>
+      </item>
+      <tag>-MD</tag>
+      <item>
+        <p>Same as <c><![CDATA[-M -MF <File>.Pbeam]]></c>.
+        </p>
+      </item>
+      <tag>-MT <em>Target</em></tag>
+      <item>
+        <p>In conjunction with <c><![CDATA[-M]]></c> or
+          <c><![CDATA[-MF]]></c>, change the name of the rule emitted
+          to <em>Target</em>.
+        </p>
+      </item>
+      <tag>-MQ <em>Target</em></tag>
+      <item>
+        <p>Like the <c><![CDATA[-MT]]></c> option above, except that
+          characters special to make(1) are quoted.
+        </p>
+      </item>
+      <tag>-MP</tag>
+      <item>
+        <p>In conjunction with <c><![CDATA[-M]]></c> or
+          <c><![CDATA[-MF]]></c>, add a phony target for each dependency.
+        </p>
+      </item>
+      <tag>-MG</tag>
+      <item>
+        <p>In conjunction with <c><![CDATA[-M]]></c> or
+          <c><![CDATA[-MF]]></c>, consider missing headers as generated
+          files and add them to the dependencies.
+        </p>
+      </item>
       <tag>--</tag>
       <item>
         <p>Signals that no more options will follow.
diff --git a/erts/doc/src/escript.xml b/erts/doc/src/escript.xml
index a89449df23..66e904f64f 100644
--- a/erts/doc/src/escript.xml
+++ b/erts/doc/src/escript.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>2007</year><year>2010</year>
+      <year>2007</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -31,7 +31,7 @@
   <com>escript</com>
   <comsummary>Erlang scripting support</comsummary>
   <description>
-    <p><c><![CDATA[escript]]></c> provides support for running short Erlang programs
+    <p><c>escript</c> provides support for running short Erlang programs
       without having to compile them first and an easy way to retrieve the
       command line arguments.</p>
   </description>
@@ -41,10 +41,10 @@
       <name>escript escript-flags script-name script-arg1 script-arg2...</name>
       <fsummary>Run a script written in Erlang</fsummary>
       <desc>
-        <p><c><![CDATA[escript]]></c> runs a script written in Erlang.</p>
+        <p><c>escript</c> runs a script written in Erlang.</p>
         <p>Here follows an example.</p>
         <pre>
-$ <input>cat factorial</input>        
+$ <input>cat factorial</input>
 #!/usr/bin/env escript
 %% -*- erlang -*-
 %%! -smp enable -sname factorial -mnesia debug verbose
@@ -59,11 +59,11 @@ main([String]) ->
     end;
 main(_) ->
     usage().
-        
+
 usage() ->
     io:format("usage: factorial integer\n"),
     halt(1).
-        
+
 fac(0) -> 1;
 fac(N) -> N * fac(N-1).
 $ <input>factorial 5</input>
@@ -74,9 +74,8 @@ $ <input>factorial five</input>
 usage: factorial integer        </pre>
         <p>The header of the Erlang script in the example differs from
         a normal Erlang module.  The first line is intended to be the
-        interpreter line, which invokes
-        <c><![CDATA[escript]]></c>. However if you invoke the
-        <c><![CDATA[escript]]></c> like this</p>
+        interpreter line, which invokes <c>escript</c>. However if you
+        invoke the <c>escript</c> like this</p>
         <pre>
 $ <input>escript factorial 5</input>        </pre>
         <p>the contents of the first line does not matter, but it
@@ -93,13 +92,13 @@ $ <input>escript factorial 5</input>        </pre>
 %%! -smp enable -sname factorial -mnesia debug verbose</pre>
         <p>Such an argument line must start with <c>%%!</c> and the
           rest of the line will interpreted as arguments to the emulator.</p>
-        <p>If you know the location of the <c><![CDATA[escript]]></c> executable, the first
-          line can directly give the path to <c><![CDATA[escript]]></c>. For instance:</p>
+        <p>If you know the location of the <c>escript</c> executable, the first
+          line can directly give the path to <c>escript</c>. For instance:</p>
         <pre>
 #!/usr/local/bin/escript        </pre>
         <p>As any other kind of scripts, Erlang scripts will not work on
           Unix platforms if the execution bit for the script file is not set.
-          (Use <c><![CDATA[chmod +x script-name]]></c> to turn on the execution bit.)
+          (Use <c>chmod +x script-name</c> to turn on the execution bit.)
           </p>
 
         <p>The rest of the Erlang script file may either contain
@@ -108,33 +107,33 @@ $ <input>escript factorial 5</input>        </pre>
 
         <p>An Erlang script file must always contain the function
         <em>main/1</em>.  When the script is run, the
-        <c><![CDATA[main/1]]></c> function will be called with a list
+        <c>main/1</c> function will be called with a list
         of strings representing the arguments given to the script (not
         changed or interpreted in any way).</p>
 
-        <p>If the <c><![CDATA[main/1]]></c> function in the script returns successfully,
+        <p>If the <c>main/1</c> function in the script returns successfully,
           the exit status for the script will be 0. If an exception is generated
           during execution, a short message will be printed and the script terminated
           with exit status 127.</p>
 
-        <p>To return your own non-zero exit code, call <c><![CDATA[halt(ExitCode)]]></c>;
+        <p>To return your own non-zero exit code, call <c>halt(ExitCode)</c>;
           for instance:</p>
         <pre>
 halt(1).</pre>
 
-        <p>Call <c><![CDATA[escript:script_name/0]]></c> from your to
-          script to retrieve the pathname of the script (the pathname
-          is usually, but not always, absolute).</p>
+        <p>Call <seealso marker="#script_name_0">escript:script_name()</seealso>
+	from your to script to retrieve the pathname of the script
+	(the pathname is usually, but not always, absolute).</p>
 
         <p>If the file contains source code (as in the example above),
           it will be processed by the preprocessor <c>epp</c>. This
           means that you for example may use pre-defined macros (such as
-          <c><![CDATA[?MODULE]]></c>) as well as include directives like
-          the <c><![CDATA[-include_lib]]></c> directive. For instance, use</p>
+          <c>?MODULE</c>) as well as include directives like
+          the <c>-include_lib</c> directive. For instance, use</p>
         <pre>
--include_lib("kernel/include/file.hrl").        </pre>
+-include_lib("kernel/include/file.hrl").</pre>
         <p>to include the record definitions for the records used by the
-          <c><![CDATA[file:read_link_info/1]]></c> function.</p>
+          <c>file:read_link_info/1</c> function.</p>
 
         <p>The script will be checked for syntactic and semantic
           correctness before being run. If there are warnings (such as
@@ -144,7 +143,7 @@ halt(1).</pre>
           127.</p>
 
         <p>Both the module declaration and the export declaration of
-          the <c><![CDATA[main/1]]></c> function are optional.</p>
+          the <c>main/1</c> function are optional.</p>
 
         <p>By default, the script will be interpreted. You can force
           it to be compiled by including the following line somewhere
@@ -154,7 +153,10 @@ halt(1).</pre>
         <p>Execution of interpreted code is slower than compiled code.
           If much of the execution takes place in interpreted code it
           may be worthwhile to compile it, even though the compilation
-          itself will take a little while.</p>
+          itself will take a little while. It is also possible to supply
+	  <c>native</c> instead of compile, this will compile the script 
+	  using the native flag, again depending on the characteristics 
+	  of the escript this could or could not be worth while.</p>
 
         <p>As mentioned earlier, it is possible to have a script which
           contains precompiled <c>beam</c> code. In a precompiled
@@ -198,6 +200,180 @@ factorial 5 = 120
 </pre>
       </desc>
     </func>
+    <func>
+      <name>escript:create(FileOrBin, Sections) -> ok | {ok, binary()} | {error, term()}</name>
+      <fsummary>Create an escript</fsummary>
+      <type>
+        <v>FileOrBin = filename() | 'binary'</v>
+        <v>Sections = [Header] Body | Body</v>
+        <v>Header = shebang | {shebang, Shebang}
+                    | comment | {comment, Comment}
+                    | {emu_args, EmuArgs}</v>
+        <v>Shebang = string() | 'default' | 'undefined'</v>
+        <v>Comment = string() | 'default' | 'undefined'</v>
+        <v>EmuArgs = string() | 'undefined'</v>
+        <v>Body = {source, SourceCode}
+                | {beam, BeamCode}
+	        | {archive, ZipArchive}</v>
+        <v>SourceCode = BeamCode = ZipArchive = binary()</v>
+      </type>
+      <desc>
+        <p>The <marker id="create_2"></marker> <c>create/2</c>
+        function creates an escript from a list of sections. The
+        sections can be given in any order. An escript begins with an
+        optional <c>Header</c> followed by a mandatory <c>Body</c>. If
+        the header is present, it does always begin with a
+        <c>shebang</c>, possibly followed by a <c>comment</c> and
+        <c>emu_args</c>. The <c>shebang</c> defaults to
+        <c>"/usr/bin/env escript"</c>. The comment defaults to
+        <c>"This is an -*- erlang -*- file"</c>. The created escript
+        can either be returned as a binary or written to file.</p>
+
+	<p>As an example of how the function can be used, we create an
+	interpreted escript which uses emu_args to set some emulator
+	flag. In this case it happens to disable the smp_support. We
+	do also extract the different sections from the newly created
+	script:</p>
+        <pre>
+&gt; <input>Source = "%% Demo\nmain(_Args) ->\n    io:format(erlang:system_info(smp_support)).\n".</input>
+"%% Demo\nmain(_Args) ->\n    io:format(erlang:system_info(smp_support)).\n"
+&gt; <input>io:format("~s\n", [Source]).</input>
+%% Demo
+main(_Args) ->
+    io:format(erlang:system_info(smp_support)).
+
+ok
+&gt; <input>{ok, Bin} = escript:create(binary, [shebang, comment, {emu_args, "-smp disable"},
+                                      {source, list_to_binary(Source)}]).</input>
+{ok,&lt;&lt;"#!/usr/bin/env escript\n%% This is an -*- erlang -*- file\n%%!-smp disabl"...&gt;&gt;}
+&gt; <input>file:write_file("demo.escript", Bin).</input>
+ok
+&gt; <input>os:cmd("escript demo.escript").</input>
+"false"
+&gt; <input>escript:extract("demo.escript", []).</input>
+{ok,[{shebang,default}, {comment,default}, {emu_args,"-smp disable"},
+     {source,&lt;&lt;"%% Demo\nmain(_Args) ->\n    io:format(erlang:system_info(smp_su"...&gt;&gt;}]}
+	</pre>
+
+	<p>An escript without header can be created like this:</p>
+<pre>
+&gt; <input>file:write_file("demo.erl",
+                  ["%% demo.erl\n-module(demo).\n-export([main/1]).\n\n", Source]).</input>
+ok
+&gt; <input>{ok, _, BeamCode} = compile:file("demo.erl", [binary, debug_info]).</input>
+{ok,demo,
+    &lt;&lt;70,79,82,49,0,0,2,208,66,69,65,77,65,116,111,109,0,0,0,
+      79,0,0,0,9,4,100,...&gt;&gt;}
+&gt; <input>escript:create("demo.beam", [{beam, BeamCode}]).</input>
+ok
+&gt; <input>escript:extract("demo.beam", []).</input>
+{ok,[{shebang,undefined}, {comment,undefined}, {emu_args,undefined},
+     {beam,&lt;&lt;70,79,82,49,0,0,3,68,66,69,65,77,65,116,
+             111,109,0,0,0,83,0,0,0,9,...&gt;&gt;}]}
+&gt; <input>os:cmd("escript demo.beam").</input>
+"true"
+</pre>
+	<p>Here we create an archive script containing both Erlang
+	code as well as beam code. Then we iterate over all files in
+	the archive and collect their contents and some info about
+	them.
+	</p>
+<pre>
+&gt; <input>{ok, SourceCode} = file:read_file("demo.erl").</input>
+{ok,&lt;&lt;"%% demo.erl\n-module(demo).\n-export([main/1]).\n\n%% Demo\nmain(_Arg"...&gt;&gt;}
+&gt; <input>escript:create("demo.escript",
+                 [shebang,
+                  {archive, [{"demo.erl", SourceCode},
+                             {"demo.beam", BeamCode}], []}]).</input>
+ok
+&gt; <input>{ok, [{shebang,default}, {comment,undefined}, {emu_args,undefined},
+     {archive, ArchiveBin}]} = escript:extract("demo.escript", []).</input>
+{ok,[{shebang,default}, {comment,undefined}, {emu_args,undefined},
+     {{archive,&lt;&lt;80,75,3,4,20,0,0,0,8,0,118,7,98,60,105,
+                152,61,93,107,0,0,0,118,0,...&gt;&gt;}]}
+&gt; <input>file:write_file("demo.zip", ArchiveBin).</input>
+ok
+&gt; <input>zip:foldl(fun(N, I, B, A) -> [{N, I(), B()} | A] end, [], "demo.zip").</input>
+{ok,[{"demo.beam",
+      {file_info,748,regular,read_write,
+                 {{2010,3,2},{0,59,22}},
+                 {{2010,3,2},{0,59,22}},
+                 {{2010,3,2},{0,59,22}},
+                 54,1,0,0,0,0,0},
+      &lt;&lt;70,79,82,49,0,0,2,228,66,69,65,77,65,116,111,109,0,0,0,
+        83,0,0,...&gt;&gt;},
+     {"demo.erl",
+      {file_info,118,regular,read_write,
+                 {{2010,3,2},{0,59,22}},
+                 {{2010,3,2},{0,59,22}},
+                 {{2010,3,2},{0,59,22}},
+                 54,1,0,0,0,0,0},
+      &lt;&lt;"%% demo.erl\n-module(demo).\n-export([main/1]).\n\n%% Demo\nmain(_Arg"...&gt;&gt;}]}</pre>
+      </desc>
+    </func>
+    <func>
+      <name>escript:extract(File, Options) -> {ok, Sections} | {error, term()}</name>
+      <fsummary>Parses an escript and extracts its sections</fsummary>
+      <type>
+        <v>File = filename()</v>
+        <v>Options = [] | [compile_source]</v>
+        <v>Sections = Headers Body</v>
+        <v>Headers =  {shebang, Shebang}
+                      {comment, Comment}
+                      {emu_args, EmuArgs}</v>
+        <v>Shebang = string() | 'default' | 'undefined'</v>
+        <v>Comment = string() | 'default' | 'undefined'</v>
+        <v>EmuArgs = string() | 'undefined'</v>
+        <v>Body = {source, SourceCode}
+                | {source, BeamCode}
+                | {beam, BeamCode}
+	        | {archive, ZipArchive}</v>
+        <v>SourceCode = BeamCode = ZipArchive = binary()</v>
+      </type>
+      <desc>
+        <p>The <marker id="extract_2"></marker> <c>extract/2</c>
+        function parses an escript and extracts its sections. This is
+        the reverse of <c>create/2</c>.</p>
+
+	<p>All sections are returned even if they do not exist in the
+	escript. If a particular section happens to have the same
+	value as the default value, the extracted value is set to the
+	atom <c>default</c>. If a section is missing, the extracted
+	value is set to the atom <c>undefined</c>.  </p>
+
+	<p>The <c>compile_source</c> option only affects the result if
+	the escript contains <c>source</c> code. In that case the
+	Erlang code is automatically compiled and <c>{source,
+	BeamCode}</c> is returned instead of <c>{source,
+	SourceCode}</c>.</p>
+
+	<pre>
+&gt; <input>escript:create("demo.escript",
+                 [shebang, {archive, [{"demo.erl", SourceCode},
+                                      {"demo.beam", BeamCode}], []}]).</input>
+ok
+&gt; <input>{ok, [{shebang,default}, {comment,undefined}, {emu_args,undefined},
+     {archive, ArchiveBin}]} =
+              escript:extract("demo.escript", []).</input>
+{ok,[{{archive,&lt;&lt;80,75,3,4,20,0,0,0,8,0,118,7,98,60,105,
+                152,61,93,107,0,0,0,118,0,...&gt;&gt;}
+     {emu_args,undefined}]}
+	</pre>
+      </desc>
+    </func>
+    <func>
+      <name>escript:script_name() -> File</name>
+      <fsummary>Returns the name of an escript</fsummary>
+      <type>
+        <v>File = filename()</v>
+      </type>
+      <desc>
+        <p>The <marker id="script_name_0"></marker>
+        <c>script_name/0</c> function returns the name of the escript
+        being executed. If the function is invoked outside the context
+        of an escript, the behavior is undefined.</p>
+      </desc>
+    </func>
   </funcs>
 
   <section>
@@ -224,6 +400,9 @@ factorial 5 = 120
        Warnings and errors (if any) are written to the standard output, but
        the script will not be run. The exit status will be 0 if there were
        no errors, and 127 otherwise.</item>
+
+       <tag>-n</tag>
+      <item>Compile the escript using the +native flag.</item>
     </taglist>
   </section>
 </comref>
diff --git a/erts/doc/src/init.xml b/erts/doc/src/init.xml
index 33364c709a..b0d0cda4fa 100644
--- a/erts/doc/src/init.xml
+++ b/erts/doc/src/init.xml
@@ -4,7 +4,7 @@
 <erlref>
   <header>
     <copyright>
-      <year>1996</year><year>2009</year>
+      <year>1996</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -67,19 +67,6 @@
       </desc>
     </func>
     <func>
-      <name>get_args() -> [Arg]</name>
-      <fsummary>Get all non-flag command line arguments</fsummary>
-      <type>
-        <v>Arg = atom()</v>
-      </type>
-      <desc>
-        <p>Returns any plain command line arguments as a list of atoms
-          (possibly empty). It is recommended that
-          <c>get_plain_arguments/1</c> is used instead, because of
-          the limited length of atoms.</p>
-      </desc>
-    </func>
-    <func>
       <name>get_argument(Flag) -> {ok, Arg} | error</name>
       <fsummary>Get the values associated with a command line user flag</fsummary>
       <type>
diff --git a/erts/doc/src/match_spec.xml b/erts/doc/src/match_spec.xml
index b9f955e4db..f0390c9db8 100644
--- a/erts/doc/src/match_spec.xml
+++ b/erts/doc/src/match_spec.xml
@@ -60,7 +60,7 @@
 
   <section>
     <title>Grammar</title>
-    <p>A match_spec can be described in this <em>informal</em> grammar:</p>
+    <p>A match_spec used in tracing can be described in this <em>informal</em> grammar:</p>
     <list type="bulleted">
       <item>MatchExpression ::= [ MatchFunction, ... ]
       </item>
@@ -117,6 +117,52 @@
       <c><![CDATA[display]]></c> | <c><![CDATA[caller]]></c> | <c><![CDATA[set_tcw]]></c> |
       <c><![CDATA[silent]]></c></item>
     </list>
+
+    <p>A match_spec used in ets can be described in this <em>informal</em> grammar:</p>
+    <list type="bulleted">
+      <item>MatchExpression ::= [ MatchFunction, ... ]
+      </item>
+      <item>MatchFunction ::= { MatchHead, MatchConditions, MatchBody }
+      </item>
+      <item>MatchHead ::=  MatchVariable | <c><![CDATA['_']]></c> | { MatchHeadPart, ... }
+      </item>
+      <item>MatchHeadPart ::= term() | MatchVariable | <c><![CDATA['_']]></c></item>
+      <item>MatchVariable ::= '$&lt;number&gt;'
+      </item>
+      <item>MatchConditions ::= [ MatchCondition, ...] | <c><![CDATA[[]]]></c></item>
+      <item>MatchCondition ::= { GuardFunction } |
+       { GuardFunction, ConditionExpression, ... }
+      </item>
+      <item>BoolFunction ::= <c><![CDATA[is_atom]]></c> | <c><![CDATA[is_constant]]></c> |
+      <c><![CDATA[is_float]]></c> | <c><![CDATA[is_integer]]></c> | <c><![CDATA[is_list]]></c> |
+      <c><![CDATA[is_number]]></c> | <c><![CDATA[is_pid]]></c> | <c><![CDATA[is_port]]></c> |
+      <c><![CDATA[is_reference]]></c> | <c><![CDATA[is_tuple]]></c> | <c><![CDATA[is_binary]]></c> |
+      <c><![CDATA[is_function]]></c> | <c><![CDATA[is_record]]></c> | <c><![CDATA[is_seq_trace]]></c> |
+      <c><![CDATA['and']]></c> | <c><![CDATA['or']]></c> | <c><![CDATA['not']]></c> | <c><![CDATA['xor']]></c> |
+      <c><![CDATA[andalso]]></c> | <c><![CDATA[orelse]]></c></item>
+      <item>ConditionExpression ::= ExprMatchVariable | { GuardFunction } |
+       { GuardFunction, ConditionExpression, ... } | TermConstruct
+      </item>
+      <item>ExprMatchVariable ::= MatchVariable (bound in the MatchHead) |
+      <c><![CDATA['$_']]></c> | <c><![CDATA['$$']]></c></item>
+      <item>TermConstruct = {{}} | {{ ConditionExpression, ... }} |
+      <c><![CDATA[[]]]></c> | [ConditionExpression, ...] | NonCompositeTerm | Constant
+      </item>
+      <item>NonCompositeTerm ::= term() (not list or tuple)
+      </item>
+      <item>Constant ::= {<c><![CDATA[const]]></c>, term()}
+      </item>
+      <item>GuardFunction ::= BoolFunction | <c><![CDATA[abs]]></c> |
+      <c><![CDATA[element]]></c> | <c><![CDATA[hd]]></c> | <c><![CDATA[length]]></c> | <c><![CDATA[node]]></c> |
+      <c><![CDATA[round]]></c> | <c><![CDATA[size]]></c> | <c><![CDATA[tl]]></c> | <c><![CDATA[trunc]]></c> |
+      <c><![CDATA['+']]></c> | <c><![CDATA['-']]></c> | <c><![CDATA['*']]></c> | <c><![CDATA['div']]></c> |
+      <c><![CDATA['rem']]></c> | <c><![CDATA['band']]></c> | <c><![CDATA['bor']]></c> | <c><![CDATA['bxor']]></c> |
+      <c><![CDATA['bnot']]></c> | <c><![CDATA['bsl']]></c> | <c><![CDATA['bsr']]></c> | <c><![CDATA['>']]></c> |
+      <c><![CDATA['>=']]></c> | <c><![CDATA['<']]></c> | <c><![CDATA['=<']]></c> | <c><![CDATA['=:=']]></c> |
+      <c><![CDATA['==']]></c> | <c><![CDATA['=/=']]></c> | <c><![CDATA['/=']]></c> | <c><![CDATA[self]]></c> |
+      <c><![CDATA[get_tcw]]></c></item>
+      <item>MatchBody ::= [ ConditionExpression, ... ]</item>
+    </list>
   </section>
 
   <section>
@@ -453,8 +499,8 @@
   <section>
     <title>Differences between match specifications in ETS and tracing</title>
     <p>ETS match specifications are there to produce a return
-      value. Usually the expression contains one single
-      <c><![CDATA[ActionTerm]]></c> which defines the return value without having
+      value. Usually the <c><![CDATA[MatchBody]]></c> contains one single
+      <c><![CDATA[ConditionExpression]]></c> which defines the return value without having
       any side effects. Calls with side effects are not allowed in the
       ETS context.</p>
     <p>When tracing there is no return value to produce, the
@@ -530,7 +576,7 @@
       the atom 'strider' and the tuple arity is 3 and return the whole
       object.</p>
     <code type="none"><![CDATA[
-[{{strider,'_'.'_'},
+[{{strider,'_','_'},
   [],
   ['$_']}]
     ]]></code>
diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml
index 824ad6d94e..3733fb2db9 100644
--- a/erts/doc/src/notes.xml
+++ b/erts/doc/src/notes.xml
@@ -4,7 +4,7 @@
 <chapter>
   <header>
     <copyright>
-      <year>2004</year><year>2010</year>
+      <year>2004</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -30,6 +30,1925 @@
   </header>
   <p>This document describes the changes made to the ERTS application.</p>
 
+<section><title>Erts 5.8.4</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+	    <p>Fix binary and iolist overflow problems. Typically
+	    problems arose in length calculation where the result
+	    would exceed (1 bsl 32 - 1).</p>
+          <p>
+	    Own Id: OTP-9118</p>
+        </item>
+        <item>
+          <p>
+	    Using the old erlang shell (i.e. erl instead on werl) on
+	    windows and doing several init:restart's would eventually
+	    hang the VM. That is no longer the case.</p>
+          <p>
+	    Own Id: OTP-9139</p>
+        </item>
+        <item>
+          <p>
+	    Removed recursive C code when printing Erlang terms to
+	    buffers, avoiding stack overflows that could cause VM to
+	    crash.</p>
+          <p>
+	    Own Id: OTP-9140</p>
+        </item>
+        <item>
+          <p>
+	    The send_timeout option in gen_tcp did not work properly
+	    in active mode or with {active,once} options. This is now
+	    corrected.</p>
+          <p>
+	    Own Id: OTP-9145</p>
+        </item>
+        <item>
+          <p>
+	    Fixed various typos across the documentation (Thanks to
+	    Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9154</p>
+        </item>
+        <item>
+	    <p>Remove duplicate stack entries which could occur after
+	    calling certain BIFs.</p>
+          <p>
+	    Own Id: OTP-9163</p>
+        </item>
+        <item>
+          <p>
+	    A race when starting two nodes simultaneously using
+	    run_erl has been removed.</p>
+          <p>
+	    Own Id: OTP-9164</p>
+        </item>
+        <item>
+          <p>
+	    Add documentation on .erlang processing back again
+	    (Thanks to Gabor Liptak)</p>
+          <p>
+	    Own Id: OTP-9189</p>
+        </item>
+        <item>
+          <p>
+	    Remove gratuitous paren in driver_entry(Thanks to Tuncer
+	    Ayaz)</p>
+          <p>
+	    Own Id: OTP-9192</p>
+        </item>
+        <item>
+          <p>
+	    Fix some wrong pointer dereferences (Thanks to Cristian
+	    Greco)</p>
+          <p>
+	    Own Id: OTP-9194</p>
+        </item>
+        <item>
+          <p>
+	    erts: Remove unused variables (Thanks to Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9205</p>
+        </item>
+        <item>
+          <p>
+	    The documentation for <c>init:get_args/0</c> has been
+	    removed. <c>init:get_args/0</c> itself was deprecated in
+	    R9C and removed in R12B. (Thanks to Eric Pailleau.)</p>
+          <p>
+	    Own Id: OTP-9209</p>
+        </item>
+        <item>
+          <p>
+	    Allow user to specify the IP address epmd binds to</p>
+          <p>
+	    The IP address(es) epmd binds to can now be specified by
+	    the user, either via epmd's new "-address" option or (if
+	    that's not used) by setting the environment variable
+	    ERL_EPMD_ADDRESS. Multiple addresses may be specified
+	    using a comma-separated list. If the loopback address is
+	    not in this list, it will be added implicitly, so that
+	    the daemon can be queried by an interactive epmd
+	    process.(Thanks to Holger Wei�)</p>
+          <p>
+	    Own Id: OTP-9213</p>
+        </item>
+        <item>
+          <p>
+	    epmd: include host address in local access check</p>
+          <p>
+	    In FreeBSD jails, the source and destination address of
+	    connections to localhost are changed to be the IP address
+	    of the jail. Consider connections from the host's IP
+	    address to itself (e.g., the source and destination
+	    address match) to be local for the access control checks.
+	    (Thanks to Michal Santos and Tom at diogunix.com)</p>
+          <p>
+	    Own Id: OTP-9214</p>
+        </item>
+        <item>
+          <p>
+	    Fix list returned by net_kernel:epmd_module</p>
+          <p>
+	    Function epmd_module of net_kernel returns a list instead
+	    of an atom, when the epmd_module-flag is used. (Thanks to
+	    Markus Knofe)</p>
+          <p>
+	    Own Id: OTP-9215</p>
+        </item>
+        <item>
+          <p>
+	    Fix epmd's dbg_perror() output</p>
+          <p>
+	    The dbg_perror() function now hands the current errno
+	    value over to dbg_gen_printf(). This fixes the problem
+	    that errno had been reset to zero by the time it was used
+	    (to print the corresponding error message) in the
+	    dbg_gen_printf() function. (Thanks to Holger Wei�)</p>
+          <p>
+	    Own Id: OTP-9223</p>
+        </item>
+        <item>
+          <p>
+	    heart: remove garbage appended to heart command</p>
+          <p>
+	    heart:get_cmd/0 is documented to return an empty string
+	    if the command is cleared. get_cmd/0 returns 2 extra
+	    bytes: 1 byte for the trailing null, 1 byte from the op
+	    (the op is an unsigned char and 2 bytes are allocated for
+	    it in the returned buffer). (Thanks to Michael Santos)</p>
+          <p>
+	    Own Id: OTP-9224</p>
+        </item>
+        <item>
+          <p>
+	    file: fix hang reading compressed files</p>
+          <p>
+	    The gzio driver goes into an infinite loop when reading
+	    past the end of a compressed file. Reported-By: Alex
+	    Morarash (Thanks to Michael Santos)</p>
+          <p>
+	    Own Id: OTP-9245</p>
+        </item>
+        <item>
+	    <p>Eliminate alias warnings from gcc 4.5.2</p>
+          <p>
+	    Own Id: OTP-9250</p>
+        </item>
+        <item>
+          <p>
+	    Unsigned integer may overflow in error message (Thanks to
+	    Michael Santos)</p>
+          <p>
+	    Own Id: OTP-9251</p>
+        </item>
+        <item>
+          <p>
+	    Driver names should be strings, not atoms</p>
+          <p>
+	    Own Id: OTP-9253</p>
+        </item>
+        <item>
+          <p>
+	    driver_entry: Remove gratuitous paren and fix typo
+	    (Thanks to Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9254</p>
+        </item>
+        <item>
+          <p>
+	    Fix format specifiers in erl_exit messages</p>
+          <p>
+	    Fix an error message by using an unsigned integer
+	    specifier as seen in a tweet by @metabrew: #erlang VM
+	    crashed with "no next heap size found: -2090496108,
+	    offset 0", suddenly allocated all available RAM</p>
+          <p>
+	    Also correct mis-typed string formats in bif.c.(Thanks to
+	    Michael Santos)</p>
+          <p>
+	    Own Id: OTP-9262</p>
+        </item>
+        <item>
+          <p>
+	    net_drv: remove unused tcp request id inet_drv: remove
+	    gratuitous assignment (Thanks to Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9263</p>
+        </item>
+        <item>
+          <p>
+	    Teach run_erl RUN_ERL_DISABLE_FLOWCNTRL for disabling
+	    flow control</p>
+          <p>
+	    Flow control can cause unwanted behaviour of the beam
+	    process, if accidentally hit Ctrl-S (instead of Ctrl-D to
+	    detach) the entire beam may be blocked.</p>
+          <p>
+	    Fix this problem by making it possible to turn off flow
+	    control by setting the environment variable
+	    RUN_ERL_DISABLE_FLOWCNTRL. (Thanks to Jonas Faklkevik)</p>
+          <p>
+	    Own Id: OTP-9270</p>
+        </item>
+        <item>
+	    <p>The following bugs due to missing memory barriers have
+	    been fixed:</p> <list> <item><p> ETS tables using the
+	    <c>write_concurrency</c> option could potentially get
+	    into an internally inconsistent state. </p></item>
+	    <item><p> ETS tables using the <c>ordered_set</c> option
+	    could potentially get into an internally inconsistent
+	    state. </p></item> <item><p> A number of memory barriers
+	    have been added when building with the
+	    <c>libatomic_ops</c> API (i.e. when passing
+	    <c>--with-libatomic_ops=PATH</c> to <c>configure</c>) and
+	    the tilera atomics API (i.e. when building for the tilera
+	    chip). Note that these bugs were due to erroneous usage
+	    of the APIs, and not in the implementations of the APIs.
+	    When using these APIs the following bugs where
+	    present:</p> <list> <item><p> The BIF
+	    <c>erlang:ports/0</c> could return an erroneous result.
+	    </p></item> <item><p> A thread blocking other threads
+	    during code loading, or setup of tracing could
+	    potentially read invalid data. </p></item> <item><p>
+	    Fixation of ETS tables could potentially get into an
+	    internally inconsistent state. </p></item> </list>
+	    </item> </list>
+          <p>
+	    Own Id: OTP-9281</p>
+        </item>
+        <item>
+          <p>
+	    Fix halfword bug for ETS ordered_set when doing
+	    <c>select/match</c> with partly bound key.</p>
+          <p>
+	    Own Id: OTP-9292</p>
+        </item>
+        <item>
+          <p>
+	    Fix bug in <c>code:is_module_native</c> that caused crash
+	    for deleted modules.</p>
+          <p>
+	    Own Id: OTP-9298</p>
+        </item>
+        <item>
+          <p>
+	    Calling <c>driver_async_cancel()</c> could cause a
+	    scheduler thread to enter an eternal loop doing no useful
+	    work. (Thanks to Anders Ramsell)</p>
+          <p>
+	    Own Id: OTP-9302</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+          <p>
+	    New <c>enif_is_exception function</c> to allow NIFs to
+	    determine whether an <c>ERL_NIF_TERM</c> represents an
+	    exception. (Thanks to Steve Vinoski)</p>
+          <p>
+	    Own Id: OTP-9150</p>
+        </item>
+        <item>
+          <p>
+	    A process being garbage collected by another process
+	    could be scheduled on another scheduler. This prevented
+	    this scheduler from doing any useful work until the
+	    garbage collection was done. This either occurred due to
+	    a explicit call to the <c>garbage_collect/1</c> BIF, or
+	    due to a garbage collection part of code loading. A
+	    process being garbage collected like this will now not be
+	    scheduled until the garbage collection has completed.</p>
+          <p>
+	    Own Id: OTP-9211</p>
+        </item>
+        <item>
+          <p>
+	    Remove unnecessary validation copy in
+	    prim_file:drv_command/3 (Thanks to Tony Rogvall)</p>
+          <p>
+	    Own Id: OTP-9276</p>
+        </item>
+        <item>
+          <p>
+	    Symbolic link handling on windows have been slightly
+	    updated to map error conditions more consequently and
+	    correctly read directory links created outside of the
+	    Erlang environment.</p>
+          <p>
+	    Own Id: OTP-9279</p>
+        </item>
+        <item>
+          <p>
+	    Due to standard library DLL mismatches between versions
+	    of OpenSSL and Erlang/OTP, OpenSSL is now linked
+	    statically to the crypto driver on Windows. This fixes
+	    problems starting crypto when running Erlang as a service
+	    on all Windows versions.</p>
+          <p>
+	    Own Id: OTP-9280</p>
+        </item>
+        <item>
+	    <p>Halfword emulator memory handling improvements:</p>
+	    <list> <item><p>Much more of internal memory structures
+	    have been made able to use "high" memory and are no
+	    longer restricted to the 4Gb limit that still applies for
+	    all process heap data.</p> </item> <item><p>Fixed faulty
+	    values from <c>erlang:memory()</c> caused by 32-bit
+	    counter overflow.</p> </item> <item><p>New counter
+	    <c>low</c> in <c>erlang:memory()</c> that sums up all
+	    memory restricted by 4Gb limit.</p> </item> </list>
+          <p>
+	    Own Id: OTP-9291 Aux Id: seq11841 </p>
+        </item>
+        <item>
+          <p>
+	    The value set in the undocumented and unsupported
+	    ERL_version_FLAGS (e.g. ERL_R14B03_FLAGS) environment
+	    variable can now be overridden by the command line
+	    (similar to ERL_AFLAGS).</p>
+          <p>
+	    Own Id: OTP-9297</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.3.2</title>
+
+    <section><title>Known Bugs and Problems</title>
+      <list>
+        <item>
+          <p>
+	    Fix halfword emulator bug in <c>ets:select_delete</c> for
+	    <c>ordered_set</c> that caused emulator to crash.</p>
+          <p>
+	    Own Id: OTP-9258 Aux Id: seq11836 </p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.3.1</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Ets table type ordered_set could order large integer keys
+	    wrongly on pure 64bit platforms. This is now corrected.</p>
+          <p>
+	    Own Id: OTP-9181</p>
+        </item>
+        <item>
+          <p>
+	    The status of a process was unnecessarily set to waiting
+	    before a process was enqueued in a run queue. This bug
+	    was harmless up until OTP-R14B01. In OTP-R14B02
+	    <c>erlang:hibernate/3</c> was fixed (OTP-9125). After the
+	    introduction of OTP-9125, the previously harmless process
+	    status bug sometimes caused erroneous badarg exceptions
+	    from <c>process_info()</c>.</p>
+          <p>
+	    OTP-9125 also introduced a thread unsafe access to the
+	    status field of a process which now also have been fixed.</p>
+          <p>
+	    *** INCOMPATIBILITY with noxs ***</p>
+          <p>
+	    Own Id: OTP-9197</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.3</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    The scroll wheel now scrolls the werl window on Windows.</p>
+          <p>
+	    Own Id: OTP-8985</p>
+        </item>
+        <item>
+          <p>
+	    Some malformed distribution messages could cause VM to
+	    crash, this is now corrected.</p>
+          <p>
+	    Own Id: OTP-8993</p>
+        </item>
+        <item>
+          <p>
+	    The OS function getifaddrs() can return NULL in some
+	    address fields for e.g PPP and tunnel devices which
+	    caused the emulator to segfault. This bug has now been
+	    corrected.</p>
+          <p>
+	    Own Id: OTP-8996</p>
+        </item>
+        <item>
+          <p>
+	    The expression &lt;&lt;A:0&gt;&gt; would always produce
+	    an empty binary, even if <c>A</c> was not an integer.
+	    Corrected to cause a <c>badarg</c> exception if the type
+	    of <c>A</c> is invalid. (Thanks to Zvi.)</p>
+          <p>
+	    Own Id: OTP-8997</p>
+        </item>
+        <item>
+          <p>
+	    A bug that potentially could cause an emulator crash when
+	    deleting an ETS-table has been fixed. A resource leak
+	    when hitting the maximum amount of ETS-tables allowed has
+	    also been fixed.</p>
+          <p>
+	    Own Id: OTP-8999</p>
+        </item>
+        <item>
+          <p>
+	    A bug in the <c>exit/2</c> BIF could potentially cause an
+	    emulator crash.</p>
+          <p>
+	    Own Id: OTP-9005</p>
+        </item>
+        <item>
+          <p>
+	    Due to a bug in glibc the runtime system could abort
+	    while trying to destroy a mutex. The runtime system will
+	    now issue a warning instead of aborting.</p>
+          <p>
+	    Own Id: OTP-9009</p>
+        </item>
+        <item>
+          <p>
+	    A bug in epmd could create strange behaviour when
+	    listen() calls failed. This is now corrected thanks to
+	    Steve Vinoski.</p>
+          <p>
+	    Own Id: OTP-9024</p>
+        </item>
+        <item>
+	    <p>When setting file_info the win32_driver will now
+	    correctly set access and modified time. Previously these
+	    entities were swapped.</p>
+          <p>
+	    Own Id: OTP-9046</p>
+        </item>
+        <item>
+          <p>
+	    Setting scheduler bind type to <c>unbound</c> failed if
+	    binding of schedulers wasn't supported, or if CPU
+	    topology wasn't present. This even though the
+	    documentation stated that it is possible to set the bind
+	    type to <c>unbound</c>.</p>
+          <p>
+	    Own Id: OTP-9056 Aux Id: Seq11779 </p>
+        </item>
+        <item>
+	    <p>Two problems were fixed in crash dump: The time left
+	    for timers are now shown as unsigned integers and the
+	    contents of ordered_set ETS tables is no longer
+	    included.</p>
+          <p>
+	    Own Id: OTP-9057</p>
+        </item>
+        <item>
+          <p>
+	    The VM could fail to set IP_TOS and SO_PRIORITY in
+	    certain situations, either because sockets were supplied
+	    as open file descriptors, or because SO_PRIORITY by
+	    default was set higher than the user can explicitly set
+	    it to. Those situations are now handled.</p>
+          <p>
+	    Own Id: OTP-9069</p>
+        </item>
+        <item>
+          <p>
+	    Wx on MacOS X generated complains on stderr about certain
+	    cocoa functions not beeing called from the "Main thread".
+	    This is now corrected.</p>
+          <p>
+	    Own Id: OTP-9081</p>
+        </item>
+        <item>
+          <p>
+	    Fix a couple typos in driver_entry(3) (thanks to Tuncer
+	    Ayaz).</p>
+          <p>
+	    Own Id: OTP-9085</p>
+        </item>
+        <item>
+          <p>
+	    Mention that "-detached" implies "-noinput"</p>
+          <p>
+	    Clarify that specifying "-noinput" is unnecessary if the
+	    "-detached" flag is given. (thanks to Holger Wei�)</p>
+          <p>
+	    Own Id: OTP-9086</p>
+        </item>
+        <item>
+          <p>
+	    A potential problem (found by code inspection) when
+	    calling a fun whose code was not loaded has been fixed.</p>
+          <p>
+	    Own Id: OTP-9095</p>
+        </item>
+        <item>
+          <p>
+	    The emulator could get into a state where it didn't check
+	    for I/O.</p>
+          <p>
+	    Own Id: OTP-9105 Aux Id: Seq11798 </p>
+        </item>
+        <item>
+          <p>
+	    Attempting to create binaries exceeding 2Gb (using for
+	    example <c>term_to_binary/1</c>) would crash the emulator
+	    with an attempt to allocate huge amounts of memory.
+	    (Thanks to Jon Meredith.)</p>
+          <p>
+	    Own Id: OTP-9117</p>
+        </item>
+        <item>
+          <p>
+	    Fix erlang:hibernate/3 on HiPE enabled emulator (Thanks
+	    to Paul Guyot)</p>
+          <p>
+	    Own Id: OTP-9125</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+	    <p>From this release, the previously experimental
+	    halfword emulator is now official. It can be enabled by
+	    giving the <c>--enable-halfword-emulator</c> option to
+	    the <c>configure</c> script.</p>
+	    <p>The halfword emulator is a 64-bit application, but
+	    uses halfwords (32-bit words) for all data in Erlang
+	    processes, therefore using less memory and being faster
+	    than the standard 64-bit emulator. The total size of all
+	    BEAM code and all process data for all processes is
+	    limited to 4Gb, but ETS tables and off-heap binaries are
+	    only limited by the amount of available memory.</p>
+          <p>
+	    Own Id: OTP-8941</p>
+        </item>
+        <item>
+          <p>
+	    32-bit atomic memory operations have been introduced
+	    internally in the run time system, and are now used where
+	    appropriate. There were previously only atomic memory
+	    operations of word size available. The 32-bit atomic
+	    memory operations slightly reduce memory consumption, and
+	    slightly improve performance on 64-bit runtime systems.</p>
+          <p>
+	    Own Id: OTP-8974</p>
+        </item>
+        <item>
+          <p>
+	    Performance enhancements for looking up timer-entries and
+	    removing timers from the wheel.</p>
+          <p>
+	    Own Id: OTP-8990</p>
+        </item>
+        <item>
+          <p>
+	    Write accesses to ETS tables have been optimized by
+	    reducing the amount of atomic memory operations needed
+	    during a write access.</p>
+          <p>
+	    Own Id: OTP-9000</p>
+        </item>
+        <item>
+          <p>
+	    Strange C coding in the VM made the -D_FORTIFY_SOURCE
+	    option to gcc-4.5 react badly. The code is now cleaned up
+	    so that it's accepted by gcc-4.5.</p>
+          <p>
+	    Own Id: OTP-9025</p>
+        </item>
+        <item>
+          <p>
+	    The memory footprint for loaded code has been somewhat
+	    reduced (especially in the 64-bit BEAM machine).</p>
+          <p>
+	    Own Id: OTP-9030</p>
+        </item>
+        <item>
+          <p>
+	    The maximum number of allowed arguments for an Erlang
+	    function has been lowered from 256 to 255, so that the
+	    number of arguments can now fit in a byte.</p>
+          <p>
+	    Own Id: OTP-9049</p>
+        </item>
+        <item>
+          <p>
+	    Dependency generation for Makefiles has been added to the
+	    compiler and erlc. See the manual pages for
+	    <c>compile</c> and <c>erlc</c>. (Thanks to Jean-Sebastien
+	    Pedron.)</p>
+          <p>
+	    Own Id: OTP-9065</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.2</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+	    <p> Fix format_man_pages so it handles all man sections
+	    and remove warnings/errors in various man pages. </p>
+          <p>
+	    Own Id: OTP-8600</p>
+        </item>
+        <item>
+          <p>
+	    The <c>configure</c> command line argument <seealso
+	    marker="doc/installation_guide:INSTALL#How-to-Build-and-Install-ErlangOTP_A-Closer-Look-at-the-individual-Steps_Configuring">--enable-ethread-pre-pentium4-compatibility</seealso>
+	    had no effect. This option is now also automatically
+	    enabled if required on the build machine.</p>
+          <p>
+	    Own Id: OTP-8847</p>
+        </item>
+        <item>
+          <p>
+	    Windows 2003 and Windows XP pre SP3 would sometimes not
+	    start the Erlang R14B VM at all due to a bug in the cpu
+	    topology detection. The bug affects Windows only, no
+	    other platform is even remotely affected. The bug is now
+	    corrected.</p>
+          <p>
+	    Own Id: OTP-8876</p>
+        </item>
+        <item>
+          <p>
+	    The HiPE run-time in the 64-bit emulator could do a
+	    64-bit write to a 32-bit struct field. It happened to be
+	    harmless on Intel/AMD processors. Corrected. (Thanks to
+	    Mikael Pettersson.)</p>
+          <p>
+	    Own Id: OTP-8877</p>
+        </item>
+        <item>
+          <p>
+	    A bug in <seealso
+	    marker="erl_driver#erl_drv_tsd_get">erl_drv_tsd_get()</seealso>
+	    and <seealso
+	    marker="erl_nif#enif_tsd_get">enif_tsd_get()</seealso>
+	    could cause an emulator crash. These functions are
+	    currently not used in OTP. That is, the crash only occur
+	    on systems with user implemented NIF libraries, or
+	    drivers that use one of these functions.</p>
+          <p>
+	    Own Id: OTP-8889</p>
+        </item>
+        <item>
+          <p>
+	    Calling <c>erlang:system_info({cpu_topology,
+	    CpuTopologyType})</c> with another <c>CpuTopologyType</c>
+	    element than one of the documented atoms <c>defined</c>,
+	    <c>detected</c>, or <c>used</c> caused an emulator crash.
+	    (Thanks to Paul Guyot)</p>
+          <p>
+	    Own Id: OTP-8914</p>
+        </item>
+        <item>
+          <p>
+	    The ERTS internal rwlock implementation could get into an
+	    inconsistent state. This bug was very seldom triggered,
+	    but could be during heavy contention. The bug was
+	    introduced in R14B (erts-5.8.1).</p>
+          <p>
+	    The bug was most likely to be triggered when using the
+	    <c>read_concurrency</c> option on an ETS table that was
+	    frequently accessed from multiple processes doing lots of
+	    writes and reads. That is, in a situation where you
+	    typically don't want to use the <c>read_concurrency</c>
+	    option in the first place.</p>
+          <p>
+	    Own Id: OTP-8925 Aux Id: OTP-8544 </p>
+        </item>
+        <item>
+          <p>
+	    Tracing to port could cause an emulator crash when
+	    unloading the trace driver.</p>
+          <p>
+	    Own Id: OTP-8932</p>
+        </item>
+        <item>
+          <p>
+	    Removed use of CancelIoEx on Windows that had been shown
+	    to cause problems with some drivers.</p>
+          <p>
+	    Own Id: OTP-8937</p>
+        </item>
+        <item>
+          <p>
+	    The fallback implementation used when no native atomic
+	    implementation was found did not compile. (Thanks to
+	    Patrick Baggett, and Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-8944</p>
+        </item>
+        <item>
+          <p>
+	    Some integer values used during load balancing could
+	    under rare circumstances wrap causing a load unbalance
+	    between schedulers.</p>
+          <p>
+	    Own Id: OTP-8950</p>
+        </item>
+        <item>
+          <p>
+	    The windows VM now correctly handles appending to large
+	    files (> 4GB).</p>
+          <p>
+	    Own Id: OTP-8958</p>
+        </item>
+        <item>
+          <p>
+	    Name resolving of IPv6 addresses has been implemented for
+	    Windows versions that support it. The use of ancient
+	    resolver flags (AI_V4MAPPED | AI_ADDRCONFIG) to the
+	    getaddrinfo() function has been removed since e.g FreeBSD
+	    regard mapped IPv4 addresses to be a security problem and
+	    the semantics of the address configured flag is
+	    uncertain.</p>
+          <p>
+	    Own Id: OTP-8969</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+          <p>
+	    The help texts produced by the <c>configure</c> scripts
+	    in the top directory and in the erts directory have been
+	    aligned and cleaned up.</p>
+          <p>
+	    Own Id: OTP-8859</p>
+        </item>
+        <item>
+          <p>
+	    When the runtime system had fewer schedulers than logical
+	    processors, the system could get an unnecessarily large
+	    amount reader groups.</p>
+          <p>
+	    Own Id: OTP-8861</p>
+        </item>
+        <item>
+          <p>
+	    <c>run_rel</c> has been updated to support Solaris's
+	    /dev/ptmx device and to load the necessary STREAMS
+	    modules so that <c>to_erl</c> can provide terminal echo
+	    of keyboard input. (Thanks to Ryan Tilder.)</p>
+          <p>
+	    Own Id: OTP-8878</p>
+        </item>
+        <item>
+          <p>
+	    The Erlang VM now supports Unicode filenames. The feature
+	    is turned on by default on systems where Unicode
+	    filenames are mandatory (Windows and MacOSX), but can be
+	    enabled on other systems with the '+fnu' emulator option.
+	    Enabling the Unicode filename feature on systems where it
+	    is not default is however considered experimental and not
+	    to be used for production. Together with the Unicode file
+	    name support, the concept of "raw filenames" is
+	    introduced, which means filenames provided without
+	    implicit unicode encoding translation. Raw filenames are
+	    provided as binaries, not lists. For further information,
+	    see stdlib users guide and the chapter about using
+	    Unicode in Erlang. Also see the file module manual page.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8887</p>
+        </item>
+        <item>
+	    <p>Buffer overflows have been prevented in <c>erlc</c>,
+	    <c>dialyzer</c>, <c>typer</c>, <c>run_test</c>,
+	    <c>heart</c>, <c>escript</c>, and <c>erlexec</c>.</p>
+	    (Thanks to Michael Santos.)
+          <p>
+	    Own Id: OTP-8892</p>
+        </item>
+        <item>
+          <p>
+	    The runtime system is now less eager to suspend processes
+	    sending messages over the distribution. The default value
+	    of the distribution buffer busy limit has also been
+	    increased from 128 KB to 1 MB. This in order to improve
+	    throughput.</p>
+          <p>
+	    Own Id: OTP-8901</p>
+        </item>
+        <item>
+          <p>
+	    The distribution buffer busy limit can now be configured
+	    at system startup. For more information see the
+	    documentation of the <c>erl</c> <seealso
+	    marker="erl#+zdbbl">+zdbbl</seealso> command line flag.
+	    (Thanks to Scott Lystig Fritchie)</p>
+          <p>
+	    Own Id: OTP-8912</p>
+        </item>
+        <item>
+          <p>
+	    The inet driver internal buffer stack implementation has
+	    been rewritten in order to reduce lock contention.</p>
+          <p>
+	    Own Id: OTP-8916</p>
+        </item>
+        <item>
+          <p>
+	    New ETS option <c>compressed</c>, to enable a more
+	    compact storage format at the expence of heavier table
+	    operations. For test and evaluation, <c>erl +ec</c> can
+	    be used to force compression on all ETS tables.</p>
+          <p>
+	    Own Id: OTP-8922 Aux Id: seq11658 </p>
+        </item>
+        <item>
+          <p>
+	    There is now a new function inet:getifaddrs/0 modeled
+	    after C library function getifaddrs() on BSD and LInux
+	    that reports existing interfaces and their addresses on
+	    the host. This replaces the undocumented and unsupported
+	    inet:getiflist/0 and inet:ifget/2.</p>
+          <p>
+	    Own Id: OTP-8926</p>
+        </item>
+        <item>
+          <p>
+	    Support for detection of CPU topology and binding of
+	    schedulers on FreeBSD 8 have been added. (Thanks to Paul
+	    Guyot)</p>
+          <p>
+	    Own Id: OTP-8939</p>
+        </item>
+        <item>
+          <p>
+	    Several bugs related to hibernate/3 and HiPE have been
+	    corrected. (Thanks to Paul Guyot.)</p>
+          <p>
+	    Own Id: OTP-8952</p>
+        </item>
+        <item>
+          <p>
+	    Support for soft and hard links on Windows versions and
+	    filesystems that support them is added.</p>
+          <p>
+	    Own Id: OTP-8955</p>
+        </item>
+        <item>
+          <p>
+	    The win32 virtual machine is now linked large address
+	    aware. his allows the Erlang VM to use up to 3 gigs of
+	    address space on Windows instead of the default of 2
+	    gigs.</p>
+          <p>
+	    Own Id: OTP-8956</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.1.2</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+	    <p> Fix that the documentation top index generator can
+	    handle an Ericsson internal application group. </p>
+          <p>
+	    Own Id: OTP-8875</p>
+        </item>
+        <item>
+	    <p>In embedded mode, on_load handlers that called
+	    <c>code:priv_dir/1</c> or other functions in <c>code</c>
+	    would hang the system. Since the <c>crypto</c>
+	    application now contains an on_loader handler that calls
+	    <c>code:priv_dir/1</c>, including the <c>crypto</c>
+	    application in the boot file would prevent the system
+	    from starting.</p>
+	    <p>Also extended the <c>-init_debug</c> option to print
+	    information about on_load handlers being run to
+	    facilitate debugging.</p>
+          <p>
+	    Own Id: OTP-8902 Aux Id: seq11703 </p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.1.1</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Windows 2003 and Windows XP pre SP3 would sometimes not
+	    start the Erlang R14B VM at all due to a bug in the cpu
+	    topology detection. The bug affects Windows only, no
+	    other platform is even remotely affected. The bug is now
+	    corrected.</p>
+          <p>
+	    Own Id: OTP-8876</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.1</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Very small floating point numbers generated errors when
+	    converting from list to float in some versions of the VM,
+	    this is now corrected so that i.e.
+	    list_to_float("1.0e-324"). returns 0.0 in all versions of
+	    Erlang.</p>
+          <p>
+	    Own Id: OTP-7178</p>
+        </item>
+        <item>
+          <p>
+	    Windows Vista and Windows 7 file system virtualization,
+	    which makes "old style" windows programs execute in a
+	    file system sandbox, was previously unintentionally
+	    turned on for the Erlang VM. This is now corrected so
+	    that i.e. writes to C:\Program Files\. without
+	    administrator privileges will fail.</p>
+          <p>
+	    Own Id: OTP-7405</p>
+        </item>
+        <item>
+          <p>
+	    Fix faulty 64-bit integer term output from drivers. Large
+	    64-bits integers did not generate correct bignums and
+	    could even cause emulator crash. Only affects drivers
+	    using ERL_DRV_INT64 or ERL_DRV_UINT64, introduced in
+	    R13B03.</p>
+          <p>
+	    Own Id: OTP-8716</p>
+        </item>
+        <item>
+          <p>
+	    Fixed: inet:setopts(S, [{linger,{true,2}}]) returned
+	    {error,einval} for SCTP sockets. The inet_drv had a bug
+	    when checking the option size.</p>
+          <p>
+	    Own Id: OTP-8726 Aux Id: seq11617 </p>
+        </item>
+        <item>
+	    <p>Fix libm linking with --as-needed flag
+          <p>
+	    When building with "--as-needed" linker flags on Linux
+	    the build will fail. This has now been fixed.</p>
+          <p>
+	    (Thanks to Christian Faulhammer)</p></p>
+          <p>
+	    Own Id: OTP-8728</p>
+        </item>
+        <item>
+          <p>
+	    gen_udp:connect/3 was broken for SCTP enabled builds. It
+	    did not detect remote end errors as it should.</p>
+          <p>
+	    Own Id: OTP-8729</p>
+        </item>
+        <item>
+	    <p>Reduce the risk of integer wrapping in bin vheap size
+	    counting.</p> <p>The vheap size series will now use the
+	    golden ratio instead of doubling and fibonacci
+	    sequences.</p>
+          <p>
+	    Own Id: OTP-8730</p>
+        </item>
+        <item>
+          <p>
+	    ETS ordered_set containing <c>[]</c> as key could cause
+	    strange thing to happen, like an infinite hanging
+	    <c>ets:select</c>.</p>
+          <p>
+	    Own Id: OTP-8732</p>
+        </item>
+        <item>
+	    <p>reference() has been substituted for ref() in the
+	    documentation.</p>
+          <p>
+	    Own Id: OTP-8733</p>
+        </item>
+        <item>
+          <p>
+	    When a native compiled module called a not loaded
+	    non-native compiled module that had an on_load function,
+	    the export entries were trashed after code loading so on
+	    the next call from the native compiled module to the
+	    non-native compiled the emulator crashed. This bug has
+	    now been fixed.</p>
+          <p>
+	    Own Id: OTP-8736</p>
+        </item>
+        <item>
+          <p>
+	    HiPE-enabled Erlang VMs running on BSD systems sometimes
+	    generated messages like "Yikes! erts_alloc() returned
+	    misaligned address 0x8016a512c". Fixed. (Thanks to Mikael
+	    Pettersson.)</p>
+          <p>
+	    Own Id: OTP-8769</p>
+        </item>
+        <item>
+          <p>
+	    A race condition in <c>erts_poll()</c> could cause delay
+	    of poll for I/O.</p>
+          <p>
+	    Own Id: OTP-8773</p>
+        </item>
+        <item>
+          <p>
+	    Removed some potential vulnerabilities from the Erlang
+	    Port Mapper Daemon (epmd) and straightened up access
+	    control. Also removed hazardous interfaces allowing
+	    anyone on a machine to forcefully unregister other nodes.
+	    This means that the ei_unregister/erl_unregister
+	    interfaces in erl_interface is rendered not only error
+	    prone and mystifying as before, but totally ineffective.
+	    The old behaviour of unchecked node unregistering can be
+	    restored if needed, see epmd documentation for details.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8780</p>
+        </item>
+        <item>
+	    <p> Building in a source tree without prebuilt platform
+	    independent build results failed on the SSL examples
+	    when: </p> <list><item> cross building. This has been
+	    solved by not building the SSL examples during a cross
+	    build. </item><item> building on Windows. </item></list>
+          <p>
+	    Own Id: OTP-8791</p>
+        </item>
+        <item>
+          <p>
+	    inet:getsockopt for SCTP sctp_default_send_param had a
+	    bug to not initialize required feilds causing random
+	    answers. It is now corrected.</p>
+          <p>
+	    Own Id: OTP-8795 Aux Id: seq11655 </p>
+        </item>
+        <item>
+          <p>
+	    The hipe_bifs:get_hrvtime/0 BIF now always returns a real
+	    value even if the "perfctr" Linux kernel extension is not
+	    available. It used to return a dummy value. (Thanks to
+	    Mikael Pettersson.)</p>
+          <p>
+	    Own Id: OTP-8798</p>
+        </item>
+        <item>
+          <p>
+	    Calling a native-code compiled module with an
+	    <c>on_load</c> function could cause a crash. (Thanks to
+	    Mikael Pettersson.)</p>
+          <p>
+	    Own Id: OTP-8799</p>
+        </item>
+        <item>
+	    <p>The emulator could crash while writing a crash dump if
+	    native-compiled modules had been loaded. (Thanks to Paul
+	    Guyot.)</p>
+          <p>
+	    Own Id: OTP-8801</p>
+        </item>
+        <item>
+          <p>
+	    The garbage collector could crash if invoked from
+	    native-compiled code after a call to a BIF. (Thanks to
+	    Paul Guyot.)</p>
+          <p>
+	    Own Id: OTP-8821</p>
+        </item>
+        <item>
+          <p>
+	    A rare memory leak in binary:matches is removed</p>
+          <p>
+	    Own Id: OTP-8823</p>
+        </item>
+        <item>
+	    <p>For a socket in the HTTP packet mode, the return value
+	    from <c>gen_tcp:recv/2,3</c> if there is an error in the
+	    header will be <c>{ok,{http_error,String}}</c> instead of
+	    <c>{error,{http_error,String}}</c> to be consistent with
+	    <c>ssl:recv/2,3</c>.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8831</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+          <p>
+	    ets:select_reverse/{1,2,3} are now documented.</p>
+          <p>
+	    Own Id: OTP-7863</p>
+        </item>
+        <item>
+          <p>
+	    External format of integers changed to make full use of
+	    all 32 bits of INTEGER_EXT. This is a compatible change
+	    as old code can read full 32-bit integers but only
+	    produce 28-bit integers as INTEGER_EXT.</p>
+          <p>
+	    Own Id: OTP-8540 Aux Id: seq11534 </p>
+        </item>
+        <item>
+          <p>
+	    Large parts of the <c>ethread</c> library have been
+	    rewritten. The <c>ethread</c> library is an Erlang
+	    runtime system internal, portable thread library used by
+	    the runtime system itself.</p>
+          <p>
+	    Most notable improvement is a reader optimized rwlock
+	    implementation which dramatically improve the performance
+	    of read-lock/read-unlock operations on multi processor
+	    systems by avoiding ping-ponging of the rwlock cache
+	    lines. The reader optimized rwlock implementation is used
+	    by miscellaneous rwlocks in the runtime system that are
+	    known to be read-locked frequently, and can be enabled on
+	    ETS tables by passing the <seealso
+	    marker="stdlib:ets#new_2_read_concurrency">{read_concurrency,
+	    true}</seealso> option upon table creation. See the
+	    documentation of <seealso
+	    marker="stdlib:ets#new/2">ets:new/2</seealso> for more
+	    information. The reader optimized rwlock implementation
+	    can be fine tuned when starting the runtime system. For
+	    more information, see the documentation of the <seealso
+	    marker="erts:erl#+rg">+rg</seealso> command line argument
+	    of <c>erl</c>.</p>
+          <p>
+	    There is also a new implementation of rwlocks that is not
+	    optimized for readers. Both implementations interleaves
+	    readers and writers during contention as opposed to,
+	    e.g., the NPTL (Linux) pthread rwlock implementation
+	    which use either a reader or writer preferred strategy.
+	    The reader/writer preferred strategy is problematic since
+	    it starves threads doing the non-preferred operation.</p>
+          <p>
+	    The new rwlock implementations in general performs better
+	    in ERTS than common pthread implementations. However, in
+	    some extremely heavily contended cases this is not the
+	    case. Such heavy contention can more or less only appear
+	    on ETS tables. This when multiple processes do very large
+	    amounts of write locked operations simultaneously on the
+	    same table. Such use of ETS is bad regardless of rwlock
+	    implementation, will never scale, and is something we
+	    strongly advise against.</p>
+          <p>
+	    The new rwlock implementations depend on atomic
+	    operations. If no native atomic implementation is found,
+	    a fallback solution will be used. Using the fallback
+	    implies a performance degradation. That is, it is more
+	    important now than before to build OTP with a native
+	    atomic implementation.</p>
+          <p>
+	    The <c>ethread</c> library contains native atomic
+	    implementations for, x86 (32 and 64 bit), powerpc (32
+	    bit), sparc V9 (32 and 64 bit), and tilera (32 bit). On
+	    other hardware gcc's builtin support for atomic memory
+	    access will be used if such exists. If no such support is
+	    found, <c>configure</c> will warn about no atomic
+	    implementation available.</p>
+          <p>
+	    The <c>ethread</c> library can now also use the
+	    <c>libatomic_ops</c> library for atomic memory accesses.
+	    This makes it possible for the Erlang runtime system to
+	    utilize optimized native atomic operations on more
+	    platforms than before. If <c>configure</c> warns about no
+	    atomic implementation available, try using the
+	    <c>libatomic_ops</c> library. Use the <seealso
+	    marker="doc/installation_guide:INSTALL#How-to-Build-and-Install-ErlangOTP_A-Closer-Look-at-the-individual-Steps_Configuring">--with-libatomic_ops=PATH</seealso>
+	    <c>configure</c> command line argument when specifying
+	    where the <c>libatomic_ops</c> installation is located.
+	    The <c>libatomic_ops</c> library can be downloaded from:
+	    <url
+	    href="http://www.hpl.hp.com/research/linux/atomic_ops/">http://www.hpl.hp.com/research/linux/atomic_ops/</url></p>
+          <p>
+	    The changed API of the <c>ethread</c> library has also
+	    caused modifications in the Erlang runtime system.
+	    Preparations for the to come "delayed deallocation"
+	    feature has also been done since it depends on the
+	    <c>ethread</c> library.</p>
+          <p>
+	    <em>Note</em>: When building for x86, the <c>ethread</c>
+	    library will now use instructions that first appeared on
+	    the pentium 4 processor. If you want the runtime system
+	    to be compatible with older processors (back to 486) you
+	    need to pass the <seealso
+	    marker="doc/installation_guide:INSTALL#How-to-Build-and-Install-ErlangOTP_A-Closer-Look-at-the-individual-Steps_Configuring">--enable-ethread-pre-pentium4-compatibility</seealso>
+	    <c>configure</c> command line argument when configuring
+	    the system.</p>
+          <p>
+	    Own Id: OTP-8544</p>
+        </item>
+        <item>
+          <p>
+	    erlang:localtime_to_universaltime({{2008, 8, 1}, {0, 0,
+	    0}},true) when TZ=UTC now behaves consistently on all
+	    Unix platforms.</p>
+          <p>
+	    The problem fixed was originally reported by Paul Guyot
+	    on erlang-bugs mailing list:</p>
+          <p>
+	    http://www.erlang.org/pipermail/erlang-bugs/2008-November/001077.html</p>
+          <p>
+	    Own Id: OTP-8580</p>
+        </item>
+        <item>
+          <p>
+	    Optimization reducing memory consumption by two words per
+	    ETS object.</p>
+          <p>
+	    Own Id: OTP-8737</p>
+        </item>
+        <item>
+          <p>
+	    Fixes for unsupported halfword-emulator</p>
+          <p>
+	    Own Id: OTP-8745</p>
+        </item>
+        <item>
+          <p>
+	    NIF 64-bit integer support; <c>enif_get_int64</c>,
+	    <c>enif_get_uint64</c>, <c>enif_make_int64</c>,
+	    <c>enif_make_uint64</c>.</p>
+          <p>
+	    Own Id: OTP-8746</p>
+        </item>
+        <item>
+          <p>
+	    Alignment of trailing data in messages has been adjusted.
+	    This in order to be able to pass data of any type as
+	    trailing data in the future.</p>
+          <p>
+	    Own Id: OTP-8754</p>
+        </item>
+        <item>
+          <p>
+	    The obsolete/driver.h header file has been removed. It
+	    has been obsolete and deprecated since R8B. Drivers that
+	    still include obsolete/driver.h must be updated to
+	    include erl_driver.h.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8758</p>
+        </item>
+        <item>
+          <p>
+	    Added erlang:system_info(build_type) which makes it
+	    easier to chose drivers, NIF libraries, etc based on
+	    build type of the runtime system.</p>
+          <p>
+	    The NIF library for crypto can now be built for valgrind
+	    and/or debug as separate NIF libraries that will be
+	    automatically loaded if the runtime system has been built
+	    with a matching build type.</p>
+          <p>
+	    Own Id: OTP-8760</p>
+        </item>
+        <item>
+          <p>
+	    Further lessened the memory requirements of ETS objects.</p>
+          <p>
+	    Own Id: OTP-8762</p>
+        </item>
+        <item>
+	    <p>The broken elib_malloc alternate memory allocator has
+	    been removed. <c>erlang:system_info(elib_malloc)</c> will
+	    always return <c>false</c>, and in R15,
+	    <c>erlang:system_info(elib_malloc)</c> will fail with a
+	    <c>badarg</c> exception.</p>
+          <p>
+	    Own Id: OTP-8764</p>
+        </item>
+        <item>
+          <p>
+	    Calling <c>erlang:system_info/1</c> with the new argument
+	    <c>update_cpu_info</c> will make the runtime system
+	    reread and update the internally stored CPU information.
+	    For more information see the documentation of <seealso
+	    marker="erlang#update_cpu_info">erlang:system_info(update_cpu_info)</seealso>.</p>
+          <p>
+	    The CPU topology is now automatically detected on Windows
+	    systems with less than 33 logical processors. The runtime
+	    system will now, also on Windows, by default bind
+	    schedulers to logical processors using the
+	    <c>default_bind</c> bind type if the amount of schedulers
+	    is at least equal to the amount of logical processors
+	    configured, binding of schedulers is supported, and a CPU
+	    topology is available at startup.</p>
+          <p>
+	    Own Id: OTP-8765</p>
+        </item>
+        <item>
+          <p>
+	    The SMP ERTS internal child waiter thread used on Linux
+	    system with NPTL was unintentionally disabled during
+	    cross compilation rewrites (OTP-8323 in R13B03). It has
+	    now been re-enabled. Enabling it again gives a slight
+	    performance improvement.</p>
+          <p>
+	    Own Id: OTP-8774</p>
+        </item>
+        <item>
+          <p>
+	    <c>epmd</c> used to generate a message to the syslog when
+	    it started up, which could be annoying. This has been
+	    changed to only generate the message if the debug swith
+	    is given. (Thanks to Michael Santos.)</p>
+          <p>
+	    Own Id: OTP-8775</p>
+        </item>
+        <item>
+          <p>
+	    The scheduler wakeup threshold is now possible to adjust
+	    at system boot. For more information see the <seealso
+	    marker="erl#+swt">+swt</seealso> command line argument of
+	    <c>erl</c>.</p>
+          <p>
+	    Own Id: OTP-8811</p>
+        </item>
+        <item>
+          <p>
+	    The undocumented function inet:ifget/2 has been improved
+	    to return interface hardware address (MAC) on platforms
+	    supporting getaddrinfo() (such as BSD unixes). Note it
+	    still does not work on all platforms for example not
+	    Windows nor Solaris, so the function is still
+	    undocumented.</p>
+          <p>
+	    Buffer overflow and field init bugs for inet:ifget/2 and
+	    inet:getservbyname/2 has also been fixed.</p>
+          <p>
+	    Thanks to Michael Santos.</p>
+          <p>
+	    Own Id: OTP-8816</p>
+        </item>
+        <item>
+          <p>
+	    Optimizations for MIPS when using gcc atomics. (Thanks to
+	    Steve Vinoski)</p>
+          <p>
+	    Own Id: OTP-8834</p>
+        </item>
+        <item>
+          <p>
+	    Lock optimization in timer functionality.</p>
+          <p>
+	    Own Id: OTP-8835</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Known Bugs and Problems</title>
+      <list>
+        <item>
+	    <p>Fix epmd and build environment to build on VxWorks</p>
+          <p>
+	    Own Id: OTP-8838</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Driver threads, such as async threads, using <seealso
+	    marker="erl_driver#ErlDrvPDL">port data locks</seealso>
+	    peeked at the port status field without proper locking
+	    when looking up the driver queue.</p>
+          <p>
+	    Own Id: OTP-8475</p>
+        </item>
+        <item>
+          <p>
+	    The use of <c>mmap()</c> was unnecessarily disabled when
+	    cross compiling.</p>
+          <p>
+	    The <c>configure</c> arguments <c>--with-ssl</c>, and
+	    <c>--with-odbc</c> refused to accept libraries outside of
+	    <c>$erl_xcomp_sysroot</c> when cross compiling for no
+	    good reason.</p>
+          <p>
+	    The <c>configure</c> argument <c>--with-odbc</c> didn't
+	    handle the value <c>yes</c> correct.</p>
+          <p>
+	    The <c>configure</c> arguments <c>--with-odbc</c>, and
+	    <c>--without-odbc</c> have also been added to the
+	    configure help.</p>
+          <p>
+	    (Thanks to Steve Vinoski for reporting these issues)</p>
+          <p>
+	    Own Id: OTP-8484</p>
+        </item>
+        <item>
+          <p>
+	    A call to the BIF <c>unregister(RegName)</c> when a port
+	    had the name <c>RegName</c> registered in the runtime
+	    system without SMP support caused a runtime system crash.
+	    (Thanks to Per Hedeland for the bugfix and test case.)</p>
+          <p>
+	    Own Id: OTP-8487</p>
+        </item>
+        <item>
+          <p>
+	    The runtime system crashed if fewer logical processors
+	    were found than reported by <c>sysconf(
+	    SC_NPROCESSORS_CONF)</c>.</p>
+          <p>
+	    Own Id: OTP-8549</p>
+        </item>
+        <item>
+          <p>
+	    Fix memory management bug causing crash of non-SMP
+	    emulator with async threads enabled. The bug did first
+	    appear in R13B03.</p>
+          <p>
+	    Own Id: OTP-8591 Aux Id: seq11554 </p>
+        </item>
+        <item>
+          <p>
+	    Port locks could be prematurely destroyed.</p>
+          <p>
+	    Own Id: OTP-8612</p>
+        </item>
+        <item>
+	    <p>The <c>empd</c> program could loop and consume 100%
+	    CPU time if an unexpected error ocurred in
+	    <c>listen()</c> or <c>accept()</c>. Now <c>epmd</c> will
+	    terminate if a non-recoverable error occurs. (Thanks to
+	    Michael Santos.)</p>
+          <p>
+	    Own Id: OTP-8618</p>
+        </item>
+        <item>
+          <p>
+	    When kernel poll has been enabled, a livelock could in
+	    rare circumstances occur. Problem reported by Chetan
+	    Ahuja, fix by Mikael Pettersson.</p>
+          <p>
+	    Own Id: OTP-8632</p>
+        </item>
+        <item>
+          <p>
+	    Windows: Closing port of program that stalled without
+	    reading all data could deadlock scheduler thread.</p>
+          <p>
+	    Own Id: OTP-8641</p>
+        </item>
+        <item>
+          <p>
+	    On some combination of Montavista Linux on Cavium Octeon
+	    processors, some socket-related system calls returned
+	    other numbers than -1 for errors. This caused a core dump
+	    in inet_drv.c. Now the code works around this problem.</p>
+          <p>
+	    Own Id: OTP-8654</p>
+        </item>
+        <item>
+          <p>
+	    Missing memory barriers in <c>erts_poll()</c> could cause
+	    the runtime system to hang indefinitely.</p>
+          <p>
+	    Own Id: OTP-8658</p>
+        </item>
+        <item>
+          <p>
+	    <c>ethr_rwmutex_tryrlock()</c> acquired and refused to
+	    acquire a lock with inverted logic. The lock was however
+	    never acquired in a thread unsafe manner. (Thanks to JR
+	    Zhang for noting this issue)</p>
+          <p>
+	    Own Id: OTP-8662</p>
+        </item>
+        <item>
+          <p>
+	    Extreme combinations of register/unregister in a highly
+	    parallell SMP application could crash the VM. The error
+	    is corrected.</p>
+          <p>
+	    Own Id: OTP-8663</p>
+        </item>
+        <item>
+          <p>
+	    On Windows, files are now opened with FILE_SHARE_DELETE
+	    to get closer to Unix semantics.</p>
+          <p>
+	    Own Id: OTP-8667</p>
+        </item>
+        <item>
+          <p>
+	    <c>erlang:system_info(multi_scheduling)</c> sometimes
+	    erroneously returned <c>enabled</c> when it should have
+	    returned <c>blocked</c>.</p>
+          <p>
+	    Own Id: OTP-8675</p>
+        </item>
+        <item>
+          <p>
+	    Fix bug causing <c>erlang:decode_packet</c> and
+	    <c>enif_make_string</c> to generate faulty strings with
+	    negative character values for ascii values larger than
+	    127. (Thanks to Paul Guyot)</p>
+          <p>
+	    Own Id: OTP-8685</p>
+        </item>
+        <item>
+          <p>
+	    <c>open_port/2</c> with the <c>spawn</c> and
+	    <c>spawn_executable</c> options can include an
+	    <c>{env,Env}</c> option. In some cases unsetting
+	    variables would not work on Unix (typically if more
+	    variables were unset than were actually present in the
+	    environment).</p>
+          <p>
+	    Own Id: OTP-8701</p>
+        </item>
+        <item>
+          <p>
+	    A user defined CPU topology set via a call to <seealso
+	    marker="erlang#system_flag_cpu_topology">erlang:system_flag(cpu_topology,
+	    CPUTopology)</seealso> was not properly verified, and
+	    could in worst case cause an emulator crash. The emulator
+	    crash could only occur when a user defined CPU topology
+	    already existed and was redefined.</p>
+          <p>
+	    Own Id: OTP-8710</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+          <p>
+	    The grammar for match specifications in ERTS users guide
+	    only described the tracing dialect of match
+	    specifications. An informal grammar for the ETS dialect
+	    is added.</p>
+          <p>
+	    Own Id: OTP-8086 Aux Id: seq11333 </p>
+        </item>
+        <item>
+          <p>
+	    The module binary from EEP31 (and EEP9) is implemented.</p>
+          <p>
+	    Own Id: OTP-8217</p>
+        </item>
+        <item>
+          <p>
+	    New NIF API function <c>enif_make_new_binary</c></p>
+          <p>
+	    Own Id: OTP-8474</p>
+        </item>
+        <item>
+          <p>
+	    The guard BIF <c>is_boolean/1</c> (introduced in R10B)
+	    has now been included in the lists of BIFs allowed in
+	    guards in the Reference Manual.</p>
+          <p>
+	    Own Id: OTP-8477</p>
+        </item>
+        <item>
+          <p>
+	    Added function <c>zip:foldl/3</c> to iterate over zip
+	    archives.</p>
+          <p>
+	    Added functions to create and extract escripts. See
+	    <c>escript:create/2</c> and <c>escript:extract/2</c>.</p>
+          <p>
+	    The undocumented function <c>escript:foldl/3</c> has been
+	    removed. The same functionality can be achieved with the
+	    more flexible functions <c>escript:extract/2</c> and
+	    <c>zip:foldl/3</c>.</p>
+          <p>
+	    Record fields has been annotated with type info. Source
+	    files as been adapted to fit within 80 chars and trailing
+	    whitespace has been removed.</p>
+          <p>
+	    Own Id: OTP-8521</p>
+        </item>
+        <item>
+	    <p>A regular expression with many levels of parenthesis
+	    could cause a buffer overflow. That has been corrected.
+	    (Thanks to Michael Santos.)</p>
+          <p>
+	    Own Id: OTP-8539</p>
+        </item>
+        <item>
+          <p>
+	    <c>erlang:decode_packet(httph_bin,..)</c> could return
+	    corrupt header strings or even crash the VM. This has
+	    been fixed. It only happened on 32-bit VM if the header
+	    name was unknown and between 16 and 20 characters long.
+	    Sockets with simular <c>packet</c> option did not suffer
+	    from this bug.</p>
+          <p>
+	    Own Id: OTP-8548</p>
+        </item>
+        <item>
+	    <p>New NIF features:</p> <list><item> Send messages from
+	    a NIF, or from thread created by NIF, to any local
+	    process (<c>enif_send</c>) </item><item> Store terms
+	    between NIF calls (<c>enif_alloc_env</c>,
+	    <c>enif_make_copy</c>) </item><item> Create binary terms
+	    with user defined memory management
+	    (<c>enif_make_resource_binary</c>) </item></list> <p>And
+	    some incompatible changes made to the API. For more
+	    information see the warning text in <seealso
+	    marker="erl_nif">erl_nif(3)</seealso>.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8555</p>
+        </item>
+        <item>
+	    <p>If the '<c>fop</c>' program (needed for building PDF
+	    files) cannot not be found, it is now possible to build
+	    the HTML and man pages anyway (there will also be dummy
+	    PDF files with no real content created). (Thanks to
+	    Tuncer Ayaz.)</p>
+          <p>
+	    Own Id: OTP-8559</p>
+        </item>
+        <item>
+	    <p>When defining macros the closing right parenthesis
+	    before the dot is now mandatory.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8562</p>
+        </item>
+        <item>
+	    <p>Local and imported functions now override the
+	    auto-imported BIFs when the names clash. The pre R14
+	    behaviour was that auto-imported BIFs would override
+	    local functions. To avoid that old programs change
+	    behaviour, the following will generate an error:</p>
+	    <list><item><p>Doing a call without explicit module name
+	    to a local function having a name clashing with the name
+	    of an auto-imported BIF that was present (and
+	    auto-imported) before OTP R14A</p></item>
+	    <item><p>Explicitly importing a function having a name
+	    clashing with the name of an autoimported BIF that was
+	    present (and autoimported) before OTP R14A</p></item>
+	    <item><p>Using any form of the old compiler directive
+	    <c>nowarn_bif_clash</c></p></item> </list> <p>If the BIF
+	    was added or auto-imported in OTP R14A or later,
+	    overriding it with an import or a local function will
+	    only result in a warning,</p> <p>To resolve clashes, you
+	    can either use the explicit module name <c>erlang</c> to
+	    call the BIF, or you can remove the auto-import of that
+	    specific BIF by using the new compiler directive
+	    <c>-compile({no_auto_import,[F/A]}).</c>, which makes all
+	    calls to the local or imported function without explicit
+	    module name pass without warnings or errors.</p> <p>The
+	    change makes it possible to add auto-imported BIFs
+	    without breaking or silently changing old code in the
+	    future. However some current code ingeniously utilizing
+	    the old behaviour or the <c>nowarn_bif_clash</c> compiler
+	    directive, might need changing to be accepted by the
+	    compiler.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8579</p>
+        </item>
+        <item>
+          <p>
+	    A bug in re that could cause certain regular expression
+	    matches never to terminate is corrected. (Thanks to
+	    Michael Santos and Gordon Guthrie.)</p>
+          <p>
+	    Own Id: OTP-8589</p>
+        </item>
+        <item>
+	    <p>The erlang:open_port spawn and spawn_executable
+	    directives can include an <c>{env,Env}</c> directive to
+	    set up environment variables for the spawned process. A
+	    bug prevented applications from using <c>{env,Env}</c> to
+	    set an environment variable whose value ended with a
+	    '<c>=</c>' (equal sign) character; the trailing equal
+	    sign was mistaken as an indication that an environment
+	    variable was to be cleared from the environment of the
+	    spawned process. (Thanks to Steve Vinoski.)</p>
+          <p>
+	    Own Id: OTP-8614</p>
+        </item>
+        <item>
+	    <p><c>receive</c> statements that can only read out a
+	    newly created reference are now specially optimized so
+	    that it will execute in constant time regardless of the
+	    number of messages in the receive queue for the process.
+	    That optimization will benefit calls to
+	    <c>gen_server:call()</c>. (See <c>gen:do_call/4</c> for
+	    an example of a receive statement that will be
+	    optimized.)</p>
+          <p>
+	    Own Id: OTP-8623</p>
+        </item>
+        <item>
+          <p>
+	    The functions file:advise/4 and file:datasync/1 have been
+	    added. (Thanks to Filipe David Manana.)</p>
+          <p>
+	    Own Id: OTP-8637</p>
+        </item>
+        <item>
+          <p>
+	    New NIF API functions: <c>enif_make_atom_len</c>,
+	    <c>enif_make_existing_atom_len</c>,
+	    <c>enif_make_string_len</c>, <c>enif_get_atom_length</c>,
+	    <c>enif_get_list_length</c>, <c>enif_is_list</c>,
+	    <c>enif_is_tuple</c> (by Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-8640</p>
+        </item>
+        <item>
+          <p>
+	    Support for using gcc's built-in functions for atomic
+	    memory access has been added. This functionallity will be
+	    used if available and no other native atomic
+	    implementation in ERTS is available.</p>
+          <p>
+	    Own Id: OTP-8659</p>
+        </item>
+        <item>
+          <p>
+	    The number of spinlocks used when implementing atomic
+	    fall-backs when no native atomic implementation is
+	    available has been increased from 16 to 1024.</p>
+          <p>
+	    Own Id: OTP-8660</p>
+        </item>
+        <item>
+          <p>
+	    Writer preferred pthread read/write locks has been
+	    enabled on Linux.</p>
+          <p>
+	    Own Id: OTP-8661</p>
+        </item>
+        <item>
+          <p>
+	    The runtime system will by default bind schedulers to
+	    logical processors using the <c>default_bind</c> bind
+	    type if the amount of schedulers are at least equal to
+	    the amount of logical processors configured, binding of
+	    schedulers is supported, and a CPU topology is available
+	    at startup. </p>
+          <p>
+	    <em>NOTE:</em> If the Erlang runtime system is the only
+	    operating system process that binds threads to logical
+	    processors, this improves the performance of the runtime
+	    system. However, if other operating system processes (as
+	    for example another Erlang runtime system) also bind
+	    threads to logical processors, there might be a
+	    performance penalty instead. If this is the case you, are
+	    are advised to unbind the schedulers using the <seealso
+	    marker="erl#+sbt">+sbtu</seealso> command line argument,
+	    or by invoking <seealso
+	    marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type,
+	    unbound)</seealso>.</p>
+          <p>
+	    Own Id: OTP-8666</p>
+        </item>
+        <item>
+          <p>
+	    The recently added BIFs erlang:min/2, erlang:max/2 and
+	    erlang:port_command/3 are now auto-imported (as they were
+	    originally intended to be). Due to the recent compiler
+	    change (OTP-8579), the only impact on old code defining
+	    it's own min/2, max/2 or port_command/3 functions will be
+	    a warning, the local functions will still be used. The
+	    warning can be removed by using
+	    -compile({no_auto_import,[min/2,max/2,port_command/3]}).
+	    in the source file.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8669 Aux Id: OTP-8579 </p>
+        </item>
+        <item>
+          <p>
+	    There is a new option 'exclusive' to file:open/2 that
+	    uses the OS O_EXCL flag where supported to open the file
+	    in exclusive mode.</p>
+          <p>
+	    Own Id: OTP-8670</p>
+        </item>
+        <item>
+          <p>
+	    Now, binary_to_term/2 is auto-imported. This will cause a
+	    compile warning if and only if a module has got a local
+	    function with that name.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8671</p>
+        </item>
+        <item>
+          <p>
+	    Alignment of scheduler data and run queues were adjusted.</p>
+          <p>
+	    Own Id: OTP-8673</p>
+        </item>
+        <item>
+	    <p>Call time breakpoint tracing</p> <list><item>Introduce
+	    a <c>call_time</c> option to
+	    <c>erlang:trace_pattern/3</c>.This option enables call
+	    time breakpoint tracing on code that is executed by
+	    processes with call tracing enabled. Call time tracing
+	    stores the number of calls and the time spent of each
+	    function with this trace pattern enabled. The information
+	    can be retrieved with <c>erlang:trace_info/2</c></item>
+	    <item>Add a scheduler array for BpData. To solve the
+	    issue of multiple schedulers constantly updating the head
+	    pointer to the bp data wheel, each scheduler now has its
+	    own entrypoint to the wheel. This head pointer can be
+	    updated without a locking being taken.</item></list>
+	    <p>Teach call count tracing to use atomics</p>
+	    <list><item>Call count previously used a global lock for
+	    accessing and writing its counter in the breakpoint. This
+	    is now changed to atomics instead.</item> <item>The
+	    change will let call count tracing and cprof to scale
+	    better when increasing the number of
+	    schedulers.</item></list>
+          <p>
+	    Own Id: OTP-8677</p>
+        </item>
+        <item>
+	    <p><c>eprof</c> has been reimplemented with support in
+	    the Erlang virtual machine and is now both faster (i.e.
+	    slows down the code being measured less) and scales much
+	    better. In measurements we saw speed-ups compared to the
+	    old eprof ranging from 6 times (for sequential code that
+	    only uses one scheduler/core) up to 84 times (for
+	    parallel code that uses 8 cores).</p>
+	    <p>Note: The API for the <c>eprof</c> has been cleaned up
+	    and extended. See the documentation.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8706</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
 <section><title>Erts 5.7.5.1</title>
 
     <section><title>Fixed Bugs and Malfunctions</title>
@@ -144,7 +2063,7 @@
         <item>
 	    <p>A number of bugs concerning re and unicode are
 	    corrected:</p>
-	    <p>re:compile no longer looses unicode option, which also
+	    <p>re:compile no longer loses unicode option, which also
 	    fixes bug in re:split.</p>
 	    <p>re:replace now handles unicode charlist replacement
 	    argument</p>
@@ -4716,7 +6635,7 @@
         </item>
         <item>
           <p>The runtime system with SMP support did not slowly adjust
-            it's view of time when the system time suddenly changed.</p>
+            its view of time when the system time suddenly changed.</p>
           <p>Timeouts could sometimes timeout too early on the runtime
             system with SMP support.</p>
           <p>Own Id: OTP-6202</p>
diff --git a/erts/doc/src/run_erl.xml b/erts/doc/src/run_erl.xml
index 7bf7f559c5..c9784299b3 100644
--- a/erts/doc/src/run_erl.xml
+++ b/erts/doc/src/run_erl.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>1999</year><year>2009</year>
+      <year>1999</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -144,6 +144,14 @@
       <item>The size (in bytes) of a log file before switching to a
        new log file. Default is 100000, minimum is 1000 and maximum is
        approximately 2^30.</item>
+      <tag>RUN_ERL_DISABLE_FLOWCNTRL</tag>
+      <item>If defined, disables input and output flow control for the pty opend by run_erl.
+       Useful if you want to remove any risk of accidentally blocking the flow control
+       by hit Ctrl-S (instead of Ctrl-D to detach).
+       Which may result in blocking of the entire beam process
+       and in the case of running heart as supervisor
+       even the heart process will be blocked when writing log message to terminal.
+       Leaving the heart process unable to do its work.</item>
     </taglist>
   </section>
 
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index a81ab28847..d9362a2a8f 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2010. All Rights Reserved.
+# Copyright Ericsson AB 1996-2011. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -61,11 +61,7 @@ ifeq ($(TYPE),quantify)
 PURIFY = quantify $(QUANTIFY_BUILD_OPTIONS)
 TYPEMARKER = .quantify
 ENABLE_ALLOC_TYPE_VARS += quantify
-ifeq ($(findstring ose,$(TARGET)),ose)
-  TYPE_FLAGS = @CFLAGS@ -DQUANTIFY
-else
   TYPE_FLAGS = @CFLAGS@ -g -O2 -DQUANTIFY -DNO_JUMP_TABLE
-endif
 else
 
 ifeq ($(TYPE),purecov)
@@ -78,7 +74,7 @@ else
 ifeq ($(TYPE),gcov)
 PURIFY = 
 TYPEMARKER = .gcov
-TYPE_FLAGS = @DEBUG_CFLAGS@ -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE=
+TYPE_FLAGS = @DEBUG_CFLAGS@ -DERTS_GCOV -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE=
 ifneq ($(findstring solaris,$(TARGET)),solaris)
 LIBS += -lgcov
 endif
@@ -203,6 +199,14 @@ MKDIR	= @MKDIR@
 
 USING_MINGW=@MIXED_CYGWIN_MINGW@
 
+ifeq ($(TARGET),win32)
+LIB_PREFIX=
+LIB_SUFFIX=.lib
+else
+LIB_PREFIX=lib
+LIB_SUFFIX=.a
+endif
+
 OMIT_OMIT_FP=no
 
 ifeq (@EMU_LOCK_CHECKING@,yes)
@@ -283,16 +287,11 @@ endif
 
 ifeq ($(TARGET),win32)
 LIBS    += -L$(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE) -lepcre
-DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/epcre.lib
 else
-LIBS    += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a
-DEPLIBS += \
-	$(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a \
-	$(ERL_TOP)/erts/lib/internal/$(TARGET)/liberts_internal.a 
-# rem liberts_internal.a
+LIBS    += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX)
 endif
 
-ELIB_FLAGS = -DENABLE_ELIB_MALLOC -DELIB_ALLOC_IS_CLIB -DELIB_HEAP_SBRK
+DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX)
 
 PERFCTR_PATH=@PERFCTR_PATH@
 USE_PERFCTR=@USE_PERFCTR@
@@ -309,7 +308,19 @@ LIBSCTP = @LIBSCTP@
 ORG_THR_LIBS=@EMU_THR_LIBS@
 THR_LIB_NAME=@EMU_THR_LIB_NAME@
 
-THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER),$(ORG_THR_LIBS))
+ifneq ($(strip $(THR_LIB_NAME)),)
+DEPLIBS +=	$(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal_r$(TYPEMARKER)$(LIB_SUFFIX) \
+		$(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)ethread$(TYPEMARKER)$(LIB_SUFFIX)
+else
+DEPLIBS +=	$(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal$(TYPEMARKER)$(LIB_SUFFIX)
+endif
+
+THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER), \
+	   $(subst -lerts_internal_r,-lerts_internal_r$(TYPEMARKER),$(ORG_THR_LIBS)))
+
+LIBS += $(THR_LIBS)
+
+ifneq ($(findstring erts_internal_r, $(THR_LIBS)),erts_internal_r)
 
 ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 ERTS_INTERNAL_LIB=erts_internal
@@ -321,7 +332,9 @@ ERTS_INTERNAL_LIB=erts_internal
 endif
 endif
 
-LIBS += $(THR_LIBS) -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER)
+LIBS += -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER)
+
+endif # erts_internal_r
 
 LIBS += @LIBRT@
 
@@ -345,7 +358,6 @@ ERLANG_OSTYPE = @ERLANG_OSTYPE@
 
 ENABLE_ALLOC_TYPE_VARS += @ERLANG_OSTYPE@
 
-EMULATOR_EXECUTABLE_SAE = beam_evm$(TF_MARKER)
 EMULATOR_EXECUTABLE_ELIB = beam.elib$(TF_MARKER)
 ifeq ($(TARGET), win32)
 EMULATOR_EXECUTABLE = beam$(TF_MARKER).dll
@@ -410,7 +422,7 @@ endif
 	@set -e ; cd zlib && $(MAKE) clean
 	@set -e ; cd pcre && $(MAKE) clean
 
-.PHONY: all sae zlib pcre clean
+.PHONY: all zlib pcre clean
 
 docs:
 
@@ -423,12 +435,6 @@ RELEASE_INCLUDES = beam/erl_driver.h sys/$(ERLANG_OSTYPE)/driver_int.h beam/erl_
 ifeq ($(TARGET),win32)
 RELEASE_INCLUDES += sys/$(ERLANG_OSTYPE)/erl_win_dyn_driver.h
 endif
-ifeq ($(findstring ose,$(TARGET)),ose)
-RELEASE_INCLUDES += sys/$(ERLANG_OSTYPE)/erl_port_signals.sig \
-		    sys/$(ERLANG_OSTYPE)/ose_erl_port_prog.h  \
-		    drivers/$(ERLANG_OSTYPE)/ose_erl_driver.h
-
-endif
 
 ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no)
 release_spec:
@@ -452,8 +458,6 @@ release_spec: all
 ifeq ($(ERLANG_OSTYPE), unix)
 	$(INSTALL_PROGRAM) $(BINDIR)/$(CS_EXECUTABLE) $(RELSYSDIR)/bin
 endif
-	$(INSTALL_DIR) $(RELEASE_PATH)/usr/include/obsolete
-	$(INSTALL_DATA) obsolete/driver.h $(RELEASE_PATH)/usr/include/obsolete
 endif
 endif
 
@@ -480,8 +484,6 @@ endif
 
 ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 else
-ifeq ($(findstring ose,$(TARGET)),ose)
-else
 ifdef HIPE_ENABLED
 GENERATE +=	$(TTF_DIR)/hipe_x86_asm.h \
 		$(TTF_DIR)/hipe_amd64_asm.h \
@@ -492,7 +494,6 @@ GENERATE +=	$(TTF_DIR)/hipe_x86_asm.h \
 		$(BINDIR)/hipe_mkliterals$(TF_MARKER)
 endif
 endif
-endif
 
 ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no)
 GENERATE=
@@ -504,19 +505,22 @@ ifdef HIPE_ENABLED
 OPCODE_TABLES += hipe/hipe_ops.tab
 endif
 
-$(TTF_DIR)/beam_opcodes.h $(TTF_DIR)/beam_opcodes.c: $(OPCODE_TABLES)
-	LANG=C $(PERL) utils/beam_makeops -outdir $(TTF_DIR) \
+$(TTF_DIR)/beam_opcodes.h $(TTF_DIR)/beam_opcodes.c: $(OPCODE_TABLES) utils/beam_makeops
+	LANG=C $(PERL) utils/beam_makeops \
+		-wordsize @EXTERNAL_WORD_SIZE@ \
+		-outdir $(TTF_DIR) \
 		-emulator $(OPCODE_TABLES)
 
 # bif and atom table
 ATOMS=	beam/atom.names
 BIFS =	beam/bif.tab
 ifdef HIPE_ENABLED
+HIPE_ARCH64_TAB=hipe/hipe_bif64.tab
 HIPE_x86_TAB=hipe/hipe_x86.tab
-HIPE_amd64_TAB=hipe/hipe_amd64.tab
+HIPE_amd64_TAB=hipe/hipe_amd64.tab $(HIPE_ARCH64_TAB)
 HIPE_ultrasparc_TAB=hipe/hipe_sparc.tab
 HIPE_ppc_TAB=hipe/hipe_ppc.tab
-HIPE_ppc64_TAB=hipe/hipe_ppc64.tab
+HIPE_ppc64_TAB=hipe/hipe_ppc64.tab $(HIPE_ARCH64_TAB)
 HIPE_arm_TAB=hipe/hipe_arm.tab
 HIPE_ARCH_TAB=$(HIPE_$(ARCH)_TAB)
 BIFS += hipe/hipe_bif0.tab hipe/hipe_bif1.tab hipe/hipe_bif2.tab $(HIPE_ARCH_TAB)
@@ -530,8 +534,9 @@ TABLES=	$(TARGET)/erl_bif_table.c $(TARGET)/erl_bif_table.h	\
 	$(TARGET)/erl_atom_table.c $(TARGET)/erl_atom_table.h   \
 	$(TARGET)/erl_pbifs.c
 
-$(TABLES):	$(ATOMS) $(BIFS)
-	LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET) $^
+$(TABLES): $(ATOMS) $(BIFS) utils/make_tables
+	LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET)\
+		$(ATOMS) $(BIFS)
 
 $(TTF_DIR)/erl_alloc_types.h: beam/erl_alloc.types utils/make_alloc_types
 	LANG=C $(PERL) utils/make_alloc_types -src $< -dst $@ $(ENABLE_ALLOC_TYPE_VARS)
@@ -592,7 +597,6 @@ ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 INCLUDES += -I$(ERL_TOP)/erts/etc/vxworks
 endif
 
-ifneq ($(findstring ose,$(TARGET)),ose)
 ifeq ($(TARGET),win32)
 # Usually the same as the default rule, but certain platforms (i.e. win32) mix
 # different compilers
@@ -618,33 +622,6 @@ endif
 $(OBJDIR)/%.o: beam/%.c
 	$(CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
 
-else
-
-INCLUDES += -Idrivers/ose
-
-ifeq ($(TYPE),debug)
-$(OBJDIR)/%.o: beam/%.c
-	  $(CC) $(CFLAGS) -DNO_JUMP_TABLE $(INCLUDES) -c $< -o $@
-else
-
-VXCC=@VXCC@
-VXCFLAGS=@VXCFLAGS@
-CFLAGS_NOOPT=@CFLAGS_NOOPT@ @DEFS@ $(WFLAGS) $(THR_DEFS) $(ARCHCFLAGS)
-
-# we want to use jump table
-$(OBJDIR)/beam_emu.o: beam/beam_emu.c
-	$(VXCC) $(VXCFLAGS) $(INCLUDES) -c $< -o $@
-
-# erl_process does not work properly with DIAB's -XO option,
-# we'll compile it with gcc instead
-$(OBJDIR)/erl_process.o: beam/erl_process.c
-	$(VXCC) $(VXCFLAGS) $(INCLUDES) -c $< -o $@
-
-$(OBJDIR)/%.o: beam/%.c
-	$(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@
-endif
-endif
-
 $(OBJDIR)/%.o: $(TARGET)/%.c
 	$(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -c $< -o $@
 
@@ -663,15 +640,11 @@ $(OBJDIR)/%.o: drivers/common/%.c
 $(OBJDIR)/%.o: drivers/$(ERLANG_OSTYPE)/%.c
 	$(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -Idrivers/$(ERLANG_OSTYPE) -I../etc/$(ERLANG_OSTYPE) -c $< -o $@
 
-# VxWorks and OSE uses unix drivers too...
+# VxWorks uses unix drivers too...
 ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 $(OBJDIR)/%.o: drivers/unix/%.c
 	$(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -c $< -o $@
 endif
-ifeq ($(findstring ose,$(TARGET)),ose)
-$(OBJDIR)/%.o: drivers/unix/%.c
-	$(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -c $< -o $@
-endif
 
 # ----------------------------------------------------------------------
 # Specials
@@ -682,13 +655,6 @@ $(BINDIR)/$(CS_EXECUTABLE): $(CS_SRC)
 	$(CS_PURIFY) $(CC) $(CS_LDFLAGS) -o $(BINDIR)/$(CS_EXECUTABLE) \
                $(CS_CFLAGS) $(COMMON_INCLUDES) $(CS_SRC) $(CS_LIBS)
 
-$(OBJDIR)/%.elib.o: beam/%.c
-	$(CC) $(ELIB_FLAGS) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
-
-# Disable vfork() for sae (then we don't need the child_setup program)
-$(OBJDIR)/sys_sae.o: sys/$(ERLANG_OSTYPE)/sys.c
-	$(CC) -DDISABLE_VFORK=1 $(CFLAGS) $(INCLUDES) -c $< -o $@
-
 $(OBJDIR)/%.kp.o: sys/common/%.c
 	$(CC) -DERTS_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
 
@@ -697,9 +663,6 @@ $(OBJDIR)/%.nkp.o: sys/common/%.c
 
 ifeq ($(GCC),yes)
 
-$(OBJDIR)/erl_obsolete.o: beam/erl_obsolete.c
-	$(CC) $(subst -Wstrict-prototypes, , $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS))) $(INCLUDES) -c $< -o $@
-
 $(OBJDIR)/erl_goodfit_alloc.o: beam/erl_goodfit_alloc.c
 	$(CC) $(subst -O2, $(GEN_OPT_FLGS) $(UNROLL_FLG), $(CFLAGS)) $(INCLUDES) -c $< -o $@
 endif
@@ -734,8 +697,6 @@ endif
 
 INIT_OBJS = $(OBJDIR)/erl_main.o		$(PRELOAD)
 
-INIT_OBJS_SAE = $(OBJDIR)/erl9_start.o
-
 EMU_OBJS = \
 	$(OBJDIR)/beam_emu.o		$(OBJDIR)/beam_opcodes.o \
 	$(OBJDIR)/beam_load.o		$(OBJDIR)/beam_bif_load.o \
@@ -776,11 +737,12 @@ RUN_OBJS = \
 	$(OBJDIR)/erl_fun.o             $(OBJDIR)/erl_bif_port.o \
 	$(OBJDIR)/erl_term.o 		$(OBJDIR)/erl_node_tables.o \
 	$(OBJDIR)/erl_monitors.o	$(OBJDIR)/erl_process_dump.o \
-	$(OBJDIR)/erl_obsolete.o	$(OBJDIR)/erl_bif_timer.o \
+	$(OBJDIR)/erl_bif_timer.o	$(OBJDIR)/erl_cpu_topology.o \
 	$(OBJDIR)/erl_drv_thread.o      $(OBJDIR)/erl_bif_chksum.o \
 	$(OBJDIR)/erl_bif_re.o		$(OBJDIR)/erl_unicode.o \
 	$(OBJDIR)/packet_parser.o	$(OBJDIR)/safe_hash.o \
-	$(OBJDIR)/erl_zlib.o		$(OBJDIR)/erl_nif.o
+	$(OBJDIR)/erl_zlib.o		$(OBJDIR)/erl_nif.o \
+	$(OBJDIR)/erl_bif_binary.o
 
 ifeq ($(TARGET),win32)
 DRV_OBJS = \
@@ -799,29 +761,21 @@ OS_OBJS = \
 	$(OBJDIR)/sys_time.o \
 	$(OBJDIR)/sys_interrupt.o \
 	$(OBJDIR)/sys_env.o \
-	$(OBJDIR)/dosmap.o \
-	$(OBJDIR)/elib_malloc.o
+	$(OBJDIR)/dosmap.o
 else
 OS_OBJS = \
 	$(OBJDIR)/sys.o \
 	$(OBJDIR)/driver_tab.o \
 	$(OBJDIR)/unix_efile.o \
 	$(OBJDIR)/gzio.o \
-	$(OBJDIR)/elib_malloc.o \
 	$(OBJDIR)/elib_memmove.o
 
-ifeq ($(findstring ose,$(TARGET)),ose)
-	OS_OBJS += $(OBJDIR)/erl_port_init.o \
-		   $(OBJDIR)/ose_inet_sock_select.o \
-		   $(OBJDIR)/ose_sfp.o
-else
 ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 	OS_OBJS += $(OBJDIR)/int64.o 
 else
 	OS_OBJS += $(OBJDIR)/sys_float.o \
                    $(OBJDIR)/sys_time.o
 endif
-endif
 DRV_OBJS = \
 	$(OBJDIR)/efile_drv.o \
 	$(OBJDIR)/inet_drv.o \
@@ -830,11 +784,7 @@ DRV_OBJS = \
 endif
 
 ifneq ($(findstring vxworks,$(TARGET)),vxworks)
-  ifeq ($(findstring ose,$(TARGET)),ose)
-	DRV_OBJS += $(OBJDIR)/ose_inet_drv.o
-  else
 	DRV_OBJS += $(OBJDIR)/ttsl_drv.o
-  endif
 endif
 
 ifeq ($(ERTS_ENABLE_KERNEL_POLL),yes)
@@ -849,14 +799,17 @@ endif
 
 OS_OBJS +=	$(OBJDIR)/erl_mseg.o \
 		$(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \
-		$(OBJDIR)/erl_mtrace_sys_wrap.o
+		$(OBJDIR)/erl_mtrace_sys_wrap.o \
+		$(OBJDIR)/erl_sys_common_misc.o
+
+HIPE_ARCH64_OBJS=$(OBJDIR)/hipe_bif64.o
 
 HIPE_x86_OS_OBJS=$(HIPE_x86_$(OPSYS)_OBJS)
 HIPE_x86_OBJS=$(OBJDIR)/hipe_x86.o $(OBJDIR)/hipe_x86_glue.o $(OBJDIR)/hipe_x86_bifs.o $(OBJDIR)/hipe_x86_signal.o $(OBJDIR)/hipe_x86_stack.o $(HIPE_x86_OS_OBJS)
-HIPE_amd64_OBJS=$(OBJDIR)/hipe_amd64.o $(OBJDIR)/hipe_amd64_glue.o $(OBJDIR)/hipe_amd64_bifs.o $(OBJDIR)/hipe_x86_signal.o $(OBJDIR)/hipe_x86_stack.o
+HIPE_amd64_OBJS=$(OBJDIR)/hipe_amd64.o $(OBJDIR)/hipe_amd64_glue.o $(OBJDIR)/hipe_amd64_bifs.o $(OBJDIR)/hipe_x86_signal.o $(OBJDIR)/hipe_x86_stack.o $(HIPE_ARCH64_OBJS)
 HIPE_ultrasparc_OBJS=$(OBJDIR)/hipe_sparc.o $(OBJDIR)/hipe_sparc_glue.o $(OBJDIR)/hipe_sparc_bifs.o $(OBJDIR)/hipe_risc_stack.o
 HIPE_ppc_OBJS=$(OBJDIR)/hipe_ppc.o $(OBJDIR)/hipe_ppc_glue.o $(OBJDIR)/hipe_ppc_bifs.o $(OBJDIR)/hipe_risc_stack.o
-HIPE_ppc64_OBJS=$(HIPE_ppc_OBJS)
+HIPE_ppc64_OBJS=$(HIPE_ppc_OBJS) $(HIPE_ARCH64_OBJS)
 HIPE_arm_OBJS=$(OBJDIR)/hipe_arm.o $(OBJDIR)/hipe_arm_glue.o $(OBJDIR)/hipe_arm_bifs.o $(OBJDIR)/hipe_risc_stack.o
 HIPE_noarch_OBJS=
 HIPE_ARCH_OBJS=$(HIPE_$(ARCH)_OBJS)
@@ -880,17 +833,6 @@ endif
 BASE_OBJS = $(RUN_OBJS) $(EMU_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS)
 
 OBJS =	$(BASE_OBJS) $(DRV_OBJS)
-OBJS_SAE =  $(subst sys.o,sys_sae.o,$(OBJS))
-
-ELIB_C_FILES =	beam/elib_malloc.c \
-		beam/elib_memmove.c \
-		beam/erl_bif_info.c \
-		beam/utils.c \
-		beam/erl_alloc.c
-
-MOD_OBJS_ELIB =	$(patsubst %.c,$(OBJDIR)/%.o,$(notdir $(ELIB_C_FILES)))
-OBJS_ELIB = 	$(patsubst %.o,%.elib.o,$(MOD_OBJS_ELIB)) \
-		$(filter-out $(MOD_OBJS_ELIB),$(OBJS))
 
 ########################################
 # HiPE section
@@ -968,33 +910,6 @@ $(TARGET)/int64.c:
 
 endif
 
-ifeq ($(findstring ose,$(TARGET)),ose)
-# Extract soft float functions from libgcc.a (for beam_emu)
-VXCC=@VXCC@
-VXCFLAGS=@VXCFLAGS@
-VXLD=@VXLD@
-VXLDFLAGS=@VXLDFLAGS@
-VXCCLIBFLAGS=@VXCCLIBFLAGS@
-STRIP=@STRIP@
-SYMPREFIX=@SYMPREFIX@
-
-NEEDFUNCTIONS=__floatsidf __adddf3 __negdf2 __muldf3 __divdf3 __subdf3
-KEEPSYMS=$(NEEDFUNCTIONS:%=-K $(SYMPREFIX)%)
-
-$(OBJDIR)/ose_sfp.o: $(TARGET)/ose_sfp.c
-	$(VXCC) $(VXCFLAGS) -o $(OBJDIR)/ose_sfp_tmp.o -c $(TARGET)/ose_sfp.c
-	$(VXLD) -o $(OBJDIR)/ose_sfp.o $(OBJDIR)/ose_sfp_tmp.o $(VXLDFLAGS) $(VXCCLIBFLAGS)
-	$(STRIP) $(KEEPSYMS) $(OBJDIR)/ose_sfp.o
-
-$(TARGET)/ose_sfp.c:
-	echo 'void dummy(void); void dummy(void) {' > $(TARGET)/ose_sfp.c
-	for x in $(NEEDFUNCTIONS); do echo 'extern void '$$x'();' \
-		>> $(TARGET)/ose_sfp.c; done 
-	for x in $(NEEDFUNCTIONS); do echo $$x'();' >> $(TARGET)/ose_sfp.c; done
-	echo '}'  >> $(TARGET)/ose_sfp.c 
-
-endif
-
 # ----------------------------------------------------------------------
 # The emulator itself
 
@@ -1010,14 +925,6 @@ $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS)
 	$(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE) \
 	$(HIPEBEAMLDFLAGS) $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS) $(LIBS)
 
-$(BINDIR)/$(EMULATOR_EXECUTABLE_ELIB): $(INIT_OBJS) $(OBJS_ELIB) $(DEPLIBS)
-	$(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE_ELIB) \
-	$(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS_ELIB) $(LIBS)
-
-$(BINDIR)/$(EMULATOR_EXECUTABLE_SAE): $(INIT_OBJS_SAE) $(OBJS_SAE) $(DEPLIBS)
-	$(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE_SAE) \
-	$(LDFLAGS) $(DEXPORT) $(INIT_OBJS_SAE) $(OBJS_SAE) $(LIBS)
-
 endif
 
 #
@@ -1112,7 +1019,7 @@ depend:
 	$(DEP_CC) $(DEP_FLAGS) $(TARGET_SRC) \
 		| $(SED_DEPEND) >> $(TARGET)/depend.mk
 ifneq ($(TARGET),win32)
-	$(DEP_CC) $(DEP_FLAGS) $(ELIB_FLAGS) $(ELIB_C_FILES) \
+	$(DEP_CC) $(DEP_FLAGS) $(ELIB_C_FILES) \
 		| $(SED_ELIB_DEPEND) >> $(TARGET)/depend.mk
 endif
 ifdef HIPE_ENABLED
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index e2a79d6e4f..b97705ed96 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -41,8 +41,7 @@ static erts_smp_rwmtx_t atom_table_lock;
 #define atom_read_unlock()	erts_smp_rwmtx_runlock(&atom_table_lock)
 #define atom_write_lock()	erts_smp_rwmtx_rwlock(&atom_table_lock)
 #define atom_write_unlock()	erts_smp_rwmtx_rwunlock(&atom_table_lock)
-#define atom_init_lock()	erts_smp_rwmtx_init(&atom_table_lock, \
-						    "atom_tab")
+
 #if 0
 #define ERTS_ATOM_PUT_OPS_STAT
 #endif
@@ -304,12 +303,17 @@ init_atom_table(void)
     HashFunctions f;
     int i;
     Atom a;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
 #ifdef ERTS_ATOM_PUT_OPS_STAT
     erts_smp_atomic_init(&atom_put_ops, 0);
 #endif
 
-    atom_init_lock();
+    erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab");
+
     f.hash = (H_FUN) atom_hash;
     f.cmp  = (HCMP_FUN) atom_cmp;
     f.alloc = (HALLOC_FUN) atom_alloc;
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 57c8b08223..68d64fb7b0 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2010. All Rights Reserved.
+# Copyright Ericsson AB 1996-2011. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -65,6 +65,7 @@ atom EXIT='EXIT'
 atom aborted
 atom abs_path
 atom absoluteURI
+atom ac
 atom active
 atom all
 atom all_but_first
@@ -75,6 +76,7 @@ atom allocator_sizes
 atom alloc_util_allocators
 atom allow_passive_connect
 atom already_loaded
+atom amd64
 atom anchored
 atom and
 atom andalso
@@ -100,8 +102,15 @@ atom band
 atom big
 atom bif_return_trap
 atom binary
+atom binary_bin_to_list_trap
+atom binary_copy_trap
+atom binary_longest_prefix_trap
+atom binary_longest_suffix_trap
+atom binary_match_trap
+atom binary_matches_trap
 atom block
 atom blocked
+atom bm
 atom bnot
 atom bor
 atom bxor
@@ -111,10 +120,12 @@ atom bsl
 atom bsr
 atom bsr_anycrlf
 atom bsr_unicode
+atom build_type
 atom busy_dist_port
 atom busy_port
 atom call
 atom call_count
+atom call_time
 atom caller
 atom capture
 atom case_clause
@@ -256,6 +267,7 @@ atom info
 atom info_msg
 atom initial_call
 atom input
+atom internal
 atom internal_error
 atom internal_status
 atom instruction_counts
@@ -368,6 +380,7 @@ atom old_heap_size
 atom on_load
 atom open
 atom open_error
+atom opt
 atom or
 atom ordered_set
 atom orelse
@@ -422,6 +435,7 @@ atom raw
 atom re
 atom re_pattern
 atom re_run_trap
+atom read_concurrency
 atom ready_input
 atom ready_output
 atom ready_async
@@ -453,6 +467,7 @@ atom scheduler
 atom scheduler_id
 atom schedulers_online
 atom scheme
+atom scope
 atom sensitive
 atom sequential_tracer
 atom sequential_trace_token
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index b1feec7074..d76a7d8e9f 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -39,10 +39,10 @@ static Eterm check_process_code(Process* rp, Module* modp);
 static void delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp);
 static void delete_export_references(Eterm module);
 static int purge_module(int module);
-static int is_native(Eterm* code);
+static int is_native(BeamInstr* code);
 static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
 static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
-static void remove_from_address_table(Eterm* code);
+static void remove_from_address_table(BeamInstr* code);
 
 Eterm
 load_module_2(BIF_ALIST_2)
@@ -142,7 +142,7 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
     if ((modp = erts_get_module(BIF_ARG_1)) == NULL) {
 	return am_undefined;
     }
-    return (is_native(modp->code) ||
+    return ((modp->code && is_native(modp->code)) ||
 	    (modp->old_code != 0 && is_native(modp->old_code))) ?
 		am_true : am_false;
 }
@@ -175,8 +175,12 @@ check_process_code_2(BIF_ALIST_2)
 	Eterm res;
 	if (internal_pid_index(BIF_ARG_1) >= erts_max_processes)
 	    goto error;
-	rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
-				       BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
+#ifdef ERTS_SMP
+	rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN,
+				   BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
+#else
+	rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0);
+#endif
 	if (!rp) {
 	    BIF_RET(am_false);
 	}
@@ -187,8 +191,10 @@ check_process_code_2(BIF_ALIST_2)
 	modp = erts_get_module(BIF_ARG_2);
 	res = check_process_code(rp, modp);
 #ifdef ERTS_SMP
-	if (BIF_P != rp)
+	if (BIF_P != rp) {
+	    erts_resume(rp, ERTS_PROC_LOCK_MAIN);
 	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+	}
 #endif
 	BIF_RET(res);
     }
@@ -337,15 +343,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 		ep->code[0] == BIF_ARG_1 &&
 		ep->code[4] != 0) {
 		ep->address = (void *) ep->code[4];
-		ep->code[3] = 0;
 		ep->code[4] = 0;
 	    }
 	}
 	modp->code[MI_ON_LOAD_FUNCTION_PTR] = 0;
 	set_default_trace_pattern(BIF_ARG_1);
     } else if (BIF_ARG_2 == am_false) {
-	Eterm* code;
-	Eterm* end;
+	BeamInstr* code;
+	BeamInstr* end;
 
 	/*
 	 * The on_load function failed. Remove the loaded code.
@@ -354,7 +359,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	 */
 	erts_total_code_size -= modp->code_length;
 	code = modp->code;
-	end = (Eterm *)((char *)code + modp->code_length);
+	end = (BeamInstr *)((char *)code + modp->code_length);
 	erts_cleanup_funs_on_purge(code, end);
 	beam_catches_delmod(modp->catches, code, modp->code_length);
 	erts_free(ERTS_ALC_T_CODE, (void *) code);
@@ -368,7 +373,6 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
     BIF_RET(am_true);
 }
 
-
 static void
 set_default_trace_pattern(Eterm module)
 {
@@ -397,13 +401,13 @@ set_default_trace_pattern(Eterm module)
 static Eterm
 check_process_code(Process* rp, Module* modp)
 {
-    Eterm* start;
+    BeamInstr* start;
     char* mod_start;
     Uint mod_size;
-    Eterm* end;
+    BeamInstr* end;
     Eterm* sp;
 #ifndef HYBRID /* FIND ME! */
-    ErlFunThing* funp;
+    struct erl_off_heap_header* oh;
     int done_gc = 0;
 #endif
 
@@ -418,7 +422,7 @@ check_process_code(Process* rp, Module* modp)
      * Pick up limits for the module.
      */
     start = modp->old_code;
-    end = (Eterm *)((char *)start + modp->old_code_length);
+    end = (BeamInstr *)((char *)start + modp->old_code_length);
     mod_start = (char *) start;
     mod_size = modp->old_code_length;
 
@@ -471,27 +475,27 @@ check_process_code(Process* rp, Module* modp)
 
 #ifndef HYBRID /* FIND ME! */
  rescan:
-    for (funp = MSO(rp).funs; funp; funp = funp->next) {
-	Eterm* fun_code;
-
-	fun_code = funp->fe->address;
-
-	if (INSIDE((Eterm *) funp->fe->address)) {
-	    if (done_gc) {
-		return am_true;
-	    } else {
-		/*
-		 * Try to get rid of this fun by garbage collecting.
-		 * Clear both fvalue and ftrace to make sure they
-		 * don't hold any funs.
-		 */
-		rp->freason = EXC_NULL;
-		rp->fvalue = NIL;
-		rp->ftrace = NIL;
-		done_gc = 1;
-                FLAGS(rp) |= F_NEED_FULLSWEEP;
-		(void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
-		goto rescan;
+    for (oh = MSO(rp).first; oh; oh = oh->next) {
+	if (thing_subtag(oh->thing_word) == FUN_SUBTAG) {
+	    ErlFunThing* funp = (ErlFunThing*) oh;
+
+	    if (INSIDE((BeamInstr *) funp->fe->address)) {
+		if (done_gc) {
+		    return am_true;
+		} else {
+		    /*
+		    * Try to get rid of this fun by garbage collecting.
+		    * Clear both fvalue and ftrace to make sure they
+		    * don't hold any funs.
+		    */
+		    rp->freason = EXC_NULL;
+		    rp->fvalue = NIL;
+		    rp->ftrace = NIL;
+		    done_gc = 1;
+		    FLAGS(rp) |= F_NEED_FULLSWEEP;
+		    (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
+		    goto rescan;
+		}
 	    }
 	}
     }
@@ -576,7 +580,7 @@ any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	case TAG_PRIMARY_LIST:
-	    if (in_area(val, mod_start, mod_size)) {
+	    if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) {
 		return 1;
 	    }
 	    break;
@@ -596,7 +600,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	case TAG_PRIMARY_LIST:
-	    if (in_area(val, mod_start, mod_size)) {
+	    if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) {
 		return 1;
 	    }
 	    break;
@@ -617,8 +621,8 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 static int
 purge_module(int module)
 {
-    Eterm* code;
-    Eterm* end;
+    BeamInstr* code;
+    BeamInstr* end;
     Module* modp;
 
     /*
@@ -653,7 +657,7 @@ purge_module(int module)
     ASSERT(erts_total_code_size >= modp->old_code_length);
     erts_total_code_size -= modp->old_code_length;
     code = modp->old_code;
-    end = (Eterm *)((char *)code + modp->old_code_length);
+    end = (BeamInstr *)((char *)code + modp->old_code_length);
     erts_cleanup_funs_on_purge(code, end);
     beam_catches_delmod(modp->old_catches, code, modp->old_code_length);
     erts_free(ERTS_ALC_T_CODE, (void *) code);
@@ -665,7 +669,7 @@ purge_module(int module)
 }
 
 static void
-remove_from_address_table(Eterm* code)
+remove_from_address_table(BeamInstr* code)
 {
     int i;
 
@@ -738,11 +742,11 @@ delete_export_references(Eterm module)
 	Export *ep = export_list(i);
         if (ep != NULL && (ep->code[0] == module)) {
 	    if (ep->address == ep->code+3 &&
-		(ep->code[3] == (Eterm) em_apply_bif)) {
+		(ep->code[3] == (BeamInstr) em_apply_bif)) {
 		continue;
 	    }
 	    ep->address = ep->code+3;
-	    ep->code[3] = (Uint) em_call_error_handler;
+	    ep->code[3] = (BeamInstr) em_call_error_handler;
 	    ep->code[4] = 0;
 	    MatchSetUnref(ep->match_prog_set);
 	    ep->match_prog_set = NULL;
@@ -774,7 +778,7 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
 }
 
 static int
-is_native(Eterm* code)
+is_native(BeamInstr* code)
 {
     return ((Eterm *)code[MI_FUNCTIONS])[1] != 0;
 }
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 1abf1dc10c..31910888d1 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
  
@@ -30,6 +30,7 @@
 #include "error.h"
 #include "erl_binary.h"
 #include "beam_bp.h"
+#include "erl_term.h"
 
 /* *************************************************************************
 ** Macros
@@ -100,6 +101,11 @@ do {                               \
     (b)->prev = (a);               \
 } while (0)
 
+
+#define BREAK_IS_BIF (1)
+#define BREAK_IS_ERL (0)
+
+
 /* *************************************************************************
 ** Local prototypes
 */
@@ -109,24 +115,42 @@ do {                               \
 */
 
 static int set_break(Eterm mfa[3], int specified,
-		     Binary *match_spec, Uint break_op, 
+		     Binary *match_spec, BeamInstr break_op,
 		     enum erts_break_op count_op, Eterm tracer_pid);
 static int set_module_break(Module *modp, Eterm mfa[3], int specified,
-			    Binary *match_spec, Uint break_op,
+			    Binary *match_spec, BeamInstr break_op,
 			    enum erts_break_op count_op, Eterm tracer_pid);
-static int set_function_break(Module *modp, Uint *pc,
-			      Binary *match_spec, Uint break_op,
+static int set_function_break(Module *modp, BeamInstr *pc, int bif,
+			      Binary *match_spec, BeamInstr break_op,
 			      enum erts_break_op count_op, Eterm tracer_pid); 
 
 static int clear_break(Eterm mfa[3], int specified, 
-		       Uint break_op);
+		       BeamInstr break_op);
 static int clear_module_break(Module *modp, Eterm mfa[3], int specified, 
-			      Uint break_op);
-static int clear_function_break(Module *modp, Uint *pc, 
-				Uint break_op);
-
-static BpData *is_break(Uint *pc, Uint break_op);
-
+			      BeamInstr break_op);
+static int clear_function_break(Module *modp, BeamInstr *pc, int bif,
+				BeamInstr break_op);
+
+static BpData *is_break(BeamInstr *pc, BeamInstr break_op);
+static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op);
+
+/* bp_hash */
+#define BP_TIME_ADD(pi0, pi1)                       \
+    do {                                            \
+	Uint r;                                     \
+	(pi0)->count   += (pi1)->count;             \
+	(pi0)->s_time  += (pi1)->s_time;            \
+	(pi0)->us_time += (pi1)->us_time;           \
+	r = (pi0)->us_time / 1000000;               \
+	(pi0)->s_time  += r;                        \
+	(pi0)->us_time  = (pi0)->us_time % 1000000; \
+    } while(0)
+
+static void bp_hash_init(bp_time_hash_t *hash, Uint n);
+static void bp_hash_rehash(bp_time_hash_t *hash, Uint n);
+static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_data_time_item_t *sitem);
+static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t *sitem);
+static void bp_hash_delete(bp_time_hash_t *hash);
 
 
 /* *************************************************************************
@@ -145,7 +169,7 @@ erts_set_trace_break(Eterm mfa[3], int specified, Binary *match_spec,
 		     Eterm tracer_pid) {
     ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
     return set_break(mfa, specified, match_spec,
-		     (Uint) BeamOp(op_i_trace_breakpoint), 0, tracer_pid);
+		     (BeamInstr) BeamOp(op_i_trace_breakpoint), 0, tracer_pid);
 }
 
 int 
@@ -153,87 +177,84 @@ erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec,
 		      Eterm tracer_pid) {
     ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
     return set_break(mfa, specified, match_spec,
-		     (Uint) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid);
+		     (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid);
 }
 
+/* set breakpoint data for on exported bif entry */
+
 void
-erts_set_mtrace_bif(Uint *pc, Binary *match_spec, Eterm tracer_pid) {
-    BpDataTrace *bdt;
+erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid) {
     ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    set_function_break(NULL, pc, BREAK_IS_BIF, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid);
+}
 
-    bdt = (BpDataTrace *) pc[-4];
-    if (bdt) {
-	MatchSetUnref(bdt->match_spec);
-	MatchSetRef(match_spec);
-	bdt->match_spec = match_spec;
-	bdt->tracer_pid = tracer_pid;
-    } else {
-	bdt = Alloc(sizeof(BpDataTrace));
-	BpInit((BpData *) bdt, 0);
-	MatchSetRef(match_spec);
-	bdt->match_spec = match_spec;
-	bdt->tracer_pid = tracer_pid;
-	pc[-4] = (Uint) bdt;
-    }
+void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) {
+    set_function_break(NULL, pc, BREAK_IS_BIF, NULL, (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL);
+}
+
+void erts_clear_time_trace_bif(BeamInstr *pc) {
+    clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_time_breakpoint));
 }
 
 int 
 erts_set_debug_break(Eterm mfa[3], int specified) {
     ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
     return set_break(mfa, specified, NULL, 
-		     (Uint) BeamOp(op_i_debug_breakpoint), 0, NIL);
+		     (BeamInstr) BeamOp(op_i_debug_breakpoint), 0, NIL);
 }
 
 int 
 erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op count_op) {
     ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
     return set_break(mfa, specified, NULL, 
-		     (Uint) BeamOp(op_i_count_breakpoint), count_op, NIL);
+		     (BeamInstr) BeamOp(op_i_count_breakpoint), count_op, NIL);
 }
 
-
+int
+erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op count_op) {
+    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    return set_break(mfa, specified, NULL,
+		     (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL);
+}
 
 int
 erts_clear_trace_break(Eterm mfa[3], int specified) {
     ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
     return clear_break(mfa, specified, 
-		       (Uint) BeamOp(op_i_trace_breakpoint));
+		       (BeamInstr) BeamOp(op_i_trace_breakpoint));
 }
 
 int
 erts_clear_mtrace_break(Eterm mfa[3], int specified) {
     ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
     return clear_break(mfa, specified, 
-		       (Uint) BeamOp(op_i_mtrace_breakpoint));
+		       (BeamInstr) BeamOp(op_i_mtrace_breakpoint));
 }
 
 void
-erts_clear_mtrace_bif(Uint *pc) {
-    BpDataTrace *bdt;
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
-    
-    bdt = (BpDataTrace *) pc[-4];
-    if (bdt) {
-	if (bdt->match_spec) {
-	    MatchSetUnref(bdt->match_spec);
-	}
-	Free(bdt);
-    }
-    pc[-4] = (Uint) NULL;
+erts_clear_mtrace_bif(BeamInstr *pc) {
+    clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_mtrace_breakpoint));
 }
 
 int
 erts_clear_debug_break(Eterm mfa[3], int specified) {
     ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
     return clear_break(mfa, specified, 
-		       (Uint) BeamOp(op_i_debug_breakpoint));
+		       (BeamInstr) BeamOp(op_i_debug_breakpoint));
 }
 
 int
 erts_clear_count_break(Eterm mfa[3], int specified) {
     ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
     return clear_break(mfa, specified, 
-		       (Uint) BeamOp(op_i_count_breakpoint));
+		       (BeamInstr) BeamOp(op_i_count_breakpoint));
+}
+
+int
+erts_clear_time_break(Eterm mfa[3], int specified) {
+    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    return clear_break(mfa, specified,
+		       (BeamInstr) BeamOp(op_i_time_breakpoint));
 }
 
 int
@@ -250,10 +271,10 @@ erts_clear_module_break(Module *modp) {
 }
 
 int
-erts_clear_function_break(Module *modp, Uint *pc) {
+erts_clear_function_break(Module *modp, BeamInstr *pc) {
     ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
     ASSERT(modp);
-    return clear_function_break(modp, pc, 0);
+    return clear_function_break(modp, pc, BREAK_IS_ERL, 0);
 }
 
 
@@ -261,13 +282,16 @@ erts_clear_function_break(Module *modp, Uint *pc) {
 /*
  * SMP NOTE: Process p may have become exiting on return!
  */
-Uint 
-erts_trace_break(Process *p, Uint *pc, Eterm *args, 
+BeamInstr
+erts_trace_break(Process *p, BeamInstr *pc, Eterm *args,
 		 Uint32 *ret_flags, Eterm *tracer_pid) {
     Eterm tpid1, tpid2;
-    BpDataTrace *bdt = (BpDataTrace *) pc[-4];
-    
-    ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI));
+    BpData **bds = (BpData **) (pc)[-4];
+    BpDataTrace *bdt = NULL;
+
+    ASSERT(bds);
+    ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    bdt = (BpDataTrace *) bds[bp_sched2ix_proc(p)];
     ASSERT(bdt);
     bdt = (BpDataTrace *) bdt->next;
     ASSERT(bdt);
@@ -286,7 +310,7 @@ erts_trace_break(Process *p, Uint *pc, Eterm *args,
 	bdt->tracer_pid = tpid2;
 	ErtsSmpBPUnlock(bdt);
     }
-    pc[-4] = (Uint) bdt;
+    bds[bp_sched2ix_proc(p)] = (BpData *) bdt;
     return bdt->orig_instr;
 }
 
@@ -296,14 +320,17 @@ erts_trace_break(Process *p, Uint *pc, Eterm *args,
  * SMP NOTE: Process p may have become exiting on return!
  */
 Uint32
-erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, int local, 
+erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args, int local,
 		Eterm *tracer_pid) {
-    BpDataTrace *bdt = (BpDataTrace *) pc[-4];
-    
+    BpData **bds = (BpData **) (pc)[-4];
+    BpDataTrace *bdt = NULL;
+
+
     ASSERT(tracer_pid);
-    if (bdt) {
+    if (bds) {
 	Eterm tpid1, tpid2;
 	Uint32 flags;
+	bdt = (BpDataTrace *)bds[bp_sched2ix_proc(p)];
 
 	ErtsSmpBPLock(bdt);
 	tpid1 = tpid2 = bdt->tracer_pid;
@@ -326,9 +353,9 @@ erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, int local,
 
 
 int 
-erts_is_trace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
+erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
     BpDataTrace *bdt = 
-	(BpDataTrace *) is_break(pc, (Uint) BeamOp(op_i_trace_breakpoint));
+	(BpDataTrace *) is_break(pc, (BeamInstr) BeamOp(op_i_trace_breakpoint));
     
     if (bdt) {
 	if (match_spec_ret) {
@@ -345,9 +372,9 @@ erts_is_trace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
 }
 
 int 
-erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
+erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
     BpDataTrace *bdt = 
-	(BpDataTrace *) is_break(pc, (Uint) BeamOp(op_i_mtrace_breakpoint));
+	(BpDataTrace *) is_break(pc, (BeamInstr) BeamOp(op_i_mtrace_breakpoint));
     
     if (bdt) {
 	if (match_spec_ret) {
@@ -364,74 +391,455 @@ erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
 }
 
 int
-erts_is_mtrace_bif(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
-    BpDataTrace *bdt = (BpDataTrace *) pc[-4];
-    
-    if (bdt) {
-	if (match_spec_ret) {
-	    *match_spec_ret = bdt->match_spec;
-	}
-	if (tracer_pid_ret) {
-	    ErtsSmpBPLock(bdt);
-	    *tracer_pid_ret = bdt->tracer_pid;
-	    ErtsSmpBPUnlock(bdt);
-	}
-	return !0;
-    }
-    return 0;
-}
-
-int
-erts_is_native_break(Uint *pc) {
+erts_is_native_break(BeamInstr *pc) {
 #ifdef HIPE
-    ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI));
-    return pc[0]  == (Uint) BeamOp(op_hipe_trap_call) 
-	|| pc[0]  == (Uint) BeamOp(op_hipe_trap_call_closure);
+    ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    return pc[0]  == (BeamInstr) BeamOp(op_hipe_trap_call)
+	|| pc[0]  == (BeamInstr) BeamOp(op_hipe_trap_call_closure);
 #else
     return 0;
 #endif
 }
 
 int 
-erts_is_count_break(Uint *pc, Sint *count_ret) {
+erts_is_count_break(BeamInstr *pc, Sint *count_ret) {
     BpDataCount *bdc = 
-	(BpDataCount *) is_break(pc, (Uint) BeamOp(op_i_count_breakpoint));
+	(BpDataCount *) is_break(pc, (BeamInstr) BeamOp(op_i_count_breakpoint));
     
     if (bdc) {
 	if (count_ret) {
-	    ErtsSmpBPLock(bdc);
-	    *count_ret = bdc->count;
-	    ErtsSmpBPUnlock(bdc);
+	    *count_ret = (Sint) erts_smp_atomic_read(&bdc->acount);
 	}
 	return !0;
     }
     return 0;
 }
 
-Uint *
+int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) {
+    Uint i, ix;
+    bp_time_hash_t hash;
+    Uint size;
+    Eterm *hp, t;
+    bp_data_time_item_t *item = NULL;
+    BpDataTime *bdt = (BpDataTime *) is_break(pc, (BeamInstr) BeamOp(op_i_time_breakpoint));
+
+    if (bdt) {
+	if (retval) {
+	    /* collect all hashes to one hash */
+	    bp_hash_init(&hash, 64);
+	    /* foreach threadspecific hash */
+	    for (i = 0; i < bdt->n; i++) {
+		bp_data_time_item_t *sitem;
+
+	        /* foreach hash bucket not NIL*/
+		for(ix = 0; ix < bdt->hash[i].n; ix++) {
+		    item = &(bdt->hash[i].item[ix]);
+		    if (item->pid != NIL) {
+			sitem = bp_hash_get(&hash, item);
+			if (sitem) {
+			    BP_TIME_ADD(sitem, item);
+			} else {
+			    bp_hash_put(&hash, item);
+			}
+		    }
+		}
+	    }
+	    /* *retval should be NIL or term from previous bif in export entry */
+
+	    if (hash.used > 0) {
+		size = (5 + 2)*hash.used;
+		hp   = HAlloc(p, size);
+
+		for(ix = 0; ix < hash.n; ix++) {
+		    item = &(hash.item[ix]);
+		    if (item->pid != NIL) {
+			t = TUPLE4(hp, item->pid,
+				make_small(item->count),
+				make_small(item->s_time),
+				make_small(item->us_time));
+			hp += 5;
+			*retval = CONS(hp, t, *retval); hp += 2;
+		    }
+		}
+	    }
+	    bp_hash_delete(&hash);
+	}
+	return !0;
+    }
+
+    return 0;
+}
+
+
+BeamInstr *
 erts_find_local_func(Eterm mfa[3]) {
     Module *modp;
-    Uint** code_base;
-    Uint* code_ptr;
+    BeamInstr** code_base;
+    BeamInstr* code_ptr;
     Uint i,n;
 
     if ((modp = erts_get_module(mfa[0])) == NULL)
 	return NULL;
-    if ((code_base = (Uint **) modp->code) == NULL)
+    if ((code_base = (BeamInstr **) modp->code) == NULL)
 	return NULL;
-    n = (Uint) code_base[MI_NUM_FUNCTIONS];
+    n = (BeamInstr) code_base[MI_NUM_FUNCTIONS];
     for (i = 0; i < n; ++i) {
 	code_ptr = code_base[MI_FUNCTIONS+i];
-	ASSERT(((Uint) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]);
+	ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]);
 	ASSERT(mfa[0] == ((Eterm) code_ptr[2]));
 	if (mfa[1] == ((Eterm) code_ptr[3]) &&
-	    ((Uint) mfa[2]) == code_ptr[4]) {
+	    ((BeamInstr) mfa[2]) == code_ptr[4]) {
 	    return code_ptr + 5;
 	}
     }
     return NULL;
 }
 
+/* bp_hash */
+ERTS_INLINE Uint bp_sched2ix() {
+#ifdef ERTS_SMP
+    ErtsSchedulerData *esdp;
+    esdp = erts_get_scheduler_data();
+    return esdp->no - 1;
+#else
+    return 0;
+#endif
+}
+static void bp_hash_init(bp_time_hash_t *hash, Uint n) {
+    Uint size = sizeof(bp_data_time_item_t)*n;
+    Uint i;
+
+    hash->n    = n;
+    hash->used = 0;
+
+    hash->item = (bp_data_time_item_t *)Alloc(size);
+    sys_memzero(hash->item, size);
+
+    for(i = 0; i < n; ++i) {
+	hash->item[i].pid = NIL;
+    }
+}
+
+static void bp_hash_rehash(bp_time_hash_t *hash, Uint n) {
+    bp_data_time_item_t *item = NULL;
+    Uint size = sizeof(bp_data_time_item_t)*n;
+    Uint ix;
+    Uint hval;
+
+    item = (bp_data_time_item_t *)Alloc(size);
+    sys_memzero(item, size);
+
+    for( ix = 0; ix < n; ++ix) {
+	item[ix].pid = NIL;
+    }
+
+    /* rehash, old hash -> new hash */
+
+    for( ix = 0; ix < hash->n; ix++) {
+	if (hash->item[ix].pid != NIL) {
+
+	    hval = ((hash->item[ix].pid) >> 4) % n; /* new n */
+
+	    while (item[hval].pid != NIL) {
+		hval = (hval + 1) % n;
+	    }
+	    item[hval].pid     = hash->item[ix].pid;
+	    item[hval].count   = hash->item[ix].count;
+	    item[hval].s_time  = hash->item[ix].s_time;
+	    item[hval].us_time = hash->item[ix].us_time;
+	}
+    }
+
+    Free(hash->item);
+    hash->n = n;
+    hash->item = item;
+}
+static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_data_time_item_t *sitem) {
+    Eterm pid = sitem->pid;
+    Uint hval = (pid >> 4) % hash->n;
+    bp_data_time_item_t *item = NULL;
+
+    item = hash->item;
+
+    while (item[hval].pid != pid) {
+	if (item[hval].pid == NIL) return NULL;
+	hval = (hval + 1) % hash->n;
+    }
+
+    return &(item[hval]);
+}
+
+static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t* sitem) {
+    Uint hval;
+    float r = 0.0;
+    bp_data_time_item_t *item;
+
+    /* make sure that the hash is not saturated */
+    /* if saturated, rehash it */
+
+    r = hash->used / (float) hash->n;
+
+    if (r > 0.7f) {
+	bp_hash_rehash(hash, hash->n * 2);
+    }
+    /* Do hval after rehash */
+    hval = (sitem->pid >> 4) % hash->n;
+
+    /* find free slot */
+    item = hash->item;
+
+    while (item[hval].pid != NIL) {
+	hval = (hval + 1) % hash->n;
+    }
+    item = &(hash->item[hval]);
+
+    item->pid     = sitem->pid;
+    item->s_time  = sitem->s_time;
+    item->us_time = sitem->us_time;
+    item->count   = sitem->count;
+    hash->used++;
+
+    return item;
+}
+
+static void bp_hash_delete(bp_time_hash_t *hash) {
+    hash->n = 0;
+    hash->used = 0;
+    Free(hash->item);
+    hash->item = NULL;
+}
+
+static void bp_time_diff(bp_data_time_item_t *item, /* out */
+	process_breakpoint_time_t *pbt,             /* in  */
+	Uint ms, Uint s, Uint us) {
+    int dms,ds,dus;
+
+    dms = ms - pbt->ms;
+    ds  = s  - pbt->s;
+    dus = us - pbt->us;
+
+    /* get_sys_now may return zero difftime,
+     * this is ok.
+     */
+
+    ASSERT(dms >= 0 || ds >= 0 || dus >= 0);
+
+    if (dus < 0) {
+	dus += 1000000;
+	ds  -= 1;
+    }
+    if (ds < 0) {
+	ds += 1000000;
+    }
+
+    item->s_time  = ds;
+    item->us_time = dus;
+}
+
+void erts_schedule_time_break(Process *p, Uint schedule) {
+    Uint ms, s, us;
+    process_breakpoint_time_t *pbt = NULL;
+    bp_data_time_item_t sitem, *item = NULL;
+    bp_time_hash_t *h = NULL;
+    BpDataTime *pbdt = NULL;
+
+    ASSERT(p);
+
+    pbt = ERTS_PROC_GET_CALL_TIME(p);
+
+    if (pbt) {
+
+	switch(schedule) {
+	case ERTS_BP_CALL_TIME_SCHEDULE_EXITING :
+	    break;
+	case ERTS_BP_CALL_TIME_SCHEDULE_OUT :
+	    /* When a process is scheduled _out_,
+	     * timestamp it and add its delta to
+	     * the previous breakpoint.
+	     */
+
+	    pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint));
+	    if (pbdt) {
+		get_sys_now(&ms,&s,&us);
+		bp_time_diff(&sitem, pbt, ms, s, us);
+		sitem.pid   = p->id;
+		sitem.count = 0;
+
+		h = &(pbdt->hash[bp_sched2ix_proc(p)]);
+
+		ASSERT(h);
+		ASSERT(h->item);
+
+		item = bp_hash_get(h, &sitem);
+		if (!item) {
+		    item = bp_hash_put(h, &sitem);
+		} else {
+		    BP_TIME_ADD(item, &sitem);
+		}
+	    }
+	    break;
+	case ERTS_BP_CALL_TIME_SCHEDULE_IN :
+	    /* When a process is scheduled _in_,
+	     * timestamp it and remove the previous
+	     * timestamp in the psd.
+	     */
+	    get_sys_now(&ms,&s,&us);
+	    pbt->ms = ms;
+	    pbt->s  = s;
+	    pbt->us = us;
+	    break;
+	default :
+	    ASSERT(0);
+		/* will never happen */
+	    break;
+	}
+    } /* pbt */
+}
+
+/* call_time breakpoint
+ * Accumulated times are added to the previous bp,
+ * not the current one. The current one is saved
+ * for future reference.
+ * The previous breakpoint is stored in the process it self, the psd.
+ * We do not need to store in a stack frame.
+ * There is no need for locking, each thread has its own
+ * area in each bp to save data.
+ * Since we need to diffrentiate between processes for each bp,
+ * every bp has a hash (per thread) to process-bp statistics.
+ * - egil
+ */
+
+void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type) {
+    Uint ms,s,us;
+    process_breakpoint_time_t *pbt = NULL;
+    bp_data_time_item_t sitem, *item = NULL;
+    bp_time_hash_t *h = NULL;
+    BpDataTime *pbdt = NULL;
+
+    ASSERT(p);
+    ASSERT(p->status == P_RUNNING);
+
+    /* get previous timestamp and breakpoint
+     * from the process psd  */
+
+    pbt = ERTS_PROC_GET_CALL_TIME(p);
+    get_sys_now(&ms,&s,&us);
+
+    switch(type) {
+	    /* get pbt
+	     * timestamp = t0
+	     * lookup bdt from code
+	     * set ts0 to pbt
+	     * add call count here?
+	     */
+	case ERTS_BP_CALL_TIME_CALL:
+	case ERTS_BP_CALL_TIME_TAIL_CALL:
+
+	    if (pbt) {
+		ASSERT(pbt->pc);
+		/* add time to previous code */
+		bp_time_diff(&sitem, pbt, ms, s, us);
+		sitem.pid   = p->id;
+		sitem.count = 0;
+
+		/* previous breakpoint */
+		pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint));
+
+		/* if null then the breakpoint was removed */
+		if (pbdt) {
+		    h = &(pbdt->hash[bp_sched2ix_proc(p)]);
+
+		    ASSERT(h);
+		    ASSERT(h->item);
+
+		    item = bp_hash_get(h, &sitem);
+		    if (!item) {
+			item = bp_hash_put(h, &sitem);
+		    } else {
+			BP_TIME_ADD(item, &sitem);
+		    }
+		}
+
+	    } else {
+		/* first call of process to instrumented function */
+		pbt = Alloc(sizeof(process_breakpoint_time_t));
+		(void *) ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCK_MAIN, pbt);
+	    }
+	    /* add count to this code */
+	    sitem.pid     = p->id;
+	    sitem.count   = 1;
+	    sitem.s_time  = 0;
+	    sitem.us_time = 0;
+
+	    /* this breakpoint */
+	    ASSERT(bdt);
+	    h = &(bdt->hash[bp_sched2ix_proc(p)]);
+
+	    ASSERT(h);
+	    ASSERT(h->item);
+
+	    item = bp_hash_get(h, &sitem);
+	    if (!item) {
+		item = bp_hash_put(h, &sitem);
+	    } else {
+		BP_TIME_ADD(item, &sitem);
+	    }
+
+	    pbt->pc = pc;
+	    pbt->ms = ms;
+	    pbt->s  = s;
+	    pbt->us = us;
+	    break;
+
+	case ERTS_BP_CALL_TIME_RETURN:
+	    /* get pbt
+	     * lookup bdt from code
+	     * timestamp = t1
+	     * get ts0 from pbt
+	     * get item from bdt->hash[bp_hash(p->id)]
+	     * ack diff (t1, t0) to item
+	     */
+
+	    if(pbt) {
+		/* might have been removed due to
+		 * trace_pattern(false)
+		 */
+		ASSERT(pbt->pc);
+
+		bp_time_diff(&sitem, pbt, ms, s, us);
+		sitem.pid   = p->id;
+		sitem.count = 0;
+
+		/* previous breakpoint */
+		pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint));
+
+		/* beware, the trace_pattern might have been removed */
+		if (pbdt) {
+		    h = &(pbdt->hash[bp_sched2ix_proc(p)]);
+
+		    ASSERT(h);
+		    ASSERT(h->item);
+
+		    item = bp_hash_get(h, &sitem);
+		    if (!item) {
+			item = bp_hash_put(h, &sitem);
+		    } else {
+			BP_TIME_ADD(item, &sitem);
+		    }
+		}
+
+		pbt->pc = pc;
+		pbt->ms = ms;
+		pbt->s  = s;
+		pbt->us = us;
+	    }
+	    break;
+	default :
+	    ASSERT(0);
+		/* will never happen */
+	    break;
+    }
+}
 
 
 /* *************************************************************************
@@ -440,7 +848,7 @@ erts_find_local_func(Eterm mfa[3]) {
 
 
 static int set_break(Eterm mfa[3], int specified, 
-		     Binary *match_spec, Eterm break_op, 
+		     Binary *match_spec, BeamInstr break_op,
 		     enum erts_break_op count_op, Eterm tracer_pid)
 {
     Module *modp;
@@ -470,45 +878,54 @@ static int set_break(Eterm mfa[3], int specified,
 }
 
 static int set_module_break(Module *modp, Eterm mfa[3], int specified,
-			    Binary *match_spec, Uint break_op, 
+			    Binary *match_spec, BeamInstr break_op,
 			    enum erts_break_op count_op, Eterm tracer_pid) {
-    Uint** code_base;
-    Uint* code_ptr;
+    BeamInstr** code_base;
+    BeamInstr* code_ptr;
     int num_processed = 0;
     Uint i,n;
 
     ASSERT(break_op);
     ASSERT(modp);
-    code_base = (Uint **) modp->code;
+    code_base = (BeamInstr **) modp->code;
     if (code_base == NULL) {
 	return 0;
     }
-    n = (Uint) code_base[MI_NUM_FUNCTIONS];
+    n = (BeamInstr) code_base[MI_NUM_FUNCTIONS];
     for (i = 0; i < n; ++i) {
 	code_ptr = code_base[MI_FUNCTIONS+i];
-	ASSERT(code_ptr[0] == (Uint) BeamOp(op_i_func_info_IaaI));
+	ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
 	if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) &&
 	    (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) {
-	    Uint   *pc = code_ptr+5;
+	    BeamInstr *pc = code_ptr+5;
 	    
 	    num_processed +=
-		set_function_break(modp, pc, match_spec, 
+		set_function_break(modp, pc, BREAK_IS_ERL, match_spec,
 				   break_op, count_op, tracer_pid);
 	}
     }
     return num_processed;
 }
 
-static int set_function_break(Module *modp, Uint *pc, 
-			      Binary *match_spec, Uint break_op, 
+static int set_function_break(Module *modp, BeamInstr *pc, int bif,
+			      Binary *match_spec, BeamInstr break_op,
 			      enum erts_break_op count_op, Eterm tracer_pid) {
-    BpData *bd, **r;
+
+    BeamInstr **code_base = NULL;
+    BpData *bd, **r, ***rs;
     size_t size;
-    Uint **code_base = (Uint **)modp->code;
+    Uint ix = 0;
     
-    ASSERT(code_base);
-    ASSERT(code_base <= (Uint **)pc);
-    ASSERT((Uint **)pc < code_base + (modp->code_length/sizeof(Uint *)));
+    if (bif == BREAK_IS_ERL) {
+	code_base = (BeamInstr **)modp->code;
+	ASSERT(code_base);
+	ASSERT(code_base <= (BeamInstr **)pc);
+	ASSERT((BeamInstr **)pc < code_base + (modp->code_length/sizeof(BeamInstr *)));
+    } else {
+	ASSERT(*pc == (BeamInstr) em_apply_bif);
+	ASSERT(modp == NULL);
+    }
+
     /*
      * Currently no trace support for native code.
      */
@@ -517,8 +934,9 @@ static int set_function_break(Module *modp, Uint *pc,
     }
     /* Do not allow two breakpoints of the same kind */
     if ( (bd = is_break(pc, break_op))) {
-	if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) 
-	    || break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) {
+	if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint)
+	    || break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) {
+
 	    BpDataTrace *bdt = (BpDataTrace *) bd;
 	    Binary *old_match_spec;
 	    
@@ -531,71 +949,115 @@ static int set_function_break(Module *modp, Uint *pc,
 	    ErtsSmpBPUnlock(bdt);
 	    MatchSetUnref(old_match_spec);
 	} else {
+	    BpDataCount *bdc = (BpDataCount *) bd;
+	    erts_aint_t count = 0;
+	    erts_aint_t res   = 0;
+
 	    ASSERT(! match_spec);
 	    ASSERT(is_nil(tracer_pid));
-	    if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) {
-		BpDataCount *bdc = (BpDataCount *) bd;
 		
-		ErtsSmpBPLock(bdc);
+	    if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) {
 		if (count_op == erts_break_stop) {
-		    if (bdc->count >= 0) {
-			bdc->count = -bdc->count-1; /* Stop call counter */
+		    count = erts_smp_atomic_read(&bdc->acount);
+		    if (count >= 0) {
+			while(1) {
+			    res = erts_smp_atomic_cmpxchg(&bdc->acount, -count - 1, count);
+			    if ((res == count) || count < 0) break;
+			    count = res;
+			}
 		    }
 		} else {
-		    bdc->count = 0; /* Reset call counter */
+		    /* Reset call counter */
+		    erts_smp_atomic_set(&bdc->acount, 0);
 		}
-		ErtsSmpBPUnlock(bdc);
+
+	    } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) {
+		BpDataTime *bdt = (BpDataTime *) bd;
+		Uint i = 0;
+
+		ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+
+		if (count_op == erts_break_stop) {
+		    bdt->pause = 1;
+		} else {
+		    bdt->pause = 0;
+		    for (i = 0; i < bdt->n; i++) {
+			bp_hash_delete(&(bdt->hash[i]));
+			bp_hash_init(&(bdt->hash[i]), 32);
+		    }
+		}
+
 	    } else {
 		ASSERT (! count_op);
 	    }
 	}
 	return 1;
     }
-    if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) ||
-	break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) {
+    if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) ||
+	break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) {
 	size = sizeof(BpDataTrace);
     } else {
 	ASSERT(! match_spec);
 	ASSERT(is_nil(tracer_pid));
-	if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) {
-	    if (count_op == erts_break_reset
-		|| count_op == erts_break_stop) {
+	if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) {
+	    if (count_op == erts_break_reset || count_op == erts_break_stop) {
 		/* Do not insert a new breakpoint */
 		return 1;
 	    }
 	    size = sizeof(BpDataCount);
+	} else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint))  {
+	    if (count_op == erts_break_reset || count_op == erts_break_stop) {
+		/* Do not insert a new breakpoint */
+		return 1;
+	    }
+	    size = sizeof(BpDataTime);
 	} else {
 	    ASSERT(! count_op);
-	    ASSERT(break_op == (Uint) BeamOp(op_i_debug_breakpoint));
+	    ASSERT(break_op == (BeamInstr) BeamOp(op_i_debug_breakpoint));
 	    size = sizeof(BpDataDebug);
 	}
     }
-    r = (BpData **) (pc-4);
+    rs = (BpData ***) (pc-4);
+    if (! *rs) {
+	size_t ssize = sizeof(BeamInstr) * erts_no_schedulers;
+	*rs = (BpData **) Alloc(ssize);
+	sys_memzero(*rs, ssize);
+    }
+
+    r = &((*rs)[0]);
+
     if (! *r) {
-	ASSERT(*pc != (Uint) BeamOp(op_i_trace_breakpoint));
-	ASSERT(*pc != (Uint) BeamOp(op_i_mtrace_breakpoint));
-	ASSERT(*pc != (Uint) BeamOp(op_i_debug_breakpoint));
-	ASSERT(*pc != (Uint) BeamOp(op_i_count_breakpoint));
+	ASSERT(*pc != (BeamInstr) BeamOp(op_i_trace_breakpoint));
+	ASSERT(*pc != (BeamInstr) BeamOp(op_i_mtrace_breakpoint));
+	ASSERT(*pc != (BeamInstr) BeamOp(op_i_debug_breakpoint));
+	ASSERT(*pc != (BeamInstr) BeamOp(op_i_count_breakpoint));
+	ASSERT(*pc != (BeamInstr) BeamOp(op_i_time_breakpoint));
 	/* First breakpoint; create singleton ring */
 	bd = Alloc(size);
 	BpInit(bd, *pc);
-	*pc = break_op;
 	*r = bd;
+	if (bif == BREAK_IS_ERL) {
+	    *pc = break_op;
+	}
     } else {
-	ASSERT(*pc == (Uint) BeamOp(op_i_trace_breakpoint) ||
-	       *pc == (Uint) BeamOp(op_i_mtrace_breakpoint) ||
-	       *pc == (Uint) BeamOp(op_i_debug_breakpoint) ||
-	       *pc == (Uint) BeamOp(op_i_count_breakpoint));
-	if (*pc == (Uint) BeamOp(op_i_debug_breakpoint)) {
+	ASSERT(*pc == (BeamInstr) BeamOp(op_i_trace_breakpoint) ||
+	       *pc == (BeamInstr) BeamOp(op_i_mtrace_breakpoint) ||
+	       *pc == (BeamInstr) BeamOp(op_i_debug_breakpoint) ||
+	       *pc == (BeamInstr) BeamOp(op_i_time_breakpoint) ||
+	       *pc == (BeamInstr) BeamOp(op_i_count_breakpoint) ||
+	       *pc == (BeamInstr) em_apply_bif);
+	if (*pc == (BeamInstr) BeamOp(op_i_debug_breakpoint)) {
 	    /* Debug bp must be last, so if it is also first; 
 	     * it must be singleton. */
-	    ASSERT(BpSingleton(*r)); 
+	    ASSERT(BpSingleton(*r));
 	    /* Insert new bp first in the ring, i.e second to last. */
 	    bd = Alloc(size);
 	    BpInitAndSpliceNext(bd, *pc, *r);
-	    *pc = break_op;
-	} else if ((*r)->prev->orig_instr 
-		   == (Uint) BeamOp(op_i_debug_breakpoint)) {
+	    if (bif == BREAK_IS_ERL) {
+		*pc = break_op;
+	    }
+	} else if ((*r)->prev->orig_instr
+		   == (BeamInstr) BeamOp(op_i_debug_breakpoint)) {
 	    /* Debug bp last in the ring; insert new second to last. */
 	    bd = Alloc(size);
 	    BpInitAndSplicePrev(bd, (*r)->prev->orig_instr, *r);
@@ -608,25 +1070,43 @@ static int set_function_break(Module *modp, Uint *pc,
 	    *r = bd;
 	}
     }
+    for (ix = 1; ix < erts_no_schedulers; ++ix) {
+	(*rs)[ix] = (*rs)[0];
+    }
+
+    bd->this_instr = break_op;
     /* Init the bp type specific data */
-    if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) ||
-	break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) {
+    if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) ||
+	break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) {
 		
 	BpDataTrace *bdt = (BpDataTrace *) bd;
 		
 	MatchSetRef(match_spec);
 	bdt->match_spec = match_spec;
 	bdt->tracer_pid = tracer_pid;
-    } else if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) {
+    } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) {
+	BpDataTime *bdt = (BpDataTime *) bd;
+	Uint i = 0;
+
+	bdt->pause = 0;
+	bdt->n     = erts_no_schedulers;
+	bdt->hash  = Alloc(sizeof(bp_time_hash_t)*(bdt->n));
+
+	for (i = 0; i < bdt->n; i++) {
+	    bp_hash_init(&(bdt->hash[i]), 32);
+	}
+    } else if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) {
 	BpDataCount *bdc = (BpDataCount *) bd;
+	erts_smp_atomic_init(&bdc->acount, 0);
+    }
 
-	bdc->count = 0;
+    if (bif == BREAK_IS_ERL) {
+	++(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]);
     }
-    ++(*(Uint*)&code_base[MI_NUM_BREAKPOINTS]);
     return 1;
 }
 
-static int clear_break(Eterm mfa[3], int specified, Uint break_op)
+static int clear_break(Eterm mfa[3], int specified, BeamInstr break_op)
 {
     int num_processed = 0;
     Module *modp;
@@ -652,52 +1132,71 @@ static int clear_break(Eterm mfa[3], int specified, Uint break_op)
 }
 
 static int clear_module_break(Module *m, Eterm mfa[3], int specified, 
-			      Uint break_op) {
-    Uint** code_base;
-    Uint* code_ptr;
+			      BeamInstr break_op) {
+    BeamInstr** code_base;
+    BeamInstr* code_ptr;
     int num_processed = 0;
-    Uint i,n;
+    Uint i;
+    BeamInstr n;
     
     ASSERT(m);
-    code_base = (Uint **) m->code;
+    code_base = (BeamInstr **) m->code;
     if (code_base == NULL) {
 	return 0;
     }
-    n = (Uint) code_base[MI_NUM_FUNCTIONS];
+    n = (BeamInstr) code_base[MI_NUM_FUNCTIONS];
     for (i = 0; i < n; ++i) {
 	code_ptr = code_base[MI_FUNCTIONS+i];
 	if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) &&
 	    (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) {
-	    Uint *pc = code_ptr + 5;
+	    BeamInstr *pc = code_ptr + 5;
 	    
 	    num_processed += 
-		clear_function_break(m, pc, break_op);
+		clear_function_break(m, pc, BREAK_IS_ERL, break_op);
 	}
     }
     return num_processed;
 }
 
-static int clear_function_break(Module *m, Uint *pc, Uint break_op) {
+static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr break_op) {
     BpData *bd;
-    Uint **code_base = (Uint **)m->code;
-    
-    ASSERT(code_base);
-    ASSERT(code_base <= (Uint **)pc);
-    ASSERT((Uint **)pc < code_base + (m->code_length/sizeof(Uint *)));
+    Uint ix = 0;
+    BeamInstr **code_base = NULL;
+
+    if (bif == BREAK_IS_ERL) {
+	code_base = (BeamInstr **)m->code;
+	ASSERT(code_base);
+	ASSERT(code_base <= (BeamInstr **)pc);
+	ASSERT((BeamInstr **)pc < code_base + (m->code_length/sizeof(BeamInstr *)));
+    } else {
+	ASSERT(*pc == (BeamInstr) em_apply_bif);
+	ASSERT(m == NULL);
+    }
+
     /*
      * Currently no trace support for native code.
      */
     if (erts_is_native_break(pc)) {
 	return 0;
     }
+
     while ( (bd = is_break(pc, break_op))) {
 	/* Remove all breakpoints of this type.
 	 * There should be only one of each type, 
 	 * but break_op may be 0 which matches any type. 
 	 */
-	Uint op;
-	BpData **r = (BpData **) (pc-4);
+	BeamInstr op;
+	BpData ***rs = (BpData ***) (pc - 4);
+	BpData   **r = NULL;
+
+#ifdef DEBUG
+	for (ix = 1; ix < erts_no_schedulers; ++ix) {
+	    ASSERT((*rs)[ix] == (*rs)[0]);
+	}
+#endif
 	
+	r = &((*rs)[0]);
+
 	ASSERT(*r);
 	/* Find opcode for this breakpoint */
 	if (break_op) {
@@ -713,8 +1212,11 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) {
 	if (BpSingleton(bd)) {
 	    ASSERT(*r == bd);
 	    /* Only one breakpoint to remove */
-	    *r  = NULL;
-	    *pc = bd->orig_instr;
+	    if (bif == BREAK_IS_ERL) {
+		*pc = bd->orig_instr;
+	    }
+	    Free(*rs);
+	    *rs = NULL;
 	} else {
 	    BpData *bd_prev = bd->prev;
 	    
@@ -726,22 +1228,64 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) {
 		bd_prev->orig_instr = bd->orig_instr;
 	    } else if (bd_prev == *r) {
 		/* We removed the first breakpoint in the ring */
-		*pc = bd->orig_instr;
+		if (bif == BREAK_IS_ERL) {
+		    *pc = bd->orig_instr;
+		}
 	    } else {
 		bd_prev->orig_instr = bd->orig_instr;
 	    }
 	}
-	if (op == (Uint) BeamOp(op_i_trace_breakpoint) ||
-	    op == (Uint) BeamOp(op_i_mtrace_breakpoint)) {
+	if (op == (BeamInstr) BeamOp(op_i_trace_breakpoint) ||
+	    op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) {
 	    
 	    BpDataTrace *bdt = (BpDataTrace *) bd;
-	    
 	    MatchSetUnref(bdt->match_spec);
 	}
+	if (op == (BeamInstr) BeamOp(op_i_time_breakpoint)) {
+	    BpDataTime *bdt = (BpDataTime *) bd;
+	    Uint i = 0;
+	    Uint j = 0;
+	    Process *h_p = NULL;
+	    bp_data_time_item_t *item = NULL;
+	    process_breakpoint_time_t *pbt = NULL;
+
+	    /* remove all psd associated with the hash
+	     * and then delete the hash.
+	     * ... sigh ...
+	     */
+
+	    for( i = 0; i < bdt->n; ++i) {
+		if (bdt->hash[i].used) {
+		    for (j = 0; j < bdt->hash[i].n; ++j) {
+			item = &(bdt->hash[i].item[j]);
+			if (item->pid != NIL) {
+			    h_p = process_tab[internal_pid_index(item->pid)];
+			    if (h_p) {
+				pbt = ERTS_PROC_SET_CALL_TIME(h_p, ERTS_PROC_LOCK_MAIN, NULL);
+				if (pbt) {
+				    Free(pbt);
+				}
+			    }
+			}
+		    }
+		}
+		bp_hash_delete(&(bdt->hash[i]));
+	    }
+	    Free(bdt->hash);
+	    bdt->hash = NULL;
+	    bdt->n = 0;
+	}
 	Free(bd);
-	ASSERT(((Uint) code_base[MI_NUM_BREAKPOINTS]) > 0);
-	--(*(Uint*)&code_base[MI_NUM_BREAKPOINTS]);
-    }
+	if (bif == BREAK_IS_ERL) {
+	    ASSERT(((BeamInstr) code_base[MI_NUM_BREAKPOINTS]) > 0);
+	    --(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]);
+	}
+	if (*rs) {
+	    for (ix = 1; ix < erts_no_schedulers; ++ix) {
+		(*rs)[ix] = (*rs)[0];
+	    }
+	}
+    } /* while bd != NULL */
     return 1;
 }
 
@@ -754,32 +1298,63 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) {
 ** returned. The program counter must point to the first executable
 ** (breakpoint) instruction of the function.
 */
-static BpData *is_break(Uint *pc, Uint break_op) {
-    ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI));
+
+BpData *erts_get_time_break(Process *p, BeamInstr *pc) {
+    return get_break(p, pc, (BeamInstr) BeamOp(op_i_time_breakpoint));
+}
+
+static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op) {
+    ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
     if (! erts_is_native_break(pc)) {
-	BpData *bd = (BpData *) pc[-4];
-	
-	if (break_op == 0) {
-	    return bd;
-	}
-	if (*pc == break_op) {
-	    ASSERT(bd);
-	    return bd->next;
-	}
-	if (! bd){
+	BpData **rs = (BpData **) pc[-4];
+	BpData  *bd = NULL, *ebd = NULL;
+
+	if (! rs) {
 	    return NULL;
 	}
+
+	bd = ebd = rs[bp_sched2ix_proc(p)];
+	ASSERT(bd);
+	if (bd->this_instr == break_op) {
+	    return bd;
+	}
+
 	bd = bd->next;
-	while (bd != (BpData *) pc[-4]) {
+	while (bd != ebd) {
 	    ASSERT(bd);
-	    if (bd->orig_instr == break_op) {
-		bd = bd->next;
+	    if (bd->this_instr == break_op) {
 		ASSERT(bd);
 		return bd;
-	    } else {
-		bd = bd->next;
 	    }
+	    bd = bd->next;
 	}
     }
     return NULL;
 }
+
+static BpData *is_break(BeamInstr *pc, BeamInstr break_op) {
+    BpData **rs = (BpData **) pc[-4];
+    BpData  *bd = NULL, *ebd = NULL;
+    ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+
+    if (! rs) {
+	return NULL;
+    }
+
+    bd = ebd = rs[bp_sched2ix()];
+    ASSERT(bd);
+    if ( (break_op == 0) || (bd->this_instr == break_op)) {
+	return bd;
+    }
+
+    bd = bd->next;
+    while (bd != ebd) {
+	ASSERT(bd);
+	if (bd->this_instr == break_op) {
+	    ASSERT(bd);
+	    return bd;
+	}
+	bd = bd->next;
+    }
+    return NULL;
+}
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 44e6b294d8..bd8a7249a7 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -27,28 +27,46 @@
 
 
 
-/*
-** Common struct to all bp_data_*
-**
-** Two gotchas: 
-**
-** 1) The type of bp_data structure in the ring is deduced from the 
-**    orig_instr field of the structure _before_ in the ring, except for 
-**    the first structure in the ring that has its instruction in
-**    pc[0] of the code to execute.
-**
-** 2) pc[-4] points to the _last_ structure in the ring before the
-**    breakpoints are being executed.
-**
-** So, as an example, when a breakpointed function starts to execute,
-** the first instruction that is a breakpoint instruction at pc[0] finds
-** its data at ((BpData *) pc[-4])->next and has to cast that pointer 
-** to the correct bp_data type.
+/* A couple of gotchas:
+ *
+ * The breakpoint structure from BeamInstr,
+ * In beam_emu where the instruction counter pointer, I (or pc),
+ * points to the *current* instruction. At that time, if the instruction
+ * is a breakpoint instruction the pc looks like the following,
+ *
+ * I[-5]   | op_i_func_info_IaaI |        scheduler specific entries
+ * I[-4]   |    BpData** bpa     |  --> | BpData * bdas1 | ... | BpData * bdasN |
+ * I[-3]   |  Tagged Module      |            |                    |
+ * I[-2]   |  Tagged Function    |            V                    V
+ * I[-1]   |  Arity              |          BpData -> BpData -> BpData -> BpData
+ * I[0]    | The bp instruction  |            ^       * the bp wheel *      |
+ *                                            |------------------------------
+ *
+ * Common struct to all bp_data_*
+ *
+ * 1) The type of bp_data structure in the ring is deduced from the
+ *    orig_instr field of the structure _before_ in the ring, except for
+ *    the first structure in the ring that has its instruction in
+ *    pc[0] of the code to execute.
+ *    This is valid as long as you don't search for the function while it is
+ *    being executed by something else. Or is in the middle of its rotation for
+ *    any other reason.
+ *    A key, the bp beam instruction, is included for this reason.
+ *
+ * 2) pc[-4][sched_id - 1] points to the _last_ structure in the ring before the
+ *    breakpoints are being executed.
+ *
+ * So, as an example, when a breakpointed function starts to execute,
+ * the first instruction that is a breakpoint instruction at pc[0] finds
+ * its data at ((BpData **) pc[-4][sched_id - 1])->next and has to cast that pointer
+ * to the correct bp_data type.
 */
+
 typedef struct bp_data {
     struct bp_data *next; /* Doubly linked ring pointers */
     struct bp_data *prev; /* -"-                         */
-    Uint orig_instr;      /* The original instruction to execute */
+    BeamInstr orig_instr;      /* The original instruction to execute */
+    BeamInstr this_instr;      /* key */
 } BpData;
 /*
 ** All the following bp_data_.. structs must begin the same way
@@ -57,7 +75,8 @@ typedef struct bp_data {
 typedef struct bp_data_trace {
     struct bp_data *next;
     struct bp_data *prev;
-    Uint            orig_instr;
+    BeamInstr       orig_instr;
+    BeamInstr       this_instr;   /* key */
     Binary         *match_spec;
     Eterm           tracer_pid;
 } BpDataTrace;
@@ -65,18 +84,58 @@ typedef struct bp_data_trace {
 typedef struct bp_data_debug {
     struct bp_data *next;
     struct bp_data *prev;
-    Uint            orig_instr;
+    BeamInstr       orig_instr;
+    BeamInstr       this_instr;   /* key */
 } BpDataDebug;
 
-typedef struct bp_data_count { /* Call count */
+typedef struct bp_data_count {    /* Call count */
     struct bp_data *next;
     struct bp_data *prev;
-    Uint            orig_instr;
-    Sint            count;
+    BeamInstr       orig_instr;
+    BeamInstr       this_instr;   /* key */
+    erts_smp_atomic_t acount;
 } BpDataCount;
 
+typedef struct {
+    Eterm pid;
+    Sint  count;
+    Uint  s_time;
+    Uint  us_time;
+} bp_data_time_item_t;
+
+typedef struct {
+    Uint n;
+    Uint used;
+    bp_data_time_item_t *item;
+} bp_time_hash_t;
+
+typedef struct bp_data_time {     /* Call time */
+    struct bp_data *next;
+    struct bp_data *prev;
+    BeamInstr       orig_instr;
+    BeamInstr       this_instr;   /* key */
+    Uint	    pause;
+    Uint	    n;
+    bp_time_hash_t  *hash;
+} BpDataTime;
+
+typedef struct {
+    Uint ms;
+    Uint s;
+    Uint us;
+    BeamInstr *pc;
+} process_breakpoint_time_t; /* used within psd */
+
 extern erts_smp_spinlock_t erts_bp_lock;
 
+#define ERTS_BP_CALL_TIME_SCHEDULE_IN      (0)
+#define ERTS_BP_CALL_TIME_SCHEDULE_OUT     (1)
+#define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)
+
+#define ERTS_BP_CALL_TIME_CALL      (0)
+#define ERTS_BP_CALL_TIME_RETURN    (1)
+#define ERTS_BP_CALL_TIME_TAIL_CALL (2)
+
 #ifdef ERTS_SMP
 #define ErtsSmpBPLock(BDC) erts_smp_spin_lock(&erts_bp_lock)
 #define ErtsSmpBPUnlock(BDC) erts_smp_spin_unlock(&erts_bp_lock)
@@ -85,31 +144,46 @@ extern erts_smp_spinlock_t erts_bp_lock;
 #define ErtsSmpBPUnlock(BDC)
 #endif
 
-#define ErtsCountBreak(pc,instr_result)                     \
-do {                                                        \
-    BpDataCount *bdc = (BpDataCount *) (pc)[-4];            \
-                                                            \
-    ASSERT((pc)[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); \
-    ASSERT(bdc);                                            \
-    bdc = (BpDataCount *) bdc->next;                        \
-    ASSERT(bdc);                                            \
-    (pc)[-4] = (Uint) bdc;                                  \
-    ErtsSmpBPLock(bdc);                                     \
-    if (bdc->count >= 0) bdc->count++;                      \
-    ErtsSmpBPUnlock(bdc);                                   \
-    *(instr_result) = bdc->orig_instr;                      \
+ERTS_INLINE Uint bp_sched2ix(void);
+
+#ifdef ERTS_SMP
+#define bp_sched2ix_proc(p) ((p)->scheduler_data->no - 1)
+#else
+#define bp_sched2ix_proc(p) (0)
+#endif
+
+#define ErtsCountBreak(p, pc,instr_result)                       \
+do {                                                             \
+    BpData **bds = (BpData **) (pc)[-4];                         \
+    BpDataCount *bdc = NULL;                                     \
+    Uint ix = bp_sched2ix_proc( (p) );                           \
+    erts_aint_t count = 0;                                       \
+                                                                 \
+    ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \
+    ASSERT(bds);                                                 \
+    bdc = (BpDataCount *) bds[ix];                               \
+    bdc = (BpDataCount *) bdc->next;                             \
+    ASSERT(bdc);                                                 \
+    bds[ix] = (BpData *) bdc;                                    \
+    count = erts_smp_atomic_read(&bdc->acount);                  \
+    if (count >= 0)  erts_smp_atomic_inc(&bdc->acount);          \
+    *(instr_result) = bdc->orig_instr;                           \
 } while (0)
 
-#define ErtsBreakSkip(pc,instr_result)                      \
-do {                                                        \
-    BpData *bd = (BpData *) (pc)[-4];                       \
-                                                            \
-    ASSERT((pc)[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); \
-    ASSERT(bd);                                             \
-    bd = bd->next;                                          \
-    ASSERT(bd);                                             \
-    (pc)[-4] = (Uint) bd;                                   \
-    *(instr_result) = bd->orig_instr;                       \
+#define ErtsBreakSkip(p, pc,instr_result)                        \
+do {                                                             \
+    BpData **bds = (BpData **) (pc)[-4];                         \
+    BpData *bd = NULL;                                           \
+    Uint ix = bp_sched2ix_proc( (p) );                           \
+                                                                 \
+    ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \
+    ASSERT(bds);                                                 \
+    bd = bds[ix];                                                \
+    ASSERT(bd);                                                  \
+    bd = bd->next;                                               \
+    ASSERT(bd);                                                  \
+    bds[ix] = bd;                                                \
+    *(instr_result) = bd->orig_instr;                            \
 } while (0)
 
 enum erts_break_op{
@@ -133,9 +207,9 @@ int erts_clear_trace_break(Eterm mfa[3], int specified);
 int erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec,
 			  Eterm tracer_pid);
 int erts_clear_mtrace_break(Eterm mfa[3], int specified);
-void erts_set_mtrace_bif(Uint *pc, Binary *match_spec, 
+void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec,
 			 Eterm tracer_pid);
-void erts_clear_mtrace_bif(Uint *pc);
+void erts_clear_mtrace_bif(BeamInstr *pc);
 int erts_set_debug_break(Eterm mfa[3], int specified);
 int erts_clear_debug_break(Eterm mfa[3], int specified);
 int erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op);
@@ -144,22 +218,33 @@ int erts_clear_count_break(Eterm mfa[3], int specified);
 
 int erts_clear_break(Eterm mfa[3], int specified);
 int erts_clear_module_break(Module *modp);
-int erts_clear_function_break(Module *modp, Uint *pc);
+int erts_clear_function_break(Module *modp, BeamInstr *pc);
 
-Uint erts_trace_break(Process *p, Uint *pc, Eterm *args, 
+BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args,
 		      Uint32 *ret_flags, Eterm *tracer_pid);
-Uint32 erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, 
+Uint32 erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args,
 		       int local, Eterm *tracer_pid);
 
-int erts_is_trace_break(Uint *pc, Binary **match_spec_ret, 
+int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret,
 			Eterm *tracer_pid_ret);
-int erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, 
+int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret,
 			 Eterm *tracer_pid_rte);
-int erts_is_mtrace_bif(Uint *pc, Binary **match_spec_ret, 
+int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret,
 		       Eterm *tracer_pid_ret);
-int erts_is_native_break(Uint *pc);
-int erts_is_count_break(Uint *pc, Sint *count_ret);
+int erts_is_native_break(BeamInstr *pc);
+int erts_is_count_break(BeamInstr *pc, Sint *count_ret);
+int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time);
+
+void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type);
+void erts_schedule_time_break(Process *p, Uint out);
+int erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op);
+int erts_clear_time_break(Eterm mfa[3], int specified);
+
+int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time);
+void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op);
+void erts_clear_time_trace_bif(BeamInstr *pc);
+BpData *erts_get_time_break(Process *p, BeamInstr *pc);
 
-Uint *erts_find_local_func(Eterm mfa[3]);
+BeamInstr *erts_find_local_func(Eterm mfa[3]);
 
 #endif /* _BEAM_BP_H */
diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c
index d5cef1cad2..e795b4efbd 100644
--- a/erts/emulator/beam/beam_catches.c
+++ b/erts/emulator/beam/beam_catches.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -26,7 +26,7 @@
 /* XXX: should use dynamic reallocation */
 #define TABSIZ (16*1024)
 static struct {
-    Eterm *cp;
+    BeamInstr *cp;
     unsigned cdr;
 } beam_catches[TABSIZ];
 
@@ -39,7 +39,7 @@ void beam_catches_init(void)
     high_mark = 0;
 }
 
-unsigned beam_catches_cons(Eterm *cp, unsigned cdr)
+unsigned beam_catches_cons(BeamInstr *cp, unsigned cdr)
 {
     int i;
 
@@ -65,7 +65,7 @@ unsigned beam_catches_cons(Eterm *cp, unsigned cdr)
     return i;
 }
 
-Eterm *beam_catches_car(unsigned i)
+BeamInstr *beam_catches_car(unsigned i)
 {
     if( i >= TABSIZ ) {
 	fprintf(stderr,
@@ -75,7 +75,7 @@ Eterm *beam_catches_car(unsigned i)
     return beam_catches[i].cp;
 }
 
-void beam_catches_delmod(unsigned head, Eterm *code, unsigned code_bytes)
+void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes)
 {
     unsigned i, cdr;
 
diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h
index ccf33d5e86..6223427f0d 100644
--- a/erts/emulator/beam/beam_catches.h
+++ b/erts/emulator/beam/beam_catches.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -23,9 +23,9 @@
 #define BEAM_CATCHES_NIL	(-1)
 
 void beam_catches_init(void);
-unsigned beam_catches_cons(Eterm* cp, unsigned cdr);
-Eterm *beam_catches_car(unsigned i);
-void beam_catches_delmod(unsigned head, Eterm* code, unsigned code_bytes);
+unsigned beam_catches_cons(BeamInstr* cp, unsigned cdr);
+BeamInstr *beam_catches_car(unsigned i);
+void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes);
 
 #define catch_pc(x)	beam_catches_car(catch_val((x)))
 
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 4242a4161e..fffb172c68 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -43,12 +43,12 @@
 #else
 # define HEXF "%08bpX"
 #endif
+#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
 void dbg_bt(Process* p, Eterm* sp);
-void dbg_where(Eterm* addr, Eterm x0, Eterm* reg);
+void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg);
 
-static void print_big(int to, void *to_arg, Eterm* addr);
-static int print_op(int to, void *to_arg, int op, int size, Eterm* addr);
+static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr);
 Eterm
 erts_debug_same_2(Process* p, Eterm term1, Eterm term2)
 {
@@ -124,6 +124,57 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool)
     BIF_ERROR(p, BADARG);
 }
 
+#if 0 /* Kept for conveninence when hard debugging. */
+void debug_dump_code(BeamInstr *I, int num)
+{
+    BeamInstr *code_ptr = I;
+    BeamInstr *end = code_ptr + num;
+    erts_dsprintf_buf_t *dsbufp;
+    BeamInstr instr;
+    int i;
+
+    dsbufp = erts_create_tmp_dsbuf(0);
+    while (code_ptr < end) {
+	erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
+	instr = (BeamInstr) code_ptr[0];
+	for (i = 0; i < NUM_SPECIFIC_OPS; i++) {
+	    if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') {
+		code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp,
+				     i, opc[i].sz-1, code_ptr+1) + 1;
+		break;
+	    }
+	}
+	if (i >= NUM_SPECIFIC_OPS) {
+	    erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp,
+		       "unknown " HEXF "\n", instr);
+	    code_ptr++;
+	}
+    }
+    dsbufp->str[dsbufp->str_len] = 0;
+    erts_fprintf(stderr,"%s", dsbufp->str);
+    erts_destroy_tmp_dsbuf(dsbufp);
+}
+#endif
+
+BIF_RETTYPE
+erts_debug_instructions_0(BIF_ALIST_0)
+{
+    int i = 0;
+    Uint needed = num_instructions * 2;
+    Eterm* hp;
+    Eterm res = NIL;
+
+    for (i = 0; i < num_instructions; i++) {
+	needed += 2*strlen(opc[i].name);
+    }
+    hp = HAlloc(BIF_P, needed);
+    for (i = num_instructions-1; i >= 0; i--) {
+	Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, strlen(opc[i].name));
+	res = erts_bld_cons(&hp, 0, s, res);
+    }
+    return res;
+}
+
 Eterm
 erts_debug_disassemble_1(Process* p, Eterm addr)
 {
@@ -132,16 +183,16 @@ erts_debug_disassemble_1(Process* p, Eterm addr)
     Eterm* tp;
     Eterm bin;
     Eterm mfa;
-    Eterm* funcinfo = NULL;	/* Initialized to eliminate warning. */
-    Uint* code_base;
-    Uint* code_ptr = NULL;	/* Initialized to eliminate warning. */
-    Uint instr;
-    Uint uaddr;
+    BeamInstr* funcinfo = NULL;	/* Initialized to eliminate warning. */
+    BeamInstr* code_base;
+    BeamInstr* code_ptr = NULL;	/* Initialized to eliminate warning. */
+    BeamInstr instr;
+    BeamInstr uaddr;
     Uint hsz;
     int i;
 
-    if (term_to_Uint(addr, &uaddr)) {
-	code_ptr = (Uint *) uaddr;
+    if (term_to_UWord(addr, &uaddr)) {
+	code_ptr = (BeamInstr *) uaddr;
 	if ((funcinfo = find_function_from_pc(code_ptr)) == NULL) {
 	    BIF_RET(am_false);
 	}
@@ -180,14 +231,14 @@ erts_debug_disassemble_1(Process* p, Eterm addr)
 	     * But this code_ptr will point to the start of the Export,
 	     * not the function's func_info instruction. BOOM !?
 	     */
-	    code_ptr = ((Eterm *) ep->address) - 5;
+	    code_ptr = ((BeamInstr *) ep->address) - 5;
 	    funcinfo = code_ptr+2;
 	} else if (modp == NULL || (code_base = modp->code) == NULL) {
 	    BIF_RET(am_undef);
 	} else {
 	    n = code_base[MI_NUM_FUNCTIONS];
 	    for (i = 0; i < n; i++) {
-		code_ptr = (Uint *) code_base[MI_FUNCTIONS+i];
+		code_ptr = (BeamInstr *) code_base[MI_FUNCTIONS+i];
 		if (code_ptr[3] == name && code_ptr[4] == arity) {
 		    funcinfo = code_ptr+2;
 		    break;
@@ -203,9 +254,9 @@ erts_debug_disassemble_1(Process* p, Eterm addr)
 
     dsbufp = erts_create_tmp_dsbuf(0);
     erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
-    instr = (Uint) code_ptr[0];
+    instr = (BeamInstr) code_ptr[0];
     for (i = 0; i < NUM_SPECIFIC_OPS; i++) {
-	if (instr == (Uint) BeamOp(i) && opc[i].name[0] != '\0') {
+	if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') {
 	    code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp,
 				 i, opc[i].sz-1, code_ptr+1) + 1;
 	    break;
@@ -216,15 +267,15 @@ erts_debug_disassemble_1(Process* p, Eterm addr)
 		   "unknown " HEXF "\n", instr);
 	code_ptr++;
     }
-    bin = new_binary(p, (byte *) dsbufp->str, (int) dsbufp->str_len);
+    bin = new_binary(p, (byte *) dsbufp->str, dsbufp->str_len);
     erts_destroy_tmp_dsbuf(dsbufp);
     hsz = 4+4;
-    (void) erts_bld_uint(NULL, &hsz, (Uint) code_ptr);
+    (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr);
     hp = HAlloc(p, hsz);
-    addr = erts_bld_uint(&hp, NULL, (Uint) code_ptr);
+    addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr);
     ASSERT(is_atom(funcinfo[0]));
     ASSERT(is_atom(funcinfo[1]));
-    mfa = TUPLE3(hp, funcinfo[0], funcinfo[1], make_small(funcinfo[2]));
+    mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2]));
     hp += 4;
     return TUPLE3(hp, addr, bin, mfa);
 }
@@ -236,20 +287,20 @@ dbg_bt(Process* p, Eterm* sp)
 
     while (sp < stack) {
 	if (is_CP(*sp)) {
-	    Eterm* addr = find_function_from_pc(cp_val(*sp));
+	    BeamInstr* addr = find_function_from_pc(cp_val(*sp));
 	    if (addr)
 		erts_fprintf(stderr,
 			     HEXF ": %T:%T/%bpu\n",
-			     addr, addr[0], addr[1], addr[2]);
+			     addr, (Eterm) addr[0], (Eterm) addr[1], addr[2]);
 	}
 	sp++;
     }
 }
 
 void
-dbg_where(Eterm* addr, Eterm x0, Eterm* reg)
+dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg)
 {
-    Eterm* f = find_function_from_pc(addr);
+    BeamInstr* f = find_function_from_pc(addr);
 
     if (f == NULL) {
 	erts_fprintf(stderr, "???\n");
@@ -259,7 +310,7 @@ dbg_where(Eterm* addr, Eterm x0, Eterm* reg)
 
 	addr = f;
 	arity = addr[2];
-	erts_fprintf(stderr, HEXF ": %T:%T(", addr, addr[0], addr[1]);
+	erts_fprintf(stderr, HEXF ": %T:%T(", addr, (Eterm) addr[0], (Eterm) addr[1]);
 	for (i = 0; i < arity; i++)
 	    erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0);
 	erts_fprintf(stderr, ")\n");
@@ -267,18 +318,19 @@ dbg_where(Eterm* addr, Eterm x0, Eterm* reg)
 }
 
 static int
-print_op(int to, void *to_arg, int op, int size, Eterm* addr)
+print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 {
     int i;
-    Uint tag;
+    BeamInstr tag;
     char* sign;
     char* start_prog;		/* Start of program for packer. */
     char* prog;			/* Current position in packer program. */
-    Uint stack[8];		/* Stack for packer. */
-    Uint* sp = stack;		/* Points to next free position. */
-    Uint packed = 0;		/* Accumulator for packed operations. */
-    Uint args[8];		/* Arguments for this instruction. */
-    Uint* ap;			/* Pointer to arguments. */
+    BeamInstr stack[8];		/* Stack for packer. */
+    BeamInstr* sp = stack;		/* Points to next free position. */
+    BeamInstr packed = 0;		/* Accumulator for packed operations. */
+    BeamInstr args[8];		/* Arguments for this instruction. */
+    BeamInstr* ap;			/* Pointer to arguments. */
+    BeamInstr* unpacked;		/* Unpacked arguments */
 
     start_prog = opc[op].pack;
 
@@ -288,7 +340,7 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 	 * Avoid copying because instructions containing bignum operands
 	 * are bigger than actually declared.
 	 */
-	ap = (Uint *) addr;
+	ap = (BeamInstr *) addr;
     } else {
 	/*
 	 * Copy all arguments to a local buffer for the unpacking.
@@ -324,9 +376,15 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 		packed >>= BEAM_TIGHT_SHIFT;
 		break;
 	    case '6':		/* Shift 16 steps */
-		*ap++ = packed & 0xffff;
-		packed >>= 16;
+		*ap++ = packed & BEAM_LOOSE_MASK;
+		packed >>= BEAM_LOOSE_SHIFT;
 		break;
+#ifdef ARCH_64
+	    case 'w':		/* Shift 32 steps */
+		*ap++ = packed & BEAM_WIDE_MASK;
+		packed >>= BEAM_WIDE_SHIFT;
+		break;
+#endif
 	    case 'p':
 		*sp++ = *--ap;
 		break;
@@ -353,7 +411,7 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 	    break;
 	case 'x':		/* x(N) */
 	    if (reg_index(ap[0]) == 0) {
-		erts_print(to, to_arg, "X[0]");
+		erts_print(to, to_arg, "x[0]");
 	    } else {
 		erts_print(to, to_arg, "x(%d)", reg_index(ap[0]));
 	    }
@@ -390,11 +448,11 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 	case 'i':		/* Tagged integer */
 	case 'c':		/* Tagged constant */
 	case 'q':		/* Tagged literal */
-	    erts_print(to, to_arg, "%T", *ap);
+	    erts_print(to, to_arg, "%T", (Eterm) *ap);
 	    ap++;
 	    break;
 	case 'A':
-	    erts_print(to, to_arg, "%d", arityval(ap[0]));
+	    erts_print(to, to_arg, "%d", arityval( (Eterm) ap[0]));
 	    ap++;
 	    break;
 	case 'd':		/* Destination (x(0), x(N), y(N)) */
@@ -421,30 +479,36 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 	    ap++;
 	    break;
 	case 'f':		/* Destination label */
-	    erts_print(to, to_arg, "f(%X)", *ap);
-	    ap++;
+	    {
+		BeamInstr* f = find_function_from_pc((BeamInstr *)*ap);
+		if (f+3 != (BeamInstr *) *ap) {
+		    erts_print(to, to_arg, "f(" HEXF ")", *ap);
+		} else {
+		    erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]);
+		}
+		ap++;
+	    }
 	    break;
 	case 'p':		/* Pointer (to label) */
 	    {
-		Eterm* f = find_function_from_pc((Eterm *)*ap);
-
-		if (f+3 != (Eterm *) *ap) {
-		    erts_print(to, to_arg, "p(%X)", *ap);
+		BeamInstr* f = find_function_from_pc((BeamInstr *)*ap);
+		if (f+3 != (BeamInstr *) *ap) {
+		    erts_print(to, to_arg, "p(" HEXF ")", *ap);
 		} else {
-		    erts_print(to, to_arg, "%T:%T/%bpu", f[0], f[1], f[2]);
+		    erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]);
 		}
 		ap++;
 	    }
 	    break;
 	case 'j':		/* Pointer (to label) */
-	    erts_print(to, to_arg, "j(%X)", *ap);
+	    erts_print(to, to_arg, "j(" HEXF ")", *ap);
 	    ap++;
 	    break;
 	case 'e':		/* Export entry */
 	    {
 		Export* ex = (Export *) *ap;
 		erts_print(to, to_arg,
-			   "%T:%T/%bpu", ex->code[0], ex->code[1], ex->code[2]);
+			   "%T:%T/%bpu", (Eterm) ex->code[0], (Eterm) ex->code[1], ex->code[2]);
 		ap++;
 	    }
 	    break;
@@ -467,7 +531,8 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 	    ap++;
 	    break;
 	case 'P':	/* Byte offset into tuple (see beam_load.c) */
-	    erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm*)) - 1);
+	case 'Q':	/* Like 'P', but packable */
+	    erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm)) - 1);
 	    ap++;
 	    break;
 	case 'l':		/* fr(N) */
@@ -487,62 +552,90 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
      * Print more information about certain instructions.
      */
 
+    unpacked = ap;
     ap = addr + size;
     switch (op) {
-    case op_i_select_val_sfI:
+    case op_i_select_val_rfI:
+    case op_i_select_val_xfI:
+    case op_i_select_val_yfI:
 	{
 	    int n = ap[-1];
 
 	    while (n > 0) {
-		erts_print(to, to_arg, "%T f(%X) ", ap[0], ap[1]);
+		erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]);
 		ap += 2;
 		size += 2;
 		n--;
 	    }
 	}
 	break;
-    case op_i_jump_on_val_sfII:
+    case op_i_select_tuple_arity_rfI:
+    case op_i_select_tuple_arity_xfI:
+    case op_i_select_tuple_arity_yfI:
+	{
+	    int n = ap[-1];
+
+	    while (n > 0) {
+		Uint arity = arityval(ap[0]);
+		erts_print(to, to_arg, " {%d} f(" HEXF ")", arity, ap[1]);
+		ap += 2;
+		size += 2;
+		n--;
+	    }
+	}
+	break;
+    case op_i_jump_on_val_rfII:
+    case op_i_jump_on_val_xfII:
+    case op_i_jump_on_val_yfII:
 	{
 	    int n;
 	    for (n = ap[-2]; n > 0; n--) {
-		erts_print(to, to_arg, "f(%X) ", ap[0]);
+		erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
 		ap++;
 		size++;
 	    }
 	}
 	break;
-    case op_i_select_big_sf:
-	while (ap[0]) {
-	    int arity = thing_arityval(ap[0]);
-	    print_big(to, to_arg, ap);
-	    size += arity+1;
-	    ap += arity+1;
-	    erts_print(to, to_arg, " f(%X) ", ap[0]);
-	    ap++;
-	    size++;
+    case op_i_jump_on_val_zero_rfI:
+    case op_i_jump_on_val_zero_xfI:
+    case op_i_jump_on_val_zero_yfI:
+	{
+	    int n;
+	    for (n = ap[-1]; n > 0; n--) {
+		erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
+		ap++;
+		size++;
+	    }
+	}
+	break;
+    case op_i_put_tuple_rI:
+    case op_i_put_tuple_xI:
+    case op_i_put_tuple_yI:
+	{
+	    int n = unpacked[-1];
+
+	    while (n > 0) {
+		if (!is_header(ap[0])) {
+		    erts_print(to, to_arg, " %T", (Eterm) ap[0]);
+		} else {
+		    switch ((ap[0] >> 2) & 0x03) {
+		    case R_REG_DEF:
+			erts_print(to, to_arg, " x(0)");
+			break;
+		    case X_REG_DEF:
+			erts_print(to, to_arg, " x(%d)", ap[0] >> 4);
+			break;
+		    case Y_REG_DEF:
+			erts_print(to, to_arg, " y(%d)", ap[0] >> 4);
+			break;
+		    }
+		}
+		ap++, size++, n--;
+	    }
 	}
-	ap++;
-	size++;
 	break;
     }
     erts_print(to, to_arg, "\n");
 
     return size;
 }
-
-static void
-print_big(int to, void *to_arg, Eterm* addr)
-{
-    int i;
-    int k;
-
-    i = BIG_SIZE(addr);
-    if (BIG_SIGN(addr))
-	erts_print(to, to_arg, "-#integer(%d) = {", i);
-    else
-	erts_print(to, to_arg, "#integer(%d) = {", i);
-    erts_print(to, to_arg, "%d", BIG_DIGIT(addr, 0));
-    for (k = 1; k < i; k++)
-	erts_print(to, to_arg, ",%d", BIG_DIGIT(addr, k));
-    erts_print(to, to_arg, "}");
-}
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 2f7f48193d..fb90a7d4f7 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -117,6 +117,7 @@ do {                                     \
 #endif
 
 #define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4]))
+#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
 
 /*
@@ -138,8 +139,8 @@ do {                                     \
 #define VALID_INSTR(IP) (0 <= (int)(IP) && ((int)(IP) < (NUMBER_OF_OPCODES*2+10)))
 #else
 #define VALID_INSTR(IP) \
-   ((Sint)LabelAddr(emulator_loop) <= (Sint)(IP) && \
-    (Sint)(IP) < (Sint)LabelAddr(end_emulator_loop))
+   ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \
+    (SWord)(IP) < (SWord)LabelAddr(end_emulator_loop))
 #endif /* NO_JUMP_TABLE */
 
 #define SET_CP(p, ip)           \
@@ -181,11 +182,11 @@ do {                                     \
  
 #define StoreBifResult(Dst, Result)                          \
   do {                                                       \
-    Eterm* stb_next;                                         \
+    BeamInstr* stb_next;                                         \
     Eterm stb_reg;                                           \
     stb_reg = Arg(Dst);                                      \
     I += (Dst) + 2;                                          \
-    stb_next = (Eterm *) *I;                                 \
+    stb_next = (BeamInstr *) *I;                                 \
     CHECK_TERM(Result);                                      \
     switch (beam_reg_tag(stb_reg)) {                         \
     case R_REG_DEF:                                          \
@@ -205,7 +206,7 @@ do {                                     \
       c_p->cp = 0;				\
    } while(0)
 
-#define RESTORE_CP(X)		SET_CP(c_p, cp_val(*(X)))
+#define RESTORE_CP(X)		SET_CP(c_p, (BeamInstr *) cp_val(*(X)))
 
 #define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y))
 
@@ -213,13 +214,13 @@ do {                                     \
  * Special Beam instructions.
  */
 
-Eterm beam_apply[2];
-Eterm beam_exit[1];
-Eterm beam_continue_exit[1];
+BeamInstr beam_apply[2];
+BeamInstr beam_exit[1];
+BeamInstr beam_continue_exit[1];
 
-Eterm* em_call_error_handler;
-Eterm* em_apply_bif;
-Eterm* em_call_traced_function;
+BeamInstr* em_call_error_handler;
+BeamInstr* em_apply_bif;
+BeamInstr* em_call_traced_function;
 
 
 /* NOTE These should be the only variables containing trace instructions.
@@ -227,9 +228,10 @@ Eterm* em_call_traced_function;
 **      for the refering variable (one of these), and rouge references
 **      will most likely cause chaos.
 */
-Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */
-Eterm beam_return_trace[1];    /* OpCode(i_return_trace) */
-Eterm beam_exception_trace[1]; /* UGLY also OpCode(i_return_trace) */
+BeamInstr beam_return_to_trace[1];   /* OpCode(i_return_to_trace) */
+BeamInstr beam_return_trace[1];      /* OpCode(i_return_trace) */
+BeamInstr beam_exception_trace[1];   /* UGLY also OpCode(i_return_trace) */
+BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
 
 /*
  * All Beam instructions in numerical order.
@@ -319,6 +321,7 @@ extern int count_instructions;
 #  define POST_BIF_GC_SWAPIN_0(_p, _res)				\
      ERTS_SMP_REQ_PROC_MAIN_LOCK((_p));					\
      PROCESS_MAIN_CHK_LOCKS((_p));					\
+     ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p));				\
      if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
        _res = erts_gc_after_bif_call((_p), (_res), NULL, 0);		\
        E = (_p)->stop;							\
@@ -326,6 +329,7 @@ extern int count_instructions;
      HTOP = HEAP_TOP((_p))
 
 #  define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity)			\
+     ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p));				\
      ERTS_SMP_REQ_PROC_MAIN_LOCK((_p));					\
      PROCESS_MAIN_CHK_LOCKS((_p));					\
      if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
@@ -342,6 +346,8 @@ extern int count_instructions;
 #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
 #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
 #define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
+#define Qb(N) (N)
+#define Ib(N) (N)
 #define x(N) reg[N]
 #define y(N) E[N]
 #define r(N) x##N
@@ -363,6 +369,7 @@ extern int count_instructions;
            reg[0] = r(0); \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
            FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
+           ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
            r(0) = reg[0]; \
            SWAPIN; \
@@ -416,6 +423,7 @@ extern int count_instructions;
        reg[0] = r(0);                                           		\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  		\
+       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);					\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
        r(0) = reg[0];                                           		\
        SWAPIN;                                                  		\
@@ -438,6 +446,7 @@ extern int count_instructions;
        reg[0] = r(0);                                           \
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  \
+       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);			\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
        r(0) = reg[0];                                           \
        SWAPIN;                                                  \
@@ -460,6 +469,7 @@ extern int count_instructions;
        reg[Live] = Extra;						\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1);	\
+       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);				\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
        if (Live > 0) {							\
 	   r(0) = reg[0];						\
@@ -470,6 +480,13 @@ extern int count_instructions;
     HEAP_SPACE_VERIFIED(need);                                      \
   } while (0)
 
+#define TestHeapPutList(Need, Reg)		\
+  do {						\
+     TestHeap((Need), 1);			\
+     PutList(Reg, r(0), r(0), StoreSimpleDest);	\
+     CHECK_TERM(r(0));				\
+  } while (0)
+
 #ifdef HYBRID
 #ifdef INCREMENTAL
 #define TestGlobalHeap(Nh, Live, hp)                                    \
@@ -514,6 +531,11 @@ extern int count_instructions;
      SWAPIN;							\
   } while (0)
 
+#define PutTuple(Dst, Arity)			\
+ do {						\
+   Dst = make_tuple(HTOP);			\
+   pt_arity = (Arity);				\
+ } while (0)
 
 /*
  * Check that we haven't used the reductions and jump to function pointed to by
@@ -522,8 +544,8 @@ extern int count_instructions;
 
 #define DispatchMacro()				\
   do {						\
-     Eterm* dis_next;				\
-     dis_next = (Eterm *) *I;			\
+     BeamInstr* dis_next;				\
+     dis_next = (BeamInstr *) *I;			\
      CHECK_ARGS(I);				\
      if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
         FCALLS--;				\
@@ -535,8 +557,8 @@ extern int count_instructions;
 
 #define DispatchMacroFun()			\
   do {						\
-     Eterm* dis_next;				\
-     dis_next = (Eterm *) *I;			\
+     BeamInstr* dis_next;				\
+     dis_next = (BeamInstr *) *I;			\
      CHECK_ARGS(I);				\
      if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
         FCALLS--;				\
@@ -590,7 +612,7 @@ extern int count_instructions;
     ASSERT(VALID_INSTR(*I));   \
     Goto(*I)
 
-#define PreFetch(N, Dst) do { Dst = (Eterm *) *(I + N + 1); } while (0)
+#define PreFetch(N, Dst) do { Dst = (BeamInstr *) *(I + N + 1); } while (0)
 #define NextPF(N, Dst)         \
     I += N + 1;                \
     ASSERT(VALID_INSTR(Dst));  \
@@ -644,7 +666,7 @@ extern int count_instructions;
 #define DeallocateReturn(Deallocate)       \
   do {                                     \
     int words_to_pop = (Deallocate);       \
-    SET_I(cp_val(*E));                     \
+    SET_I((BeamInstr *) cp_val(*E));                     \
     E = ADD_BYTE_OFFSET(E, words_to_pop);  \
     CHECK_TERM(r(0));                      \
     Goto(*I);                              \
@@ -657,79 +679,77 @@ extern int count_instructions;
 #define MoveCall(Src, Dest, CallDest, Size)	\
     (Dest) = (Src);				\
     SET_CP(c_p, I+Size+1);			\
-    SET_I((Eterm *) CallDest);			\
+    SET_I((BeamInstr *) CallDest);			\
     Dispatch();
 
 #define MoveCallLast(Src, Dest, CallDest, Deallocate)	\
     (Dest) = (Src);					\
     RESTORE_CP(E);					\
     E = ADD_BYTE_OFFSET(E, (Deallocate));		\
-    SET_I((Eterm *) CallDest);				\
+    SET_I((BeamInstr *) CallDest);				\
     Dispatch();
 
 #define MoveCallOnly(Src, Dest, CallDest)	\
     (Dest) = (Src);				\
-    SET_I((Eterm *) CallDest);			\
+    SET_I((BeamInstr *) CallDest);			\
     Dispatch();
 
+#define MoveJump(Src)				\
+     r(0) = (Src);				\
+     SET_I((BeamInstr *) Arg(0));		\
+     Goto(*I);
+
 #define GetList(Src, H, T) do {			\
    Eterm* tmp_ptr = list_val(Src);		\
    H = CAR(tmp_ptr);				\
    T = CDR(tmp_ptr); } while (0)
 
-#define GetTupleElement(Src, Element, Dest)				  \
-  do {									  \
-    tmp_arg1 = (Eterm) (((unsigned char *) tuple_val(Src)) + (Element));  \
-    (Dest) = (*(Eterm *)tmp_arg1);					  \
+#define GetTupleElement(Src, Element, Dest)				\
+  do {									\
+    tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + 	\
+				(Element));				\
+    (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1));			\
   } while (0)
 
-#define ExtractNextElement(Dest)				\
-    tmp_arg1 += sizeof(Eterm);					\
-    (Dest) = (* (Eterm *) (((unsigned char *) tmp_arg1)))
+#define ExtractNextElement(Dest)					  \
+    tmp_arg1 += sizeof(Eterm);						  \
+    (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1))))
 
-#define ExtractNextElement2(Dest)		\
-  do {						\
-    Eterm* ene_dstp = &(Dest);			\
-    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
-    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
-    tmp_arg1 += sizeof(Eterm) + sizeof(Eterm);	\
+#define ExtractNextElement2(Dest)				\
+  do {								\
+    Eterm* ene_dstp = &(Dest);					\
+    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
+    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
+    tmp_arg1 += sizeof(Eterm) + sizeof(Eterm);			\
   } while (0)
 
 #define ExtractNextElement3(Dest)		\
   do {						\
     Eterm* ene_dstp = &(Dest);			\
-    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
-    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
-    ene_dstp[2] = ((Eterm *) tmp_arg1)[3];	\
+    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
+    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
+    ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3];	\
     tmp_arg1 += 3*sizeof(Eterm);		\
   } while (0)
 
 #define ExtractNextElement4(Dest)		\
   do {						\
     Eterm* ene_dstp = &(Dest);			\
-    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
-    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
-    ene_dstp[2] = ((Eterm *) tmp_arg1)[3];	\
-    ene_dstp[3] = ((Eterm *) tmp_arg1)[4];	\
+    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
+    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
+    ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3];	\
+    ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4];	\
     tmp_arg1 += 4*sizeof(Eterm);		\
   } while (0)
 
 #define ExtractElement(Element, Dest)		\
   do {						\
      tmp_arg1 += (Element);			\
-     (Dest) = (* (Eterm *) tmp_arg1);		\
+     (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1));		\
   } while (0)
 
-#define PutTuple(Arity, Src, Dest)		\
-     ASSERT(is_arity_value(Arity));		\
-     Dest = make_tuple(HTOP);			\
-     HTOP[0] = (Arity);				\
-     HTOP[1] = (Src);				\
-     HTOP += 2
-
-#define Put(Word) *HTOP++ = (Word)
-
 #define EqualImmed(X, Y, Action) if (X != Y) { Action; }
+#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
 
 #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }
 
@@ -759,8 +779,13 @@ extern int count_instructions;
 
 #define IsTuple(X, Action) if (is_not_tuple(X)) Action
 
-#define IsArity(Pointer, Arity, Fail) \
-    if (*(Eterm *)(tmp_arg1 = (Eterm)tuple_val(Pointer)) != (Arity)) { Fail; }
+#define IsArity(Pointer, Arity, Fail)					  \
+    if (*(Eterm *)							  \
+	EXPAND_POINTER(tmp_arg1 = (Eterm) 				  \
+		       COMPRESS_POINTER(tuple_val(Pointer))) != (Arity))  \
+    { 									  \
+        Fail; 								  \
+    }
 
 #define IsFunction(X, Action)			\
   do {						\
@@ -776,11 +801,14 @@ extern int count_instructions;
      }						\
   } while (0)
 
-#define IsTupleOfArity(Src, Arity, Fail) \
-  do { \
-    if (is_not_tuple(Src) || *(Eterm *)(tmp_arg1 = (Eterm) tuple_val(Src)) != Arity) { \
-        Fail; \
-    } \
+#define IsTupleOfArity(Src, Arity, Fail)				      \
+  do {									      \
+    if (is_not_tuple(Src) || 						      \
+	*(Eterm *)							      \
+	EXPAND_POINTER(tmp_arg1 = 					      \
+		       (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \
+        Fail;								      \
+    }									      \
   } while (0)
 
 #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; }
@@ -791,7 +819,7 @@ extern int count_instructions;
 #define IsBitstring(Src, Fail) \
   if (is_not_binary(Src)) { Fail; }
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define BsSafeMul(A, B, Fail, Target)		\
    do { Uint64 _res = (A) * (B);		\
       if (_res / B != A) { Fail; }		\
@@ -973,34 +1001,55 @@ extern int count_instructions;
 #define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; }
 #define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; }
 
-static BifFunction translate_gc_bif(void* gcf);
-static Eterm* handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf);
-static Eterm* next_catch(Process* c_p, Eterm *reg);
+/*
+ * process_main() is already huge, so we want to avoid inlining
+ * into it. Especially functions that are seldom used.
+ */
+#ifdef __GNUC__
+#  define NOINLINE __attribute__((__noinline__))
+#else
+#  define NOINLINE
+#endif
+
+/*
+ * The following functions are called directly by process_main().
+ * Don't inline them.
+ */
+static BifFunction translate_gc_bif(void* gcf) NOINLINE;
+static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
+			       Eterm* reg, BifFunction bf) NOINLINE;
+static BeamInstr* call_error_handler(Process* p, BeamInstr* ip,
+				     Eterm* reg, Eterm func) NOINLINE;
+static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE;
+static BeamInstr* apply(Process* p, Eterm module, Eterm function,
+			Eterm args, Eterm* reg) NOINLINE;
+static BeamInstr* call_fun(Process* p, int arity,
+			   Eterm* reg, Eterm args) NOINLINE;
+static BeamInstr* apply_fun(Process* p, Eterm fun,
+			    Eterm args, Eterm* reg) NOINLINE;
+static Eterm new_fun(Process* p, Eterm* reg,
+		     ErlFunEntry* fe, int num_free) NOINLINE;
+
+
+/*
+ * Functions not directly called by process_main(). OK to inline.
+ */
+static BeamInstr* next_catch(Process* c_p, Eterm *reg);
 static void terminate_proc(Process* c_p, Eterm Value);
 static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc);
-static void save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg,
+static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
 			     BifFunction bf, Eterm args);
 static struct StackTrace * get_trace_from_exc(Eterm exc);
 static Eterm make_arglist(Process* c_p, Eterm* reg, int a);
-static Eterm call_error_handler(Process* p, Eterm* ip, Eterm* reg);
-static Eterm call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg);
-static Uint* fixed_apply(Process* p, Eterm* reg, Uint arity);
-static Eterm* apply(Process* p, Eterm module, Eterm function,
-		     Eterm args, Eterm* reg);
-static int hibernate(Process* c_p, Eterm module, Eterm function,
-		     Eterm args, Eterm* reg);
-static Eterm* call_fun(Process* p, int arity, Eterm* reg, Eterm args);
-static Eterm* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg);
-static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free);
-
-#if defined(_OSE_) || defined(VXWORKS)
+
+#if defined(VXWORKS)
 static int init_done;
 #endif
 
 void
 init_emulator(void)
 {
-#if defined(_OSE_) || defined(VXWORKS)
+#if defined(VXWORKS)
     init_done = 0;
 #endif
     process_main();
@@ -1039,7 +1088,7 @@ init_emulator(void)
  */
 void process_main(void)
 {
-#if !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(VXWORKS)
     static int init_done = 0;
 #endif
     Process* c_p = NULL;
@@ -1078,7 +1127,7 @@ void process_main(void)
     /*
      * Pointer to next threaded instruction.
      */
-    register Eterm *I REG_I = NULL;
+    register BeamInstr *I REG_I = NULL;
 
     /* Number of reductions left.  This function
      * returns to the scheduler when FCALLS reaches zero.
@@ -1090,9 +1139,14 @@ void process_main(void)
      */
     register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
     register Eterm tmp_arg2 REG_tmp_arg2 = NIL;
-    Eterm tmp_big[2];		/* Temporary buffer for small bignums. */
+#if HEAP_ON_C_STACK
+    Eterm tmp_big[2];           /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */
+#else
+    Eterm *tmp_big;		/* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */
+#endif
 
 #ifndef ERTS_SMP
+#if !HALFWORD_HEAP
     static Eterm save_reg[ERTS_X_REGS_ALLOCATED];
     /* X registers -- not used directly, but
      * through 'reg', because using it directly
@@ -1100,7 +1154,7 @@ void process_main(void)
      * while using it through reg needs only
      * one.
      */
-
+#endif
     /*
      * Floating point registers.
      */
@@ -1130,6 +1184,8 @@ void process_main(void)
 
     Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */
 
+    Eterm pt_arity;		/* Used by do_put_tuple */
+
     ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
 
 
@@ -1141,13 +1197,17 @@ void process_main(void)
      * Note: c_p->arity must be set to reflect the number of useful terms in
      * c_p->arg_reg before calling the scheduler.
      */
-
     if (!init_done) {
 	init_done = 1;
 	goto init_emulator;
     }
 #ifndef ERTS_SMP
+#if !HALFWORD_HEAP
     reg = save_reg;	/* XXX: probably wastes a register on x86 */
+#else
+    /* Registers need to be heap allocated (correct memory range) for tracing to work */
+    reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm));
+#endif
 #endif
     c_p = NULL;
     reds_used = 0;
@@ -1158,7 +1218,12 @@ void process_main(void)
  do_schedule1:
     PROCESS_MAIN_CHK_LOCKS(c_p);
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+#if HALFWORD_HEAP
+    ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0);
+#endif
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     c_p = schedule(c_p, reds_used);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 #ifdef DEBUG
     pid = c_p->id;
 #endif
@@ -1168,11 +1233,14 @@ void process_main(void)
     reg = c_p->scheduler_data->save_reg;
     freg = c_p->scheduler_data->freg;
 #endif
+#if !HEAP_ON_C_STACK
+    tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap;
+#endif
     ERL_BITS_RELOAD_STATEP(c_p);
     {
 	int reds;
 	Eterm* argp;
-	Eterm* next;
+	BeamInstr *next;
 	int i;
 
 	argp = c_p->arg_reg;
@@ -1199,7 +1267,7 @@ void process_main(void)
 	    FCALLS = REDS_IN(c_p) = reds;
 	}
 
-	next = (Eterm *) *I;
+	next = (BeamInstr *) *I;
 	r(0) = c_p->arg_reg[0];
 #ifdef HARDDEBUG
 	if (c_p->arity > 0) {
@@ -1223,6 +1291,52 @@ void process_main(void)
 #define STORE_ARITH_RESULT(res) StoreBifResult(2, (res));
 #define ARITH_FUNC(name) erts_gc_##name
 
+	{
+	    Eterm increment_reg_val;
+	    Eterm increment_val;
+	    Uint live;
+	    Eterm result;
+
+	OpCase(i_increment_yIId):
+	    increment_reg_val = yb(Arg(0));
+	    goto do_increment;
+
+	OpCase(i_increment_xIId):
+	    increment_reg_val = xb(Arg(0));
+	    goto do_increment;
+
+	OpCase(i_increment_rIId):
+	    increment_reg_val = r(0);
+	    I--;
+
+	do_increment:
+	    increment_val = Arg(1);
+	    if (is_small(increment_reg_val)) {
+		Sint i = signed_val(increment_reg_val) + increment_val;
+		ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
+		if (MY_IS_SSMALL(i)) {
+		    result = make_small(i);
+		store_result:
+		    StoreBifResult(3, result);
+		}
+	    }
+
+	    live = Arg(2);
+	    SWAPOUT;
+	    reg[0] = r(0);
+	    reg[live] = increment_reg_val;
+	    reg[live+1] = make_small(increment_val);
+	    result = erts_gc_mixed_plus(c_p, reg, live);
+	    r(0) = reg[0];
+	    SWAPIN;
+	    ERTS_HOLE_CHECK(c_p);
+	    if (is_value(result)) {
+		goto store_result;
+	    }
+	    ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
+	    goto find_func_info;
+	}
+	    
  OpCase(i_plus_jId):
  {
      Eterm result;
@@ -1286,12 +1400,58 @@ void process_main(void)
     }
     Next(1);
 
+    {
+	Eterm is_eq_exact_lit_val;
+
+    OpCase(i_is_eq_exact_literal_xfc):
+	is_eq_exact_lit_val = xb(Arg(0));
+	I++;
+	goto do_is_eq_exact_literal;
+
+    OpCase(i_is_eq_exact_literal_yfc):
+	is_eq_exact_lit_val = yb(Arg(0));
+	I++;
+	goto do_is_eq_exact_literal;
+
+    OpCase(i_is_eq_exact_literal_rfc):
+	is_eq_exact_lit_val = r(0);
+
+    do_is_eq_exact_literal:
+	if (!eq(Arg(1), is_eq_exact_lit_val)) {
+	    ClauseFail();
+	}
+	Next(2);
+    }
+
+    {
+	Eterm is_ne_exact_lit_val;
+
+    OpCase(i_is_ne_exact_literal_xfc):
+	is_ne_exact_lit_val = xb(Arg(0));
+	I++;
+	goto do_is_ne_exact_literal;
+
+    OpCase(i_is_ne_exact_literal_yfc):
+	is_ne_exact_lit_val = yb(Arg(0));
+	I++;
+	goto do_is_ne_exact_literal;
+
+    OpCase(i_is_ne_exact_literal_rfc):
+	is_ne_exact_lit_val = r(0);
+
+    do_is_ne_exact_literal:
+	if (eq(Arg(1), is_ne_exact_lit_val)) {
+	    ClauseFail();
+	}
+	Next(2);
+    }
+
  OpCase(i_move_call_only_fcr): {
      r(0) = Arg(1);
  }
  /* FALL THROUGH */
  OpCase(i_call_only_f): {
-     SET_I((Eterm *) Arg(0));
+     SET_I((BeamInstr *) Arg(0));
      Dispatch();
  }
 
@@ -1302,7 +1462,7 @@ void process_main(void)
  OpCase(i_call_last_fP): {
      RESTORE_CP(E);
      E = ADD_BYTE_OFFSET(E, Arg(1));
-     SET_I((Eterm *) Arg(0));
+     SET_I((BeamInstr *) Arg(0));
      Dispatch();
  }
 
@@ -1313,7 +1473,7 @@ void process_main(void)
  /* FALL THROUGH */
  OpCase(i_call_f): {
      SET_CP(c_p, I+2);
-     SET_I((Eterm *) Arg(0));
+     SET_I((BeamInstr *) Arg(0));
      Dispatch();
  }
 
@@ -1349,7 +1509,7 @@ void process_main(void)
     Dispatchx();
 
  OpCase(init_y): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(1, next);
      make_blank(yb(Arg(0)));
@@ -1357,7 +1517,7 @@ void process_main(void)
  }
 
  OpCase(i_trim_I): {
-     Eterm* next;
+     BeamInstr *next;
      Uint words;
      Uint cp;
 
@@ -1369,6 +1529,17 @@ void process_main(void)
      NextPF(1, next);
  }
 
+ OpCase(move_x1_c): {
+	x(1) = Arg(0);
+	Next(1);
+    }
+
+ OpCase(move_x2_c): {
+	x(2) = Arg(0);
+	Next(1);
+    }
+
+
  OpCase(return): {
     SET_I(c_p->cp);
     /*
@@ -1382,30 +1553,6 @@ void process_main(void)
     Goto(*I);
  }
 
- OpCase(test_heap_1_put_list_Iy): {
-     Eterm* next;
-
-     PreFetch(2, next);
-     TestHeap(Arg(0), 1);
-     PutList(yb(Arg(1)), r(0), r(0), StoreSimpleDest);
-     CHECK_TERM(r(0));
-     NextPF(2, next);
- }
-
- OpCase(put_string_IId):
-    {
-      unsigned char* s;
-      int len;
-      Eterm result;
-
-      len = Arg(0);		/* Length. */
-      result = NIL;
-      for (s = (unsigned char *) Arg(1); len > 0; s--, len--) {
-	  PutList(make_small(*s), result, result, StoreSimpleDest);
-      }
-      StoreBifResult(2, result);
-    }
-
     /*
      * Send is almost a standard call-BIF with two arguments, except for:
      *    1) It cannot be traced.
@@ -1414,7 +1561,7 @@ void process_main(void)
      */
 
  OpCase(send): {
-     Eterm* next;
+     BeamInstr *next;
      Eterm result;
 
      PRE_BIF_SWAPOUT(c_p);
@@ -1429,7 +1576,7 @@ void process_main(void)
 	 NextPF(0, next);
      } else if (c_p->freason == TRAP) {
 	 SET_CP(c_p, I+1);
-	 SET_I((Eterm *) c_p->def_arg_reg[3]);
+	 SET_I(*((BeamInstr **) (BeamInstr) ((c_p)->def_arg_reg + 3)));
 	 SWAPIN;
 	 r(0) = c_p->def_arg_reg[0];
 	 x(1) = c_p->def_arg_reg[1];
@@ -1438,24 +1585,36 @@ void process_main(void)
      goto find_func_info;
  }
 
- OpCase(i_element_jssd): {
-     Eterm index;
-     Eterm tuple;
-
-     /*
-      * Inlined version of element/2 for speed.
-      */
-     GetArg2(1, index, tuple);
-     if (is_small(index) && is_tuple(tuple)) {
-	 Eterm* tp = tuple_val(tuple);
-
-	 if ((signed_val(index) >= 1) &&
-	     (signed_val(index) <= arityval(*tp))) {
-	     Eterm result = tp[signed_val(index)];
-	     StoreBifResult(3, result);
-	 }
-     }
- }
+    {
+	Eterm element_index;
+	Eterm element_tuple;
+
+    OpCase(i_element_xjsd):
+	element_tuple = xb(Arg(0));
+	I++;
+	goto do_element;
+
+    OpCase(i_element_yjsd):
+	element_tuple = yb(Arg(0));
+	I++;
+	goto do_element;
+
+    OpCase(i_element_rjsd):
+	element_tuple = r(0);
+	/* Fall through */
+
+    do_element:
+	GetArg1(1, element_index);
+	if (is_small(element_index) && is_tuple(element_tuple)) {
+	    Eterm* tp = tuple_val(element_tuple);
+
+	    if ((signed_val(element_index) >= 1) &&
+		(signed_val(element_index) <= arityval(*tp))) {
+		Eterm result = tp[signed_val(element_index)];
+		StoreBifResult(2, result);
+	    }
+	}
+    }
  /* Fall through */
 
  OpCase(badarg_j):
@@ -1463,24 +1622,32 @@ void process_main(void)
     c_p->freason = BADARG;
     goto lb_Cl_error;
 
- OpCase(i_fast_element_jIsd): {
-     Eterm tuple;
-
-     /*
-      * Inlined version of element/2 for even more speed.
-      * The first argument is an untagged integer >= 1.
-      * The second argument is guaranteed to be a register operand.
-      */
-     GetArg1(2, tuple);
-     if (is_tuple(tuple)) {
-	 Eterm* tp = tuple_val(tuple);
-	 tmp_arg2 = Arg(1);
-	 if (tmp_arg2 <= arityval(*tp)) {
-	     Eterm result = tp[tmp_arg2];
-	     StoreBifResult(3, result);
-	 }
-     }
+    {
+	Eterm fast_element_tuple;
+
+    OpCase(i_fast_element_rjId):
+	fast_element_tuple = r(0);
+
+    do_fast_element:
+	if (is_tuple(fast_element_tuple)) {
+	    Eterm* tp = tuple_val(fast_element_tuple);
+	    Eterm pos = Arg(1);	/* Untagged integer >= 1 */
+	    if (pos <= arityval(*tp)) {
+		Eterm result = tp[pos];
+		StoreBifResult(2, result);
+	    }
+	}
      goto badarg;
+
+    OpCase(i_fast_element_xjId):
+     fast_element_tuple = xb(Arg(0));
+     I++;
+     goto do_fast_element;
+
+    OpCase(i_fast_element_yjId):
+     fast_element_tuple = yb(Arg(0));
+     I++;
+     goto do_fast_element;
  }
 
  OpCase(catch_yf):
@@ -1506,6 +1673,7 @@ void process_main(void)
 		 SWAPOUT;
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
+		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 SWAPIN;
 	     }
@@ -1531,6 +1699,10 @@ void process_main(void)
  /*
   * Skeleton for receive statement:
   *
+  *             recv_mark L1                     Optional
+  *             call make_ref/monitor            Optional
+  *             ...
+  *             recv_set L1                      Optional
   *      L1:          <-------------------+
   *                   <-----------+       |
   *     	     	       	  |   	  |
@@ -1549,13 +1721,41 @@ void process_main(void)
   *
   */
 
+ OpCase(recv_mark_f): {
+     /*
+      * Save the current position in message buffer and the
+      * the label for the loop_rec/2 instruction for the
+      * the receive statement.
+      */
+     c_p->msg.mark = (BeamInstr *) Arg(0);
+     c_p->msg.saved_last = c_p->msg.last;
+     Next(1);
+ }
+
+ OpCase(i_recv_set): {
+     /*
+      * If the mark is valid (points to the loop_rec/2
+      * instruction that follows), we know that the saved
+      * position points to the first message that could
+      * possibly be matched out.
+      *
+      * If the mark is invalid, we do nothing, meaning that
+      * we will look through all messages in the message queue.
+      */
+     if (c_p->msg.mark == (BeamInstr *) (I+1)) {
+	 c_p->msg.save = c_p->msg.saved_last;
+     }
+     I++;
+     /* Fall through to the loop_rec/2 instruction */
+ }
+
     /*
      * Pick up the next message and place it in x(0).
      * If no message, jump to a wait or wait_timeout instruction.
      */
  OpCase(i_loop_rec_fr):
  {
-     Eterm* next;
+     BeamInstr *next;
      ErlMessage* msgp;
 
  loop_rec__:
@@ -1579,7 +1779,7 @@ void process_main(void)
 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	 else {
 #endif
-	     SET_I((Eterm *) Arg(0));
+	     SET_I((BeamInstr *) Arg(0));
 	     Goto(*I);		/* Jump to a wait or wait_timeout instruction */
 #ifdef ERTS_SMP
 	 }
@@ -1592,6 +1792,7 @@ void process_main(void)
 				       PROCESS_MAIN_CHK_LOCKS(c_p);
 				   },
 				   {
+				       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 				       PROCESS_MAIN_CHK_LOCKS(c_p);
 				       r(0) = reg[0];
 				       SWAPIN;
@@ -1615,7 +1816,7 @@ void process_main(void)
   * Remove a (matched) message from the message queue.
   */
  OpCase(remove_message): {
-     Eterm* next;
+     BeamInstr *next;
      ErlMessage* msgp;
 
      PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -1650,6 +1851,7 @@ void process_main(void)
      CANCEL_TIMER(c_p);
      free_message(msgp);
 
+     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
      PROCESS_MAIN_CHK_LOCKS(c_p);
 
      NextPF(0, next);
@@ -1660,7 +1862,7 @@ void process_main(void)
      * message didn't match), then jump to the loop_rec instruction.
      */
  OpCase(loop_rec_end_f): {
-     SET_I((Eterm *) Arg(0));
+     SET_I((BeamInstr *) Arg(0));
      SAVE_MESSAGE(c_p);
      goto loop_rec__;
  }
@@ -1690,12 +1892,12 @@ void process_main(void)
      }
      GetArg1(1, timeout_value);
      if (timeout_value != make_small(0)) {
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
 	 Uint time_val;
 #endif
 
 	 if (is_small(timeout_value) && signed_val(timeout_value) > 0 &&
-#if defined(ARCH_64)
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	     ((unsigned_val(timeout_value) >> 32) == 0)
 #else
 	     1
@@ -1706,14 +1908,18 @@ void process_main(void)
 	      * c_p->def_arg_reg[0].  Note that it is safe to use this
 	      * location because there are no living x registers in
 	      * a receive statement.
+	      * Note that for the halfword emulator, the two first elements
+	      * of the array are used.
 	      */
-	     c_p->def_arg_reg[0] = (Eterm) (I+3);
+	     BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+	     *pi = I+3;
 	     set_timer(c_p, unsigned_val(timeout_value));
 	 } else if (timeout_value == am_infinity) {
 	     c_p->flags |= F_TIMO;
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
 	 } else if (term_to_Uint(timeout_value, &time_val)) {
-	     c_p->def_arg_reg[0] = (Eterm) (I+3);
+	     BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+	     *pi = I+3;
 	     set_timer(c_p, time_val);
 #endif
 	 } else {		/* Wrong time */
@@ -1742,7 +1948,7 @@ void process_main(void)
 
 	 wait2: {
 	     ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	     c_p->i = (Eterm *) Arg(0); /* L1 */
+	     c_p->i = (BeamInstr *) Arg(0); /* L1 */
 	     SWAPOUT;
 	     c_p->arity = 0;
 	     c_p->status = P_WAITING;
@@ -1770,7 +1976,8 @@ void process_main(void)
       * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
       */
      if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
-	 c_p->def_arg_reg[0] = (Eterm) (I+3);
+	 BeamInstr** p = (BeamInstr **) c_p->def_arg_reg;
+	 *p = I+3;
 	 set_timer(c_p, Arg(1));
      }
      goto wait2;
@@ -1785,7 +1992,7 @@ void process_main(void)
  }
 
  OpCase(timeout): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(0, next);
      if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
@@ -1799,14 +2006,93 @@ void process_main(void)
      NextPF(0, next);
  }
 
- OpCase(i_select_val_sfI):
-     GetArg1(0, tmp_arg1);
+
+ {
+     Eterm select_val2;
+
+ OpCase(i_select_tuple_arity2_yfAfAf):
+     select_val2 = yb(Arg(0));
+     goto do_select_tuple_arity2;
+
+ OpCase(i_select_tuple_arity2_xfAfAf):
+     select_val2 = xb(Arg(0));
+     goto do_select_tuple_arity2;
+
+ OpCase(i_select_tuple_arity2_rfAfAf):
+     select_val2 = r(0);
+     I--;
+
+ do_select_tuple_arity2:
+     if (is_not_tuple(select_val2)) {
+	 goto select_val2_fail;
+     }
+     select_val2 = *tuple_val(select_val2);
+     goto do_select_val2;
+
+ OpCase(i_select_val2_yfcfcf):
+     select_val2 = yb(Arg(0));
+     goto do_select_val2;
+
+ OpCase(i_select_val2_xfcfcf):
+     select_val2 = xb(Arg(0));
+     goto do_select_val2;
+
+ OpCase(i_select_val2_rfcfcf):
+     select_val2 = r(0);
+     I--;
+
+ do_select_val2:
+     if (select_val2 == Arg(2)) {
+	 I += 2;
+     } else if (select_val2 == Arg(4)) {
+	 I += 4;
+     }
+
+ select_val2_fail:
+     SET_I((BeamInstr *) Arg(1));
+     Goto(*I);
+ }
+
+ {
+     Eterm select_val;
+
+ OpCase(i_select_tuple_arity_xfI):
+     select_val = xb(Arg(0));
+     goto do_select_tuple_arity;
+
+ OpCase(i_select_tuple_arity_yfI):
+     select_val = yb(Arg(0));
+     goto do_select_tuple_arity;
+
+ OpCase(i_select_tuple_arity_rfI):
+     select_val = r(0);
+     I--;
+
+ do_select_tuple_arity:
+     if (is_tuple(select_val)) {
+	 select_val = *tuple_val(select_val);
+	 goto do_binary_search;
+     }
+     SET_I((BeamInstr *) Arg(1));
+     Goto(*I);
+
+ OpCase(i_select_val_xfI):
+     select_val = xb(Arg(0));
+     goto do_binary_search;
+
+ OpCase(i_select_val_yfI):
+     select_val = yb(Arg(0));
+     goto do_binary_search;
+     
+ OpCase(i_select_val_rfI):
+     select_val = r(0);
+     I--;
 
  do_binary_search:
  {
      struct Pairs {
-	 Eterm val;
-	 Eterm* addr;
+	 BeamInstr val;
+	 BeamInstr* addr;
      };
      struct Pairs* low;
      struct Pairs* high;
@@ -1837,48 +2123,98 @@ void process_main(void)
 	 unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1);
 
 	 mid = (struct Pairs*)((char*)low + boffset);
-	 if (tmp_arg1 < mid->val) {
+	 if (select_val < mid->val) {
 	     high = mid;
-	 } else if (tmp_arg1 > mid->val) {
+	 } else if (select_val > mid->val) {
 	     low = mid + 1;
 	 } else {
 	     SET_I(mid->addr);
 	     Goto(*I);
 	 }
      }
-     SET_I((Eterm *) Arg(1));
+     SET_I((BeamInstr *) Arg(1));
      Goto(*I);
  }
+ }
 
- OpCase(i_jump_on_val_zero_sfI):
  {
-     Eterm index;
-
-     GetArg1(0, index);
-     if (is_small(index)) {
-	 index = signed_val(index);
-	 if (index < Arg(2)) {
-	     SET_I((Eterm *) (&Arg(3))[index]);
+     Eterm jump_on_val_zero_index;
+     
+ OpCase(i_jump_on_val_zero_yfI):
+     jump_on_val_zero_index = yb(Arg(0));
+     goto do_jump_on_val_zero_index;
+
+ OpCase(i_jump_on_val_zero_xfI):
+     jump_on_val_zero_index = xb(Arg(0));
+     goto do_jump_on_val_zero_index;
+
+ OpCase(i_jump_on_val_zero_rfI):
+     jump_on_val_zero_index = r(0);
+     I--;
+
+ do_jump_on_val_zero_index:
+     if (is_small(jump_on_val_zero_index)) {
+	 jump_on_val_zero_index = signed_val(jump_on_val_zero_index);
+	 if (jump_on_val_zero_index < Arg(2)) {
+	     SET_I((BeamInstr *) (&Arg(3))[jump_on_val_zero_index]);
 	     Goto(*I);
 	 }
      }
-     SET_I((Eterm *) Arg(1));
+     SET_I((BeamInstr *) Arg(1));
      Goto(*I);
  }
 
- OpCase(i_jump_on_val_sfII):
  {
-     Eterm index;
+     Eterm jump_on_val_index;
 
-     GetArg1(0, index);
-     if (is_small(index)) {
-	 index = (Uint) (signed_val(index) - Arg(3));
-	 if (index < Arg(2)) {
-	     SET_I((Eterm *) (&Arg(4))[index]);
+ 
+ OpCase(i_jump_on_val_yfII):
+     jump_on_val_index = yb(Arg(0));
+     goto do_jump_on_val_index;
+
+ OpCase(i_jump_on_val_xfII):
+     jump_on_val_index = xb(Arg(0));
+     goto do_jump_on_val_index;
+
+ OpCase(i_jump_on_val_rfII):
+     jump_on_val_index = r(0);
+     I--;
+
+ do_jump_on_val_index:
+     if (is_small(jump_on_val_index)) {
+	 jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3));
+	 if (jump_on_val_index < Arg(2)) {
+	     SET_I((BeamInstr *) (&Arg(4))[jump_on_val_index]);
 	     Goto(*I);
 	 }
      }
-     SET_I((Eterm *) Arg(1));
+     SET_I((BeamInstr *) Arg(1));
+     Goto(*I);
+ }
+
+ do_put_tuple: {
+     Eterm* hp = HTOP;
+
+     *hp++ = make_arityval(pt_arity);
+
+     do {
+	 Eterm term = *I++;
+	 switch (term & _TAG_IMMED1_MASK) {
+	 case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
+	     *hp++ = r(0);
+	     break;
+	 case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
+	     *hp++ = x(term >> _TAG_IMMED1_SIZE);
+	     break;
+	 case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
+	     *hp++ = y(term >> _TAG_IMMED1_SIZE);
+	     break;
+	 default:
+	     *hp++ = term;
+	     break;
+	 }
+     } while (--pt_arity != 0);
+     HTOP = hp;
      Goto(*I);
  }
 
@@ -1909,13 +2245,14 @@ void process_main(void)
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	result = (*bf)(c_p, arg);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
 	    StoreBifResult(3, result);
 	}
-	SET_I((Eterm *) Arg(0));
+	SET_I((BeamInstr *) Arg(0));
 	Goto(*I);
     }
 
@@ -1937,6 +2274,7 @@ void process_main(void)
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	result = (*bf)(c_p, arg);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
@@ -1955,7 +2293,7 @@ void process_main(void)
 	GcBifFunction bf;
 	Eterm arg;
 	Eterm result;
-	Uint live = Arg(3);
+	Uint live = (Uint) Arg(3);
 
 	GetArg1(2, arg);
 	reg[0] = r(0);
@@ -1966,6 +2304,7 @@ void process_main(void)
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	result = (*bf)(c_p, reg, live);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	SWAPIN;
@@ -1976,7 +2315,7 @@ void process_main(void)
 	    StoreBifResult(4, result);
 	}
 	if (Arg(0) != 0) {
-	    SET_I((Eterm *) Arg(0));
+	    SET_I((BeamInstr *) Arg(0));
 	    Goto(*I);
 	}
 	reg[0] = arg;
@@ -1984,6 +2323,83 @@ void process_main(void)
 	goto post_error_handling;
     }
 
+ OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1
+			    and i_gc_bif3 */
+    {
+	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
+	GcBifFunction bf;
+	Eterm result;
+	Uint live = (Uint) Arg(2);
+
+	reg[0] = r(0);
+	reg[live++] = tmp_arg1;
+	reg[live] = tmp_arg2;
+	bf = (GcBifFunction) Arg(1);
+	c_p->fcalls = FCALLS;
+	SWAPOUT;
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	result = (*bf)(c_p, reg, live);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	SWAPIN;
+	r(0) = reg[0];
+	ERTS_HOLE_CHECK(c_p);
+	FCALLS = c_p->fcalls;
+	if (is_value(result)) {
+	    StoreBifResult(3, result);
+	}
+	if (Arg(0) != 0) {
+	    SET_I((BeamInstr *) Arg(0));
+	    Goto(*I);
+	}
+	reg[0] = tmp_arg1;
+	reg[1] = tmp_arg2;
+	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
+	goto post_error_handling;
+    }
+
+ OpCase(i_gc_bif3_jIsId):
+    {
+	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
+	GcBifFunction bf;
+	Eterm arg;
+	Eterm result;
+	Uint live = (Uint) Arg(3);
+
+	GetArg1(2, arg);
+	reg[0] = r(0);
+	reg[live++] = arg;
+	reg[live++] = tmp_arg1;
+	reg[live] = tmp_arg2;
+	bf = (GcBifFunction) Arg(1);
+	c_p->fcalls = FCALLS;
+	SWAPOUT;
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	result = (*bf)(c_p, reg, live);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	SWAPIN;
+	r(0) = reg[0];
+	ERTS_HOLE_CHECK(c_p);
+	FCALLS = c_p->fcalls;
+	if (is_value(result)) {
+	    StoreBifResult(4, result);
+	}
+	if (Arg(0) != 0) {
+	    SET_I((BeamInstr *) Arg(0));
+	    Goto(*I);
+	}
+	reg[0] = arg;
+	reg[1] = tmp_arg1;
+	reg[2] = tmp_arg2;
+	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
+	goto post_error_handling;
+    }
+
  /*
   * Guards bifs and, or, xor in guards.
   */
@@ -1998,13 +2414,14 @@ void process_main(void)
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	result = (*bf)(c_p, tmp_arg1, tmp_arg2);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
 	    StoreBifResult(2, result);
 	}
-	SET_I((Eterm *) Arg(0));
+	SET_I((BeamInstr *) Arg(0));
 	Goto(*I);
     }
 
@@ -2021,6 +2438,7 @@ void process_main(void)
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	result = (*bf)(c_p, tmp_arg1, tmp_arg2);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	if (is_value(result)) {
@@ -2040,7 +2458,7 @@ void process_main(void)
      */
  OpCase(call_bif0_e):
     {
-	Eterm (*bf)(Process*, Uint*) = GET_BIF_ADDRESS(Arg(0));
+	Eterm (*bf)(Process*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
 
 	PRE_BIF_SWAPOUT(c_p);
 	c_p->fcalls = FCALLS - 1;
@@ -2073,9 +2491,9 @@ void process_main(void)
 
  OpCase(call_bif1_e):
     {
-	Eterm (*bf)(Process*, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0));
+	Eterm (*bf)(Process*, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
 	Eterm result;
-	Eterm* next;
+	BeamInstr *next;
 
 	c_p->fcalls = FCALLS - 1;
 	if (FCALLS <= 0) {
@@ -2108,9 +2526,9 @@ void process_main(void)
 
  OpCase(call_bif2_e):
     {
-	Eterm (*bf)(Process*, Eterm, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0));
+	Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
 	Eterm result;
-	Eterm* next;
+	BeamInstr *next;
 
 	PRE_BIF_SWAPOUT(c_p);
 	c_p->fcalls = FCALLS - 1;
@@ -2145,9 +2563,9 @@ void process_main(void)
 
  OpCase(call_bif3_e):
     {
-	Eterm (*bf)(Process*, Eterm, Eterm, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0));
+	Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
 	Eterm result;
-	Eterm* next;
+	BeamInstr *next;
 
 	PRE_BIF_SWAPOUT(c_p);
 	c_p->fcalls = FCALLS - 1;
@@ -2168,7 +2586,7 @@ void process_main(void)
 	} else if (c_p->freason == TRAP) {
 	call_bif_trap3:
 	    SET_CP(c_p, I+2);
-	    SET_I((Eterm *)c_p->def_arg_reg[3]);
+	    SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3)));
 	    SWAPIN;
 	    r(0) = c_p->def_arg_reg[0];
 	    x(1) = c_p->def_arg_reg[1];
@@ -2276,7 +2694,7 @@ void process_main(void)
  lb_Cl_error: {
      if (Arg(0) != 0) {
 	 OpCase(jump_f): {
-	     SET_I((Eterm *) Arg(0));
+	     SET_I((BeamInstr *) Arg(0));
 	     Goto(*I);
 	 }
      }
@@ -2444,23 +2862,25 @@ void process_main(void)
 
  OpCase(i_int_bnot_jsId):
  {
-     GetArg1(1, tmp_arg1);
-     if (is_small(tmp_arg1)) {
-	 tmp_arg1 = make_small(~signed_val(tmp_arg1));
+     Eterm bnot_val;
+
+     GetArg1(1, bnot_val);
+     if (is_small(bnot_val)) {
+	 bnot_val = make_small(~signed_val(bnot_val));
      } else {
 	 Uint live = Arg(2);
 	 SWAPOUT;
 	 reg[0] = r(0);
-	 reg[live] = tmp_arg1;
-	 tmp_arg1 = erts_gc_bnot(c_p, reg, live);
+	 reg[live] = bnot_val;
+	 bnot_val = erts_gc_bnot(c_p, reg, live);
 	 r(0) = reg[0];
 	 SWAPIN;
 	 ERTS_HOLE_CHECK(c_p);
-	 if (is_nil(tmp_arg1)) {
+	 if (is_nil(bnot_val)) {
 	     goto lb_Cl_error;
 	 }
      }
-     StoreBifResult(3, tmp_arg1);
+     StoreBifResult(3, bnot_val);
  }
 
  badarith:
@@ -2468,7 +2888,7 @@ void process_main(void)
     goto lb_Cl_error;
 
  OpCase(i_apply): {
-     Eterm* next;
+     BeamInstr *next;
      SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
      SWAPIN;
@@ -2483,13 +2903,13 @@ void process_main(void)
  }
 
  OpCase(i_apply_last_P): {
-     Eterm* next;
+     BeamInstr *next;
      SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
      SWAPIN;
      if (next != NULL) {
 	 r(0) = reg[0];
-	 SET_CP(c_p, (Eterm *) E[0]);
+	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
 	 SET_I(next);
 	 Dispatch();
@@ -2499,7 +2919,7 @@ void process_main(void)
  }
 
  OpCase(i_apply_only): {
-     Eterm* next;
+     BeamInstr *next;
      SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
      SWAPIN;
@@ -2513,7 +2933,7 @@ void process_main(void)
  }
 
  OpCase(apply_I): {
-     Eterm* next;
+     BeamInstr *next;
 
      reg[0] = r(0);
      SWAPOUT;
@@ -2530,7 +2950,7 @@ void process_main(void)
  }
 
  OpCase(apply_last_IP): {
-     Eterm* next;
+     BeamInstr *next;
 
      reg[0] = r(0);
      SWAPOUT;
@@ -2538,7 +2958,7 @@ void process_main(void)
      SWAPIN;
      if (next != NULL) {
 	 r(0) = reg[0];
-	 SET_CP(c_p, (Eterm *) E[0]);
+	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
 	 E = ADD_BYTE_OFFSET(E, Arg(1));
 	 SET_I(next);
 	 Dispatch();
@@ -2548,7 +2968,7 @@ void process_main(void)
  }
 
  OpCase(i_apply_fun): {
-     Eterm* next;
+     BeamInstr *next;
 
      SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
@@ -2563,14 +2983,14 @@ void process_main(void)
  }
 
  OpCase(i_apply_fun_last_P): {
-     Eterm* next;
+     BeamInstr *next;
 
      SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
      SWAPIN;
      if (next != NULL) {
 	 r(0) = reg[0];
-	 SET_CP(c_p, (Eterm *) E[0]);
+	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
 	 SET_I(next);
 	 Dispatchfun();
@@ -2579,7 +2999,7 @@ void process_main(void)
  }
 
  OpCase(i_apply_fun_only): {
-     Eterm* next;
+     BeamInstr *next;
 
      SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
@@ -2593,10 +3013,11 @@ void process_main(void)
  }
 
  OpCase(i_call_fun_I): {
-     Eterm* next;
+     BeamInstr *next;
 
      SWAPOUT;
      reg[0] = r(0);
+
      next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
      SWAPIN;
      if (next != NULL) {
@@ -2609,7 +3030,7 @@ void process_main(void)
  }
 
  OpCase(i_call_fun_last_IP): {
-     Eterm* next;
+     BeamInstr *next;
 
      SWAPOUT;
      reg[0] = r(0);
@@ -2617,7 +3038,7 @@ void process_main(void)
      SWAPIN;
      if (next != NULL) {
 	r(0) = reg[0];
-	SET_CP(c_p, (Eterm *) E[0]);
+	SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
 	E = ADD_BYTE_OFFSET(E, Arg(1));
 	SET_I(next);
 	Dispatchfun();
@@ -2714,123 +3135,10 @@ void process_main(void)
      goto do_schedule1;
  }
 
- OpCase(i_select_tuple_arity_sfI):
- {
-     GetArg1(0, tmp_arg1);
-
-     if (is_tuple(tmp_arg1)) {
-	 tmp_arg1 = *tuple_val(tmp_arg1);
-	 goto do_binary_search;
-     }
-     SET_I((Eterm *) Arg(1));
-     Goto(*I);
- }     
-
- OpCase(i_select_big_sf):
-    {
-	Eterm* bigp;
-	Uint arity;
-	Eterm* given;
-	Uint given_arity;
-	Uint given_size;
-
-	GetArg1(0, tmp_arg1);
-	if (is_big(tmp_arg1)) {
-
-	    /*
-	     * The loader has sorted the bignumbers in descending order
-	     * on the arity word.  Therefore, we know that the search
-	     * has failed as soon as we encounter an arity word less than
-	     * the arity word of the given number.  There is a zero word
-	     * (less than any valid arity word) stored after the last bignumber.
-	     */
-
- 	    given = big_val(tmp_arg1);
-	    given_arity = given[0];
-	    given_size = thing_arityval(given_arity);
-	    bigp = &Arg(2);
-	    while ((arity = bigp[0]) > given_arity) {
-		bigp += thing_arityval(arity) + 2;
-	    }
-	    while (bigp[0] == given_arity) {
-		if (memcmp(bigp+1, given+1, sizeof(Eterm)*given_size) == 0) {
-		    SET_I((Eterm *) bigp[given_size+1]);
-		    Goto(*I);
-		}
-		bigp += thing_arityval(arity) + 2;
-	    }
-	}
-
-	/*
-	 * Failed.
-	 */
-
-	SET_I((Eterm *) Arg(1));
-	Goto(*I);
-    }
-
-#ifdef ARCH_64
- OpCase(i_select_float_sfI):
- {
-     Uint f;
-     int n;
-     struct ValLabel {
-	 Uint f;
-	 Eterm* addr;
-     };
-     struct ValLabel* ptr;
-
-     GetArg1(0, tmp_arg1);
-     ASSERT(is_float(tmp_arg1));
-     f = float_val(tmp_arg1)[1];
-     n = Arg(2);
-     ptr = (struct ValLabel *) &Arg(3);
-     while (n-- > 0) {
-	 if (ptr->f == f) {
-	     SET_I(ptr->addr);
-	     Goto(*I);
-	 }
-	 ptr++;
-     }
-     SET_I((Eterm *) Arg(1));
-     Goto(*I);
- }
-#else
- OpCase(i_select_float_sfI):
- {
-     Uint fpart1;
-     Uint fpart2;
-     int n;
-     struct ValLabel {
-	 Uint fpart1;
-	 Uint fpart2;
-	 Eterm* addr;
-     };
-     struct ValLabel* ptr;
-
-     GetArg1(0, tmp_arg1);
-     ASSERT(is_float(tmp_arg1));
-     fpart1 = float_val(tmp_arg1)[1];
-     fpart2 = float_val(tmp_arg1)[2];
-
-     n = Arg(2);
-     ptr = (struct ValLabel *) &Arg(3);
-     while (n-- > 0) {
-	 if (ptr->fpart1 == fpart1 && ptr->fpart2 == fpart2) {
-	     SET_I(ptr->addr);
-	     Goto(*I);
-	 }
-	 ptr++;
-     }
-     SET_I((Eterm *) Arg(1));
-     Goto(*I);
- }
-#endif
-
  OpCase(set_tuple_element_sdP): {
      Eterm element;
      Eterm tuple;
-     Eterm* next;
+     BeamInstr *next;
      Eterm* p;
      
      PreFetch(3, next);
@@ -2872,15 +3180,17 @@ void process_main(void)
 	the first argument. We also handle atom tags in the first
 	argument for backwards compatibility.
      */
-     GetArg2(0, tmp_arg1, tmp_arg2);
-     c_p->fvalue = tmp_arg2;
+     Eterm raise_val1;
+     Eterm raise_val2;
+     GetArg2(0, raise_val1, raise_val2);
+     c_p->fvalue = raise_val2;
      if (c_p->freason == EXC_NULL) {
        /* a safety check for the R10-0 case; should not happen */
        c_p->ftrace = NIL;
        c_p->freason = EXC_ERROR;
      }
      /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */
-     switch (tmp_arg1) {
+     switch (raise_val1) {
      case am_throw:
        c_p->freason = EXC_THROWN & ~EXF_SAVETRACE;
        break;
@@ -2896,8 +3206,8 @@ void process_main(void)
 	   passed from a user! Currently only expecting generated calls.
 	*/
 	 struct StackTrace *s;
-	 c_p->ftrace = tmp_arg1;
-	 s = get_trace_from_exc(tmp_arg1);
+	 c_p->ftrace = raise_val1;
+	 s = get_trace_from_exc(raise_val1);
 	 if (s == NULL) {
 	   c_p->freason = EXC_ERROR;
 	 } else {
@@ -2908,11 +3218,24 @@ void process_main(void)
      goto find_func_info;
  }
 
- OpCase(badmatch_s): {
-     GetArg1(0, tmp_arg1);
-     c_p->fvalue = tmp_arg1;
-     c_p->freason = BADMATCH;
- }
+    {
+	Eterm badmatch_val;
+
+    OpCase(badmatch_y):
+	badmatch_val = yb(Arg(0));
+	goto do_badmatch;
+
+    OpCase(badmatch_x):
+	badmatch_val = xb(Arg(0));
+	goto do_badmatch;
+
+    OpCase(badmatch_r):
+	badmatch_val = r(0);
+
+    do_badmatch:
+	c_p->fvalue = badmatch_val;
+	c_p->freason = BADMATCH;
+    }
  /* Fall through here */
 
  find_func_info: {
@@ -2935,12 +3258,11 @@ void process_main(void)
      */
     SWAPOUT;
     reg[0] = r(0);
-    tmp_arg1 = call_error_handler(c_p, I-3, reg);
+    I = call_error_handler(c_p, I-3, reg, am_undefined_function);
     r(0) = reg[0];
     SWAPIN;
-    if (tmp_arg1) {
-	SET_I(c_p->i);
-	Dispatch();
+    if (I) {
+	Goto(*I);
     }
 
  /* Fall through */
@@ -2963,128 +3285,154 @@ void process_main(void)
      }
  }
 
- OpCase(call_nif):
-     {
-          /*
-	   * call_nif is always first instruction in function:
-	   *
-	   * I[-3]: Module
-	   * I[-2]: Function
-	   * I[-1]: Arity
-	   * I[0]: &&call_nif
-	   * I[1]: Function pointer to NIF function
-	   * I[2]: Pointer to erl_module_nif
-	   */
-     	 BifFunction vbf;
-
-	 c_p->current = I-3; /* current and vbf set to please handle_error */ 
-	 SWAPOUT;
-	 c_p->fcalls = FCALLS - 1;
-	 PROCESS_MAIN_CHK_LOCKS(c_p);
-	 tmp_arg2 = I[-1];
-	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+    {
+	Eterm nif_bif_result;
+	Eterm bif_nif_arity;
 
-	 ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	 {
-	     typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
-	     NifF* fp = vbf = (NifF*) I[1];
-	     struct enif_environment_t env;
-	     erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
-	     reg[0] = r(0);
-	     tmp_arg1 = (*fp)(&env, tmp_arg2, reg);
-	     erts_post_nif(&env);
-	 }
-	 ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
-	 PROCESS_MAIN_CHK_LOCKS(c_p);
-	 goto apply_bif_or_nif_epilogue;
-	 
- OpCase(apply_bif):
-	/*
-	 * At this point, I points to the code[3] in the export entry for
-	 * the BIF:
-	 *
-	 * code[0]: Module
-	 * code[1]: Function
-	 * code[2]: Arity
-	 * code[3]: &&apply_bif
-	 * code[4]: Function pointer to BIF function
-	 */
+    OpCase(call_nif):
+	{
+	    /*
+	     * call_nif is always first instruction in function:
+	     *
+	     * I[-3]: Module
+	     * I[-2]: Function
+	     * I[-1]: Arity
+	     * I[0]: &&call_nif
+	     * I[1]: Function pointer to NIF function
+	     * I[2]: Pointer to erl_module_nif
+	     */
+	    BifFunction vbf;
 
-	c_p->current = I-3;	/* In case we apply process_info/1,2 or load_nif/1 */
-	c_p->i = I;		/* In case we apply check_process_code/2. */
-	c_p->arity = 0;		/* To allow garbage collection on ourselves
-				 * (check_process_code/2).
-				 */
-	SWAPOUT;
-	c_p->fcalls = FCALLS - 1;
-	vbf = (BifFunction) Arg(0);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	tmp_arg2 = I[-1];
-	ASSERT(tmp_arg2 <= 3);
-	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-	switch (tmp_arg2) {
-	case 3:
+	    c_p->current = I-3; /* current and vbf set to please handle_error */ 
+	    SWAPOUT;
+	    c_p->fcalls = FCALLS - 1;
+	    PROCESS_MAIN_CHK_LOCKS(c_p);
+	    bif_nif_arity = I[-1];
+	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+
+	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	    {
-		Eterm (*bf)(Process*, Eterm, Eterm, Eterm, Uint*) = vbf;
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		tmp_arg1 = (*bf)(c_p, r(0), x(1), x(2), I);
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
-		PROCESS_MAIN_CHK_LOCKS(c_p);
+		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
+		NifF* fp = vbf = (NifF*) I[1];
+		struct enif_environment_t env;
+		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
+		reg[0] = r(0);
+		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
+		erts_post_nif(&env);
 	    }
-	    break;
-	case 2:
-	    {
-		Eterm (*bf)(Process*, Eterm, Eterm, Uint*) = vbf;
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		tmp_arg1 = (*bf)(c_p, r(0), x(1), I);
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
-		PROCESS_MAIN_CHK_LOCKS(c_p);
+	    ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
+	    PROCESS_MAIN_CHK_LOCKS(c_p);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	    goto apply_bif_or_nif_epilogue;
+	 
+	OpCase(apply_bif):
+	    /*
+	     * At this point, I points to the code[3] in the export entry for
+	     * the BIF:
+	     *
+	     * code[0]: Module
+	     * code[1]: Function
+	     * code[2]: Arity
+	     * code[3]: &&apply_bif
+	     * code[4]: Function pointer to BIF function
+	     */
+
+	    c_p->current = I-3;	/* In case we apply process_info/1,2 or load_nif/1 */
+	    c_p->i = I;		/* In case we apply check_process_code/2. */
+	    c_p->arity = 0;		/* To allow garbage collection on ourselves
+					 * (check_process_code/2).
+					 */
+	    SWAPOUT;
+	    c_p->fcalls = FCALLS - 1;
+	    vbf = (BifFunction) Arg(0);
+	    PROCESS_MAIN_CHK_LOCKS(c_p);
+	    bif_nif_arity = I[-1];
+	    ASSERT(bif_nif_arity <= 3);
+	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	    switch (bif_nif_arity) {
+	    case 3:
+		{
+		    Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf;
+		    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+		    nif_bif_result = (*bf)(c_p, r(0), x(1), x(2), I);
+		    ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+			   is_non_value(nif_bif_result));
+		    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+		    PROCESS_MAIN_CHK_LOCKS(c_p);
+		}
+		break;
+	    case 2:
+		{
+		    Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf;
+		    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+		    nif_bif_result = (*bf)(c_p, r(0), x(1), I);
+		    ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+			   is_non_value(nif_bif_result));
+		    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+		    PROCESS_MAIN_CHK_LOCKS(c_p);
+		}
+		break;
+	    case 1:
+		{
+		    Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf;
+		    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+		    nif_bif_result = (*bf)(c_p, r(0), I);
+		    ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+			   is_non_value(nif_bif_result));
+		    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+		    PROCESS_MAIN_CHK_LOCKS(c_p);
+		}
+		break;
+	    case 0:
+		{
+		    Eterm (*bf)(Process*, BeamInstr*) = vbf;
+		    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+		    nif_bif_result = (*bf)(c_p, I);
+		    ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+			   is_non_value(nif_bif_result));
+		    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+		    PROCESS_MAIN_CHK_LOCKS(c_p);
+		    break;
+		}
+	    default:
+		erl_exit(1, "apply_bif: invalid arity: %u\n",
+			 bif_nif_arity);
 	    }
-	    break;
-	case 1:
-	    {
-		Eterm (*bf)(Process*, Eterm, Uint*) = vbf;
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		tmp_arg1 = (*bf)(c_p, r(0), I);
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
-		PROCESS_MAIN_CHK_LOCKS(c_p);
+
+	apply_bif_or_nif_epilogue:
+	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_HOLE_CHECK(c_p);
+	    if (c_p->mbuf) {
+		reg[0] = r(0);
+		nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result,
+						  reg, bif_nif_arity);
+		r(0) = reg[0];
 	    }
-	    break;
-	case 0:
-	    {
-		Eterm (*bf)(Process*, Uint*) = vbf;
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		tmp_arg1 = (*bf)(c_p, I);
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
-		PROCESS_MAIN_CHK_LOCKS(c_p);
-		break;
+	    SWAPIN;  /* There might have been a garbage collection. */
+	    FCALLS = c_p->fcalls;
+	    if (is_value(nif_bif_result)) {
+		r(0) = nif_bif_result;
+		CHECK_TERM(r(0));
+		SET_I(c_p->cp);
+		c_p->cp = 0;
+		Goto(*I);
+	    } else if (c_p->freason == TRAP) {
+		SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3)));
+		r(0) = c_p->def_arg_reg[0];
+		x(1) = c_p->def_arg_reg[1];
+		x(2) = c_p->def_arg_reg[2];
+		if (c_p->flags & F_HIBERNATE_SCHED) {
+		    c_p->flags &= ~F_HIBERNATE_SCHED;
+		    goto do_schedule;
+		}
+		Dispatch();
 	    }
-	}
-apply_bif_or_nif_epilogue:
-	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
-	ERTS_HOLE_CHECK(c_p);
-	if (c_p->mbuf) {
 	    reg[0] = r(0);
-	    tmp_arg1 = erts_gc_after_bif_call(c_p, tmp_arg1, reg, tmp_arg2);
-	    r(0) = reg[0];
+	    I = handle_error(c_p, c_p->cp, reg, vbf);
+	    goto post_error_handling;
 	}
-	SWAPIN;			/* There might have been a garbage collection. */
-	FCALLS = c_p->fcalls;
-	if (is_value(tmp_arg1)) {
-	    r(0) = tmp_arg1;
-	    CHECK_TERM(r(0));
-	    SET_I(c_p->cp);
-	    Goto(*I);
-	} else if (c_p->freason == TRAP) {
-	    SET_I((Eterm *)c_p->def_arg_reg[3]);
-	    r(0) = c_p->def_arg_reg[0];
-	    x(1) = c_p->def_arg_reg[1];
-	    x(2) = c_p->def_arg_reg[2];
-	    Dispatch();
-	}
-	reg[0] = r(0);
-	I = handle_error(c_p, c_p->cp, reg, vbf);
-	goto post_error_handling;
     }
 
  OpCase(i_get_sd):
@@ -3097,17 +3445,25 @@ apply_bif_or_nif_epilogue:
 	StoreBifResult(1, result);
     }
 
- OpCase(i_put_tuple_only_Ad): {
-     tmp_arg1 = make_tuple(HTOP);
-     *HTOP++ = Arg(0);
-     StoreBifResult(1, tmp_arg1);
- }
+    {
+	Eterm case_end_val;
 
- OpCase(case_end_s):
-    GetArg1(0, tmp_arg1);
-    c_p->fvalue = tmp_arg1;
-    c_p->freason = EXC_CASE_CLAUSE;
-    goto find_func_info;
+    OpCase(case_end_x):
+	case_end_val = xb(Arg(0));
+	goto do_case_end;
+
+    OpCase(case_end_y):
+	case_end_val = yb(Arg(0));
+	goto do_case_end;
+
+    OpCase(case_end_r):
+	case_end_val = r(0);
+
+    do_case_end:
+	c_p->fvalue = case_end_val;
+	c_p->freason = EXC_CASE_CLAUSE;
+	goto find_func_info;
+    }
 
  OpCase(if_end):
     c_p->freason = EXC_IF_CLAUSE;
@@ -3120,10 +3476,13 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(try_case_end_s):
-    GetArg1(0, tmp_arg1);
-    c_p->fvalue = tmp_arg1;
-    c_p->freason = EXC_TRY_CLAUSE;
-    goto find_func_info;
+    {
+	Eterm try_case_end_val;
+	GetArg1(0, try_case_end_val);
+	c_p->fvalue = try_case_end_val;
+	c_p->freason = EXC_TRY_CLAUSE;
+	goto find_func_info;
+    }
 
  /*
   * Construction of binaries using new instructions.
@@ -3270,12 +3629,12 @@ apply_bif_or_nif_epilogue:
 	 HTOP += PROC_BIN_SIZE;
 	 pb->thing_word = HEADER_PROC_BIN;
 	 pb->size = num_bytes;
-	 pb->next = MSO(c_p).mso;
-	 MSO(c_p).mso = pb;
+	 pb->next = MSO(c_p).first;
+	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
 	 pb->val = bptr;
 	 pb->bytes = (byte*) bptr->orig_bytes;
 	 pb->flags = 0;
-	 MSO(c_p).overhead += pb->size / sizeof(Eterm);
+	 OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
 	 new_binary = make_binary(pb);
 	 goto do_bits_sub_bin;
      }
@@ -3371,13 +3730,13 @@ apply_bif_or_nif_epilogue:
 	 HTOP += PROC_BIN_SIZE;
 	 pb->thing_word = HEADER_PROC_BIN;
 	 pb->size = tmp_arg1;
-	 pb->next = MSO(c_p).mso;
-	 MSO(c_p).mso = pb;
+	 pb->next = MSO(c_p).first;
+	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
 	 pb->val = bptr;
 	 pb->bytes = (byte*) bptr->orig_bytes;
 	 pb->flags = 0;
 	 
-	 MSO(c_p).overhead += tmp_arg1 / sizeof(Eterm);
+	 OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm));
 
 	 StoreBifResult(2, make_binary(pb));
      }
@@ -3413,42 +3772,6 @@ apply_bif_or_nif_epilogue:
 	 }
  }
 
- OpCase(i_bs_bits_to_bytes_rjd): {
-     tmp_arg1 = r(0);
-     goto do_bits_to_bytes;
- }
-
- OpCase(i_bs_bits_to_bytes_yjd): {
-     tmp_arg1 = yb(Arg(0));
-     I++;
-     goto do_bits_to_bytes;
-
- OpCase(i_bs_bits_to_bytes_xjd): {
-     tmp_arg1 = xb(Arg(0));
-     I++;
- }
-
- do_bits_to_bytes:
-     {
-	 if (is_valid_bit_size(tmp_arg1)) {
-	     tmp_arg1 = make_small(unsigned_val(tmp_arg1) >> 3);
-	 } else {
-	     Uint bytes;
-	     if (!term_to_Uint(tmp_arg1, &bytes)) {
-		 goto badarg;
-	     }
-	     tmp_arg1 = bytes;
-	     if ((tmp_arg1 & 0x07) != 0) {
-		 goto badarg;
-	     }
-	     SWAPOUT;
-	     tmp_arg1 = erts_make_integer(tmp_arg1 >> 3, c_p);
-	     HTOP = HEAP_TOP(c_p);
-	 }
-	 StoreBifResult(1, tmp_arg1);
-     }
- }
-
  OpCase(i_bs_add_jId): {
      Uint Unit = Arg(1);
      if (is_both_small(tmp_arg1, tmp_arg2)) {
@@ -3486,7 +3809,7 @@ apply_bif_or_nif_epilogue:
 
 	 /*
 	  * Now we know that one of the arguments is
-	  * not at small. We must convert both arguments
+	  * not a small. We must convert both arguments
 	  * to Uints and check for errors at the same time.
 	  *
 	  * Error checking is tricky.
@@ -3534,7 +3857,7 @@ apply_bif_or_nif_epilogue:
 
  OpCase(bs_put_string_II):
     {
-	Eterm* next;
+	BeamInstr *next;
 	PreFetch(2, next);
 	erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) Arg(1), Arg(0)));
 	NextPF(2, next);
@@ -3705,21 +4028,22 @@ apply_bif_or_nif_epilogue:
 
  {
      Eterm header;
-     Eterm* next;
+     BeamInstr *next;
      Uint slots;
+     Eterm context;
 
      OpCase(i_bs_start_match2_rfIId): {
-	 tmp_arg1 = r(0);
+	 context = r(0);
 
      do_start_match:
 	 slots = Arg(2);
-	 if (!is_boxed(tmp_arg1)) {
+	 if (!is_boxed(context)) {
 	     ClauseFail();
 	 }
 	 PreFetch(4, next);
-	 header = *boxed_val(tmp_arg1);
+	 header = *boxed_val(context);
 	 if (header_is_bin_matchstate(header)) {
-	     ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
+	     ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context);
 	     Uint actual_slots = HEADER_NUM_SLOTS(header);
 	     ms->save_offset[0] = ms->mb.offset;
 	     if (actual_slots < slots) {
@@ -3727,8 +4051,8 @@ apply_bif_or_nif_epilogue:
 		 Uint live = Arg(1);
 		 Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
 
-		 TestHeapPreserve(wordsneeded, live, tmp_arg1);
-		 ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
+		 TestHeapPreserve(wordsneeded, live, context);
+		 ms = (ErlBinMatchState *) boxed_val(context);
 		 dst = (ErlBinMatchState *) HTOP;
 		 *dst = *ms;
 		 *HTOP = HEADER_BIN_MATCHSTATE(slots);
@@ -3740,12 +4064,12 @@ apply_bif_or_nif_epilogue:
 	     Eterm result;
 	     Uint live = Arg(1);
 	     Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
-	     TestHeapPreserve(wordsneeded, live, tmp_arg1);
+	     TestHeapPreserve(wordsneeded, live, context);
 	     HEAP_TOP(c_p) = HTOP;
 #ifdef DEBUG
 	     c_p->stop = E;	/* Needed for checking in HeapOnlyAlloc(). */
 #endif
-	     result = erts_bs_start_match_2(c_p, tmp_arg1, slots);
+	     result = erts_bs_start_match_2(c_p, context, slots);
 	     HTOP = HEAP_TOP(c_p);
 	     HEAP_SPACE_VERIFIED(0);
 	     if (is_non_value(result)) {
@@ -3759,19 +4083,19 @@ apply_bif_or_nif_epilogue:
 	 NextPF(4, next);
      }
      OpCase(i_bs_start_match2_xfIId): {
-	 tmp_arg1 = xb(Arg(0));
+	 context = xb(Arg(0));
 	 I++;
 	 goto do_start_match;
      }
      OpCase(i_bs_start_match2_yfIId): {
-	 tmp_arg1 = yb(Arg(0));
+	 context = yb(Arg(0));
 	 I++;
 	 goto do_start_match;
      }
  }
 
  OpCase(bs_test_zero_tail2_fr): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      
      PreFetch(1, next);
@@ -3783,7 +4107,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(bs_test_zero_tail2_fx): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      
      PreFetch(2, next);
@@ -3795,7 +4119,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(bs_test_tail_imm2_frI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(2, next);
      _mb = ms_matchbuffer(r(0));
@@ -3805,7 +4129,7 @@ apply_bif_or_nif_epilogue:
      NextPF(2, next);
  }
  OpCase(bs_test_tail_imm2_fxI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(3, next);
      _mb = ms_matchbuffer(xb(Arg(1)));
@@ -3816,7 +4140,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(bs_test_unit_frI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(2, next);
      _mb = ms_matchbuffer(r(0));
@@ -3826,7 +4150,7 @@ apply_bif_or_nif_epilogue:
      NextPF(2, next);
  }
  OpCase(bs_test_unit_fxI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(3, next);
      _mb = ms_matchbuffer(xb(Arg(1)));
@@ -3837,7 +4161,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(bs_test_unit8_fr): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(1, next);
      _mb = ms_matchbuffer(r(0));
@@ -3847,7 +4171,7 @@ apply_bif_or_nif_epilogue:
      NextPF(1, next);
  }
  OpCase(bs_test_unit8_fx): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(2, next);
      _mb = ms_matchbuffer(xb(Arg(1)));
@@ -3857,93 +4181,105 @@ apply_bif_or_nif_epilogue:
      NextPF(2, next);
  }
 
+ {
+     Eterm bs_get_integer8_context;
+
  OpCase(i_bs_get_integer_8_rfd): {
-     tmp_arg1 = r(0);
-     goto do_bs_get_integer_8;
- }
+	 bs_get_integer8_context = r(0);
+	 goto do_bs_get_integer_8;
+     }
 
  OpCase(i_bs_get_integer_8_xfd): {
-     tmp_arg1 = xb(Arg(0));
-     I++;
- }
+	 bs_get_integer8_context = xb(Arg(0));
+	 I++;
+     }
 
  do_bs_get_integer_8: {
-     ErlBinMatchBuffer *_mb;
-     Eterm _result;
-     _mb = ms_matchbuffer(tmp_arg1);
-     if (_mb->size - _mb->offset < 8) {
-	 ClauseFail();
-     }
-     if (BIT_OFFSET(_mb->offset) != 0) {
-	 _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
-     } else {
-	 _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
-	 _mb->offset += 8;
+	 ErlBinMatchBuffer *_mb;
+	 Eterm _result;
+	 _mb = ms_matchbuffer(bs_get_integer8_context);
+	 if (_mb->size - _mb->offset < 8) {
+	     ClauseFail();
+	 }
+	 if (BIT_OFFSET(_mb->offset) != 0) {
+	     _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
+	 } else {
+	     _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
+	     _mb->offset += 8;
+	 }
+	 StoreBifResult(1, _result);
      }
-     StoreBifResult(1, _result);
  }
 
- OpCase(i_bs_get_integer_16_rfd): {
-     tmp_arg1 = r(0);
+ {
+     Eterm bs_get_integer_16_context;
+
+ OpCase(i_bs_get_integer_16_rfd):
+     bs_get_integer_16_context = r(0);
      goto do_bs_get_integer_16;
- }
 
- OpCase(i_bs_get_integer_16_xfd): {
-     tmp_arg1 = xb(Arg(0));
+ OpCase(i_bs_get_integer_16_xfd):
+     bs_get_integer_16_context = xb(Arg(0));
      I++;
- }
 
- do_bs_get_integer_16: {
-     ErlBinMatchBuffer *_mb;
-     Eterm _result;
-     _mb = ms_matchbuffer(tmp_arg1);
-     if (_mb->size - _mb->offset < 16) {
-	 ClauseFail();
-     }
-     if (BIT_OFFSET(_mb->offset) != 0) {
-	 _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
-     } else {
-	 _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
-	 _mb->offset += 16;
+ do_bs_get_integer_16:
+     {
+	 ErlBinMatchBuffer *_mb;
+	 Eterm _result;
+	 _mb = ms_matchbuffer(bs_get_integer_16_context);
+	 if (_mb->size - _mb->offset < 16) {
+	     ClauseFail(); 
+	 }
+	 if (BIT_OFFSET(_mb->offset) != 0) {
+	     _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
+	 } else {
+	     _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
+	     _mb->offset += 16;
+	 }
+	 StoreBifResult(1, _result);
      }
-     StoreBifResult(1, _result);
  }
 
- OpCase(i_bs_get_integer_32_rfId): {
-     tmp_arg1 = r(0);
+ {
+     Eterm bs_get_integer_32_context;
+
+ OpCase(i_bs_get_integer_32_rfId):
+     bs_get_integer_32_context = r(0);
      goto do_bs_get_integer_32;
- }
+
      
- OpCase(i_bs_get_integer_32_xfId): {
-     tmp_arg1 = xb(Arg(0));
+ OpCase(i_bs_get_integer_32_xfId):
+     bs_get_integer_32_context = xb(Arg(0));
      I++;
- }
 
- do_bs_get_integer_32: {
-     ErlBinMatchBuffer *_mb;
-     Uint32 _integer;
-     Eterm _result;
-     _mb = ms_matchbuffer(tmp_arg1);
-     if (_mb->size - _mb->offset < 32) { ClauseFail(); }
-     if (BIT_OFFSET(_mb->offset) != 0) {
-	 _integer = erts_bs_get_unaligned_uint32(_mb);
-     } else {
-	 _integer = get_int32(_mb->base + _mb->offset/8);
-     }
-     _mb->offset += 32;
-#ifndef ARCH_64
-     if (IS_USMALL(0, _integer)) {
+
+ do_bs_get_integer_32:
+     {
+	 ErlBinMatchBuffer *_mb;
+	 Uint32 _integer;
+	 Eterm _result;
+	 _mb = ms_matchbuffer(bs_get_integer_32_context);
+	 if (_mb->size - _mb->offset < 32) { ClauseFail(); }
+	 if (BIT_OFFSET(_mb->offset) != 0) {
+	     _integer = erts_bs_get_unaligned_uint32(_mb);
+	 } else {
+	     _integer = get_int32(_mb->base + _mb->offset/8);
+	 }
+	 _mb->offset += 32;
+#if !defined(ARCH_64) || HALFWORD_HEAP
+	 if (IS_USMALL(0, _integer)) {
 #endif
-	 _result = make_small(_integer);
-#ifndef ARCH_64
-     } else {
-	 TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
-	 _result = uint_to_big((Uint) _integer, HTOP);
-	 HTOP += BIG_UINT_HEAP_SIZE;
-	 HEAP_SPACE_VERIFIED(0);
-     }
+	     _result = make_small(_integer);
+#if !defined(ARCH_64) || HALFWORD_HEAP
+	 } else {
+	     TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
+	     _result = uint_to_big((Uint) _integer, HTOP);
+	     HTOP += BIG_UINT_HEAP_SIZE;
+	     HEAP_SPACE_VERIFIED(0);
+	 }
 #endif
-     StoreBifResult(2, _result);
+	 StoreBifResult(2, _result);
+     }
  }
 
  /* Operands: Size Live Fail Flags Dst */
@@ -4041,54 +4377,64 @@ apply_bif_or_nif_epilogue:
      StoreBifResult(3, result);
  }
 
- /* Operands: MatchContext Fail Dst */
+ {
+     Eterm get_utf8_context;
+
+     /* Operands: MatchContext Fail Dst */
  OpCase(i_bs_get_utf8_rfd): {
-     tmp_arg1 = r(0);
-     goto do_bs_get_utf8;
- }
+	 get_utf8_context = r(0);
+	 goto do_bs_get_utf8;
+     }
 
  OpCase(i_bs_get_utf8_xfd): {
-     tmp_arg1 = xb(Arg(0));
-     I++;
- }
+	 get_utf8_context = xb(Arg(0));
+	 I++;
+     }
 
- /*
-  * tmp_arg1 = match_context
-  * Operands: Fail Dst
-  */
+     /*
+      * get_utf8_context = match_context
+      * Operands: Fail Dst
+      */
 
-    do_bs_get_utf8: {
-     Eterm result = erts_bs_get_utf8(ms_matchbuffer(tmp_arg1));
-     if (is_non_value(result)) {
-	 ClauseFail();
+ do_bs_get_utf8: {
+	 Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context));
+	 if (is_non_value(result)) {
+	     ClauseFail();
+	 }
+	 StoreBifResult(1, result);
      }
-     StoreBifResult(1, result);
  }
 
- /* Operands: MatchContext Fail Flags Dst */
+ {
+     Eterm get_utf16_context;
+
+     /* Operands: MatchContext Fail Flags Dst */
  OpCase(i_bs_get_utf16_rfId): {
-     tmp_arg1 = r(0);
-     goto do_bs_get_utf16;
- }
+	 get_utf16_context = r(0);
+	 goto do_bs_get_utf16;
+     }
 
  OpCase(i_bs_get_utf16_xfId): {
-     tmp_arg1 = xb(Arg(0));
-     I++;
- }
+	 get_utf16_context = xb(Arg(0));
+	 I++;
+     }
 
- /*
-  * tmp_arg1 = match_context
-  * Operands: Fail Flags Dst
-  */
-    do_bs_get_utf16: {
-     Eterm result = erts_bs_get_utf16(ms_matchbuffer(tmp_arg1), Arg(1));
-     if (is_non_value(result)) {
-	 ClauseFail();
+     /*
+      * get_utf16_context = match_context
+      * Operands: Fail Flags Dst
+      */
+ do_bs_get_utf16: {
+	 Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context),
+					  Arg(1));
+	 if (is_non_value(result)) {
+	     ClauseFail();
+	 }
+	 StoreBifResult(2, result);
      }
-     StoreBifResult(2, result);
  }
 
  {
+     Eterm context_to_binary_context;
      ErlBinMatchBuffer* mb;
      ErlSubBin* sb;
      Uint size;
@@ -4097,27 +4443,29 @@ apply_bif_or_nif_epilogue:
      Uint hole_size;
 
      OpCase(bs_context_to_binary_r): {
-	 tmp_arg1 = x0;
+	 context_to_binary_context = x0;
 	 I -= 2;
 	 goto do_context_to_binary;
      }
 
      /* Unfortunately, inlining can generate this instruction. */
      OpCase(bs_context_to_binary_y): {
-	 tmp_arg1 = yb(Arg(0));
+	 context_to_binary_context = yb(Arg(0));
 	 goto do_context_to_binary0;
      }
 
      OpCase(bs_context_to_binary_x): {
-	 tmp_arg1 = xb(Arg(0));
+	 context_to_binary_context = xb(Arg(0));
      
      do_context_to_binary0:
 	 I--;
      }
 
  do_context_to_binary:
-     if (is_boxed(tmp_arg1) && header_is_bin_matchstate(*boxed_val(tmp_arg1))) {
-	 ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
+     if (is_boxed(context_to_binary_context) &&
+	 header_is_bin_matchstate(*boxed_val(context_to_binary_context))) {
+	 ErlBinMatchState* ms;
+	 ms = (ErlBinMatchState *) boxed_val(context_to_binary_context);
 	 mb = &ms->mb;
 	 offs = ms->save_offset[0];
 	 size = mb->size - offs;
@@ -4126,17 +4474,17 @@ apply_bif_or_nif_epilogue:
      Next(2);
 
      OpCase(i_bs_get_binary_all_reuse_rfI): {
-	 tmp_arg1 = x0;
+	 context_to_binary_context = x0;
 	 goto do_bs_get_binary_all_reuse;
      }
 
      OpCase(i_bs_get_binary_all_reuse_xfI): {
-	 tmp_arg1 = xb(Arg(0));
+	 context_to_binary_context = xb(Arg(0));
 	 I++;
      }
 
  do_bs_get_binary_all_reuse:
-     mb = ms_matchbuffer(tmp_arg1);
+     mb = ms_matchbuffer(context_to_binary_context);
      size = mb->size - mb->offset;
      if (size % Arg(1) != 0) {
 	 ClauseFail();
@@ -4145,7 +4493,7 @@ apply_bif_or_nif_epilogue:
 
  do_bs_get_binary_all_reuse_common:
      orig = mb->orig;
-     sb = (ErlSubBin *) boxed_val(tmp_arg1);
+     sb = (ErlSubBin *) boxed_val(context_to_binary_context);
      hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
      sb->thing_word = HEADER_SUB_BIN;
      sb->size = BYTE_OFFSET(size);
@@ -4161,18 +4509,20 @@ apply_bif_or_nif_epilogue:
  }
 
  {
+     Eterm match_string_context;
+
      OpCase(i_bs_match_string_rfII): {
-	 tmp_arg1 = r(0);
+	 match_string_context = r(0);
 	 goto do_bs_match_string;
      }
      OpCase(i_bs_match_string_xfII): {
-	 tmp_arg1 = xb(Arg(0));
+	 match_string_context = xb(Arg(0));
 	 I++;
      }
 
  do_bs_match_string:
      {
-	 Eterm* next;
+	 BeamInstr *next;
 	 byte* bytes;
 	 Uint bits;
 	 ErlBinMatchBuffer* mb;
@@ -4181,7 +4531,7 @@ apply_bif_or_nif_epilogue:
 	 PreFetch(3, next);
 	 bits = Arg(1);
 	 bytes = (byte *) Arg(2);
-	 mb = ms_matchbuffer(tmp_arg1);
+	 mb = ms_matchbuffer(match_string_context);
 	 if (mb->size - mb->offset < bits) {
 	     ClauseFail();
 	 }
@@ -4199,7 +4549,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(i_bs_save2_rI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchState *_ms;
      PreFetch(1, next);
      _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
@@ -4207,7 +4557,7 @@ apply_bif_or_nif_epilogue:
      NextPF(1, next);
  }
  OpCase(i_bs_save2_xI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchState *_ms;
      PreFetch(2, next);
      _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0)));
@@ -4216,7 +4566,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(i_bs_restore2_rI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchState *_ms;
      PreFetch(1, next);
      _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
@@ -4224,7 +4574,7 @@ apply_bif_or_nif_epilogue:
      NextPF(1, next);
  }
  OpCase(i_bs_restore2_xI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchState *_ms;
      PreFetch(2, next);
      _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0)));
@@ -4241,7 +4591,7 @@ apply_bif_or_nif_epilogue:
   * deallocate not followed by a return, and that should work.
   */
  OpCase(deallocate_I): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(1, next);
      D(Arg(0));
@@ -4264,7 +4614,7 @@ apply_bif_or_nif_epilogue:
      */
  OpCase(call_traced_function): {
      if (IS_TRACED_FL(c_p, F_TRACE_CALLS)) {
-	 unsigned offset = offsetof(Export, code) + 3*sizeof(Eterm);
+	 unsigned offset = offsetof(Export, code) + 3*sizeof(BeamInstr);
 	 Export* ep = (Export *) (((char *)I)-offset);
 	 Uint32 flags;
 
@@ -4274,6 +4624,7 @@ apply_bif_or_nif_epilogue:
 	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	 flags = erts_call_trace(c_p, ep->code, ep->match_prog_set, reg,
 				 0, &c_p->tracer_proc);
+	 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	 ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	 PROCESS_MAIN_CHK_LOCKS(c_p);
 	 ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -4285,32 +4636,32 @@ apply_bif_or_nif_epilogue:
 		 /* SWAPOUT, SWAPIN was done and r(0) was saved above */
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 FCALLS -= erts_garbage_collect(c_p, 3, reg, ep->code[2]);
+		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 r(0) = reg[0];
 		 SWAPIN;
 	     }
 	     E -= 3;
 	     ASSERT(c_p->htop <= E && E <= c_p->hend);
-	     ASSERT(is_CP((Eterm)(ep->code)));
+	     ASSERT(is_CP((BeamInstr)(ep->code)));
 	     ASSERT(is_internal_pid(c_p->tracer_proc) || 
 		    is_internal_port(c_p->tracer_proc));
-	     E[2] = make_cp(c_p->cp);
+	     E[2] = make_cp(c_p->cp); /* Code in lower range on halfword */
 	     E[1] = am_true; /* Process tracer */
 	     E[0] = make_cp(ep->code);
-	     c_p->cp = (Eterm*)
-		 make_cp(flags & MATCH_SET_EXCEPTION_TRACE
-			 ? beam_exception_trace : beam_return_trace);
+	     c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE)
+		 ? beam_exception_trace : beam_return_trace;
 	     erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	     c_p->trace_flags |= F_EXCEPTION_TRACE;
 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	 }
      }
-     SET_I((Uint *) Arg(0));
+     SET_I((BeamInstr *)Arg(0));
      Dispatch();
  }
 
  OpCase(return_trace): {
-     Uint* code = (Uint *) E[0];
+     BeamInstr* code = (BeamInstr *) (UWord) E[0];
      
      SWAPOUT;		/* Needed for shared heap */
      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
@@ -4318,44 +4669,118 @@ apply_bif_or_nif_epilogue:
      ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
      SWAPIN;
      c_p->cp = NULL;
-     SET_I((Eterm *) E[2]);
+     SET_I((BeamInstr *) cp_val(E[2]));
      E += 3;
      Goto(*I);
  }
 
  OpCase(i_count_breakpoint): {
-     Uint real_I;
+     BeamInstr real_I;
      
-     ErtsCountBreak((Uint *) I, &real_I);
+     ErtsCountBreak(c_p, (BeamInstr *) I, &real_I);
      ASSERT(VALID_INSTR(real_I));
      Goto(real_I);
  }
 
+ /* need to send mfa instead of bdt pointer
+  * the pointer might be deallocated.
+  */
+
+ OpCase(i_time_breakpoint): {
+     BeamInstr real_I;
+     BpData **bds = (BpData **) (I)[-4];
+     BpDataTime *bdt = NULL;
+     Uint ix = 0;
+#ifdef ERTS_SMP
+     ix = c_p->scheduler_data->no - 1;
+#else
+     ix = 0;
+#endif
+     bdt = (BpDataTime *)bds[ix];
+
+     ASSERT((I)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+     ASSERT(bdt);
+     bdt = (BpDataTime *) bdt->next;
+     ASSERT(bdt);
+     bds[ix] = (BpData *) bdt;
+     real_I = bdt->orig_instr;
+     ASSERT(VALID_INSTR(real_I));
+
+     if (IS_TRACED_FL(c_p, F_TRACE_CALLS) && !(bdt->pause)) {
+	 if (	(*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) ||
+		(*(c_p->cp) == (BeamInstr) OpCode(return_trace)) ||
+		(*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace))) {
+	     /* This _IS_ a tail recursive call */
+	     SWAPOUT;
+	     erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_TAIL_CALL);
+	     SWAPIN;
+	 } else {
+	     SWAPOUT;
+	     erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_CALL);
+
+	     /* r register needs to be copied to the array
+	      * for the garbage collector
+	      */
+	     ASSERT(c_p->htop <= E && E <= c_p->hend);
+	     if (E - 2 < HTOP) {
+		 reg[0] = r(0);
+		 PROCESS_MAIN_CHK_LOCKS(c_p);
+		 FCALLS -= erts_garbage_collect(c_p, 2, reg, I[-1]);
+		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+		 PROCESS_MAIN_CHK_LOCKS(c_p);
+		 r(0) = reg[0];
+	     }
+	     SWAPIN;
+
+	     ASSERT(c_p->htop <= E && E <= c_p->hend);
+
+	     E -= 2;
+	     E[0] = make_cp(I);
+	     E[1] = make_cp(c_p->cp);     /* original return address */
+	     c_p->cp = beam_return_time_trace;
+	 }
+     }
+
+     Goto(real_I);
+ }
+
+ OpCase(i_return_time_trace): {
+     BeamInstr *pc = (BeamInstr *) (UWord) E[0];
+     SWAPOUT;
+     erts_trace_time_break(c_p, pc, NULL, ERTS_BP_CALL_TIME_RETURN);
+     SWAPIN;
+     c_p->cp = NULL;
+     SET_I((BeamInstr *) cp_val(E[1]));
+     E += 2;
+     Goto(*I);
+ }
+
  OpCase(i_trace_breakpoint):
      if (! IS_TRACED_FL(c_p, F_TRACE_CALLS)) {
-	 Uint real_I;
+	 BeamInstr real_I;
 	 
-	 ErtsBreakSkip((Uint *) I, &real_I);
+	 ErtsBreakSkip(c_p, (BeamInstr *) I, &real_I);
 	 Goto(real_I);
      }
  /* Fall through to next case */
  OpCase(i_mtrace_breakpoint): {
-     Uint real_I;
+     BeamInstr real_I;
      Uint32 flags;
      Eterm tracer_pid;
-     Uint *cpp;
+     Uint* cpp;
      int return_to_trace = 0, need = 0;
      flags = 0;
      SWAPOUT;
      reg[0] = r(0);
 
-     if (*cp_val((Eterm)c_p->cp) 
-	 == (Uint) OpCode(return_trace)) {
-	 cpp = (Uint*)&E[2];
-     } else if (*cp_val((Eterm)c_p->cp) 
-		== (Uint) OpCode(i_return_to_trace)) {
+     if (*(c_p->cp) == (BeamInstr) OpCode(return_trace)) {
+	 cpp = &E[2];
+     } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace)) {
+	 return_to_trace = !0;
+	 cpp = &E[0];
+     } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) {
 	 return_to_trace = !0;
-	 cpp = (Uint*)&E[0];
+	 cpp = &E[0];
      } else {
 	 cpp = NULL;
      }
@@ -4364,19 +4789,21 @@ apply_bif_or_nif_epilogue:
 	  * return_trace and/or i_return_to_trace stackframes
 	  * on the stack, they are not intermixed with y registers
 	  */
-	 Eterm *cp_save = c_p->cp;
+	 BeamInstr *cp_save = c_p->cp;
 	 for (;;) {
 	     ASSERT(is_CP(*cpp));
-	     if (*cp_val(*cpp) == (Uint) OpCode(return_trace)) {
+	     if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) {
 		 cpp += 3;
-	     } else if (*cp_val(*cpp) == (Uint) OpCode(i_return_to_trace)) {
+	     } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) {
 		 return_to_trace = !0;
 		 cpp += 1;
+	     } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_time_trace)) {
+		 cpp += 2;
 	     } else
 		 break;
 	 }
-	 c_p->cp = (Eterm *) *cpp;
-	 ASSERT(is_CP((Eterm)c_p->cp));
+	 c_p->cp = (BeamInstr *) cp_val(*cpp);
+	 ASSERT(is_CP(*cpp));
 	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	 real_I = erts_trace_break(c_p, I, reg, &flags, &tracer_pid);
 	 ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
@@ -4403,6 +4830,7 @@ apply_bif_or_nif_epilogue:
 	     /* SWAPOUT was done and r(0) was saved above */
 	     PROCESS_MAIN_CHK_LOCKS(c_p);
 	     FCALLS -= erts_garbage_collect(c_p, need, reg, I[-1]);
+	     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	     PROCESS_MAIN_CHK_LOCKS(c_p);
 	     r(0) = reg[0];
 	     SWAPIN;
@@ -4412,12 +4840,12 @@ apply_bif_or_nif_epilogue:
 	 E -= 1;
 	 ASSERT(c_p->htop <= E && E <= c_p->hend);
 	 E[0] = make_cp(c_p->cp);
-	 c_p->cp = (Eterm *) make_cp(beam_return_to_trace);
+	 c_p->cp = (BeamInstr *) beam_return_to_trace;
      }
      if (flags & MATCH_SET_RX_TRACE) {
 	 E -= 3;
 	 ASSERT(c_p->htop <= E && E <= c_p->hend);
-	 ASSERT(is_CP((Eterm) (I - 3)));
+	 ASSERT(is_CP((Eterm) (UWord) (I - 3)));
 	 ASSERT(am_true == tracer_pid || 
 		is_internal_pid(tracer_pid) || is_internal_port(tracer_pid));
 	 E[2] = make_cp(c_p->cp);
@@ -4425,9 +4853,9 @@ apply_bif_or_nif_epilogue:
 	 E[0] = make_cp(I - 3); /* We ARE at the beginning of an 
 				   instruction,
 				   the funcinfo is above i. */
-	 c_p->cp = (Eterm*)
-	     make_cp(flags & MATCH_SET_EXCEPTION_TRACE
-		     ? beam_exception_trace : beam_return_trace);
+	 c_p->cp =
+	     (flags & MATCH_SET_EXCEPTION_TRACE)
+		     ? beam_exception_trace : beam_return_trace;
 	 erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	 c_p->trace_flags |= F_EXCEPTION_TRACE;
 	 erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
@@ -4440,10 +4868,10 @@ apply_bif_or_nif_epilogue:
 	 Uint *cpp = (Uint*) E;
 	 for(;;) {
 	     ASSERT(is_CP(*cpp));
-	     if (*cp_val(*cpp) == (Uint) OpCode(return_trace)) {
+	     if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) {
 		 do ++cpp; while(is_not_CP(*cpp));
 		 cpp += 2;
-	     } else if (*cp_val(*cpp) == (Uint) OpCode(i_return_to_trace)) {
+	     } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) {
 		 do ++cpp; while(is_not_CP(*cpp));
 	     } else break;
 	 }
@@ -4454,7 +4882,7 @@ apply_bif_or_nif_epilogue:
 	 SWAPIN;
      }
      c_p->cp = NULL;
-     SET_I((Eterm *) E[0]);
+     SET_I((BeamInstr *) cp_val(E[0]));
      E += 1;
      Goto(*I);
  }
@@ -4465,7 +4893,7 @@ apply_bif_or_nif_epilogue:
 
  OpCase(i_global_cons):
  {
-     Eterm *next;
+     BeamInstr *next;
 #ifdef HYBRID
      Eterm *hp;
 
@@ -4487,7 +4915,7 @@ apply_bif_or_nif_epilogue:
 
  OpCase(i_global_tuple):
  {
-     Eterm *next;
+     BeamInstr *next;
      int len;
 #ifdef HYBRID
      Eterm list;
@@ -4522,7 +4950,7 @@ apply_bif_or_nif_epilogue:
 
  OpCase(i_global_copy):
  {
-     Eterm *next;
+     BeamInstr *next;
      PreFetch(0,next);
 #ifdef HYBRID
      if (!IS_CONST(r(0)))
@@ -4551,7 +4979,7 @@ apply_bif_or_nif_epilogue:
 
  OpCase(fmove_ql): {
      Eterm fr = Arg(1);
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(2, next);
      GET_DOUBLE(Arg(0), *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
@@ -4561,7 +4989,7 @@ apply_bif_or_nif_epilogue:
  OpCase(fmove_dl): {
      Eterm targ1;
      Eterm fr = Arg(1);
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(2, next);
      GetR(0, targ1);
@@ -4570,7 +4998,7 @@ apply_bif_or_nif_epilogue:
      NextPF(2, next);
  }
 
- OpCase(fmove_new_ld): {
+ OpCase(fmove_ld): {
      Eterm fr = Arg(0);
      Eterm dest = make_float(HTOP);
 
@@ -4582,7 +5010,7 @@ apply_bif_or_nif_epilogue:
  OpCase(fconv_dl): {
      Eterm targ1;
      Eterm fr = Arg(1);
-     Eterm* next;
+     BeamInstr *next;
 
      GetR(0, targ1);
      PreFetch(2, next);
@@ -4600,18 +5028,13 @@ apply_bif_or_nif_epilogue:
      NextPF(2, next);
  }
 
- /*
-  * Old allocating fmove.
-  */
-
-
 #ifdef NO_FPE_SIGNALS
      OpCase(fclearerror):
      OpCase(i_fcheckerror):
 	 erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
 #else
      OpCase(fclearerror): {
-	 Eterm* next;
+	 BeamInstr *next;
 
 	 PreFetch(0, next);
 	 ERTS_FP_CHECK_INIT(c_p);
@@ -4619,7 +5042,7 @@ apply_bif_or_nif_epilogue:
      }
 
      OpCase(i_fcheckerror): {
-	 Eterm* next;
+	 BeamInstr *next;
 
 	 PreFetch(0, next);
 	 ERTS_FP_ERROR(c_p, freg[0].fd, goto fbadarith);
@@ -4633,7 +5056,7 @@ apply_bif_or_nif_epilogue:
 
 
  OpCase(i_fadd_lll): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(3, next);
      ERTS_FP_CHECK_INIT(c_p);
@@ -4642,7 +5065,7 @@ apply_bif_or_nif_epilogue:
      NextPF(3, next);
  }
  OpCase(i_fsub_lll): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(3, next);
      ERTS_FP_CHECK_INIT(c_p);
@@ -4651,7 +5074,7 @@ apply_bif_or_nif_epilogue:
      NextPF(3, next);
  }
  OpCase(i_fmul_lll): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(3, next);
      ERTS_FP_CHECK_INIT(c_p);
@@ -4660,7 +5083,7 @@ apply_bif_or_nif_epilogue:
      NextPF(3, next);
  }
  OpCase(i_fdiv_lll): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(3, next);
      ERTS_FP_CHECK_INIT(c_p);
@@ -4669,7 +5092,7 @@ apply_bif_or_nif_epilogue:
      NextPF(3, next);
  }
  OpCase(i_fnegate_ll): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(2, next);
      ERTS_FP_CHECK_INIT(c_p);
@@ -4736,7 +5159,7 @@ apply_bif_or_nif_epilogue:
      neg_o_reds = -c_p->def_arg_reg[4];
      FCALLS = c_p->fcalls;
      SWAPIN;
-     switch( c_p->def_arg_reg[3] ) {
+     switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */
        case HIPE_MODE_SWITCH_RES_RETURN:
 	 ASSERT(is_value(reg[0]));
 	 MoveReturn(reg[0], r(0));
@@ -4748,7 +5171,7 @@ apply_bif_or_nif_epilogue:
 	 /* This can be used to call any function value, but currently it's
 	    only used to call closures referring to unloaded modules. */
 	 {
-	     Eterm *next;
+	     BeamInstr *next;
 
 	     next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
 	     SWAPIN;
@@ -4805,7 +5228,8 @@ apply_bif_or_nif_epilogue:
 
  OpCase(i_hibernate): {
      SWAPOUT;
-     if (hibernate(c_p, r(0), x(1), x(2), reg)) {
+     if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
+	 c_p->flags &= ~F_HIBERNATE_SCHED;
 	 goto do_schedule;
      } else {
 	 I = handle_error(c_p, I, reg, hibernate_3);
@@ -4816,12 +5240,11 @@ apply_bif_or_nif_epilogue:
  OpCase(i_debug_breakpoint): {
      SWAPOUT;
      reg[0] = r(0);
-     tmp_arg1 = call_breakpoint_handler(c_p, I-3, reg);
+     I = call_error_handler(c_p, I-3, reg, am_breakpoint);
      r(0) = reg[0];
      SWAPIN;
-     if (tmp_arg1) {
-	 SET_I(c_p->i);
-	 Dispatch();
+     if (I) {
+	 Goto(*I);
      }
      goto no_error_handler;
  }
@@ -4881,13 +5304,15 @@ apply_bif_or_nif_epilogue:
      em_call_error_handler = OpCode(call_error_handler);
      em_call_traced_function = OpCode(call_traced_function);
      em_apply_bif = OpCode(apply_bif);
-     beam_apply[0] = (Eterm) OpCode(i_apply);
-     beam_apply[1] = (Eterm) OpCode(normal_exit);
-     beam_exit[0] = (Eterm) OpCode(error_action_code);
-     beam_continue_exit[0] = (Eterm) OpCode(continue_exit);
-     beam_return_to_trace[0] = (Eterm) OpCode(i_return_to_trace);
-     beam_return_trace[0] = (Eterm) OpCode(return_trace);
-     beam_exception_trace[0] = (Eterm) OpCode(return_trace); /* UGLY */
+
+     beam_apply[0]             = (BeamInstr) OpCode(i_apply);
+     beam_apply[1]             = (BeamInstr) OpCode(normal_exit);
+     beam_exit[0]              = (BeamInstr) OpCode(error_action_code);
+     beam_continue_exit[0]     = (BeamInstr) OpCode(continue_exit);
+     beam_return_to_trace[0]   = (BeamInstr) OpCode(i_return_to_trace);
+     beam_return_trace[0]      = (BeamInstr) OpCode(return_trace);
+     beam_exception_trace[0]   = (BeamInstr) OpCode(return_trace); /* UGLY */
+     beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace);
 
      /*
       * Enter all BIFs into the export table.
@@ -4897,8 +5322,10 @@ apply_bif_or_nif_epilogue:
 			      bif_table[i].name,
 			      bif_table[i].arity);
 	 bif_export[i] = ep;
-	 ep->code[3] = (Eterm) OpCode(apply_bif);
-	 ep->code[4] = (Eterm) bif_table[i].f;
+	 ep->code[3] = (BeamInstr) OpCode(apply_bif);
+	 ep->code[4] = (BeamInstr) bif_table[i].f;
+	 /* XXX: set func info for bifs */
+	 ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
      }
 
      return;
@@ -4943,6 +5370,10 @@ translate_gc_bif(void* gcf)
 	return round_1;
     } else if (gcf == erts_gc_trunc_1) {
 	return round_1;
+    } else if (gcf == erts_gc_binary_part_2) {
+	return binary_part_2;
+    } else if (gcf == erts_gc_binary_part_3) {
+	return binary_part_3;
     } else {
 	erl_exit(1, "bad gc bif");
     }
@@ -5001,8 +5432,8 @@ Eterm error_atom[NUMBER_EXIT_CODES] = {
  * at the point of the original exception.
  */
 
-static Eterm*
-handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf)
+static BeamInstr*
+handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
 {
     Eterm* hp;
     Eterm Value = c_p->fvalue;
@@ -5056,7 +5487,7 @@ handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf)
     /* Find a handler or die */
     if ((c_p->catches > 0 || IS_TRACED_FL(c_p, F_EXCEPTION_TRACE))
 	&& !(c_p->freason & EXF_PANIC)) {
-	Eterm *new_pc;
+	BeamInstr *new_pc;
         /* The Beam handler code (catch_end or try_end) checks reg[0]
 	   for THE_NON_VALUE to see if the previous code finished
 	   abnormally. If so, reg[1], reg[2] and reg[3] should hold the
@@ -5082,30 +5513,37 @@ handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf)
 /*
  * Find the nearest catch handler
  */
-static Eterm*
+static BeamInstr*
 next_catch(Process* c_p, Eterm *reg) {
     int active_catches = c_p->catches > 0;
     int have_return_to_trace = 0;
     Eterm *ptr, *prev, *return_to_trace_ptr = NULL;
-    Uint i_return_trace = beam_return_trace[0];
-    Uint i_return_to_trace = beam_return_to_trace[0];
+
+    BeamInstr i_return_trace      = beam_return_trace[0];
+    BeamInstr i_return_to_trace   = beam_return_to_trace[0];
+    BeamInstr i_return_time_trace = beam_return_time_trace[0];
+
     ptr = prev = c_p->stop;
     ASSERT(is_CP(*ptr));
     ASSERT(ptr <= STACK_START(c_p));
     if (ptr == STACK_START(c_p)) return NULL;
     if ((is_not_CP(*ptr) || (*cp_val(*ptr) != i_return_trace &&
-			     *cp_val(*ptr) != i_return_to_trace))
+			     *cp_val(*ptr) != i_return_to_trace &&
+			     *cp_val(*ptr) != i_return_time_trace ))
 	&& c_p->cp) {
 	/* Can not follow cp here - code may be unloaded */
-	Uint *cpp = cp_val((Eterm) c_p->cp);
+	BeamInstr *cpp = c_p->cp;
 	if (cpp == beam_exception_trace) {
-	    erts_trace_exception(c_p, (Eterm*) ptr[0],
+	    erts_trace_exception(c_p, cp_val(ptr[0]),
 				 reg[1], reg[2], ptr+1);
 	    /* Skip return_trace parameters */
 	    ptr += 2;
 	} else if (cpp == beam_return_trace) {
 	    /* Skip return_trace parameters */
 	    ptr += 2;
+	} else if (cpp == beam_return_time_trace) {
+	    /* Skip return_trace parameters */
+	    ptr += 1;
 	} else if (cpp == beam_return_to_trace) {
 	    have_return_to_trace = !0; /* Record next cp */
 	}
@@ -5123,7 +5561,7 @@ next_catch(Process* c_p, Eterm *reg) {
 		    if (is_catch(*ptr) && active_catches) goto found_catch;
 		}
 		if (cp_val(*prev) == beam_exception_trace) {
-		    erts_trace_exception(c_p, (Eterm*) ptr[0],
+		    erts_trace_exception(c_p, cp_val(ptr[0]),
 					 reg[1], reg[2], ptr+1);
 		}
 		/* Skip return_trace parameters */
@@ -5135,6 +5573,13 @@ next_catch(Process* c_p, Eterm *reg) {
 		}
 		have_return_to_trace = !0; /* Record next cp */
 		return_to_trace_ptr = NULL;
+	    } else if (*cp_val(*prev) == i_return_time_trace) {
+		/* Skip stack frame variables */
+		while (++ptr, ptr < STACK_START(c_p) && is_not_CP(*ptr)) {
+		    if (is_catch(*ptr) && active_catches) goto found_catch;
+		}
+		/* Skip return_trace parameters */
+		ptr += 1;
 	    } else {
 		if (have_return_to_trace) {
 		    /* Record this cp as possible return_to trace cp */
@@ -5252,7 +5697,7 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
  */
 
 static void
-save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
+save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 		Eterm args) {
     struct StackTrace* s;
     int sz;
@@ -5263,7 +5708,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
     }
 
     /* Create a container for the exception data */
-    sz = (offsetof(struct StackTrace, trace) + sizeof(Eterm)*depth
+    sz = (offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth
           + sizeof(Eterm) - 1) / sizeof(Eterm);
     s = (struct StackTrace *) HAlloc(c_p, 1 + sz);
     /* The following fields are inside the bignum */
@@ -5298,7 +5743,6 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
 	    ASSERT(c_p->current);
 	    s->current = c_p->current;
 	    a = s->current[2];
-	    ASSERT(s->current[2] <= 3);
 	}
 	/* Save first stack entry */
 	ASSERT(pc);
@@ -5350,9 +5794,10 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
 
     /* Save the actual stack trace */
     if (depth > 0) {
-	Eterm *ptr, *prev = s->depth ? s->trace[s->depth-1] : NULL;
-	Uint i_return_trace = beam_return_trace[0];
-	Uint i_return_to_trace = beam_return_to_trace[0];
+	Eterm *ptr;
+	BeamInstr *prev = s->depth ? s->trace[s->depth-1] : NULL;
+	BeamInstr i_return_trace = beam_return_trace[0];
+	BeamInstr i_return_to_trace = beam_return_to_trace[0];
 	/*
 	 * Traverse the stack backwards and add all unique continuation
 	 * pointers to the buffer, up to the maximum stack trace size.
@@ -5365,7 +5810,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
 				   *cp_val(*ptr) != i_return_to_trace))
 	    && c_p->cp) {
 	    /* Can not follow cp here - code may be unloaded */
-	    Uint *cpp = cp_val((Eterm) c_p->cp);
+	    BeamInstr *cpp = c_p->cp;
 	    if (cpp == beam_exception_trace || cpp == beam_return_trace) {
 		/* Skip return_trace parameters */
 		ptr += 2;
@@ -5385,7 +5830,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
 		    /* Skip stack frame variables */
 		    do ++ptr; while (is_not_CP(*ptr));
 		} else {
-		    Eterm *cp = (Eterm *)(*ptr);
+		    BeamInstr *cp = cp_val(*ptr);
 		    if (cp != prev) {
 			/* Record non-duplicates only */
 			prev = cp;
@@ -5457,9 +5902,9 @@ build_stacktrace(Process* c_p, Eterm exc) {
     struct StackTrace* s;
     Eterm  args;
     int    depth;
-    Eterm* current;
+    BeamInstr* current;
     Eterm  Where = NIL;
-    Eterm* next_p = &Where;
+    Eterm *next_p = &Where;
 
     if (! (s = get_trace_from_exc(exc))) {
         return NIL;
@@ -5523,7 +5968,7 @@ build_stacktrace(Process* c_p, Eterm exc) {
 	 * Finally, we go through the saved continuation pointers.
 	 */
 	for (i = 0; i < depth; i++) {
-	    Eterm *fi = find_function_from_pc((Eterm *) s->trace[i]);
+	    BeamInstr *fi = find_function_from_pc((BeamInstr *) s->trace[i]);
 	    if (fi == NULL) continue;
 	    mfa = TUPLE3(hp, fi[0], fi[1], make_small(fi[2]));
 	    hp += 4;
@@ -5539,8 +5984,8 @@ build_stacktrace(Process* c_p, Eterm exc) {
 }
 
 
-static Eterm
-call_error_handler(Process* p, Eterm* fi, Eterm* reg)
+static BeamInstr*
+call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
 {
     Eterm* hp;
     Export* ep;
@@ -5552,62 +5997,12 @@ call_error_handler(Process* p, Eterm* fi, Eterm* reg)
     /*
      * Search for the error_handler module.
      */
-    ep = erts_find_function(erts_proc_get_error_handler(p),
-			    am_undefined_function, 3);
-    if (ep == NULL) {		/* No error handler */
-	p->current = fi;
-	p->freason = EXC_UNDEF;
-	return 0;
-    }
-    p->i = ep->address;
-
-    /*
-     * Create a list with all arguments in the x registers.
-     */
-
-    arity = fi[2];
-    sz = 2 * arity;
-    if (HeapWordsLeft(p) < sz) {
-	erts_garbage_collect(p, sz, reg, arity);
-    }
-    hp = HEAP_TOP(p);
-    HEAP_TOP(p) += sz;
-    args = NIL;
-    for (i = arity-1; i >= 0; i--) {
-	args = CONS(hp, reg[i], args);
-	hp += 2;
-    }
-
-    /*
-     * Set up registers for call to error_handler:undefined_function/3.
-     */
-    reg[0] = fi[0];
-    reg[1] = fi[1];
-    reg[2] = args;
-    return 1;
-}
-
-static Eterm
-call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg)
-{
-    Eterm* hp;
-    Export* ep;
-    int arity;
-    Eterm args;
-    Uint sz;
-    int i;
-
-    /*
-     * Search for error handler module.
-     */
-    ep = erts_find_function(erts_proc_get_error_handler(p),
-			    am_breakpoint, 3);
+    ep = erts_find_function(erts_proc_get_error_handler(p), func, 3);
     if (ep == NULL) {		/* No error handler */
 	p->current = fi;
 	p->freason = EXC_UNDEF;
 	return 0;
     }
-    p->i = ep->address;
 
     /*
      * Create a list with all arguments in the x registers.
@@ -5627,15 +6022,14 @@ call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg)
     }
 
     /*
-     * Set up registers for call to error_handler:breakpoint/3.
+     * Set up registers for call to error_handler:<func>/3.
      */
     reg[0] = fi[0];
     reg[1] = fi[1];
     reg[2] = args;
-    return 1;
+    return ep->address;
 }
 
-
 
 static Export*
 apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg)
@@ -5681,7 +6075,7 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity,
     return ep;
 }
 
-static Uint*
+static BeamInstr*
 apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
 {
     int arity;
@@ -5763,7 +6157,7 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
     return ep->address;
 }
 
-static Uint*
+static BeamInstr*
 fixed_apply(Process* p, Eterm* reg, Uint arity)
 {
     Export* ep;
@@ -5812,8 +6206,8 @@ fixed_apply(Process* p, Eterm* reg, Uint arity)
     return ep->address;
 }
 
-static int
-hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
+int
+erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
 {
     int arity;
     Eterm tmp;
@@ -5869,7 +6263,7 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
     c_p->stop = STACK_START(c_p);
     c_p->catches = 0;
     c_p->i = beam_apply;
-    c_p->cp = (Eterm *) beam_apply+1;
+    c_p->cp = (BeamInstr *) beam_apply+1;
 
     /*
      * If there are no waiting messages, garbage collect and
@@ -5884,22 +6278,25 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
 	c_p->fvalue = NIL;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_garbage_collect_hibernate(c_p);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	c_p->status = P_WAITING;
 #ifdef ERTS_SMP
 	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
 	if (c_p->msg.len > 0)
 	    erts_add_to_runq(c_p);
+	else
 #endif
+	    c_p->status = P_WAITING;
     }
     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
     c_p->current = bif_export[BIF_hibernate_3]->code;
+    c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */
     return 1;
 }
 
-static Uint*
+static BeamInstr*
 call_fun(Process* p,		/* Current process. */
 	 int arity,		/* Number of arguments for Fun. */
 	 Eterm* reg,		/* Contents of registers. */
@@ -5919,7 +6316,7 @@ call_fun(Process* p,		/* Current process. */
     if (is_fun_header(hdr)) {
 	ErlFunThing* funp = (ErlFunThing *) fun_val(fun);
 	ErlFunEntry* fe;
-	Eterm* code_ptr;
+	BeamInstr* code_ptr;
 	Eterm* var_ptr;
 	int actual_arity;
 	unsigned num_free;
@@ -6010,12 +6407,17 @@ call_fun(Process* p,		/* Current process. */
 		reg[0] = module;
 		reg[1] = fun;
 		reg[2] = args;
+		reg[3] = NIL;
 		return ep->address;
 	    }
 	}
     } else if (is_export_header(hdr)) {
-	Export* ep = (Export *) (export_val(fun))[1];
-	int actual_arity = (int) ep->code[2];
+	Export *ep;
+	int actual_arity;
+
+	ep = *((Export **) (export_val(fun) + 1));
+	actual_arity = (int) ep->code[2];
+
 	if (arity == actual_arity) {
 	    return ep->address;
 	} else {
@@ -6082,7 +6484,7 @@ call_fun(Process* p,		/* Current process. */
     }
 }
 
-static Eterm*
+static BeamInstr*
 apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg)
 {
     int arity;
@@ -6125,6 +6527,7 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
     if (HEAP_LIMIT(p) - HEAP_TOP(p) <= needed) {
 	PROCESS_MAIN_CHK_LOCKS(p);
 	erts_garbage_collect(p, needed, reg, num_free);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
 	PROCESS_MAIN_CHK_LOCKS(p);
     }
     hp = p->htop;
@@ -6134,8 +6537,8 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
     erts_refc_inc(&fe->refc, 2);
     funp->thing_word = HEADER_FUN;
 #ifndef HYBRID /* FIND ME! */
-    funp->next = MSO(p).funs;
-    MSO(p).funs = funp;
+    funp->next = MSO(p).first;
+    MSO(p).first = (struct erl_off_heap_header*) funp;
 #endif
     funp->fe = fe;
     funp->num_free = num_free;
@@ -6176,7 +6579,7 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
     if ((ep = export_get(&e)) == NULL) {
 	return 0;
     }
-    return ep->address == ep->code+3 && (ep->code[3] == (Uint) em_apply_bif);
+    return ep->address == ep->code+3 && (ep->code[3] == (BeamInstr) em_apply_bif);
 }
 
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 99fab28dce..57fe25453d 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -51,9 +51,9 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 #define EXPORTED  2
 
 #ifdef NO_JUMP_TABLE
-#  define BeamOpCode(Op) ((Uint)(Op))
+#  define BeamOpCode(Op) ((BeamInstr)(Op))
 #else
-#  define BeamOpCode(Op) ((Eterm)beam_ops[Op])
+#  define BeamOpCode(Op) ((BeamInstr)beam_ops[Op])
 #endif
 
 #if defined(WORDS_BIGENDIAN)
@@ -89,13 +89,12 @@ typedef struct {
 } Label;
 
 /*
- * Type for a operand for a generic instruction.
+ * Type for an operand for a generic instruction.
  */
 
 typedef struct {
     unsigned type;		/* Type of operand. */
-    Uint val;			/* Value of operand. */
-    Uint bigarity;		/* Arity for bignumbers (only). */
+    BeamInstr val;		/* Value of operand. */
 } GenOpArg;
 
 /*
@@ -142,7 +141,7 @@ typedef struct {
 typedef struct {
     Eterm function;		/* Tagged atom for function. */
     int arity;			/* Arity. */
-    Eterm* address;		/* Address to function in code. */
+    BeamInstr* address;		/* Address to function in code. */
 } ExportEntry;
 
 #define MakeIffId(a, b, c, d) \
@@ -274,13 +273,12 @@ typedef struct {
     int num_functions;		/* Number of functions in module. */
     int num_labels;		/* Number of labels. */
     int code_buffer_size;	/* Size of code buffer in words.  */
-    Eterm* code;		/* Loaded code. */
+    BeamInstr* code;		/* Loaded code. */
     int ci;			/* Current index into loaded code. */
     Label* labels;
-    Uint put_strings;		/* Linked list of put_string instructions. */
-    Uint new_bs_put_strings;	/* Linked list of i_new_bs_put_string instructions. */
+    BeamInstr new_bs_put_strings;	/* Linked list of i_new_bs_put_string instructions. */
     StringPatch* string_patches; /* Linked list of position into string table to patch. */
-    Uint catches;		/* Linked list of catch_yf instructions. */
+    BeamInstr catches;		/* Linked list of catch_yf instructions. */
     unsigned loaded_size;	/* Final size of code when loaded. */
     byte mod_md5[16];		/* MD5 for module code. */
     int may_load_nif;           /* true if NIFs may later be loaded for this module */  
@@ -327,11 +325,6 @@ typedef struct {
     Literal* literals;		/* Array of literals. */
     LiteralPatch* literal_patches; /* Operands that need to be patched. */
     Uint total_literal_size;	/* Total heap size for all literals. */
-
-    /*
-     * Floating point.
-     */
-    int new_float_instructions;	/* New allocation scheme for floating point. */
 } LoaderState;
 
 typedef struct {
@@ -341,7 +334,7 @@ typedef struct {
 
 #define GetTagAndValue(Stp, Tag, Val) \
    do { \
-      Uint __w; \
+      BeamInstr __w; \
       GetByte(Stp, __w); \
       Tag = __w & 0x07; \
       if ((__w & 0x08) == 0) { \
@@ -388,7 +381,7 @@ typedef struct {
        goto load_error; \
     } else { \
        int __n = (N); \
-       Uint __result = 0; \
+       BeamInstr __result = 0; \
        Stp->file_left -= (unsigned) __n; \
        while (__n-- > 0) { \
           __result = __result << 8 | *Stp->file_p++; \
@@ -465,7 +458,7 @@ static int bin_load(Process *c_p, ErtsProcLocks c_p_locks,
 static void init_state(LoaderState* stp);
 static int insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 			   Eterm group_leader, Eterm module,
-			   Eterm* code, Uint size, Uint catches);
+			   BeamInstr* code, Uint size, BeamInstr catches);
 static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
 			 Uint num_types, Uint num_mandatory);
 static int load_atom_table(LoaderState* stp);
@@ -477,19 +470,16 @@ static int read_code_header(LoaderState* stp);
 static int load_code(LoaderState* stp);
 static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
 			  GenOpArg Tuple, GenOpArg Dst);
-static GenOp* gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
+static GenOp* gen_split_values(LoaderState* stp, GenOpArg S,
+			       GenOpArg TypeFail, GenOpArg Fail,
 			       GenOpArg Size, GenOpArg* Rest);
 static GenOp* gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 			     GenOpArg Size, GenOpArg* Rest);
-static GenOp* gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail,
-			     GenOpArg Size, GenOpArg* Rest);
+static GenOp* gen_select_literals(LoaderState* stp, GenOpArg S,
+				  GenOpArg Fail, GenOpArg Size,
+				  GenOpArg* Rest);
 static GenOp* const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 			       GenOpArg Size, GenOpArg* Rest);
-static GenOp* gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg Func,
-			    GenOpArg arity, GenOpArg label);
-static GenOp*
-gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
-	      GenOpArg Src, GenOpArg Dst);
 
 static int freeze_code(LoaderState* stp);
 
@@ -499,8 +489,8 @@ static void load_printf(int line, LoaderState* context, char *fmt, ...);
 static int transform_engine(LoaderState* st);
 static void id_to_string(Uint id, char* s);
 static void new_genop(LoaderState* stp);
-static int get_int_val(LoaderState* stp, Uint len_code, Uint* result);
-static int get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result);
+static int get_int_val(LoaderState* stp, Uint len_code, BeamInstr* result);
+static int get_erlang_integer(LoaderState* stp, Uint len_code, BeamInstr* result);
 static int new_label(LoaderState* stp);
 static void new_literal_patch(LoaderState* stp, int pos);
 static void new_string_patch(LoaderState* stp, int pos);
@@ -513,7 +503,7 @@ static Eterm compilation_info_for_module(Process* p, Eterm mod);
 static Eterm native_addresses(Process* p, Eterm mod);
 int patch_funentries(Eterm Patchlist);
 int patch(Eterm Addresses, Uint fe);
-static int safe_mul(Uint a, Uint b, Uint* resp);
+static int safe_mul(UWord a, UWord b, UWord* resp);
 
 
 static int must_swap_floats;
@@ -591,7 +581,18 @@ erts_load_module(Process *c_p,
     }
     return result;
 }
-
+/* #define LOAD_MEMORY_HARD_DEBUG 1*/
+
+#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG)
+/* Requires allocators ERTS_ALLOC_UTIL_HARD_DEBUG also set in erl_alloc_util.h */
+extern void check_allocators(void);
+extern void check_allocated_block(Uint type, void *blk);
+#define CHKALLOC() check_allocators()
+#define CHKBLK(TYPE,BLK) if ((BLK) != NULL) check_allocated_block((TYPE),(BLK))
+#else
+#define CHKALLOC() /* nothing */
+#define CHKBLK(TYPE,BLK) /* nothing */
+#endif
 
 static int
 bin_load(Process *c_p, ErtsProcLocks c_p_locks,
@@ -608,6 +609,12 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Scan the IFF file.
      */
 
+#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG)
+    erts_fprintf(stderr,"Loading a module\n");
+#endif
+
+    CHKALLOC();
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     state.file_name = "IFF header for Beam file";
     state.file_p = bytes;
     state.file_left = unloaded_size;
@@ -619,6 +626,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the header for the code chunk.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     define_file(&state, "code chunk header", CODE_CHUNK);
     if (!read_code_header(&state)) {
 	goto load_error;
@@ -628,6 +636,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the atom table.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     define_file(&state, "atom table", ATOM_CHUNK);
     if (!load_atom_table(&state)) {
 	goto load_error;
@@ -637,6 +646,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the import table.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     define_file(&state, "import table", IMP_CHUNK);
     if (!load_import_table(&state)) {
 	goto load_error;
@@ -646,6 +656,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the lambda (fun) table.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     if (state.chunks[LAMBDA_CHUNK].size > 0) {
 	define_file(&state, "lambda (fun) table", LAMBDA_CHUNK);
 	if (!read_lambda_table(&state)) {
@@ -657,6 +668,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the literal table.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     if (state.chunks[LITERAL_CHUNK].size > 0) {
 	define_file(&state, "literals table (constant pool)", LITERAL_CHUNK);
 	if (!read_literal_table(&state)) {
@@ -668,18 +680,25 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Load the code chunk.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     state.file_name = "code chunk";
     state.file_p = state.code_start;
     state.file_left = state.code_size;
-    if (!load_code(&state) || !freeze_code(&state)) {
+    if (!load_code(&state)) {
 	goto load_error;
     }
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
+    if (!freeze_code(&state)) {
+	goto load_error;
+    }
+
 
     /*
      * Read and validate the export table.  (This must be done after
      * loading the code, because it contains labels.)
      */
     
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     define_file(&state, "export table", EXP_CHUNK);
     if (!read_export_table(&state)) {
 	goto load_error;
@@ -690,16 +709,25 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * exported and imported functions.  This can't fail.
      */
     
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     rval = insert_new_code(c_p, c_p_locks, state.group_leader, state.module,
 			   state.code, state.loaded_size, state.catches);
     if (rval < 0) {
 	goto load_error;
     }
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     final_touch(&state);
 
     /*
      * Loading succeded.
      */
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
+#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG)
+    erts_fprintf(stderr,"Loaded %T\n",*modp);
+#if 0
+    debug_dump_code(state.code,state.ci);
+#endif
+#endif
     rval = 0;
     state.code = NULL;		/* Prevent code from being freed. */
     *modp = state.module;
@@ -784,14 +812,13 @@ init_state(LoaderState* stp)
     stp->total_literal_size = 0;
     stp->literal_patches = 0;
     stp->string_patches = 0;
-    stp->new_float_instructions = 0;
     stp->may_load_nif = 0;
     stp->on_load = 0;
 }
 
 static int
 insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
-		Eterm group_leader, Eterm module, Eterm* code, Uint size, Uint catches)
+		Eterm group_leader, Eterm module, BeamInstr* code, Uint size, BeamInstr catches)
 {
     Module* modp;
     int rval;
@@ -833,7 +860,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 	modules[i] = modules[i-1];
     }
     modules[i].start = code;
-    modules[i].end = (Eterm *) (((byte *)code) + size);
+    modules[i].end = (BeamInstr *) (((byte *)code) + size);
     num_loaded_modules++;
     mid_module = &modules[num_loaded_modules/2];
     return 0;
@@ -1083,7 +1110,7 @@ load_import_table(LoaderState* stp)
 	 * the BIF function.
 	 */
 	if ((e = erts_find_export_entry(mod, func, arity)) != NULL) {
-	    if (e->code[3] == (Uint) em_apply_bif) {
+	    if (e->code[3] == (BeamInstr) em_apply_bif) {
 		stp->import[i].bf = (BifFunction) e->code[4];
 		if (func == am_load_nif && mod == am_erlang && arity == 2) {
 		    stp->may_load_nif = 1;
@@ -1151,7 +1178,7 @@ read_export_table(LoaderState* stp)
 	 * redefine).
 	 */
 	if ((e = erts_find_export_entry(stp->module, func, arity)) != NULL) {
-	    if (e->code[3] == (Uint) em_apply_bif) {
+	    if (e->code[3] == (BeamInstr) em_apply_bif) {
 		int j;
 
 		for (j = 0; j < sizeof(allow_redef)/sizeof(allow_redef[0]); j++) {
@@ -1220,7 +1247,7 @@ static int
 read_literal_table(LoaderState* stp)
 {
     int i;
-    Uint uncompressed_sz;
+    BeamInstr uncompressed_sz;
     byte* uncompressed = 0;
 
     GetInt(stp, 4, uncompressed_sz);
@@ -1338,19 +1365,20 @@ read_code_header(LoaderState* stp)
      * Initialize code area.
      */
     stp->code_buffer_size = erts_next_heap_size(2048 + stp->num_functions, 0);
-    stp->code = (Eterm*) erts_alloc(ERTS_ALC_T_CODE,
-				    sizeof(Eterm) * stp->code_buffer_size);
+    stp->code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE,
+				    sizeof(BeamInstr) * stp->code_buffer_size);
 
     stp->code[MI_NUM_FUNCTIONS] = stp->num_functions;
     stp->ci = MI_FUNCTIONS + stp->num_functions + 1;
 
     stp->code[MI_ATTR_PTR] = 0;
+    stp->code[MI_ATTR_SIZE] = 0;
     stp->code[MI_ATTR_SIZE_ON_HEAP] = 0;
     stp->code[MI_COMPILE_PTR] = 0;
+    stp->code[MI_COMPILE_SIZE] = 0;
     stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0;
     stp->code[MI_NUM_BREAKPOINTS] = 0;
 
-    stp->put_strings = 0;
     stp->new_bs_put_strings = 0;
     stp->catches = 0;
     return 1;
@@ -1365,29 +1393,30 @@ read_code_header(LoaderState* stp)
        LoadError2(Stp, "bad tag %d; expected %d", Actual, Expected); \
     } else {}
 
-#define Need(w) \
-    ASSERT(ci <= code_buffer_size); \
-    if (code_buffer_size < ci+(w)) { \
-        code_buffer_size = erts_next_heap_size(ci+(w), 0); \
-	stp->code = code \
-	    = (Eterm *) erts_realloc(ERTS_ALC_T_CODE, \
-				     (void *) code, \
-				     code_buffer_size * sizeof(Eterm)); \
-    }
+#define CodeNeed(w) do {						\
+    ASSERT(ci <= code_buffer_size);					\
+    if (code_buffer_size < ci+(w)) {					\
+        code_buffer_size = erts_next_heap_size(ci+(w), 0);		\
+	stp->code = code						\
+	    = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE,			\
+				     (void *) code,			\
+				     code_buffer_size * sizeof(BeamInstr));	\
+    } 									\
+} while (0)
     
+#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
 
 static int
 load_code(LoaderState* stp)
 {
     int i;
-    int tmp;
     int ci;
     int last_func_start = 0;
     char* sign;
     int arg;			/* Number of current argument. */
     int num_specific;		/* Number of specific ops for current. */
-    Eterm* code;
+    BeamInstr* code;
     int code_buffer_size;
     int specific;
     Uint last_label = 0;	/* Number of last label. */
@@ -1446,7 +1475,7 @@ load_code(LoaderState* stp)
       if (((First) & 0x08) == 0) { \
 	 Val = (First) >> 4; \
       } else if (((First) & 0x10) == 0) { \
-         Uint __w; \
+         BeamInstr __w; \
 	 GetByte(Stp, __w); \
 	 Val = (((First) >> 5) << 8) | __w; \
       } else { \
@@ -1455,7 +1484,7 @@ load_code(LoaderState* stp)
    } while (0)
 
 	for (arg = 0; arg < arity; arg++) {
-	    Uint first;
+	    BeamInstr first;
 
 	    GetByte(stp, first);
 	    last_op->a[arg].type = first & 0x07;
@@ -1464,7 +1493,7 @@ load_code(LoaderState* stp)
 		if ((first & 0x08) == 0) {
 		    last_op->a[arg].val = first >> 4;
 		} else if ((first & 0x10) == 0) {
-		    Uint w;
+		    BeamInstr w;
 		    GetByte(stp, w);
 		    ASSERT(first < 0x800);
 		    last_op->a[arg].val = ((first >> 5) << 8) | w;
@@ -1523,7 +1552,7 @@ load_code(LoaderState* stp)
 		break;
 	    case TAG_z:
 		{
-		    Uint ext_tag;
+		    BeamInstr ext_tag;
 		    unsigned tag;
 
 		    GetValue(stp, first, ext_tag);
@@ -1531,14 +1560,15 @@ load_code(LoaderState* stp)
 		    case 0:	/* Floating point number */
 			{
 			    Eterm* hp;
-# ifndef ARCH_64
+/* XXX:PaN - Halfword should use ARCH_64 variant instead */
+#if !defined(ARCH_64) || HALFWORD_HEAP
 			    Uint high, low;
 # endif
 			    last_op->a[arg].val = new_literal(stp, &hp,
 							      FLOAT_SIZE_OBJECT);
 			    hp[0] = HEADER_FLONUM;
 			    last_op->a[arg].type = TAG_q;
-# ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 			    GetInt(stp, 8, hp[1]);
 # else
 			    GetInt(stp, 4, high);
@@ -1575,12 +1605,11 @@ load_code(LoaderState* stp)
 			break;
 		    case 3: 	/* Allocation list. */
 			{
-			    Uint n;
-			    Uint type;
-			    Uint val;
-			    Uint words = 0;
+			    BeamInstr n;
+			    BeamInstr type;
+			    BeamInstr val;
+			    BeamInstr words = 0;
 			    
-			    stp->new_float_instructions = 1;
 			    GetTagAndValue(stp, tag, n);
 			    VerifyTag(stp, tag, TAG_u);
 			    while (n-- > 0) {
@@ -1607,7 +1636,7 @@ load_code(LoaderState* stp)
 			}
 		    case 4:	/* Literal. */
 			{
-			    Uint val;
+			    BeamInstr val;
 
 			    GetTagAndValue(stp, tag, val);
 			    VerifyTag(stp, tag, TAG_u);
@@ -1734,7 +1763,7 @@ load_code(LoaderState* stp)
 	    }
 
 	    stp->specific_op = specific;
-	    Need(opc[stp->specific_op].sz+2); /* Extra margin for packing */
+	    CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */
 	    code[ci++] = BeamOpCode(stp->specific_op);
 	}
 	
@@ -1772,7 +1801,7 @@ load_code(LoaderState* stp)
 	    case 'c':		/* Tagged constant */
 		switch (tag) {
 		case TAG_i:
-		    code[ci++] = make_small(tmp_op->a[arg].val);
+		    code[ci++] = (BeamInstr) make_small((Uint) tmp_op->a[arg].val);
 		    break;
 		case TAG_a:
 		    code[ci++] = tmp_op->a[arg].val;
@@ -1802,7 +1831,7 @@ load_code(LoaderState* stp)
 		    code[ci++] = make_yreg(tmp_op->a[arg].val);
 		    break;
 		case TAG_i:
-		    code[ci++] = make_small(tmp_op->a[arg].val);
+		    code[ci++] = (BeamInstr) make_small((Uint)tmp_op->a[arg].val);
 		    break;
 		case TAG_a:
 		    code[ci++] = tmp_op->a[arg].val;
@@ -1896,12 +1925,12 @@ load_code(LoaderState* stp)
 		if (stp->import[i].bf == NULL) {
 		    LoadError1(stp, "not a BIF: import table index %d", i);
 		}
-		code[ci++] = (Eterm) stp->import[i].bf;
+		code[ci++] = (BeamInstr) stp->import[i].bf;
 		break;
-	    case 'P':		/* Byte offset into tuple */
+	    case 'P':		/* Byte offset into tuple or stack */
+	    case 'Q':		/* Like 'P', but packable */
 		VerifyTag(stp, tag, TAG_u);
-		tmp = tmp_op->a[arg].val;
-		code[ci++] = (Eterm) ((tmp_op->a[arg].val+1) * sizeof(Eterm *));
+		code[ci++] = (BeamInstr) ((tmp_op->a[arg].val+1) * sizeof(Eterm));
 		break;
 	    case 'l':		/* Floating point register. */
 		VerifyTag(stp, tag_to_letter[tag], *sign);
@@ -1919,75 +1948,14 @@ load_code(LoaderState* stp)
 	}
 
 	/*
-	 * Load any list arguments using the primitive tags.
-	 */
-
-	for ( ; arg < tmp_op->arity; arg++) {
-	    switch (tmp_op->a[arg].type) {
-	    case TAG_i:
-		Need(1);
-		code[ci++] = make_small(tmp_op->a[arg].val);
-		break;
-	    case TAG_u:
-	    case TAG_a:
-	    case TAG_v:
-		Need(1);
-		code[ci++] = tmp_op->a[arg].val;
-		break;
-	    case TAG_f:
-		Need(1);
-		code[ci] = stp->labels[tmp_op->a[arg].val].patches;
-		stp->labels[tmp_op->a[arg].val].patches = ci;
-		ci++;
-		break;
-	    case TAG_q:
-		{
-		    Eterm lit;
-
-		    lit = stp->literals[tmp_op->a[arg].val].term;
-		    if (is_big(lit)) {
-			Eterm* bigp;
-			Uint size;
-
-			bigp = big_val(lit);
-			size = bignum_header_arity(*bigp);
-			Need(size+1);
-			code[ci++] = *bigp++;
-			while (size-- > 0) {
-			    code[ci++] = *bigp++;
-			}
-		    } else if (is_float(lit)) {
-#ifdef ARCH_64
-			Need(1);
-			code[ci++] = float_val(stp->literals[tmp_op->a[arg].val].term)[1];
-#else
-			Eterm* fptr;
-
-			fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1;
-			Need(2);
-			code[ci++] = *fptr++;
-			code[ci++] = *fptr;
-#endif
-		    } else {
-			LoadError0(stp, "literal is neither float nor big");
-		    }
-		}
-		break;
-	    default:
-		LoadError1(stp, "unsupported primitive type '%c'",
-			   tag_to_letter[tmp_op->a[arg].type]);
-	    }
-	}
-
-	/*
 	 * The packing engine.
 	 */
 	if (opc[stp->specific_op].pack[0]) {
 	    char* prog;		/* Program for packing engine. */
-	    Uint stack[8];	/* Stack. */
-	    Uint* sp = stack;	/* Points to next free position. */
-	    Uint packed = 0;	/* Accumulator for packed operations. */
-	    
+	    BeamInstr stack[8];	/* Stack. */
+	    BeamInstr* sp = stack;	/* Points to next free position. */
+	    BeamInstr packed = 0;	/* Accumulator for packed operations. */
+
 	    for (prog = opc[stp->specific_op].pack; *prog; prog++) {
 		switch (*prog) {
 		case 'g':	/* Get instruction; push on stack. */
@@ -2000,8 +1968,13 @@ load_code(LoaderState* stp)
 		    packed = (packed << BEAM_TIGHT_SHIFT) | code[--ci];
 		    break;
 		case '6':	/* Shift 16 steps */
-		    packed = (packed << 16) | code[--ci];
+		    packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci];
+		    break;
+#ifdef ARCH_64
+		case 'w':	/* Shift 32 steps */
+		    packed = (packed << BEAM_WIDE_SHIFT) | code[--ci];
 		    break;
+#endif
 		case 'p':	/* Put instruction (from stack). */
 		    code[ci++] = *--sp;
 		    break;
@@ -2017,6 +1990,58 @@ load_code(LoaderState* stp)
 	}
 
 	/*
+	 * Load any list arguments using the primitive tags.
+	 */
+
+	for ( ; arg < tmp_op->arity; arg++) {
+	    switch (tmp_op->a[arg].type) {
+	    case TAG_i:
+		CodeNeed(1);
+		code[ci++] = make_small(tmp_op->a[arg].val);
+		break;
+	    case TAG_u:
+	    case TAG_a:
+	    case TAG_v:
+		CodeNeed(1);
+		code[ci++] = tmp_op->a[arg].val;
+		break;
+	    case TAG_f:
+		CodeNeed(1);
+		code[ci] = stp->labels[tmp_op->a[arg].val].patches;
+		stp->labels[tmp_op->a[arg].val].patches = ci;
+		ci++;
+		break;
+	    case TAG_r:
+		CodeNeed(1);
+		code[ci++] = (R_REG_DEF << _TAG_PRIMARY_SIZE) |
+		    TAG_PRIMARY_HEADER;
+		break;
+	    case TAG_x:
+		CodeNeed(1);
+		code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) |
+		    (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER;
+		break;
+	    case TAG_y:
+		CodeNeed(1);
+		code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) |
+		    (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER;
+		break;
+	    case TAG_n:
+		CodeNeed(1);
+		code[ci++] = NIL;
+		break;
+	    case TAG_q:
+		CodeNeed(1);
+		new_literal_patch(stp, ci);
+		code[ci++] = tmp_op->a[arg].val;
+		break;
+	    default:
+		LoadError1(stp, "unsupported primitive type '%c'",
+			   tag_to_letter[tmp_op->a[arg].type]);
+	    }
+	}
+
+	/*
 	 * Handle a few special cases.
 	 */
 	switch (stp->specific_op) {
@@ -2037,9 +2062,9 @@ load_code(LoaderState* stp)
 			/* Must make room for call_nif op */
 			int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start);
 			ASSERT(pad > 0 && pad < MIN_FUNC_SZ);
-			Need(pad);
-			sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(Eterm));
-			sys_memset(&code[finfo_ix], 0, pad*sizeof(Eterm));
+			CodeNeed(pad);
+			sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(BeamInstr));
+			sys_memset(&code[finfo_ix], 0, pad*sizeof(BeamInstr));
 			ci += pad;
 			stp->labels[last_label].value += pad;
 		    }
@@ -2050,6 +2075,7 @@ load_code(LoaderState* stp)
 		 */
 		stp->function = code[ci-2];
 		stp->arity = code[ci-1];
+
 		ASSERT(stp->labels[last_label].value == ci - FINFO_SZ);
 		offset = MI_FUNCTIONS + function_number;
 		code[offset] = stp->labels[last_label].patches;
@@ -2072,34 +2098,6 @@ load_code(LoaderState* stp)
 	    /* Remember offset for the on_load function. */
 	    stp->on_load = ci;
 	    break;
-	case op_put_string_IId:
-	    {
-		/*
-		 * At entry:
-		 *
-		 * code[ci-4]	&&lb_put_string_IId
-		 * code[ci-3]	length of string
-		 * code[ci-2]   offset into string table
-		 * code[ci-1]   destination register
-		 *
-		 * Since we don't know the address of the string table yet,
-		 * just check the offset and length for validity, and use
-		 * the instruction field as a link field to link all put_string
-		 * instructions into a single linked list.  At exit:
-		 *
-		 * code[ci-4]	pointer to next put_string instruction (or 0
-		 *		if this is the last)
-		 */
-		Uint offset = code[ci-2];
-		Uint len = code[ci-3];
-		unsigned strtab_size = stp->chunks[STR_CHUNK].size;
-		if (offset > strtab_size || offset + len > strtab_size) {
-		    LoadError2(stp, "invalid string reference %d, size %d", offset, len);
-		}
-		code[ci-4] = stp->put_strings;
-		stp->put_strings = ci - 4;
-	    }
-	    break;
 	case op_bs_put_string_II:
 	    {
 		/*
@@ -2161,7 +2159,6 @@ load_code(LoaderState* stp)
 	}
     }
     
-#undef Need
 
  load_error:
     return 0;
@@ -2212,11 +2209,12 @@ use_jump_tab(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
 }
 
 /*
- * Predicate to test whether all values in a table are big numbers.
+ * Predicate to test whether all values in a table are either
+ * floats or bignums.
  */
 
 static int
-all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
+floats_or_bignums(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
 {
     int i;
 
@@ -2228,9 +2226,6 @@ all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
 	if (Rest[i].type != TAG_q) {
 	    return 0;
 	}
-	if (is_not_big(stp->literals[Rest[i].val].term)) {
-	    return 0;
-	}
 	if (Rest[i+1].type != TAG_f) {
 	    return 0;
 	}
@@ -2290,6 +2285,14 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
     return 0;
 }
 
+static int
+same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label)
+{
+    return Target.type = TAG_f && Label.type == TAG_u &&
+	Target.val == Label.val;
+}
+
+
 /*
  * Generate an instruction for element/2.
  */
@@ -2301,23 +2304,23 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
     GenOp* op;
 
     NEW_GENOP(stp, op);
-    op->op = genop_i_element_4;
     op->arity = 4;
-    op->a[0] = Fail;
-    op->a[1] = Index;
-    op->a[2] = Tuple;
-    op->a[3] = Dst;
     op->next = NULL;
 
-    /*
-     * If safe, generate a faster instruction.
-     */
-
     if (Index.type == TAG_i && Index.val > 0 &&
 	(Tuple.type == TAG_r || Tuple.type == TAG_x || Tuple.type == TAG_y)) {
 	op->op = genop_i_fast_element_4;
-	op->a[1].type = TAG_u;
-	op->a[1].val = Index.val;
+	op->a[0] = Tuple;
+	op->a[1] = Fail;
+	op->a[2].type = TAG_u;
+	op->a[2].val = Index.val;
+	op->a[3] = Dst;
+    } else {
+	op->op = genop_i_element_4;
+	op->a[0] = Tuple;
+	op->a[1] = Fail;
+	op->a[2] = Index;
+	op->a[3] = Dst;
     }
 
     return op;
@@ -2373,7 +2376,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
 		 GenOpArg Flags, GenOpArg Dst)
 {
     GenOp* op;
-    Uint bits;
+    UWord bits;
 
     NEW_GENOP(stp, op);
 
@@ -2568,8 +2571,6 @@ binary_too_big_bits(LoaderState* stp, GenOpArg Size)
     return Size.type == TAG_u && (((Size.val+7)/8) >> (8*sizeof(Uint)-3) != 0);
 }
 
-#define new_float_allocation(Stp) ((Stp)->new_float_instructions)
-
 static GenOp*
 gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size,
 	       GenOpArg Unit, GenOpArg Flags, GenOpArg Src)
@@ -2782,6 +2783,52 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms,
     return op;
 }
 
+static GenOp*
+gen_increment(LoaderState* stp, GenOpArg Reg, GenOpArg Integer,
+	      GenOpArg Live, GenOpArg Dst)
+{
+    GenOp* op;
+
+    NEW_GENOP(stp, op);
+    op->op = genop_i_increment_4;
+    op->arity = 4;
+    op->next = NULL;
+    op->a[0] = Reg;
+    op->a[1].type = TAG_u;
+    op->a[1].val = Integer.val;
+    op->a[2] = Live;
+    op->a[3] = Dst;
+    return op;
+}
+
+static GenOp*
+gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer,
+			 GenOpArg Live, GenOpArg Dst)
+{
+    GenOp* op;
+
+    NEW_GENOP(stp, op);
+    op->op = genop_i_increment_4;
+    op->arity = 4;
+    op->next = NULL;
+    op->a[0] = Reg;
+    op->a[1].type = TAG_u;
+    op->a[1].val = -Integer.val;
+    op->a[2] = Live;
+    op->a[3] = Dst;
+    return op;
+}
+
+/*
+ * Test whether the negation of the given number is small.
+ */
+static int
+negation_is_small(LoaderState* stp, GenOpArg Int)
+{
+    return Int.type == TAG_i && IS_SSMALL(-Int.val);
+}
+
+
 static int
 smp(LoaderState* stp)
 {
@@ -2838,14 +2885,14 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     op->a[1].type = TAG_u;
     
     if (Time.type == TAG_i && (timeout = Time.val) >= 0 &&
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	(timeout >> 32) == 0
 #else
 	1
 #endif
 	) {
 	op->a[1].val = timeout;
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
     } else if (Time.type == TAG_q) {
 	Eterm big;
 
@@ -2856,11 +2903,13 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	if (big_arity(big) > 1 || big_sign(big)) {
 	    goto error;
 	} else {
-	    (void) term_to_Uint(big, &op->a[1].val);
+	    Uint u;
+	    (void) term_to_Uint(big, &u);
+	    op->a[1].val = (BeamInstr) u;
 	}
 #endif
     } else {
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
     error:
 #endif
 	op->op = genop_i_wait_error_0;
@@ -2883,14 +2932,14 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     op->a[1].type = TAG_u;
     
     if (Time.type == TAG_i && (timeout = Time.val) >= 0 &&
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	(timeout >> 32) == 0
 #else
 	1
 #endif
 	) {
 	op->a[1].val = timeout;
-#ifndef ARCH_64
+#if !defined(ARCH_64) || HALFWORD_HEAP
     } else if (Time.type == TAG_q) {
 	Eterm big;
 
@@ -2901,11 +2950,13 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	if (big_arity(big) > 1 || big_sign(big)) {
 	    goto error;
 	} else {
-	    (void) term_to_Uint(big, &op->a[1].val);
+	    Uint u;
+	    (void) term_to_Uint(big, &u);
+	    op->a[1].val = (BeamInstr) u;
 	}
 #endif
     } else {
-#ifndef ARCH_64
+#if !defined(ARCH_64) || HALFWORD_HEAP
     error:
 #endif
 	op->op = genop_i_wait_error_locked_0;
@@ -2969,6 +3020,21 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 	ASSERT(op->a[i].val < op->a[i+2].val);
     }
 #endif
+
+    /*
+     * Use a special-cased instruction if there are only two values.
+     */
+    if (size == 2) {
+	op->op = genop_i_select_tuple_arity2_6;
+	op->arity--;
+	op->a[2].type = TAG_u;
+	op->a[2].val = arityval(op->a[3].val);
+	op->a[3] = op->a[4];
+	op->a[4].type = TAG_u;
+	op->a[4].val = arityval(op->a[5].val);
+	op->a[5] = op->a[6];
+    }
+
     return op;
 }
 
@@ -2978,18 +3044,24 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
  */
 
 static GenOp*
-gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
-		 GenOpArg Size, GenOpArg* Rest)
+gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg TypeFail,
+		 GenOpArg Fail, GenOpArg Size, GenOpArg* Rest)
 
 {
     GenOp* op1;
     GenOp* op2;
     GenOp* label;
-    Uint type;
+    GenOp* is_integer;
     int i;
 
     ASSERT(Size.val >= 2 && Size.val % 2 == 0);
 
+    NEW_GENOP(stp, is_integer);
+    is_integer->op = genop_is_integer_2;
+    is_integer->arity = 2;
+    is_integer->a[0] = TypeFail;
+    is_integer->a[1] = S;
+
     NEW_GENOP(stp, label);
     label->op = genop_label_1;
     label->arity = 1;
@@ -3015,15 +3087,13 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     op2->a[2].type = TAG_u;
     op2->a[2].val = 0;
 
-    op1->next = label;
-    label->next = op2;
-    op2->next = NULL;
-
-    type = Rest[0].type;
+    /*
+     * Split the list.
+     */
 
     ASSERT(Size.type == TAG_u);
     for (i = 0; i < Size.val; i += 2) {
-	GenOp* op = (Rest[i].type == type) ? op1 : op2;
+	GenOp* op = (Rest[i].type == TAG_q) ? op2 : op1;
 	int dst = 3 + op->a[2].val;
 
 	ASSERT(Rest[i+1].type == TAG_f);
@@ -3032,13 +3102,36 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 	op->arity += 2;
 	op->a[2].val += 2;
     }
+    ASSERT(op1->a[2].val > 0);
+    ASSERT(op2->a[2].val > 0);
 
     /*
-     * None of the instructions should have zero elements in the list.
+     * Order the instruction sequence appropriately.
      */
 
-    ASSERT(op1->a[2].val > 0);
-    ASSERT(op2->a[2].val > 0);
+    if (TypeFail.val == Fail.val) {
+	/*
+	 * select_val L1 S ... (small numbers)
+	 * label L1
+	 * is_integer Fail S
+	 * select_val Fail S ... (bignums)
+	 */
+	op1->next = label;
+	label->next = is_integer;
+	is_integer->next = op2;
+    } else {
+	/*
+	 * is_integer TypeFail S
+	 * select_val L1 S ... (small numbers)
+	 * label L1
+	 * select_val Fail S ... (bignums)
+	 */
+	is_integer->next = op1;
+	op1->next = label;
+	label->next = op2;
+	op1 = is_integer;
+    }
+    op2->next = NULL;
 
     return op1;
 }
@@ -3060,6 +3153,29 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr
     ASSERT(Size.val >= 2 && Size.val % 2 == 0);
 
     /*
+     * If there is only one choice, don't generate a jump table.
+     */
+    if (Size.val == 2) {
+	GenOp* jump;
+
+	NEW_GENOP(stp, op);
+	op->arity = 3;
+	op->op = genop_is_ne_exact_3;
+	op->a[0] = Rest[1];
+	op->a[1] = S;
+	op->a[2] = Rest[0];
+
+	NEW_GENOP(stp, jump);
+	jump->next = NULL;
+	jump->arity = 1;
+	jump->op = genop_jump_1;
+	jump->a[0] = Fail;
+
+	op->next = jump;
+	return op;
+    }
+
+    /*
      * Calculate the minimum and maximum values and size of jump table.
      */
 
@@ -3131,8 +3247,9 @@ genopargcompare(GenOpArg* a, GenOpArg* b)
 }
 
 /*
- * Generate a select_val instruction.  We know that a jump table is not suitable,
- * and that all values are of the same type (integer, atoms, floats; never bignums).
+ * Generate a select_val instruction.  We know that a jump table
+ * is not suitable, and that all values are of the same type
+ * (integer or atoms).
  */
 
 static GenOp*
@@ -3146,12 +3263,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 
     NEW_GENOP(stp, op);
     op->next = NULL;
-    if (Rest[0].type != TAG_q) {
-	op->op = genop_i_select_val_3;
-    } else {
-	ASSERT(is_float(stp->literals[Rest[0].val].term));
-	op->op = genop_i_select_float_3;
-    }
+    op->op = genop_i_select_val_3;
     GENOP_ARITY(op, arity);
     op->a[0] = S;
     op->a[1] = Fail;
@@ -3173,19 +3285,19 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     }
 #endif
 
-    return op;
-}
-
-/* 
- *  Compare function for qsort().
- */
+    /*
+     * Use a special-cased instruction if there are only two values.
+     */
+    if (size == 2) {
+	op->op = genop_i_select_val2_6;
+	op->arity--;
+	op->a[2] = op->a[3];
+	op->a[3] = op->a[4];
+	op->a[4] = op->a[5];
+	op->a[5] = op->a[6];
+    }
 
-static int
-genbigcompare(GenOpArg* a, GenOpArg* b)
-{
-    int val = (int)(b->bigarity - a->bigarity);
-    
-    return val != 0 ? val : ((int) (a->val - b->val));
+    return op;
 }
 
 /*
@@ -3193,37 +3305,35 @@ genbigcompare(GenOpArg* a, GenOpArg* b)
  */
 
 static GenOp*
-gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail,
+gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 	       GenOpArg Size, GenOpArg* Rest)
 {
     GenOp* op;
-    int arity = Size.val + 2 + 1;
-    int size = Size.val / 2;
+    GenOp* jump;
+    GenOp** prev_next = &op;
+
     int i;
 
-    NEW_GENOP(stp, op);
-    op->next = NULL;
-    op->op = genop_i_select_big_2;
-    GENOP_ARITY(op, arity);
-    op->a[0] = S;
-    op->a[1] = Fail;
     for (i = 0; i < Size.val; i += 2) {
+	GenOp* op;
 	ASSERT(Rest[i].type == TAG_q);
-	op->a[i+2] = Rest[i];
-	op->a[i+2].bigarity = *big_val(stp->literals[op->a[i+2].val].term);
-	op->a[i+3] = Rest[i+1];
-    }
-    ASSERT(i+2 == arity-1);
-    op->a[arity-1].type = TAG_u;
-    op->a[arity-1].val = 0;
-
-    /*
-     * Sort the values in descending arity order.
-     */
-
-    qsort(op->a+2, size, 2*sizeof(GenOpArg), 
-	  (int (*)(const void *, const void *)) genbigcompare);
 
+	NEW_GENOP(stp, op);
+	op->op = genop_is_ne_exact_3;
+	op->arity = 3;
+	op->a[0] = Rest[i+1];
+	op->a[1] = S;
+	op->a[2] = Rest[i];
+	*prev_next = op;
+	prev_next = &op->next;
+    }
+
+    NEW_GENOP(stp, jump);
+    jump->next = NULL;
+    jump->op = genop_jump_1;
+    jump->arity = 1;
+    jump->a[0] = Fail;
+    *prev_next = jump;
     return op;
 }
 
@@ -3241,7 +3351,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     int i;
 
     ASSERT(Size.type == TAG_u);
-    ASSERT(S.type == TAG_q);
 
     NEW_GENOP(stp, op);
     op->next = NULL;
@@ -3252,18 +3361,32 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
      * Search for a literal matching the controlling expression.
      */
 
-    if (S.type == TAG_q) {
-	Eterm expr = stp->literals[S.val].term;
-	for (i = 0; i < Size.val; i += 2) {
-	    if (Rest[i].type == TAG_q) {
-		Eterm term = stp->literals[Rest[i].val].term;
-		if (eq(term, expr)) {
-		    ASSERT(Rest[i+1].type == TAG_f);
-		    op->a[0] = Rest[i+1];
-		    return op;
+    switch (S.type) {
+    case TAG_q:
+	{
+	    Eterm expr = stp->literals[S.val].term;
+	    for (i = 0; i < Size.val; i += 2) {
+		if (Rest[i].type == TAG_q) {
+		    Eterm term = stp->literals[Rest[i].val].term;
+		    if (eq(term, expr)) {
+			ASSERT(Rest[i+1].type == TAG_f);
+			op->a[0] = Rest[i+1];
+			return op;
+		    }
 		}
 	    }
 	}
+	break;
+    case TAG_i:
+    case TAG_a:
+	for (i = 0; i < Size.val; i += 2) {
+	    if (Rest[i].val == S.val && Rest[i].type == S.type) {
+		ASSERT(Rest[i+1].type == TAG_f);
+		op->a[0] = Rest[i+1];
+		return op;
+	    }
+	}
+	break;
     }
 
     /*
@@ -3274,36 +3397,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     return op;
 }
 
-
-static GenOp*
-gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg func,
-	      GenOpArg arity, GenOpArg label)
-{
-    GenOp* fi;
-    GenOp* op;
-
-    NEW_GENOP(stp, fi);
-    fi->op = genop_i_func_info_4;
-    fi->arity = 4;
-    fi->a[0].type = TAG_u;	/* untagged Zero */
-    fi->a[0].val = 0;
-    fi->a[1] = mod;
-    fi->a[2] = func;
-    fi->a[3] = arity;
-
-    NEW_GENOP(stp, op);
-    op->op = genop_label_1;
-    op->arity = 1;
-    op->a[0] = label;
-    
-    fi->next = op;
-    op->next = NULL;
-
-    return fi;
-}
-
-
-
 static GenOp*
 gen_make_fun2(LoaderState* stp, GenOpArg idx)
 {
@@ -3321,15 +3414,21 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx)
     op->op = genop_i_make_fun_2;
     op->arity = 2;
     op->a[0].type = TAG_u;
-    op->a[0].val = (Uint) fe;
+    op->a[0].val = (BeamInstr) fe;
     op->a[1].type = TAG_u;
     op->a[1].val = stp->lambdas[idx.val].num_free;
     op->next = NULL;
     return op;
 }
-
+/*
+ * Rewrite gc_bifs with one parameter (the common case). Utilized
+ * in ops.tab to rewrite instructions calling bif's in guards
+ * to use a garbage collecting implementation. The instructions
+ * are sometimes once again rewritten to handle literals (putting the
+ * parameter in the mostly unused r[0] before the instruction is executed).
+ */
 static GenOp*
-gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
+gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 	      GenOpArg Src, GenOpArg Dst)
 {
     GenOp* op;
@@ -3341,22 +3440,24 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
     op->a[0] = Fail;
     op->a[1].type = TAG_u;
     bf = stp->import[Bif.val].bf;
+    /* The translations here need to have a reverse counterpart in
+       beam_emu.c:translate_gc_bif for error handling to work properly. */
     if (bf == length_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_length_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_length_1;
     } else if (bf == size_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_size_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_size_1;
     } else if (bf == bit_size_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_bit_size_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_bit_size_1;
     } else if (bf == byte_size_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_byte_size_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_byte_size_1;
     } else if (bf == abs_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_abs_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_abs_1;
     } else if (bf == float_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_float_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_float_1;
     } else if (bf == round_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_round_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_round_1;
     } else if (bf == trunc_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_trunc_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_trunc_1;
     } else {
 	abort();
     }
@@ -3367,6 +3468,127 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
     return op;
 }
 
+/*
+ * This is used by the ops.tab rule that rewrites gc_bifs with two parameters
+ * The instruction returned is then again rewritten to an i_load instruction
+ * folowed by i_gc_bif2_jIId, to handle literals properly.
+ * As opposed to the i_gc_bif1_jIsId, the instruction  i_gc_bif2_jIId is
+ * always rewritten, regardless of if there actually are any literals.
+ */
+static GenOp*
+gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
+	      GenOpArg S1, GenOpArg S2, GenOpArg Dst)
+{
+    GenOp* op;
+    BifFunction bf;
+
+    NEW_GENOP(stp, op);
+    op->op = genop_ii_gc_bif2_6;
+    op->arity = 6;
+    op->a[0] = Fail;
+    op->a[1].type = TAG_u;
+    bf = stp->import[Bif.val].bf;
+    /* The translations here need to have a reverse counterpart in
+       beam_emu.c:translate_gc_bif for error handling to work properly. */
+    if (bf == binary_part_2) {
+	op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2;
+    } else {
+	abort();
+    }
+    op->a[2] = S1;
+    op->a[3] = S2;
+    op->a[4] = Live;
+    op->a[5] = Dst;
+    op->next = NULL;
+    return op;
+}
+
+/*
+ * This is used by the ops.tab rule that rewrites gc_bifs with three parameters
+ * The instruction returned is then again rewritten to a move instruction that
+ * uses r[0] for temp storage, followed by an i_load instruction,
+ * folowed by i_gc_bif3_jIsId, to handle literals properly. Rewriting
+ * always occur, as with the gc_bif2 counterpart.
+ */
+static GenOp*
+gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
+	      GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst)
+{
+    GenOp* op;
+    BifFunction bf;
+
+    NEW_GENOP(stp, op);
+    op->op = genop_ii_gc_bif3_7;
+    op->arity = 7;
+    op->a[0] = Fail;
+    op->a[1].type = TAG_u;
+    bf = stp->import[Bif.val].bf;
+    /* The translations here need to have a reverse counterpart in
+       beam_emu.c:translate_gc_bif for error handling to work properly. */
+    if (bf == binary_part_3) {
+	op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3;
+    } else {
+	abort();
+    }
+    op->a[2] = S1;
+    op->a[3] = S2;
+    op->a[4] = S3;
+    op->a[5] = Live;
+    op->a[6] = Dst;
+    op->next = NULL;
+    return op;
+}
+
+static GenOp*
+tuple_append_put5(LoaderState* stp, GenOpArg Arity, GenOpArg Dst,
+		  GenOpArg* Puts, GenOpArg S1, GenOpArg S2, GenOpArg S3,
+		  GenOpArg S4, GenOpArg S5)
+{
+    GenOp* op;
+    int arity = Arity.val;	/* Arity of tuple, not the instruction */
+    int i;
+
+    NEW_GENOP(stp, op);
+    op->next = NULL;
+    GENOP_ARITY(op, arity+2+5);
+    op->op = genop_i_put_tuple_2;
+    op->a[0] = Dst;
+    op->a[1].type = TAG_u;
+    op->a[1].val = arity + 5;
+    for (i = 0; i < arity; i++) {
+	op->a[i+2] = Puts[i];
+    }
+    op->a[arity+2] = S1;
+    op->a[arity+3] = S2;
+    op->a[arity+4] = S3;
+    op->a[arity+5] = S4;
+    op->a[arity+6] = S5;
+    return op;
+}
+
+static GenOp*
+tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst,
+		 GenOpArg* Puts, GenOpArg S)
+{
+    GenOp* op;
+    int arity = Arity.val;	/* Arity of tuple, not the instruction */
+    int i;
+
+    NEW_GENOP(stp, op);
+    op->next = NULL;
+    GENOP_ARITY(op, arity+2+1);
+    op->op = genop_i_put_tuple_2;
+    op->a[0] = Dst;
+    op->a[1].type = TAG_u;
+    op->a[1].val = arity + 1;
+    for (i = 0; i < arity; i++) {
+	op->a[i+2] = Puts[i];
+    }
+    op->a[arity+2] = S;
+    return op;
+}
+
+
 
 /*
  * Freeze the code in memory, move the string table into place,
@@ -3376,7 +3598,8 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 static int
 freeze_code(LoaderState* stp)
 {
-    Eterm* code = stp->code;
+    BeamInstr* code = stp->code;
+    Uint *literal_end = NULL;
     Uint index;
     int i;
     byte* str_table;
@@ -3401,46 +3624,49 @@ freeze_code(LoaderState* stp)
      * Calculate the final size of the code.
      */
 
-    size = (stp->ci + stp->total_literal_size) * sizeof(Eterm) +
+    size = (stp->ci * sizeof(BeamInstr)) + (stp->total_literal_size * sizeof(Eterm)) +
 	strtab_size + attr_size + compile_size;
 
     /*
      * Move the code to its final location.
      */
 
-    code = (Eterm *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size);
-
+    code = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size);
+    CHKBLK(ERTS_ALC_T_CODE,code);
     /*
      * Place a pointer to the op_int_code_end instruction in the
      * function table in the beginning of the file.
      */
 
-    code[MI_FUNCTIONS+stp->num_functions] = (Eterm) (code + stp->ci - 1);
+    code[MI_FUNCTIONS+stp->num_functions] = (BeamInstr) (code + stp->ci - 1);
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Store the pointer to the on_load function.
      */
 
     if (stp->on_load) {
-	code[MI_ON_LOAD_FUNCTION_PTR] = (Eterm) (code + stp->on_load);
+	code[MI_ON_LOAD_FUNCTION_PTR] = (BeamInstr) (code + stp->on_load);
     } else {
 	code[MI_ON_LOAD_FUNCTION_PTR] = 0;
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
+    literal_end = (Uint *) (code+stp->ci);
     /*
      * Place the literal heap directly after the code and fix up all
-     * put_literal instructions that refer to it.
+     * instructions that refer to it.
      */
     {
-	Eterm* ptr;
-	Eterm* low;
-	Eterm* high;
+	Uint* ptr;
+	Uint* low;
+	Uint* high;
 	LiteralPatch* lp;
 
-	low = code+stp->ci;
+	low = (Uint *) (code+stp->ci);
 	high = low + stp->total_literal_size;
-	code[MI_LITERALS_START] = (Eterm) low;
-	code[MI_LITERALS_END] = (Eterm) high;
+	code[MI_LITERALS_START] = (BeamInstr) low;
+	code[MI_LITERALS_END] = (BeamInstr) high;
 	ptr = low;
 	for (i = 0; i < stp->num_literals; i++) {
 	    Uint offset;
@@ -3472,7 +3698,7 @@ freeze_code(LoaderState* stp)
 	}
 	lp = stp->literal_patches;
 	while (lp != 0) {
-	    Uint* op_ptr;
+	    BeamInstr* op_ptr;
 	    Uint literal;
 	    Literal* lit;
 
@@ -3485,53 +3711,55 @@ freeze_code(LoaderState* stp)
 	    op_ptr[0] = literal;
 	    lp = lp->next;
 	}
-	stp->ci += stp->total_literal_size;
+	literal_end += stp->total_literal_size;
     }
     
     /*
      * Place the string table and, optionally, attributes, after the literal heap.
      */
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
-    sys_memcpy(code+stp->ci, stp->chunks[STR_CHUNK].start, strtab_size);
-    str_table = (byte *) (code+stp->ci);
+    sys_memcpy(literal_end, stp->chunks[STR_CHUNK].start, strtab_size);
+    CHKBLK(ERTS_ALC_T_CODE,code);
+    str_table = (byte *) literal_end;
     if (attr_size) {
 	byte* attr = str_table + strtab_size;
 	sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size);
-	code[MI_ATTR_PTR] = (Eterm) attr;
-	code[MI_ATTR_SIZE] = (Eterm) stp->chunks[ATTR_CHUNK].size;
+	code[MI_ATTR_PTR] = (BeamInstr) attr;
+	code[MI_ATTR_SIZE] = (BeamInstr) stp->chunks[ATTR_CHUNK].size;
 	decoded_size = erts_decode_ext_size(attr, attr_size, 0);
 	if (decoded_size < 0) {
  	    LoadError0(stp, "bad external term representation of module attributes");
  	}
 	code[MI_ATTR_SIZE_ON_HEAP] = decoded_size;
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
     if (compile_size) {
 	byte* compile_info = str_table + strtab_size + attr_size;
+	CHKBLK(ERTS_ALC_T_CODE,code);
 	sys_memcpy(compile_info, stp->chunks[COMPILE_CHUNK].start,
 	       stp->chunks[COMPILE_CHUNK].size);
-	code[MI_COMPILE_PTR] = (Eterm) compile_info;
-	code[MI_COMPILE_SIZE] = (Eterm) stp->chunks[COMPILE_CHUNK].size;
+
+	CHKBLK(ERTS_ALC_T_CODE,code);
+	code[MI_COMPILE_PTR] = (BeamInstr) compile_info;
+	CHKBLK(ERTS_ALC_T_CODE,code);
+	code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size;
+	CHKBLK(ERTS_ALC_T_CODE,code);
 	decoded_size = erts_decode_ext_size(compile_info, compile_size, 0);
+	CHKBLK(ERTS_ALC_T_CODE,code);
 	if (decoded_size < 0) {
  	    LoadError0(stp, "bad external term representation of compilation information");
  	}
+	CHKBLK(ERTS_ALC_T_CODE,code);
 	code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size;
     }
-
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
-     * Go through all put_strings instructions, restore the pointer to
-     * the instruction and convert string offsets to pointers (to the
-     * LAST character).
+     * Make sure that we have not overflowed the allocated code space.
      */
-
-    index = stp->put_strings;
-    while (index != 0) {
-	Uint next = code[index];
-	code[index] = BeamOpCode(op_put_string_IId);
-	code[index+2] = (Uint) (str_table + code[index+2] + code[index+1] - 1);
-	index = next;
-    }
+    ASSERT(str_table + strtab_size + attr_size + compile_size ==
+	   ((byte *) code) + size);
 
     /*
      * Go through all i_new_bs_put_strings instructions, restore the pointer to
@@ -3543,23 +3771,25 @@ freeze_code(LoaderState* stp)
     while (index != 0) {
 	Uint next = code[index];
 	code[index] = BeamOpCode(op_bs_put_string_II);
-	code[index+2] = (Uint) (str_table + code[index+2]);
+	code[index+2] = (BeamInstr) (str_table + code[index+2]);
 	index = next;
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     {
 	StringPatch* sp = stp->string_patches;
 
 	while (sp != 0) {
-	    Uint* op_ptr;
+	    BeamInstr* op_ptr;
 	    byte* strp;
 
 	    op_ptr = code + sp->pos;
 	    strp = str_table + op_ptr[0];
-	    op_ptr[0] = (Eterm) strp;
+	    op_ptr[0] = (BeamInstr) strp;
 	    sp = sp->next;
 	}
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Resolve all labels.
@@ -3579,10 +3809,11 @@ freeze_code(LoaderState* stp)
 	    ASSERT(this_patch < stp->ci);
 	    next_patch = code[this_patch];
 	    ASSERT(next_patch < stp->ci);
-	    code[this_patch] = (Uint) (code + value);
+	    code[this_patch] = (BeamInstr) (code + value);
 	    this_patch = next_patch;
 	}
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Fix all catch_yf instructions.
@@ -3590,13 +3821,14 @@ freeze_code(LoaderState* stp)
     index = stp->catches;
     catches = BEAM_CATCHES_NIL;
     while (index != 0) {
-	Uint next = code[index];
+	BeamInstr next = code[index];
 	code[index] = BeamOpCode(op_catch_yf);
-	catches = beam_catches_cons((Uint*)code[index+2], catches);
+	catches = beam_catches_cons((BeamInstr *)code[index+2], catches);
 	code[index+2] = make_catch(catches);
 	index = next;
     }
     stp->catches = catches;
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Save the updated code pointer and code size.
@@ -3605,6 +3837,7 @@ freeze_code(LoaderState* stp)
     stp->code = code;
     stp->loaded_size = size;
 
+    CHKBLK(ERTS_ALC_T_CODE,code);
     return 1;
 
  load_error:
@@ -3638,7 +3871,7 @@ final_touch(LoaderState* stp)
 	     * callable yet.
 	     */
 	    ep->address = ep->code+3;
-	    ep->code[4] = (Eterm) stp->export[i].address;
+	    ep->code[4] = (BeamInstr) stp->export[i].address;
 	}
     }
 
@@ -3650,14 +3883,14 @@ final_touch(LoaderState* stp)
 	Eterm mod;
 	Eterm func;
 	Uint arity;
-	Uint import;
+	BeamInstr import;
 	Uint current;
 	Uint next;
 
 	mod = stp->import[i].module;
 	func = stp->import[i].function;
 	arity = stp->import[i].arity;
-	import = (Uint) erts_export_put(mod, func, arity);
+	import = (BeamInstr) erts_export_put(mod, func, arity);
 	current = stp->import[i].patches;
 	while (current != 0) {
 	    ASSERT(current < stp->ci);
@@ -3675,7 +3908,7 @@ final_touch(LoaderState* stp)
 	for (i = 0; i < stp->num_lambdas; i++) {
 	    unsigned entry_label = stp->lambdas[i].label;
 	    ErlFunEntry* fe = stp->lambdas[i].fe;
-	    Eterm* code_ptr = (Eterm *) (stp->code + stp->labels[entry_label].value);
+	    BeamInstr* code_ptr = (BeamInstr *) (stp->code + stp->labels[entry_label].value);
 
 	    if (fe->address[0] != 0) {
 		/*
@@ -3762,11 +3995,23 @@ transform_engine(LoaderState* st)
 	    if (i == 0)
 		goto restart;
 	    break;
+#if defined(TOP_is_eq)
 	case TOP_is_eq:
 	    ASSERT(ap < instr->arity);
 	    if (*pc++ != instr->a[ap].val)
 		goto restart;
 	    break;
+#endif
+	case TOP_is_type_eq:
+	    mask = *pc++;
+
+	    ASSERT(ap < instr->arity);
+	    ASSERT(instr->a[ap].type < BEAM_NUM_TAGS);
+	    if (((1 << instr->a[ap].type) & mask) == 0)
+		goto restart;
+	    if (*pc++ != instr->a[ap].val)
+		goto restart;
+	    break;
 	case TOP_is_same_var:
 	    ASSERT(ap < instr->arity);
 	    i = *pc++;
@@ -3807,7 +4052,7 @@ transform_engine(LoaderState* st)
 		if (i >= st->num_imports || st->import[i].bf == NULL)
 		    goto restart;
 		if (bif_number != -1 &&
-		    bif_export[bif_number]->code[4] != (Uint) st->import[i].bf) {
+		    bif_export[bif_number]->code[4] != (BeamInstr) st->import[i].bf) {
 		    goto restart;
 		}
 	    }
@@ -3887,14 +4132,17 @@ transform_engine(LoaderState* st)
 	case TOP_rest_args:
 	    {
 		int n = *pc++;
+		int formal_arity = gen_opc[instr->op].arity;
+		int num_vars = n + (instr->arity - formal_arity);
+		int j = formal_arity;
+
 		var = erts_alloc(ERTS_ALC_T_LOADER_TMP,
-				 instr->arity * sizeof(GenOpArg));
+				 num_vars * sizeof(GenOpArg));
 		for (i = 0; i < n; i++) {
 		    var[i] = def_vars[i];
 		}
-		while (i < instr->arity) {
-		    var[i] = instr->a[i];
-		    i++;
+		while (i < num_vars) {
+		    var[i++] = instr->a[j++];
 		}
 	    }
 	    break;
@@ -4071,7 +4319,7 @@ load_printf(int line, LoaderState* context, char *fmt,...)
 
 
 static int
-get_int_val(LoaderState* stp, Uint len_code, Uint* result)
+get_int_val(LoaderState* stp, Uint len_code, BeamInstr* result)
 {
     Uint count;
     Uint val;
@@ -4103,7 +4351,7 @@ get_int_val(LoaderState* stp, Uint len_code, Uint* result)
 
 
 static int
-get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result)
+get_erlang_integer(LoaderState* stp, Uint len_code, BeamInstr* result)
 {
     Uint count;
     Sint val;
@@ -4124,11 +4372,12 @@ get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result)
 	count = len_code + 2;
     } else {
 	Uint tag;
+	UWord len_word;
 
 	ASSERT(len_code == 7);
-	GetTagAndValue(stp, tag, len_code);
+	GetTagAndValue(stp, tag, len_word);
 	VerifyTag(stp, TAG_u, tag);
-	count = len_code + 9;
+	count = len_word + 9;
     }
 
     /*
@@ -4376,7 +4625,7 @@ functions_in_module(Process* p, /* Process whose heap to use. */
 		     Eterm mod) /* Tagged atom for module. */
 {
     Module* modp;
-    Eterm* code;
+    BeamInstr* code;
     int i;
     Uint num_functions;
     Eterm* hp;
@@ -4394,9 +4643,9 @@ functions_in_module(Process* p, /* Process whose heap to use. */
     num_functions = code[MI_NUM_FUNCTIONS];
     hp = HAlloc(p, 5*num_functions);
     for (i = num_functions-1; i >= 0 ; i--) {
-	Eterm* func_info = (Eterm *) code[MI_FUNCTIONS+i];
-	Eterm name = func_info[3];
-	int arity = func_info[4];
+	BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i];
+	Eterm name = (Eterm) func_info[3];
+	int arity = (int) func_info[4];
 	Eterm tuple;
 
 	ASSERT(is_atom(name));
@@ -4419,7 +4668,7 @@ static Eterm
 native_addresses(Process* p, Eterm mod)
 {
     Module* modp;
-    Eterm* code;
+    BeamInstr* code;
     int i;
     Eterm* hp;
     Uint num_functions;
@@ -4442,9 +4691,9 @@ native_addresses(Process* p, Eterm mod)
     hp = HAlloc(p, need);
     hp_end = hp + need;
     for (i = num_functions-1; i >= 0 ; i--) {
-	Eterm* func_info = (Eterm *) code[MI_FUNCTIONS+i];
-	Eterm name = func_info[3];
-	int arity = func_info[4];
+	BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i];
+	Eterm name = (Eterm) func_info[3];
+	int arity = (int) func_info[4];
 	Eterm tuple;
 
 	ASSERT(is_atom(name));
@@ -4486,7 +4735,7 @@ exported_from_module(Process* p, /* Process whose heap to use. */
 	    Eterm tuple;
 	    
 	    if (ep->address == ep->code+3 &&
-		ep->code[3] == (Eterm) em_call_error_handler) {
+		ep->code[3] == (BeamInstr) em_call_error_handler) {
 		/* There is a call to the function, but it does not exist. */ 
 		continue;
 	    }
@@ -4519,7 +4768,7 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
 
 {
     Module* modp;
-    Eterm* code;
+    BeamInstr* code;
     Eterm* hp;
     byte* ext;
     Eterm result = NIL;
@@ -4559,7 +4808,7 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
 			    Eterm mod) /* Tagged atom for module. */
 {
     Module* modp;
-    Eterm* code;
+    BeamInstr* code;
     Eterm* hp;
     byte* ext;
     Eterm result = NIL;
@@ -4591,8 +4840,8 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
 /*
  * Returns a pointer to {module, function, arity}, or NULL if not found.
  */
-Eterm*
-find_function_from_pc(Eterm* pc)
+BeamInstr *
+find_function_from_pc(BeamInstr* pc)
 {
     Range* low = modules;
     Range* high = low + num_loaded_modules;
@@ -4604,9 +4853,9 @@ find_function_from_pc(Eterm* pc)
 	} else if (pc > mid->end) {
 	    low = mid + 1;
 	} else {
-	    Eterm** low1 = (Eterm **) (mid->start + MI_FUNCTIONS);
-	    Eterm** high1 = low1 + mid->start[MI_NUM_FUNCTIONS];
-	    Eterm** mid1;
+	    BeamInstr** low1 = (BeamInstr **) (mid->start + MI_FUNCTIONS);
+	    BeamInstr** high1 = low1 + mid->start[MI_NUM_FUNCTIONS];
+	    BeamInstr** mid1;
 
 	    while (low1 < high1) {
 		mid1 = low1 + (high1-low1) / 2;
@@ -4719,10 +4968,10 @@ code_module_md5_1(Process* p, Eterm Bin)
 
 #define WORDS_PER_FUNCTION 6
 
-static Eterm*
-make_stub(Eterm* fp, Eterm mod, Eterm func, Uint arity, Uint native, Eterm OpCode)
+static BeamInstr*
+make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode)
 {
-    fp[0] = (Eterm) BeamOp(op_i_func_info_IaaI);
+    fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
     fp[1] = native;
     fp[2] = mod;
     fp[3] = func;
@@ -4741,14 +4990,14 @@ static byte*
 stub_copy_info(LoaderState* stp,
 	       int chunk,	/* Chunk: ATTR_CHUNK or COMPILE_CHUNK */
 	       byte* info,	/* Where to store info. */
-	       Eterm* ptr_word,	/* Where to store pointer into info. */
-	       Eterm* size_word) /* Where to store size of info. */
+	       BeamInstr* ptr_word,	/* Where to store pointer into info. */
+	       BeamInstr* size_word) /* Where to store size of info. */
 {
     Sint decoded_size;
     Uint size = stp->chunks[chunk].size;
     if (size != 0) {
 	memcpy(info, stp->chunks[chunk].start, size);
-	*ptr_word = (Eterm) info;
+	*ptr_word = (BeamInstr) info;
 	decoded_size = erts_decode_ext_size(info, size, 0);
 	if (decoded_size < 0) {
  	    return 0;
@@ -4791,7 +5040,7 @@ stub_read_export_table(LoaderState* stp)
 }
 
 static void
-stub_final_touch(LoaderState* stp, Eterm* fp)
+stub_final_touch(LoaderState* stp, BeamInstr* fp)
 {
     int i;
     int n = stp->num_exps;
@@ -4978,12 +5227,12 @@ Eterm
 erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 {
     LoaderState state;
-    Eterm Funcs;
-    Eterm Patchlist;
+    BeamInstr Funcs;
+    BeamInstr Patchlist;
     Eterm* tp;
-    Eterm* code = NULL;
-    Eterm* ptrs;
-    Eterm* fp;
+    BeamInstr* code = NULL;
+    BeamInstr* ptrs;
+    BeamInstr* fp;
     byte* info;
     Uint ci;
     int n;
@@ -5072,7 +5321,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Allocate memory for the stub module.
      */
 
-    code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(Eterm);
+    code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr);
     code_size += state.chunks[ATTR_CHUNK].size;
     code_size += state.chunks[COMPILE_CHUNK].size;
     code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size);
@@ -5086,8 +5335,10 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 
     code[MI_NUM_FUNCTIONS] = n;
     code[MI_ATTR_PTR] = 0;
+    code[MI_ATTR_SIZE] = 0;
     code[MI_ATTR_SIZE_ON_HEAP] = 0;
     code[MI_COMPILE_PTR] = 0;
+    code[MI_COMPILE_SIZE] = 0;
     code[MI_COMPILE_SIZE_ON_HEAP] = 0;
     code[MI_NUM_BREAKPOINTS] = 0;
     code[MI_ON_LOAD_FUNCTION_PTR] = 0;
@@ -5141,7 +5392,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 	 * Set the pointer and make the stub. Put a return instruction
 	 * as the body until we know what kind of trap we should put there.
 	 */
-	ptrs[i] = (Eterm) fp;
+	ptrs[i] = (BeamInstr) fp;
 #ifdef HIPE
 	op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */
 #else
@@ -5154,8 +5405,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Insert the last pointer and the int_code_end instruction.
      */
 
-    ptrs[i] = (Eterm) fp;
-    *fp++ = (Eterm) BeamOp(op_int_code_end);
+    ptrs[i] = (BeamInstr) fp;
+    *fp++ = (BeamInstr) BeamOp(op_int_code_end);
 
     /*
      * Copy attributes and compilation information.
@@ -5198,6 +5449,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 	if (state.lambdas != state.def_lambdas) {
 	    erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas);
 	}
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels);
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom);
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export);
 	if (bin != NULL) {
 	    driver_free_binary(bin);
 	}
@@ -5209,9 +5463,18 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     if (code != NULL) {
 	erts_free(ERTS_ALC_T_CODE, code);
     }
+    if (state.labels != NULL) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels);
+    }
     if (state.lambdas != state.def_lambdas) {
 	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas);
     }
+    if (state.atom != NULL) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom);
+    }
+    if (state.export != NULL) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export);
+    }
     if (bin != NULL) {
 	driver_free_binary(bin);
     }
@@ -5222,9 +5485,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 
 #undef WORDS_PER_FUNCTION
 
-static int safe_mul(Uint a, Uint b, Uint* resp)
+static int safe_mul(UWord a, UWord b, UWord* resp)
 {
-    Uint res = a * b;
+    Uint res = a * b; /* XXX:Pan - used in bit syntax, the multiplication has to be stored in Uint */
     *resp = res;
 
     if (b == 0) {
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index c17844a553..26e3054c4b 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -44,13 +44,13 @@ extern void** beam_ops;
 #endif
 
 
-extern Eterm beam_debug_apply[];
-extern Eterm* em_call_error_handler;
-extern Eterm* em_apply_bif;
-extern Eterm* em_call_traced_function;
+extern BeamInstr beam_debug_apply[];
+extern BeamInstr* em_call_error_handler;
+extern BeamInstr* em_apply_bif;
+extern BeamInstr* em_call_traced_function;
 typedef struct {
-    Eterm* start;		/* Pointer to start of module. */
-    Eterm* end;			/* Points one word beyond last function in module. */
+    BeamInstr* start;		/* Pointer to start of module. */
+    BeamInstr* end;			/* Points one word beyond last function in module. */
 } Range;
 
 /*
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 1b670585a7..68b3350d7f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -368,7 +368,6 @@ static int demonitor(Process *c_p, Eterm ref)
    ErtsMonitor *mon = NULL;  /* The monitor entry to delete */
    Process  *rp;    /* Local target process */
    Eterm     to = NIL;    /* Monitor link traget */
-   Eterm     ref_p; /* Pid of this end */
    DistEntry *dep = NULL;  /* Target's distribution entry */
    int deref_de = 0;
    int res;
@@ -381,7 +380,6 @@ static int demonitor(Process *c_p, Eterm ref)
        res = ERTS_DEMONITOR_BADARG;
        goto done; /* Cannot be this monitor's ref */
    }
-   ref_p = c_p->id;
 
    mon = erts_lookup_monitor(c_p->monitors, ref);
    if (!mon) {
@@ -616,13 +614,15 @@ local_name_monitor(Process *p, Eterm target_name)
     rp = erts_whereis_process(p, p_locks, target_name, ERTS_PROC_LOCK_LINK,
 			      ERTS_P2P_FLG_ALLOW_OTHER_X);
     if (!rp) {
-	Eterm lhp[3];
+	DeclareTmpHeap(lhp,3,p);
 	Eterm item;
+	UseTmpHeap(3,p);
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	p_locks &= ~ERTS_PROC_LOCK_LINK;
 	item = TUPLE2(lhp, target_name, erts_this_dist_entry->sysname);
 	erts_queue_monitor_message(p, &p_locks,
 				   mon_ref, am_process, item, am_noproc);
+	UnUseTmpHeap(3,p);
     }
     else if (rp != p) {
 	erts_add_monitor(&(p->monitors), MON_ORIGIN, mon_ref, rp->id,
@@ -811,7 +811,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
     so.min_heap_size  = H_MIN_SIZE;
     so.min_vheap_size = BIN_VH_MIN_SIZE;
     so.priority       = PRIORITY_NORMAL;
-    so.max_gen_gcs    = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs);
+    so.max_gen_gcs    = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs);
     so.scheduler      = 0;
 
     /*
@@ -1089,10 +1089,20 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 BIF_RETTYPE hibernate_3(BIF_ALIST_3)
 {
     /*
-     * hibernate/3 is implemented as an instruction; therefore
-     * this function will never be called.
+     * hibernate/3 is usually translated to an instruction; therefore
+     * this function is only called from HiPE or when the call could not
+     * be translated.
      */
-    BIF_ERROR(BIF_P, BADARG);
+    Eterm reg[3];
+
+    if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) {
+        /*
+         * If hibernate succeeded, TRAP. The process will be suspended
+         * if status is P_WAITING or continue (if any message was in the queue).
+         */
+        BIF_TRAP_CODE_PTR_(BIF_P, BIF_P->i);
+    }
+    return THE_NON_VALUE;
 }
 
 /**********************************************************************/
@@ -1130,6 +1140,34 @@ BIF_RETTYPE error_2(Process* p, Eterm value, Eterm args)
 }
 
 /**********************************************************************/
+/*
+ * This is like exactly like error/1. The only difference is
+ * that Dialyzer thinks that it it will return an arbitrary term.
+ * It is useful in stub functions for NIFs.
+ */
+
+BIF_RETTYPE nif_error_1(Process* p, Eterm term)
+{
+    p->fvalue = term;
+    BIF_ERROR(p, EXC_ERROR);
+}
+
+/**********************************************************************/
+/*
+ * This is like exactly like error/2. The only difference is
+ * that Dialyzer thinks that it it will return an arbitrary term.
+ * It is useful in stub functions for NIFs.
+ */
+
+BIF_RETTYPE nif_error_2(Process* p, Eterm value, Eterm args)
+{
+    Eterm* hp = HAlloc(p, 3);
+
+    p->fvalue = TUPLE2(hp, value, args);
+    BIF_ERROR(p, EXC_ERROR_2);
+}
+
+/**********************************************************************/
 /* this is like throw/1 except that we set freason to EXC_EXIT */
 
 BIF_RETTYPE exit_1(BIF_ALIST_1)
@@ -1321,9 +1359,10 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
 #ifdef ERTS_SMP
 	 if (rp == BIF_P)
 	     rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-	 else
+	 if (rp_locks)
+	     erts_smp_proc_unlock(rp, rp_locks);
+	 if (rp != BIF_P)
 	     erts_smp_proc_dec_refc(rp);
-	 erts_smp_proc_unlock(rp, rp_locks);
 #endif
 	 /*
 	  * We may have exited ourselves and may have to take action.
@@ -2129,20 +2168,146 @@ BIF_RETTYPE tl_1(BIF_ALIST_1)
 /**********************************************************************/
 /* return the size of an I/O list */
 
-BIF_RETTYPE iolist_size_1(BIF_ALIST_1)
+static Eterm
+accumulate(Eterm acc, Uint size)
 {
-    Sint size = io_list_len(BIF_ARG_1);
+    if (is_non_value(acc)) {
+	/*
+	 * There is no pre-existing accumulator. Allocate a
+	 * bignum buffer with one extra word to be used if
+	 * the bignum grows in the future.
+	 */
+	Eterm* hp = (Eterm *) erts_alloc(ERTS_ALC_T_TEMP_TERM,
+					 (BIG_UINT_HEAP_SIZE+1) *
+					 sizeof(Eterm));
+	return uint_to_big(size, hp);
+    } else {
+	Eterm* big;
+	int need_heap;
 
-    if (size == -1) {
-	BIF_ERROR(BIF_P, BADARG);
-    } else if (IS_USMALL(0, (Uint) size)) {
-	BIF_RET(make_small(size));
+	/*
+	 * Add 'size' to 'acc' in place. There is always one
+	 * extra word allocated in case the bignum grows by one word.
+	 */
+	big = big_val(acc);
+	need_heap = BIG_NEED_SIZE(BIG_SIZE(big));
+	acc = big_plus_small(acc, size, big);
+	if (BIG_NEED_SIZE(big_size(acc)) > need_heap) {
+	    /*
+	     * The extra word has been consumed. Grow the
+	     * allocation by one word.
+	     */
+	    big = (Eterm *) erts_realloc(ERTS_ALC_T_TEMP_TERM,
+					 big_val(acc),
+					 (need_heap+1) * sizeof(Eterm));
+	    acc = make_big(big);
+	}
+	return acc;
+    }
+}
+
+static Eterm
+consolidate(Process* p, Eterm acc, Uint size)
+{
+    Eterm* hp;
+
+    if (is_non_value(acc)) {
+	return erts_make_integer(size, p);
     } else {
-	Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
-	BIF_RET(uint_to_big(size, hp));
+	Eterm* big;
+	Uint sz;
+	Eterm res;
+	
+	acc = accumulate(acc, size);
+	big = big_val(acc);
+	sz = BIG_NEED_SIZE(BIG_SIZE(big));
+	hp = HAlloc(p, sz);
+	res = make_big(hp);
+	while (sz--) {
+	    *hp++ = *big++;
+	}
+	erts_free(ERTS_ALC_T_TEMP_TERM, (void *) big_val(acc));
+	return res;
     }
 }
 
+BIF_RETTYPE iolist_size_1(BIF_ALIST_1)
+{
+    Eterm obj, hd;
+    Eterm* objp;
+    Uint size = 0;
+    Uint cur_size;
+    Uint new_size;
+    Eterm acc = THE_NON_VALUE;
+    DECLARE_ESTACK(s);
+
+    obj = BIF_ARG_1;
+    goto L_again;
+
+    while (!ESTACK_ISEMPTY(s)) {
+	obj = ESTACK_POP(s);
+    L_again:
+	if (is_list(obj)) {
+	L_iter_list:
+	    objp = list_val(obj);
+	    hd = CAR(objp);
+	    obj = CDR(objp);
+	    /* Head */
+	    if (is_byte(hd)) {
+		size++;
+		if (size == 0) {
+		    acc = accumulate(acc, (Uint) -1);
+		    size = 1;
+		}
+	    } else if (is_binary(hd) && binary_bitsize(hd) == 0) {
+		cur_size = binary_size(hd);
+		if ((new_size = size + cur_size) >= size) {
+		    size = new_size;
+		} else {
+		    acc = accumulate(acc, size);
+		    size = cur_size;
+		}
+	    } else if (is_list(hd)) {
+		ESTACK_PUSH(s, obj);
+		obj = hd;
+		goto L_iter_list;
+	    } else if (is_not_nil(hd)) {
+		goto L_type_error;
+	    }
+	    /* Tail */
+	    if (is_list(obj)) {
+		goto L_iter_list;
+	    } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+		cur_size = binary_size(obj);
+		if ((new_size = size + cur_size) >= size) {
+		    size = new_size;
+		} else {
+		    acc = accumulate(acc, size);
+		    size = cur_size;
+		}
+	    } else if (is_not_nil(obj)) {
+		goto L_type_error;
+	    }
+	} else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+	    cur_size = binary_size(obj);
+	    if ((new_size = size + cur_size) >= size) {
+		size = new_size;
+	    } else {
+		acc = accumulate(acc, size);
+		size = cur_size;
+	    }
+	} else if (is_not_nil(obj)) {
+	    goto L_type_error;
+	}
+    }
+
+    DESTROY_ESTACK(s);
+    BIF_RET(consolidate(BIF_P, acc, size));
+
+ L_type_error:
+    DESTROY_ESTACK(s);
+    BIF_ERROR(BIF_P, BADARG);
+}
 
 /**********************************************************************/
 
@@ -3176,20 +3341,32 @@ BIF_RETTYPE garbage_collect_1(BIF_ALIST_1)
 	BIF_ERROR(BIF_P, BADARG);
     }
 
-    rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
+    if (BIF_P->id == BIF_ARG_1)
+	rp = BIF_P;
+    else {
+#ifdef ERTS_SMP
+	rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN,
 				   BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
-    if (!rp)
-	BIF_RET(am_false);
-    if (rp == ERTS_PROC_LOCK_BUSY)
-	ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1);
+	if (rp == ERTS_PROC_LOCK_BUSY)
+	    ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1);
+#else
+	rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0);
+#endif
+	if (!rp)
+	    BIF_RET(am_false);
+    }
 
     /* The GC cost is taken for the process executing this BIF. */
 
     FLAGS(rp) |= F_NEED_FULLSWEEP;
     reds = erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
 
-    if (BIF_P != rp)
+#ifdef ERTS_SMP
+    if (BIF_P != rp) {
+	erts_resume(rp, ERTS_PROC_LOCK_MAIN);
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+    }
+#endif
 
     BIF_RET2(am_true, reds);
 }
@@ -3231,6 +3408,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
     Eterm* dead_ports;
     int alive, dead;
     Uint32 next_ss;
+    int i;
 
     /* To get a consistent snapshot... 
      * We add alive ports from start of the buffer
@@ -3239,27 +3417,25 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
 
     erts_smp_mtx_lock(&ports_snapshot_mtx); /* One snapshot at a time */
 
-    erts_smp_atomic_set(&erts_dead_ports_ptr, (long) (port_buf + erts_max_ports));
+    erts_smp_atomic_set(&erts_dead_ports_ptr,
+			(erts_aint_t) (port_buf + erts_max_ports));
 
-    next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot);
+    next_ss = erts_smp_atomic32_inctest(&erts_ports_snapshot);
 
-    if (erts_smp_atomic_read(&erts_ports_alive) > 0) {
-	long i;
-	for (i = erts_max_ports-1; i >= 0; i--) {
-	    Port* prt = &erts_port[i];
-	    erts_smp_port_state_lock(prt);
-	    if (!(prt->status & ERTS_PORT_SFLGS_DEAD)
-		&& prt->snapshot != next_ss) {
-		ASSERT(prt->snapshot == next_ss - 1);
-		*pp++ = prt->id;		
-		prt->snapshot = next_ss; /* Consumed by this snapshot */
-	    }
-	    erts_smp_port_state_unlock(prt);
+    for (i = erts_max_ports-1; i >= 0; i--) {
+	Port* prt = &erts_port[i];
+	erts_smp_port_state_lock(prt);
+	if (!(prt->status & ERTS_PORT_SFLGS_DEAD)
+	    && prt->snapshot != next_ss) {
+	    ASSERT(prt->snapshot == next_ss - 1);
+	    *pp++ = prt->id;		
+	    prt->snapshot = next_ss; /* Consumed by this snapshot */
 	}
+	erts_smp_port_state_unlock(prt);
     }
 
     dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr,
-					      (long)NULL);
+					      (erts_aint_t) NULL);
     erts_smp_mtx_unlock(&ports_snapshot_mtx);
 
     ASSERT(pp <= dead_ports);
@@ -3270,7 +3446,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
     ASSERT((alive+dead) <= erts_max_ports);
 
     if (alive+dead > 0) {
-	long i;
+	erts_aint_t i;
 	Eterm *hp = HAlloc(BIF_P, (alive+dead)*2);
 
 	for (i = 0; i < alive; i++) {
@@ -3320,7 +3496,7 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1)
     erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64);       
     pres = erts_dsprintf(dsbufp, "%.*T\n", INT_MAX, BIF_ARG_1);
     if (pres < 0)
-	erl_exit(1, "Failed to convert term to string: %d (s)\n",
+	erl_exit(1, "Failed to convert term to string: %d (%s)\n",
 		 -pres, erl_errno_id(-pres));
     hp = HAlloc(BIF_P, 2*dsbufp->str_len); /* we need length * 2 heap words */
     res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL);
@@ -3438,7 +3614,7 @@ term2list_dsprintf(Process *p, Eterm term)
     erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64);       
     pres = erts_dsprintf(dsbufp, "%T", term);
     if (pres < 0)
-	erl_exit(1, "Failed to convert term to list: %d (s)\n",
+	erl_exit(1, "Failed to convert term to list: %d (%s)\n",
 		 -pres, erl_errno_id(-pres));
     hp = HAlloc(p, 2*dsbufp->str_len); /* we need length * 2 heap words */
     res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL);
@@ -3466,9 +3642,16 @@ BIF_RETTYPE make_fun_3(BIF_ALIST_3)
     if (arity < 0) {
 	goto error;
     }
+#if HALFWORD_HEAP
+    hp = HAlloc(BIF_P, 3);
+    hp[0] = HEADER_EXPORT;
+    /* Yes, May be misaligned, but X86_64 will fix it... */
+    *((Export **) (hp+1)) = erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity);
+#else
     hp = HAlloc(BIF_P, 2);
     hp[0] = HEADER_EXPORT;
     hp[1] = (Eterm) erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity);
+#endif
     BIF_RET(make_export(hp));
 }
 
@@ -3574,11 +3757,11 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1)
 
       etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1);
       etp->header = make_external_pid_header(1);
-      etp->next = MSO(BIF_P).externals;
+      etp->next = MSO(BIF_P).first;
       etp->node = enp;
       etp->data.ui[0] = make_pid_data(c, b);
 
-      MSO(BIF_P).externals = etp;
+      MSO(BIF_P).first = (struct erl_off_heap_header*) etp;
       erts_deref_dist_entry(dep);
       BIF_RET(make_external_pid(etp));
     }
@@ -3759,7 +3942,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    goto error;
 	}
 	nval = (n > (Sint) ((Uint16) -1)) ? ((Uint16) -1) : ((Uint16) n);
-	oval = (Uint) erts_smp_atomic_xchg(&erts_max_gen_gcs, (long) nval);
+	oval = (Uint) erts_smp_atomic32_xchg(&erts_max_gen_gcs,
+					     (erts_aint32_t) nval);
 	BIF_RET(make_small(oval));
     } else if (BIF_ARG_1 == am_min_heap_size) {
 	int oval = H_MIN_SIZE;
@@ -3884,7 +4068,7 @@ BIF_RETTYPE hash_2(BIF_ALIST_2)
     if ((range = signed_val(BIF_ARG_2)) <= 0) {  /* [1..MAX_SMALL] */
 	BIF_ERROR(BIF_P, BADARG);
     }
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     if (range > ((1L << 27) - 1))
 	BIF_ERROR(BIF_P, BADARG);
 #endif
@@ -3956,7 +4140,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2)
     /*
      * Return either a small or a big. Use the heap for bigs if there is room.
      */
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     BIF_RET(make_small(final_hash));
 #else
     if (IS_USMALL(0, final_hash)) {
@@ -4102,7 +4286,7 @@ void erts_init_bif(void)
 
     erts_smp_spinlock_init(&make_ref_lock, "make_ref");
     erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot");
-    erts_smp_atomic_init(&erts_dead_ports_ptr, (long)NULL);
+    erts_smp_atomic_init(&erts_dead_ports_ptr, (erts_aint_t) NULL);
 
     /*
      * bif_return_trap/1 is a hidden BIF that bifs that need to
@@ -4118,8 +4302,8 @@ void erts_init_bif(void)
 #else
     bif_return_trap_export.code[2] = 1;
 #endif
-    bif_return_trap_export.code[3] = (Eterm) em_apply_bif;
-    bif_return_trap_export.code[4] = (Eterm) &bif_return_trap;
+    bif_return_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    bif_return_trap_export.code[4] = (BeamInstr) &bif_return_trap;
 
     flush_monitor_message_trap = erts_export_put(am_erlang,
 						 am_flush_monitor_message,
@@ -4134,54 +4318,6 @@ void erts_init_bif(void)
     await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3);
 }
 
-BIF_RETTYPE blocking_read_file_1(BIF_ALIST_1)
-{
-    Eterm bin;
-    Eterm* hp;
-    byte *buff;
-    int i, buff_size;
-    FILE *file;
-    struct stat file_info;
-    char *filename = NULL;
-    size_t size;
- 
-    i = list_length(BIF_ARG_1);
-    if (i < 0) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    filename = erts_alloc(ERTS_ALC_T_TMP, i + 1);
-    if (intlist_to_buf(BIF_ARG_1, filename, i) != i)
-	erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
-    filename[i] = '\0';
- 
-    hp = HAlloc(BIF_P, 3);
- 
-    file = fopen(filename, "r");
-    if(file == NULL){
-	erts_free(ERTS_ALC_T_TMP, (void *) filename);
-	BIF_RET(TUPLE2(hp, am_error, am_nofile));
-    }
- 
-    stat(filename, &file_info);
-    erts_free(ERTS_ALC_T_TMP, (void *) filename);
-
-    buff_size = file_info.st_size;
-    buff = (byte *) erts_alloc_fnf(ERTS_ALC_T_TMP, buff_size);
-    if (!buff) {
-	fclose(file);
-	BIF_RET(TUPLE2(hp, am_error, am_allocator));
-    }
-    size = fread(buff, 1, buff_size, file);
-    fclose(file);
-    if (size < 0)
-	size = 0;
-    else if (size > buff_size)
-	size = (size_t) buff_size;
-    bin = new_binary(BIF_P, buff, (int) size);
-    erts_free(ERTS_ALC_T_TMP, (void *) buff);
- 
-    BIF_RET(TUPLE2(hp, am_ok, bin));
-}
 #ifdef HARDDEBUG
 /*
 You'll need this line in bif.tab to be able to use this debug bif
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 05e9b78c28..8faa09feb8 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -125,7 +125,7 @@ do {						\
 #define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc)		\
 do {							\
     (Proc)->arity = 0;					\
-    (Proc)->def_arg_reg[3] = (Eterm) (Trap->address);	\
+    *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address);	\
     (Proc)->freason = TRAP;				\
     (Ret) = THE_NON_VALUE;				\
 } while (0)
@@ -134,7 +134,7 @@ do {							\
 do {							\
     (Proc)->arity = 1;					\
     (Proc)->def_arg_reg[0] = (Eterm) (A0);		\
-    (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address);	\
+    *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address);	\
     (Proc)->freason = TRAP;				\
     (Ret) = THE_NON_VALUE;				\
 } while (0)
@@ -144,7 +144,7 @@ do {							\
     (Proc)->arity = 2;					\
     (Proc)->def_arg_reg[0] = (Eterm) (A0);		\
     (Proc)->def_arg_reg[1] = (Eterm) (A1);		\
-    (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address);	\
+    *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address);	\
     (Proc)->freason = TRAP;				\
     (Ret) = THE_NON_VALUE;				\
 } while (0)
@@ -155,14 +155,14 @@ do {							\
     (Proc)->def_arg_reg[0] = (Eterm) (A0);		\
     (Proc)->def_arg_reg[1] = (Eterm) (A1);		\
     (Proc)->def_arg_reg[2] = (Eterm) (A2);		\
-    (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address);	\
+    *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address);	\
     (Proc)->freason = TRAP;				\
     (Ret) = THE_NON_VALUE;				\
 } while (0)
 
 #define BIF_TRAP0(p, Trap_) do {			\
       (p)->arity = 0;					\
-      (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address);	\
+      *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address);	\
       (p)->freason = TRAP;				\
       return THE_NON_VALUE;				\
  } while(0)
@@ -170,7 +170,7 @@ do {							\
 #define BIF_TRAP1(Trap_, p, A0) do {			\
       (p)->arity = 1;					\
       (p)->def_arg_reg[0] = (A0);			\
-      (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address);	\
+      *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address);	\
       (p)->freason = TRAP;				\
       return THE_NON_VALUE;				\
  } while(0)
@@ -179,7 +179,7 @@ do {							\
       (p)->arity = 2;					\
       (p)->def_arg_reg[0] = (A0);			\
       (p)->def_arg_reg[1] = (A1);			\
-      (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address);	\
+      *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address);	\
       (p)->freason = TRAP;				\
       return THE_NON_VALUE;				\
  } while(0)
@@ -189,14 +189,20 @@ do {							\
       (p)->def_arg_reg[0] = (A0);			\
       (p)->def_arg_reg[1] = (A1);			\
       (p)->def_arg_reg[2] = (A2);			\
-      (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address);	\
+      *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address);	\
       (p)->freason = TRAP;				\
       return THE_NON_VALUE;				\
  } while(0)
 
 #define BIF_TRAP_CODE_PTR_0(p, Code_) do {		\
       (p)->arity = 0;					\
-      (p)->def_arg_reg[3] = (Eterm) (Code_);		\
+      *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_);	\
+      (p)->freason = TRAP;				\
+      return THE_NON_VALUE;				\
+ } while(0)
+
+#define BIF_TRAP_CODE_PTR_(p, Code_) do {		\
+      *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_);	\
       (p)->freason = TRAP;				\
       return THE_NON_VALUE;				\
  } while(0)
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index b6fa06354a..d9dd80fa8b 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -660,6 +660,7 @@ bif erts_debug:display/1
 bif 'erl.system.debug':display/1 ebif_erts_debug_display_1
 bif erts_debug:dist_ext_to_term/2
 bif 'erl.system.debug':dist_ext_to_term/2 ebif_erts_debug_dist_ext_to_term_2
+bif erts_debug:instructions/0
 
 #
 # Monitor testing bif's...
@@ -686,8 +687,6 @@ bif 'erl.system.code':make_stub_module/3	ebif_code_make_stub_module_3
 bif code:is_module_native/1
 bif 'erl.system.code':is_native/1	ebif_code_is_native_1 code_is_module_native_1
 
-bif erlang:blocking_read_file/1
-
 #
 # New Bifs in R9C.
 #
@@ -760,6 +759,50 @@ bif erlang:finish_after_on_load/2
 bif erlang:binary_to_term/2
 
 #
+# The binary match bifs (New in R14A - EEP9)
+#
+
+#
+# The searching/splitting/substituting thingies
+#
+ubif erlang:binary_part/2
+ubif erlang:binary_part/3
+
+bif binary:compile_pattern/1
+bif binary:match/2
+bif binary:match/3
+bif binary:matches/2
+bif binary:matches/3
+bif binary:longest_common_prefix/1
+bif binary:longest_common_suffix/1
+bif binary:first/1
+bif binary:last/1
+bif binary:at/2
+bif binary:part/2 binary_binary_part_2
+bif binary:part/3 binary_binary_part_3
+bif binary:bin_to_list/1
+bif binary:bin_to_list/2
+bif binary:bin_to_list/3
+bif binary:list_to_bin/1
+bif binary:copy/1
+bif binary:copy/2
+bif binary:referenced_byte_size/1
+bif binary:encode_unsigned/1
+bif binary:encode_unsigned/2
+bif binary:decode_unsigned/1
+bif binary:decode_unsigned/2
+
+bif erlang:nif_error/1
+bif erlang:nif_error/2
+
+#
+# Helpers for unicode filenames
+#
+bif prim_file:internal_name2native/1
+bif prim_file:internal_native2name/1
+bif prim_file:internal_normalize_utf8/1
+bif file:native_name_encoding/0
+#
 # Obsolete
 #
 
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 03c88da8c6..d18de9ae5d 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -1459,7 +1459,31 @@ Eterm uint_to_big(Uint x, Eterm *y)
     BIG_DIGIT(y, 0) = x;
     return make_big(y);
 }
+/*
+** convert UWord to bigint
+** (must only be used if x is to big to be stored as a small)
+** Allocation is tricky, the heap need has to be calculated
+** with the macro BIG_UWORD_HEAP_SIZE(x)
+*/
 
+Eterm uword_to_big(UWord x, Eterm *y)
+{
+#if HALFWORD_HEAP
+    Uint upper = x >> 32;
+    Uint lower = x & 0xFFFFFFFFUL;
+    if (upper == 0) {
+	*y = make_pos_bignum_header(1);
+    } else {
+	*y = make_pos_bignum_header(2);
+	BIG_DIGIT(y, 1) = upper;
+    }
+    BIG_DIGIT(y, 0) = lower;
+#else
+    *y = make_pos_bignum_header(1);
+    BIG_DIGIT(y, 0) = x;
+#endif
+    return make_big(y);
+}
 
 /*
 ** convert signed int to bigint
@@ -1480,19 +1504,19 @@ Eterm small_to_big(Sint x, Eterm *y)
 Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp)
 {
     Eterm *hp = *hpp;
-#ifdef ARCH_32
+#if defined(ARCH_32) || HALFWORD_HEAP
     if (x >= (((Uint64) 1) << 32)) {
 	*hp = make_pos_bignum_header(2);
 	BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff));
 	BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff));
-	*hpp += 2;
+	*hpp += 3;
     }
     else
 #endif
     {
 	*hp = make_pos_bignum_header(1);
 	BIG_DIGIT(hp, 0) = (Uint) x;
-	*hpp += 1;
+	*hpp += 2;
     }
     return make_big(hp);
 }
@@ -1507,7 +1531,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 	neg = 1;
 	x = -x;
     }
-#ifdef ARCH_32
+#if defined(ARCH_32) || HALFWORD_HEAP
     if (x >= (((Uint64) 1) << 32)) {
 	if (neg)
 	    *hp = make_neg_bignum_header(2);
@@ -1515,7 +1539,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 	    *hp = make_pos_bignum_header(2);
 	BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff));
 	BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff));
-	*hpp += 2;
+	*hpp += 3;
     }
     else
 #endif
@@ -1525,7 +1549,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 	else
 	    *hp = make_pos_bignum_header(1);
 	BIG_DIGIT(hp, 0) = (Uint) x;
-	*hpp += 1;
+	*hpp += 2;
     }
     return make_big(hp);
 }
@@ -1534,7 +1558,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 ** Convert a bignum to a double float
 */
 int
-big_to_double(Eterm x, double* resp)
+big_to_double(Wterm x, double* resp)
 {
     double d = 0.0;
     Eterm* xp = big_val(x);
@@ -1564,7 +1588,7 @@ big_to_double(Eterm x, double* resp)
 /*
  ** Estimate the number of decimal digits (include sign)
  */
-int big_decimal_estimate(Eterm x)
+int big_decimal_estimate(Wterm x)
 {
     Eterm* xp = big_val(x);
     int lg = I_lg(BIG_V(xp), BIG_SIZE(xp));
@@ -1578,7 +1602,7 @@ int big_decimal_estimate(Eterm x)
 ** Convert a bignum into a string of decimal numbers
 */
 
-static void write_big(Eterm x, void (*write_func)(void *, char), void *arg)
+static void write_big(Wterm x, void (*write_func)(void *, char), void *arg)
 {
     Eterm* xp = big_val(x);
     ErtsDigit* dx = BIG_V(xp);
@@ -1657,7 +1681,7 @@ write_string(void *arg, char c)
     *(--(*((char **) arg))) = c;
 }
 
-char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz)
+char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz)
 {
     char *big_str = buf + buf_sz - 1;
     *big_str = '\0';
@@ -1701,7 +1725,7 @@ static Eterm big_norm(Eterm *x, dsize_t xl, short sign)
 /*
 ** Compare bignums
 */
-int big_comp(Eterm x, Eterm y)
+int big_comp(Wterm x, Wterm y)
 {
     Eterm* xp = big_val(x);
     Eterm* yp = big_val(y);
@@ -1854,6 +1878,87 @@ term_to_Uint(Eterm term, Uint *up)
     }
 }
 
+int
+term_to_UWord(Eterm term, UWord *up)
+{
+#if SIZEOF_VOID_P == ERTS_SIZEOF_ETERM
+    return term_to_Uint(term,up);
+#else
+    if (is_small(term)) {
+	Sint i = signed_val(term);
+	if (i < 0) {
+	    *up = BADARG;
+	    return 0;
+	}
+	*up = (UWord) i;
+	return 1;
+    } else if (is_big(term)) {
+	ErtsDigit* xr = big_v(term);
+	dsize_t  xl = big_size(term);
+	UWord uval = 0;
+	int n = 0;
+
+	if (big_sign(term)) {
+	    *up = BADARG;
+	    return 0;
+	} else if (xl*D_EXP > sizeof(UWord)*8) {
+	    *up = SYSTEM_LIMIT;
+	    return 0;
+	}
+	while (xl-- > 0) {
+	    uval |= ((UWord)(*xr++)) << n;
+	    n += D_EXP;
+	}
+	*up = uval;
+	return 1;
+    } else {
+	*up = BADARG;
+	return 0;
+    }
+#endif
+}
+
+int
+term_to_Uint64(Eterm term, Uint64 *up)
+{
+#if SIZEOF_VOID_P == 8
+    return term_to_UWord(term,up);
+#else
+    if (is_small(term)) {
+	Sint i = signed_val(term);
+	if (i < 0) {
+	    *up = BADARG;
+	    return 0;
+	}
+	*up = (Uint64) i;
+	return 1;
+    } else if (is_big(term)) {
+	ErtsDigit* xr = big_v(term);
+	dsize_t  xl = big_size(term);
+	Uint64 uval = 0;
+	int n = 0;
+
+	if (big_sign(term)) {
+	    *up = BADARG;
+	    return 0;
+	} else if (xl*D_EXP > sizeof(Uint64)*8) {
+	    *up = SYSTEM_LIMIT;
+	    return 0;
+	}
+	while (xl-- > 0) {
+	    uval |= ((Uint64)(*xr++)) << n;
+	    n += D_EXP;
+	}
+	*up = uval;
+	return 1;
+    } else {
+	*up = BADARG;
+	return 0;
+    }
+#endif
+}
+
+
 int term_to_Sint(Eterm term, Sint *sp)
 {
     if (is_small(term)) {
@@ -1888,6 +1993,47 @@ int term_to_Sint(Eterm term, Sint *sp)
     }
 }
 
+#if HAVE_INT64
+int term_to_Sint64(Eterm term, Sint64 *sp)
+{
+#if ERTS_SIZEOF_ETERM == 8
+    return term_to_Sint(term, sp);
+#else
+    if (is_small(term)) {
+	*sp = signed_val(term);
+	return 1;
+    } else if (is_big(term)) {
+	ErtsDigit* xr = big_v(term);
+	dsize_t xl = big_size(term);
+	int sign = big_sign(term);
+	Uint64 uval = 0;
+	int n = 0;
+
+	if (xl*D_EXP > sizeof(Uint64)*8) {
+	    return 0;
+	}
+	while (xl-- > 0) {
+	    uval |= ((Uint64)(*xr++)) << n;
+	    n += D_EXP;
+	}
+	if (sign) {
+	    uval = -uval;
+	    if ((Sint64)uval > 0)
+		return 0;
+	} else {
+	    if ((Sint64)uval < 0)
+		return 0;
+	}
+	*sp = uval;
+	return 1;
+    } else {
+	return 0;
+    }
+#endif
+}
+#endif /* HAVE_INT64 */
+
+
 /*
 ** Add and subtract
 */
@@ -1914,7 +2060,7 @@ static Eterm B_plus_minus(ErtsDigit *x, dsize_t xl, short xsgn,
 /*
 ** Add bignums
 */
-Eterm big_plus(Eterm x, Eterm y, Eterm *r)
+Eterm big_plus(Wterm x, Wterm y, Eterm *r)
 {
     Eterm* xp = big_val(x);
     Eterm* yp = big_val(y);
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index b8e38d482c..2afc37004f 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -34,7 +34,7 @@
 
 typedef Uint     ErtsDigit;
 
-#if (SIZEOF_VOID_P == 4) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8)
+#if ((SIZEOF_VOID_P == 4) || HALFWORD_HEAP) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8)
 /* Assume 32-bit machine with long long support */
 typedef Uint64   ErtsDoubleDigit;
 typedef Uint16   ErtsHalfDigit;
@@ -58,7 +58,7 @@ typedef Uint32   ErtsHalfDigit;
 
 typedef Uint  dsize_t;	 /* Vector size type */
 
-#define D_EXP (SIZEOF_VOID_P*8)
+#define D_EXP (ERTS_SIZEOF_ETERM*8)
 #define D_MASK     ((ErtsDigit)(-1))      /* D_BASE-1 */
 
 /* macros for bignum objects */
@@ -88,7 +88,13 @@ typedef Uint  dsize_t;	 /* Vector size type */
 
 #define BIG_UINT_HEAP_SIZE (1 + 1)	/* always, since sizeof(Uint) <= sizeof(Eterm) */
 
-#ifdef ARCH_32
+#if HALFWORD_HEAP
+#define BIG_UWORD_HEAP_SIZE(UW) (((UW) >> (sizeof(Uint) * 8)) ? 3 : 2)
+#else
+#define BIG_UWORD_HEAP_SIZE(UW) BIG_UINT_HEAP_SIZE
+#endif
+
+#if defined(ARCH_32) || HALFWORD_HEAP
 
 #define ERTS_UINT64_BIG_HEAP_SIZE__(X) \
   ((X) >= (((Uint64) 1) << 32) ? (1 + 2) : (1 + 1))
@@ -108,13 +114,13 @@ typedef Uint  dsize_t;	 /* Vector size type */
 
 #endif
 
-int big_decimal_estimate(Eterm);
+int big_decimal_estimate(Wterm);
 Eterm erts_big_to_list(Eterm, Eterm**);
-char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz);
+char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz);
 
 Eterm small_times(Sint, Sint, Eterm*);
 
-Eterm big_plus(Eterm, Eterm, Eterm*);
+Eterm big_plus(Wterm, Wterm, Eterm*);
 Eterm big_minus(Eterm, Eterm, Eterm*);
 Eterm big_times(Eterm, Eterm, Eterm*);
 Eterm big_div(Eterm, Eterm, Eterm*);
@@ -131,11 +137,12 @@ Eterm big_bxor(Eterm, Eterm, Eterm*);
 Eterm big_bnot(Eterm, Eterm*);
 
 Eterm big_lshift(Eterm, Sint, Eterm*);
-int big_comp (Eterm, Eterm);
+int big_comp (Wterm, Wterm);
 int big_ucomp (Eterm, Eterm);
-int big_to_double(Eterm x, double* resp);
+int big_to_double(Wterm x, double* resp);
 Eterm small_to_big(Sint, Eterm*);
 Eterm uint_to_big(Uint, Eterm*);
+Eterm uword_to_big(UWord, Eterm*);
 Eterm erts_make_integer(Uint, Process *);
 
 dsize_t big_bytes(Eterm);
@@ -143,7 +150,12 @@ Eterm bytes_to_big(byte*, dsize_t, int, Eterm*);
 byte* big_to_bytes(Eterm, byte*);
 
 int term_to_Uint(Eterm, Uint*);
+int term_to_UWord(Eterm, UWord*);
 int term_to_Sint(Eterm, Sint*);
+#if HAVE_INT64
+int term_to_Uint64(Eterm, Uint64*);
+int term_to_Sint64(Eterm, Sint64*);
+#endif
 
 Uint32 big_to_uint32(Eterm b);
 int term_equals_2pow32(Eterm);
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 08c64610a2..1fb39c6c67 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -32,17 +32,17 @@
 #include "erl_bits.h"
 
 #ifdef DEBUG
-static int list_to_bitstr_buf(Eterm obj, char* buf, int len);
+static int list_to_bitstr_buf(Eterm obj, char* buf, Uint len);
 #else
 static int list_to_bitstr_buf(Eterm obj, char* buf);
 #endif
-static Sint bitstr_list_len(Eterm obj);
+static int bitstr_list_len(Eterm obj, Uint* num_bytes);
 
 void
 erts_init_binary(void)
 {
     /* Verify Binary alignment... */
-    if ((((Uint) &((Binary *) 0)->orig_bytes[0]) % ((Uint) 8)) != 0) {
+    if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) {
 	/* I assume that any compiler should be able to optimize this
 	   away. If not, this test is not very expensive... */
 	erl_exit(ERTS_ABORT_EXIT,
@@ -56,7 +56,7 @@ erts_init_binary(void)
  */
 
 Eterm
-new_binary(Process *p, byte *buf, int len)
+new_binary(Process *p, byte *buf, Uint len)
 {
     ProcBin* pb;
     Binary* bptr;
@@ -88,8 +88,8 @@ new_binary(Process *p, byte *buf, int len)
     pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
     pb->thing_word = HEADER_PROC_BIN;
     pb->size = len;
-    pb->next = MSO(p).mso;
-    MSO(p).mso = pb;
+    pb->next = MSO(p).first;
+    MSO(p).first = (struct erl_off_heap_header*)pb;
     pb->val = bptr;
     pb->bytes = (byte*) bptr->orig_bytes;
     pb->flags = 0;
@@ -97,7 +97,7 @@ new_binary(Process *p, byte *buf, int len)
     /*
      * Miscellanous updates. Return the tagged binary.
      */
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     return make_binary(pb);
 }
 
@@ -127,8 +127,8 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len)
     pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
     pb->thing_word = HEADER_PROC_BIN;
     pb->size = len;
-    pb->next = MSO(p).mso;
-    MSO(p).mso = pb;
+    pb->next = MSO(p).first;
+    MSO(p).first = (struct erl_off_heap_header*)pb;
     pb->val = bptr;
     pb->bytes = (byte*) bptr->orig_bytes;
     pb->flags = 0;
@@ -136,7 +136,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len)
     /*
      * Miscellanous updates. Return the tagged binary.
      */
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     return make_binary(pb);
 }
 
@@ -180,7 +180,7 @@ erts_realloc_binary(Eterm bin, size_t size)
 }
 
 byte*
-erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra)
+erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, ErtsAlcType_t allocator, unsigned extra)
 {
     byte* bytes;
     Eterm* real_bin;
@@ -208,7 +208,7 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra)
 	bytes = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + offs;
     }
     if (bit_offs) {
-	byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, byte_size + extra);
+	byte* buf = (byte *) erts_alloc(allocator, byte_size + extra);
 	*base_ptr = buf;
 	buf += extra;
 	erts_copy_bits(bytes, bit_offs, 1, buf, 0, 1, byte_size*8);	
@@ -217,8 +217,8 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra)
     return bytes;
 }
 
-static Eterm
-bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs)
+Eterm
+erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs)
 {
     if (bitoffs == 0) {
 	while (size) {
@@ -263,7 +263,7 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1)
 	Eterm* hp = HAlloc(BIF_P, 2 * size);
 	byte* bytes = binary_bytes(real_bin)+offset;
 
-	BIF_RET(bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
+	BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
     }
 
     error:
@@ -295,7 +295,7 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3)
     }
     i = stop-start+1;
     hp = HAlloc(BIF_P, 2*i);
-    BIF_RET(bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs));
+    BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs));
     
     error:
 	BIF_ERROR(BIF_P, BADARG);
@@ -339,36 +339,50 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
 	previous = CONS(hp, make_binary(last), previous);
 	hp += 2;
     }
-    BIF_RET(bin_bytes_to_list(previous, hp, bytes, size, bitoffs));
+    BIF_RET(erts_bin_bytes_to_list(previous, hp, bytes, size, bitoffs));
 }
 
 
 /* Turn a possibly deep list of ints (and binaries) into */
 /* One large binary object                               */
 
-BIF_RETTYPE list_to_binary_1(BIF_ALIST_1)
+/*
+ * This bif also exists in the binary module, under the name
+ * binary:list_to_bin/1, why it's divided into interface and
+ * implementation. Also the backend for iolist_to_binary_1.
+ */
+
+BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg)
 {
     Eterm bin;
-    int i;
+    Uint size;
     int offset;
     byte* bytes;
-    if (is_nil(BIF_ARG_1)) {
-	BIF_RET(new_binary(BIF_P,(byte*)"",0));
+
+    if (is_nil(arg)) {
+	BIF_RET(new_binary(p,(byte*)"",0));
     }
-    if (is_not_list(BIF_ARG_1)) {
+    if (is_not_list(arg)) {
 	goto error;
     }
-    if ((i = io_list_len(BIF_ARG_1)) < 0) {
-	goto error;
+    switch (erts_iolist_size(arg, &size)) {
+    case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT);
+    case ERTS_IOLIST_TYPE: goto error;
+    default: ;
     }
-    bin = new_binary(BIF_P, (byte *)NULL, i);
+    bin = new_binary(p, (byte *)NULL, size);
     bytes = binary_bytes(bin);
-    offset = io_list_to_buf(BIF_ARG_1, (char*) bytes, i);
+    offset = io_list_to_buf(arg, (char*) bytes, size);
     ASSERT(offset == 0);
     BIF_RET(bin);
     
-    error:
-	BIF_ERROR(BIF_P, BADARG);
+ error:
+    BIF_ERROR(p, BADARG);
+}
+
+BIF_RETTYPE list_to_binary_1(BIF_ALIST_1)
+{
+    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1);
 }
 
 /* Turn a possibly deep list of ints (and binaries) into */
@@ -376,37 +390,17 @@ BIF_RETTYPE list_to_binary_1(BIF_ALIST_1)
 
 BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1)
 {
-    Eterm bin;
-    int i;
-    int offset;
-    byte* bytes;
-
     if (is_binary(BIF_ARG_1)) {
 	BIF_RET(BIF_ARG_1);
     }
-    if (is_nil(BIF_ARG_1)) {
-	BIF_RET(new_binary(BIF_P,(byte*)"",0));
-    }
-    if (is_not_list(BIF_ARG_1)) {
-	goto error;
-    }
-    if ((i = io_list_len(BIF_ARG_1)) < 0) {
-	goto error;
-    }
-    bin = new_binary(BIF_P, (byte *)NULL, i);
-    bytes = binary_bytes(bin);
-    offset = io_list_to_buf(BIF_ARG_1, (char*) bytes, i);
-    ASSERT(offset == 0);
-    BIF_RET(bin);
-    
-    error:
-	BIF_ERROR(BIF_P, BADARG);
+    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1);
 }
 
 BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1)
 {
     Eterm bin;
-    int i,offset;
+    Uint sz;
+    int offset;
     byte* bytes;
     ErlSubBin* sb1; 
     Eterm* hp;
@@ -415,15 +409,19 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1)
 	BIF_RET(new_binary(BIF_P,(byte*)"",0));
     }
     if (is_not_list(BIF_ARG_1)) {
-	goto error;
+    error:
+	BIF_ERROR(BIF_P, BADARG);
     }
-    if ((i = bitstr_list_len(BIF_ARG_1)) < 0) {
+    switch (bitstr_list_len(BIF_ARG_1, &sz)) {
+    case ERTS_IOLIST_TYPE:
 	goto error;
+    case ERTS_IOLIST_OVERFLOW:
+	BIF_ERROR(BIF_P, SYSTEM_LIMIT);
     }
-    bin = new_binary(BIF_P, (byte *)NULL, i);
+    bin = new_binary(BIF_P, (byte *)NULL, sz);
     bytes = binary_bytes(bin);
 #ifdef DEBUG
-    offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, i);
+    offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, sz);
 #else
     offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes);
 #endif
@@ -432,20 +430,16 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1)
 	hp = HAlloc(BIF_P, ERL_SUB_BIN_SIZE);
 	sb1 = (ErlSubBin *) hp;
 	sb1->thing_word = HEADER_SUB_BIN;
-	sb1->size = i-1;
+	sb1->size = sz-1;
 	sb1->offs = 0;
 	sb1->orig = bin;
 	sb1->bitoffs = 0;
 	sb1->bitsize = offset;
 	sb1->is_writable = 0;
-	hp += ERL_SUB_BIN_SIZE;
 	bin = make_binary(sb1);
     }
     
     BIF_RET(bin);
-    
-    error:
-	BIF_ERROR(BIF_P, BADARG);
 }
 
 BIF_RETTYPE split_binary_2(BIF_ALIST_2)
@@ -497,16 +491,6 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2)
 	BIF_ERROR(BIF_P, BADARG);
 }
 
-void
-erts_cleanup_mso(ProcBin* pb)
-{
-    while (pb != NULL) {
-	ProcBin* next = pb->next;
-	if (erts_refc_dectest(&pb->val->refc, 0) == 0)
-	    erts_bin_free(pb->val);
-	pb = next;
-    }
-}
 
 /*
  * Local functions.
@@ -519,7 +503,7 @@ erts_cleanup_mso(ProcBin* pb)
  */
 static int
 #ifdef DEBUG
-list_to_bitstr_buf(Eterm obj, char* buf, int len)
+list_to_bitstr_buf(Eterm obj, char* buf, Uint len)
 #else
 list_to_bitstr_buf(Eterm obj, char* buf)
 #endif
@@ -622,8 +606,8 @@ list_to_bitstr_buf(Eterm obj, char* buf)
     return offset;
 }
 
-static Sint
-bitstr_list_len(Eterm obj)
+static int
+bitstr_list_len(Eterm obj, Uint* num_bytes)
 {
     Eterm* objp;
     Uint len = 0;
@@ -631,6 +615,26 @@ bitstr_list_len(Eterm obj)
     DECLARE_ESTACK(s);
     goto L_again;
 
+#define SAFE_ADD(Var, Val)			\
+    do {					\
+        Uint valvar = (Val);			\
+	Var += valvar;				\
+	if (Var < valvar) {			\
+	    goto L_overflow_error;		\
+	}					\
+    } while (0)
+
+#define SAFE_ADD_BITSIZE(Var, Bin)					\
+    do {								\
+	if (*binary_val(Bin) == HEADER_SUB_BIN) {			\
+            Uint valvar = ((ErlSubBin *) binary_val(Bin))->bitsize;	\
+	    Var += valvar;						\
+	    if (Var < valvar) {						\
+	         goto L_overflow_error;					\
+	    }								\
+        }								\
+    } while (0)
+
     while (!ESTACK_ISEMPTY(s)) {
 	obj = ESTACK_POP(s);
     L_again:
@@ -641,9 +645,12 @@ bitstr_list_len(Eterm obj)
 	    obj = CAR(objp);
 	    if (is_byte(obj)) {
 		len++;
+		if (len == 0) {
+		    goto L_overflow_error;
+		}
 	    } else if (is_binary(obj)) {
-		len += binary_size(obj);
-		offs += binary_bitsize(obj);
+		SAFE_ADD(len, binary_size(obj));
+		SAFE_ADD_BITSIZE(offs, obj);
 	    } else if (is_list(obj)) {
 		ESTACK_PUSH(s, CDR(objp));
 		goto L_iter_list; /* on head */
@@ -655,23 +662,44 @@ bitstr_list_len(Eterm obj)
 	    if (is_list(obj))
 		goto L_iter_list; /* on tail */
 	    else if (is_binary(obj)) {
-		len += binary_size(obj);
-		offs += binary_bitsize(obj);
+		SAFE_ADD(len, binary_size(obj));
+		SAFE_ADD_BITSIZE(offs, obj);
 	    } else if (is_not_nil(obj)) {
 		goto L_type_error;
 	    }
 	} else if (is_binary(obj)) {
-	    len += binary_size(obj);
-	    offs += binary_bitsize(obj);
+	    SAFE_ADD(len, binary_size(obj));
+	    SAFE_ADD_BITSIZE(offs, obj);
 	} else if (is_not_nil(obj)) {
 	    goto L_type_error;
 	}
     }
+#undef SAFE_ADD
+#undef SAFE_ADD_BITSIZE
 
     DESTROY_ESTACK(s);
-    return (Sint) (len + (offs/8) + ((offs % 8) != 0));
+
+    /*
+     * Make sure that the number of bits in the bitstring will fit
+     * in an Uint to ensure that the binary can be matched using
+     * the binary syntax.
+     */
+    if (len << 3 < len) {
+	goto L_overflow_error;
+    }
+    len += (offs >> 3) + ((offs & 7) != 0);
+    if (len << 3 < len) {
+	goto L_overflow_error;
+    }
+    *num_bytes = len;
+    return ERTS_IOLIST_OK;
 
  L_type_error:
     DESTROY_ESTACK(s);
-    return (Sint) -1;
+    return ERTS_IOLIST_TYPE;
+
+ L_overflow_error:
+    DESTROY_ESTACK(s);
+    return ERTS_IOLIST_OVERFLOW;
 }
+
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index cc69977b79..b8889e6206 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /* This File contains functions which are called if a user hits ^C */
@@ -38,10 +38,6 @@
 #include "erl_instrument.h"
 #include "erl_bif_timer.h"
 
-#ifdef _OSE_
-#include "time.h"
-#endif
-
 /* Forward declarations -- should really appear somewhere else */
 static void process_killer(void);
 void do_break(void);
@@ -102,7 +98,7 @@ process_killer(void)
 		switch(j) {
 		case 'k':
 		    if (rp->status == P_WAITING) {
-			Uint32 rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
+			ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
 			erts_smp_proc_inc_refc(rp);
 			erts_smp_proc_lock(rp, rp_locks);
 			(void) erts_send_exit_signal(NULL,
@@ -262,17 +258,15 @@ print_process_info(int to, void *to_arg, Process *p)
     }
 
     {
-       long s = 0;
        int frags = 0;
        ErlHeapFragment *m = p->mbuf;
        while (m != NULL) {
 	   frags++;
-	   s += m->size;
 	   m = m->next;
        }
        erts_print(to, to_arg, "Number of heap fragments: %d\n", frags);
     }
-    erts_print(to, to_arg, "Heap fragment data: %bpu\n", MBUF_SIZE(p));
+    erts_print(to, to_arg, "Heap fragment data: %beu\n", MBUF_SIZE(p));
 
     scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
     if (scb) {
@@ -319,15 +313,14 @@ print_process_info(int to, void *to_arg, Process *p)
     }
     
     /* print the number of reductions etc */
-    erts_print(to, to_arg, "Reductions: %bpu\n", p->reds);
+    erts_print(to, to_arg, "Reductions: %beu\n", p->reds);
 
-    erts_print(to, to_arg, "Stack+heap: %bpu\n", p->heap_sz);
+    erts_print(to, to_arg, "Stack+heap: %beu\n", p->heap_sz);
     erts_print(to, to_arg, "OldHeap: %bpu\n",
-               (OLD_HEAP(p) == NULL) ? 0 :
-               (unsigned)(OLD_HEND(p) - OLD_HEAP(p)) );
+               (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HEAP(p)) );
     erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop));
     erts_print(to, to_arg, "OldHeap unused: %bpu\n",
-	       (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HEAP(p)) );
+	       (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) );
 
     if (garbing) {
 	print_garb_info(to, to_arg, p);
@@ -381,7 +374,7 @@ loaded(int to, void *to_arg)
     int i;
     int old = 0;
     int cur = 0;
-    Eterm* code;
+    BeamInstr* code;
 
     /*
      * Calculate and print totals.
@@ -564,7 +557,7 @@ do_break(void)
 #endif
 #ifdef DEBUG
 	case 't':
-	    p_slpq();
+	    erts_p_slpq();
 	    return;
 	case 'b':
 	    bin_check();
@@ -617,29 +610,29 @@ static void
 bin_check(void)
 {
     Process  *rp;
-    ProcBin *bp;
-    int i, printed;
+    struct erl_off_heap_header* hdr;
+    int i, printed = 0;
 
     for (i=0; i < erts_max_processes; i++) {
 	if ((rp = process_tab[i]) == NULL)
 	    continue;
-	if (!(bp = rp->off_heap.mso))
-	    continue;
-	printed = 0;
-	while (bp) {
-	    if (printed == 0) {
-		erts_printf("Process %T holding binary data \n", rp->id);
-		printed = 1;
+	for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) {
+	    if (hdr->thing_word == HEADER_PROC_BIN) {
+		ProcBin *bp = (ProcBin*) hdr;
+		if (!printed) {
+		    erts_printf("Process %T holding binary data \n", rp->id);
+		    printed = 1;
+		}
+		erts_printf("%p orig_size: %bpd, norefs = %bpd\n",
+			    bp->val, 
+			    bp->val->orig_size, 
+			    erts_smp_atomic_read(&bp->val->refc));
 	    }
-	    erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n",
-		       (unsigned long)bp->val, 
-		       (long)bp->val->orig_size, 
-		       erts_smp_atomic_read(&bp->val->refc));
-
-	    bp = bp->next;
 	}
-	if (printed == 1)
+	if (printed) {
 	    erts_printf("--------------------------------------\n");
+	    printed = 0;
+	}
     }
     /* db_bin_check() has to be rewritten for the AVL trees... */
     /*db_bin_check();*/ 
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 0a5050b1fe..90201f3a90 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -37,6 +37,8 @@ MA_STACK_DECLARE(dst);
 MA_STACK_DECLARE(offset);
 #endif
 
+static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*);
+
 void
 init_copy(void)
 {
@@ -70,8 +72,11 @@ copy_object(Eterm obj, Process* to)
  * Return the "flat" size of the object.
  */
 
-Uint
-size_object(Eterm obj)
+#if HALFWORD_HEAP
+Uint size_object_rel(Eterm obj, Eterm* base)
+#else
+Uint size_object(Eterm obj)
+#endif
 {
     Uint sum = 0;
     Eterm* ptr;
@@ -82,24 +87,24 @@ size_object(Eterm obj)
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST:
 	    sum += 2;
-	    ptr = list_val(obj);
+	    ptr = list_val_rel(obj,base);
 	    obj = *ptr++;
 	    if (!IS_CONST(obj)) {
 		ESTACK_PUSH(s, obj);
-	    }
+	    }	    
 	    obj = *ptr;
 	    break;
 	case TAG_PRIMARY_BOXED:
 	    {
-		Eterm hdr = *boxed_val(obj);
+		Eterm hdr = *boxed_val_rel(obj,base);
 		ASSERT(is_header(hdr));
 		switch (hdr & _TAG_HEADER_MASK) {
 		case ARITYVAL_SUBTAG:
-		    ptr = tuple_val(obj);
+		    ptr = tuple_val_rel(obj,base);
 		    arity = header_arity(hdr);
 		    sum += arity + 1;
 		    if (arity == 0) { /* Empty tuple -- unusual. */
-			goto size_common;
+			goto pop_next;
 		    }
 		    while (arity-- > 1) {
 			obj = *++ptr;
@@ -111,11 +116,10 @@ size_object(Eterm obj)
 		    break;
 		case FUN_SUBTAG:
 		    {
-			Eterm* bptr = fun_val(obj);
+			Eterm* bptr = fun_val_rel(obj,base);
 			ErlFunThing* funp = (ErlFunThing *) bptr;
 			unsigned eterms = 1 /* creator */ + funp->num_free;
 			unsigned sz = thing_arityval(hdr);
-
 			sum += 1 /* header */ + sz + eterms;
 			bptr += 1 /* header */ + sz;
 			while (eterms-- > 1) {
@@ -135,7 +139,7 @@ size_object(Eterm obj)
 			Uint bitoffs;
 			Uint extra_bytes;
 			Eterm hdr;
-			ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize);
+			ERTS_GET_REAL_BIN_REL(obj, real_bin, offset, bitoffs, bitsize, base);
 			if ((bitsize + bitoffs) > 8) {
 			    sum += ERL_SUB_BIN_SIZE;
 			    extra_bytes = 2;
@@ -145,13 +149,13 @@ size_object(Eterm obj)
 			} else {
 			    extra_bytes = 0;
 			}
-			hdr = *binary_val(real_bin);
+			hdr = *binary_val_rel(real_bin,base);
 			if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) {
 			    sum += PROC_BIN_SIZE;
 			} else {
-			    sum += heap_bin_size(binary_size(obj)+extra_bytes);
+			    sum += heap_bin_size(binary_size_rel(obj,base)+extra_bytes);
 			}
-			goto size_common;
+			goto pop_next;
 		    }
 		    break;
 		case BIN_MATCHSTATE_SUBTAG:
@@ -159,18 +163,12 @@ size_object(Eterm obj)
 			     "size_object: matchstate term not allowed");
 		default:
 		    sum += thing_arityval(hdr) + 1;
-		    /* Fall through */
-		size_common:
-		    if (ESTACK_ISEMPTY(s)) {
-			DESTROY_ESTACK(s);
-			return sum;
-		    }
-		    obj = ESTACK_POP(s);
-		    break;
+		    goto pop_next;
 		}
 	    }
 	    break;
 	case TAG_PRIMARY_IMMED1:
+	pop_next:
 	    if (ESTACK_ISEMPTY(s)) {
 		DESTROY_ESTACK(s);
 		return sum;
@@ -186,8 +184,12 @@ size_object(Eterm obj)
 /*
  *  Copy a structure to a heap.
  */
-Eterm
-copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#if HALFWORD_HEAP
+Eterm copy_struct_rel(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap,
+                      Eterm* src_base, Eterm* dst_base)
+#else
+Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#endif
 {
     char* hstart;
     Uint hsize;
@@ -219,7 +221,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 
     /* Copy the object onto the heap */
     switch (primary_tag(obj)) {
-    case TAG_PRIMARY_LIST: argp = &res; goto L_copy_list;
+    case TAG_PRIMARY_LIST:
+	argp = &res;
+	objp = list_val_rel(obj,src_base);
+	goto L_copy_list;
     case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed;
     default:
 	erl_exit(ERTS_ABORT_EXIT,
@@ -236,32 +241,46 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    hp++;
 	    break;
 	case TAG_PRIMARY_LIST:
-	    objp = list_val(obj);
+	    objp = list_val_rel(obj,src_base);
+	#if !HALFWORD_HEAP || defined(DEBUG)
 	    if (in_area(objp,hstart,hsize)) {
+		ASSERT(!HALFWORD_HEAP);
 		hp++;
 		break;
 	    }
+	#endif
 	    argp = hp++;
 	    /* Fall through */
 
 	L_copy_list:
 	    tailp = argp;
-	    while (is_list(obj)) {
-		objp = list_val(obj);
+	    for (;;) {
 		tp = tailp;
-		elem = *objp;
+		elem = CAR(objp);
 		if (IS_CONST(elem)) {
-		    *(hbot-2) = elem;
-		    tailp = hbot-1;
 		    hbot -= 2;
+		    CAR(hbot) = elem;
+		    tailp = &CDR(hbot);
 		}
 		else {
-		    *htop = elem;
-		    tailp = htop+1;
+		    CAR(htop) = elem;
+		#if HALFWORD_HEAP
+		    CDR(htop) = CDR(objp);
+		    *tailp = make_list_rel(htop,dst_base);
 		    htop += 2;
+		    goto L_copy;
+		#else
+		    tailp = &CDR(htop);
+		    htop += 2;
+		#endif
+		}
+		ASSERT(!HALFWORD_HEAP || tp < hp || tp >= hbot);
+		*tp = make_list_rel(tailp - 1, dst_base);
+		obj = CDR(objp);
+		if (!is_list(obj)) {
+		    break;
 		}
-		*tp = make_list(tailp - 1);
-		obj = *(objp+1);
+		objp = list_val_rel(obj,src_base);
 	    }
 	    switch (primary_tag(obj)) {
 	    case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy;
@@ -273,21 +292,24 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    }
 	    
 	case TAG_PRIMARY_BOXED:
-	    if (in_area(boxed_val(obj),hstart,hsize)) {
+	#if !HALFWORD_HEAP || defined(DEBUG)
+	    if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) {
+		ASSERT(!HALFWORD_HEAP);
 		hp++;
 		break;
 	    }
+	#endif
 	    argp = hp++;
 
 	L_copy_boxed:
-	    objp = boxed_val(obj);
+	    objp = boxed_val_rel(obj, src_base);
 	    hdr = *objp;
 	    switch (hdr & _TAG_HEADER_MASK) {
 	    case ARITYVAL_SUBTAG:
 		{
 		    int const_flag = 1; /* assume constant tuple */
 		    i = arityval(hdr);
-		    *argp = make_tuple(htop);
+		    *argp = make_tuple_rel(htop, dst_base);
 		    tp = htop;	/* tp is pointer to new arity value */
 		    *htop++ = *objp++; /* copy arity value */
 		    while (i--) {
@@ -316,13 +338,13 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    while (i--)  {
 			*tp++ = *objp++;
 		    }
-		    *argp = make_binary(hbot);
+		    *argp = make_binary_rel(hbot, dst_base);
 		    pb = (ProcBin*) hbot;
 		    erts_refc_inc(&pb->val->refc, 2);
-		    pb->next = off_heap->mso;
+		    pb->next = off_heap->first;
 		    pb->flags = 0;
-		    off_heap->mso = pb;
-		    off_heap->overhead += pb->size / sizeof(Eterm);
+		    off_heap->first = (struct erl_off_heap_header*) pb;
+		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
 		break;
 	    case SUB_BINARY_SUBTAG:
@@ -343,7 +365,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			extra_bytes = 0;
 		    } 
 		    real_size = size+extra_bytes;
-		    objp = binary_val(real_bin);
+		    objp = binary_val_rel(real_bin,src_base);
 		    if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) {
 			ErlHeapBin* from = (ErlHeapBin *) objp;
 			ErlHeapBin* to;
@@ -368,12 +390,12 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			to->val = from->val;
 			erts_refc_inc(&to->val->refc, 2);
 			to->bytes = from->bytes + offset;
-			to->next = off_heap->mso;
+			to->next = off_heap->first;
 			to->flags = 0;
-			off_heap->mso = to;
-			off_heap->overhead += to->size / sizeof(Eterm);
+			off_heap->first = (struct erl_off_heap_header*) to;
+			OH_OVERHEAD(off_heap, to->size / sizeof(Eterm));
 		    }
-		    *argp = make_binary(hbot);
+		    *argp = make_binary_rel(hbot, dst_base);
 		    if (extra_bytes != 0) {
 			ErlSubBin* res;
 			hbot -= ERL_SUB_BIN_SIZE;
@@ -385,7 +407,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			res->offs = 0;
 			res->is_writable = 0;
 			res->orig = *argp;
-			*argp = make_binary(hbot);
+			*argp = make_binary_rel(hbot, dst_base);
 		    }
 		    break;
 		}
@@ -401,11 +423,11 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    }
 #ifndef HYBRID /* FIND ME! */
 		    funp = (ErlFunThing *) tp;
-		    funp->next = off_heap->funs;
-		    off_heap->funs = funp;
+		    funp->next = off_heap->first;
+		    off_heap->first = (struct erl_off_heap_header*) funp;
 		    erts_refc_inc(&funp->fe->refc, 2);
 #endif
-		    *argp = make_fun(tp);
+		    *argp = make_fun_rel(tp, dst_base);
 		}
 		break;
 	    case EXTERNAL_PID_SUBTAG:
@@ -421,11 +443,11 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    *htop++ = *objp++;
 		  }
 
-		  etp->next = off_heap->externals;
-		  off_heap->externals = etp;
+		  etp->next = off_heap->first;
+		  off_heap->first = (struct erl_off_heap_header*)etp;
 		  erts_refc_inc(&etp->node->refc, 2);
 
-		  *argp = make_external(tp);
+		  *argp = make_external_rel(tp, dst_base);
 		}
 		break;
 	    case BIN_MATCHSTATE_SUBTAG:
@@ -435,7 +457,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		i = thing_arityval(hdr)+1;
 		hbot -= i;
 		tp = hbot;
-		*argp = make_boxed(hbot);
+		*argp = make_boxed_rel(hbot, dst_base);
 		while (i--) {
 		    *tp++ = *objp++;
 		}
@@ -455,7 +477,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     if (htop != hbot)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "Internal error in copy_struct() when copying %T:"
-		 " htop=%p != hbot=%p (sz=%bpu)\n",
+		 " htop=%p != hbot=%p (sz=%beu)\n",
 		 org_obj, htop, hbot, org_sz); 
 #else
     if (htop > hbot) {
@@ -655,9 +677,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     *hp++ = *objp++;
                 }
                 erts_refc_inc(&pb->val->refc, 2);
-                pb->next = erts_global_offheap.mso;
-                erts_global_offheap.mso = pb;
-                erts_global_offheap.overhead += pb->size / sizeof(Eterm);
+                pb->next = erts_global_offheap.first;
+                erts_global_offheap.first = pb;
+		OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
                 continue;
             }
 
@@ -677,9 +699,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                 while (i--) {
                     *hp++ = *objp++;
                 }
-#ifndef HYBRID // FIND ME!
-                funp->next = erts_global_offheap.funs;
-                erts_global_offheap.funs = funp;
+#ifndef HYBRID /* FIND ME! */
+                funp->next = erts_global_offheap.first;
+                erts_global_offheap.first = funp;
                 erts_refc_inc(&funp->fe->refc, 2);
 #endif
                 for (i = k; i < j; i++) {
@@ -723,8 +745,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     *hp++ = *objp++;
                 }
 
-                etp->next = erts_global_offheap.externals;
-                erts_global_offheap.externals = etp;
+                etp->next = erts_global_offheap.first;
+                erts_global_offheap.first = etp;
 		erts_refc_inc(&etp->node->refc, 2);
                 continue;
             }
@@ -780,9 +802,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     to_bin->size = real_size;
                     to_bin->val = from_bin->val;
                     to_bin->bytes = from_bin->bytes + sub_offset;
-                    to_bin->next = erts_global_offheap.mso;
-                    erts_global_offheap.mso = to_bin;
-                    erts_global_offheap.overhead += to_bin->size / sizeof(Eterm);
+                    to_bin->next = erts_global_offheap.first;
+                    erts_global_offheap.first = to_bin;
+		    OH_OVERHEAD(&erts_global_offheap, to_bin->size / sizeof(Eterm));
 		    res_binary=make_binary(to_bin);
 		    hp += PROC_BIN_SIZE;
                 }
@@ -890,12 +912,21 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
  *
  * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr).
  */
-Eterm
-copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#if HALFWORD_HEAP
+Eterm copy_shallow_rel(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap,
+		       Eterm* src_base)
+#else
+Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#endif
 {
     Eterm* tp = ptr;
     Eterm* hp = *hpp;
-    Sint offs = hp - tp;
+    const Eterm res = make_tuple(hp);
+#if HALFWORD_HEAP
+    const Sint offs = COMPRESS_POINTER(hp - (tp - src_base));
+#else
+    const Sint offs = (hp - tp) * sizeof(Eterm);
+#endif
 
     while (sz--) {
 	Eterm val = *tp++;
@@ -906,7 +937,7 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    break;
 	case TAG_PRIMARY_LIST:
 	case TAG_PRIMARY_BOXED:
-	    *hp++ = offset_ptr(val, offs);
+	    *hp++ = byte_offset_ptr(val, offs);
 	    break;
 	case TAG_PRIMARY_HEADER:
 	    *hp++ = val;
@@ -915,57 +946,43 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		break;
 	    case REFC_BINARY_SUBTAG:
 		{
-		    ProcBin* pb = (ProcBin *) (hp-1);
-		    int tari = thing_arityval(val);
-
-		    sz -= tari;
-		    while (tari--) {
-			*hp++ = *tp++;
-		    }
+		    ProcBin* pb = (ProcBin *) (tp-1);
 		    erts_refc_inc(&pb->val->refc, 2);
-		    pb->next = off_heap->mso;
-		    off_heap->mso = pb;
-		    off_heap->overhead += pb->size / sizeof(Eterm);
+		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
-		break;
+		goto off_heap_common;
+
 	    case FUN_SUBTAG:
 		{
-#ifndef HYBRID /* FIND ME! */
-		    ErlFunThing* funp = (ErlFunThing *) (hp-1);
-#endif
-		    int tari = thing_arityval(val);
-
-		    sz -= tari;
-		    while (tari--) {
-			*hp++ = *tp++;
-		    }
-#ifndef HYBRID /* FIND ME! */
-		    funp->next = off_heap->funs;
-		    off_heap->funs = funp;
+		    ErlFunThing* funp = (ErlFunThing *) (tp-1);
 		    erts_refc_inc(&funp->fe->refc, 2);
-#endif
 		}
-		break;
+		goto off_heap_common;
+
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
 		{
-		    ExternalThing* etp = (ExternalThing *) (hp-1);
+		    ExternalThing* etp = (ExternalThing *) (tp-1);
+		    erts_refc_inc(&etp->node->refc, 2);
+		}
+	    off_heap_common:
+		{
+		    struct erl_off_heap_header* ohh = (struct erl_off_heap_header*)(hp-1);
 		    int tari = thing_arityval(val);
-
+		    
 		    sz -= tari;
 		    while (tari--) {
 			*hp++ = *tp++;
 		    }
-		    etp->next = off_heap->externals;
-		    off_heap->externals = etp;
-		    erts_refc_inc(&etp->node->refc, 2);
+		    ohh->next = off_heap->first;
+		    off_heap->first = ohh;
 		}
 		break;
 	    default:
 		{
 		    int tari = header_arity(val);
-
+    
 		    sz -= tari;
 		    while (tari--) {
 			*hp++ = *tp++;
@@ -977,5 +994,95 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	}
     }
     *hpp = hp;
-    return make_tuple(ptr + offs); 
+
+    return res;
+}
+
+/* Move all terms in heap fragments into heap. The terms must be guaranteed to 
+ * be contained within the fragments. The source terms are destructed with
+ * move markers.
+ * Typically used to copy a multi-fragmented message (from NIF).
+ */
+void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
+		      Eterm* refs, unsigned nrefs)
+{
+    ErlHeapFragment* bp;
+    Eterm* hp_start = *hpp;
+    Eterm* hp_end;
+    Eterm* hp;
+    unsigned i;
+
+    for (bp=first; bp!=NULL; bp=bp->next) {
+	move_one_frag(hpp, bp->mem, bp->used_size, off_heap);
+	OH_OVERHEAD(off_heap, bp->off_heap.overhead);
+    }
+    hp_end = *hpp;
+    for (hp=hp_start; hp<hp_end; ++hp) {
+	Eterm* ptr;
+	Eterm val;
+	Eterm gval = *hp;
+	switch (primary_tag(gval)) {
+	case TAG_PRIMARY_BOXED:
+	    ptr = boxed_val(gval);
+	    val = *ptr;
+	    if (IS_MOVED_BOXED(val)) {
+		ASSERT(is_boxed(val));
+		*hp = val;
+	    }
+	    break;
+	case TAG_PRIMARY_LIST:
+	    ptr = list_val(gval);
+	    val = *ptr;
+	    if (IS_MOVED_CONS(val)) {
+		*hp = ptr[1];
+	    }
+	    break;
+	case TAG_PRIMARY_HEADER:
+	    if (header_is_thing(gval)) {
+		hp += thing_arityval(gval);
+	    }
+	    break;
+	}
+    }
+    for (i=0; i<nrefs; ++i) {
+	refs[i] = follow_moved(refs[i]);
+    }
+}
+
+static void
+move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
+{
+    Eterm* ptr = src;
+    Eterm* end = ptr + src_sz;
+    Eterm dummy_ref;
+    Eterm* hp = *hpp;
+
+    while (ptr != end) {
+	Eterm val;
+	ASSERT(ptr < end);
+	val = *ptr;
+	ASSERT(val != ERTS_HOLE_MARKER);
+	if (is_header(val)) {
+	    struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp;
+	    ASSERT(ptr + header_arity(val) < end);
+	    MOVE_BOXED(ptr, val, hp, &dummy_ref);	    
+	    switch (val & _HEADER_SUBTAG_MASK) {
+	    case REFC_BINARY_SUBTAG:
+	    case FUN_SUBTAG:
+	    case EXTERNAL_PID_SUBTAG:
+	    case EXTERNAL_PORT_SUBTAG:
+	    case EXTERNAL_REF_SUBTAG:
+		hdr->next = off_heap->first;
+		off_heap->first = hdr;
+		break;
+	    }
+	}
+	else { /* must be a cons cell */
+	    ASSERT(ptr+1 < end);
+	    MOVE_CONS(ptr, val, hp, &dummy_ref);
+	    ptr += 2;
+	}
+    }
+    *hpp = hp;
 }
+
diff --git a/erts/emulator/beam/decl.h b/erts/emulator/beam/decl.h
deleted file mode 100644
index da1be29d53..0000000000
--- a/erts/emulator/beam/decl.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-#ifndef __DECL_H__
-#define __DECL_H__
-
-#if defined(__STDC__) || defined(_MSC_VER)
-#define EXTERN_FUNCTION(t, f, x)  extern t f x
-#define FUNCTION(t, f, x) t f x
-#define _DOTS_ ...
-#define _VOID_      void
-#elif defined(__cplusplus)
-#define EXTERN_FUNCTION(f, x) extern "C" { f x }
-#define FUNCTION(t, f, x) t f x
-#define _DOTS_ ...
-#define _VOID_    void
-#else
-#define EXTERN_FUNCTION(t, f, x) extern t f (/*x*/)
-#define FUNCTION(t, f, x) t f (/*x*/)
-#define _DOTS_
-#define _VOID_
-#endif
-
-/*
-** Example of declarations
-**
-** EXTERN_FUNCTION(void, foo, (int, int, char));
-** FUNCTION(void, bar, (int, char));
-**
-** struct funcs {
-**    FUNCTION(int*, (*f1), (int, int));
-**    FUNCTION(void, (*f2), (int, char));
-**    FUNCTION(void, (*f3), (_VOID_));
-**    FUNCTION(int,  (*f4), (char*, _DOTS_));
-** };
-**
-*/
-
-#endif
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index e3094404e2..b1cdd0660a 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -97,6 +97,8 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz)
 #define PASS_THROUGH 'p'        /* This code should go */
 
 int erts_is_alive; /* System must be blocked on change */
+int erts_dist_buf_busy_limit;
+
 
 /* distribution trap functions */
 Export* dsend2_trap = NULL;
@@ -160,7 +162,7 @@ Uint erts_dist_cache_size(void)
 static ErtsProcList *
 get_suspended_on_de(DistEntry *dep, Uint32 unset_qflgs)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&dep->qlock));
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&dep->qlock));
     dep->qflgs &= ~unset_qflgs;
     if (dep->qflgs & ERTS_DE_QFLG_EXIT) {
 	/* No resume when exit has been scheduled */
@@ -228,6 +230,7 @@ int is_node_name_atom(Eterm a)
 
 typedef struct {
     DistEntry *dep;
+    Eterm *lhp;
 } NetExitsContext;
 
 /* 
@@ -253,8 +256,9 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
 	    erts_destroy_monitor(rmon);
 	}
     } else {
-	Eterm lhp[3];
+	DeclareTmpHeapNoproc(lhp,3);
 	Eterm watched;
+	UseTmpHeapNoproc(3);
 	ASSERT(mon->type == MON_TARGET);
 	rmon = erts_remove_monitor(&(rp->monitors),mon->ref);
 	/* ASSERT(rmon != NULL); can happen during process exit */
@@ -271,6 +275,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
 				       watched, am_noconnection);
 	    erts_destroy_monitor(rmon);
 	}
+	UnUseTmpHeapNoproc(3);
     }
     erts_smp_proc_unlock(rp, rp_locks);
  done:
@@ -450,17 +455,17 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 
 	if (dep->status & ERTS_DE_SFLG_EXITING) {
 #ifdef DEBUG
-	    erts_smp_spin_lock(&dep->qlock);
+	    erts_smp_mtx_lock(&dep->qlock);
 	    ASSERT(dep->qflgs & ERTS_DE_QFLG_EXIT);
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
 #endif
 	}
 	else {
 	    dep->status |= ERTS_DE_SFLG_EXITING;
-	    erts_smp_spin_lock(&dep->qlock);
+	    erts_smp_mtx_lock(&dep->qlock);
 	    ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT));
 	    dep->qflgs |= ERTS_DE_QFLG_EXIT;
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
 	}
 
 	erts_smp_de_links_lock(dep);
@@ -574,7 +579,7 @@ static void clear_dist_entry(DistEntry *dep)
     erts_smp_de_links_unlock(dep);
 #endif
 
-    erts_smp_spin_lock(&dep->qlock);
+    erts_smp_mtx_lock(&dep->qlock);
 
     if (!dep->out_queue.last)
 	obuf = dep->finalized_out_queue.first;
@@ -590,7 +595,7 @@ static void clear_dist_entry(DistEntry *dep)
     dep->status = 0;
     suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL);
 
-    erts_smp_spin_unlock(&dep->qlock);
+    erts_smp_mtx_unlock(&dep->qlock);
     erts_smp_atomic_set(&dep->dist_cmd_scheduled, 0);
     dep->send = NULL;
     erts_smp_de_rwunlock(dep);
@@ -608,10 +613,10 @@ static void clear_dist_entry(DistEntry *dep)
     }
 
     if (obufsize) {
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize >= obufsize);
 	dep->qsize -= obufsize;
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
     }
 }
 
@@ -632,19 +637,27 @@ static void clear_dist_entry(DistEntry *dep)
 int
 erts_dsig_send_link(ErtsDSigData *dsdp, Eterm local, Eterm remote)
 {
-    Eterm ctl_heap[4];
+    DeclareTmpHeapNoproc(ctl_heap,4);
     Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_LINK), local, remote);
+    int res;
+    UseTmpHeapNoproc(4);
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UnUseTmpHeapNoproc(4);
+    return res;
 }
 
 int
 erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote)
 {
-    Eterm ctl_heap[4];
+    DeclareTmpHeapNoproc(ctl_heap,4);
     Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_UNLINK), local, remote);
+    int res;
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UseTmpHeapNoproc(4);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UnUseTmpHeapNoproc(4);
+    return res;
 }
 
 
@@ -656,7 +669,10 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
 			  Eterm ref, Eterm reason)
 {
     Eterm ctl;
-    Eterm ctl_heap[6];
+    DeclareTmpHeapNoproc(ctl_heap,6);
+    int res;
+
+    UseTmpHeapNoproc(6);
 
     ctl = TUPLE5(&ctl_heap[0], make_small(DOP_MONITOR_P_EXIT),
 		 watched, watcher, ref, reason);
@@ -667,7 +683,9 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
     erts_smp_de_links_unlock(dsdp->dep);
 #endif
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    UnUseTmpHeapNoproc(6);
+    return res;
 }
 
 /* We want to monitor a process (named or unnamed) on another node, we send:
@@ -678,13 +696,17 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
 		       Eterm ref)
 {
     Eterm ctl;
-    Eterm ctl_heap[5];
+    DeclareTmpHeapNoproc(ctl_heap,5);
+    int res;
 
+    UseTmpHeapNoproc(5);
     ctl = TUPLE4(&ctl_heap[0],
 		 make_small(DOP_MONITOR_P),
 		 watcher, watched, ref);
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UnUseTmpHeapNoproc(5);
+    return res;
 }
 
 /* A local process monitoring a remote one wants to stop monitoring, either 
@@ -696,23 +718,29 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher,
 			 Eterm watched, Eterm ref, int force)
 {
     Eterm ctl;
-    Eterm ctl_heap[5];
+    DeclareTmpHeapNoproc(ctl_heap,5);
+    int res;
 
+    UseTmpHeapNoproc(5);
     ctl = TUPLE4(&ctl_heap[0],
 		 make_small(DOP_DEMONITOR_P),
 		 watcher, watched, ref);
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, force);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, force);
+    UnUseTmpHeapNoproc(5);
+    return res;
 }
 
 int
 erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message)
 {
     Eterm ctl;
-    Eterm ctl_heap[5];
+    DeclareTmpHeapNoproc(ctl_heap,5);
     Eterm token = NIL;
     Process *sender = dsdp->proc;
+    int res;
 
+    UseTmpHeapNoproc(5);
     if (SEQ_TRACE_TOKEN(sender) != NIL) {
 	seq_trace_update_send(sender);
 	token = SEQ_TRACE_TOKEN(sender);
@@ -724,17 +752,21 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message)
 		     make_small(DOP_SEND_TT), am_Cookie, remote, token);
     else
 	ctl = TUPLE3(&ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote);
-    return dsig_send(dsdp, ctl, message, 0);
+    res = dsig_send(dsdp, ctl, message, 0);
+    UnUseTmpHeapNoproc(5);
+    return res;
 }
 
 int
 erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
 {
     Eterm ctl;
-    Eterm ctl_heap[6];
+    DeclareTmpHeapNoproc(ctl_heap,6);
     Eterm token = NIL;
     Process *sender = dsdp->proc;
+    int res;
 
+    UseTmpHeapNoproc(6);
     if (SEQ_TRACE_TOKEN(sender) != NIL) {
 	seq_trace_update_send(sender);
 	token = SEQ_TRACE_TOKEN(sender);
@@ -747,7 +779,9 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
     else
 	ctl = TUPLE4(&ctl_heap[0], make_small(DOP_REG_SEND),
 		     sender->id, am_Cookie, remote_name);
-    return dsig_send(dsdp, ctl, message, 0);
+    res = dsig_send(dsdp, ctl, message, 0);
+    UnUseTmpHeapNoproc(6);
+    return res;
 }
 
 /* local has died, deliver the exit signal to remote */
@@ -756,8 +790,10 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
 		       Eterm reason, Eterm token)
 {
     Eterm ctl;
-    Eterm ctl_heap[6];
+    DeclareTmpHeapNoproc(ctl_heap,6);
+    int res;
 
+    UseTmpHeapNoproc(6);
     if (token != NIL) {	
 	seq_trace_update_send(dsdp->proc);
 	seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local);
@@ -767,38 +803,58 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
 	ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT), local, remote, reason);
     }
     /* forced, i.e ignore busy */
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    UnUseTmpHeapNoproc(6);
+    return res;
 }
 
 int
 erts_dsig_send_exit(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason)
 {
-    Eterm ctl_heap[5];
-    Eterm ctl = TUPLE4(&ctl_heap[0],
-		       make_small(DOP_EXIT), local, remote, reason);
+    DeclareTmpHeapNoproc(ctl_heap,5);
+    int res;
+    Eterm ctl;
+
+    UseTmpHeapNoproc(5);
+    ctl = TUPLE4(&ctl_heap[0],
+		 make_small(DOP_EXIT), local, remote, reason);
     /* forced, i.e ignore busy */
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    res =  dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    UnUseTmpHeapNoproc(5);
+    return res;
 }
 
 int
 erts_dsig_send_exit2(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason)
 {
-    Eterm ctl_heap[5];
-    Eterm ctl = TUPLE4(&ctl_heap[0],
-		       make_small(DOP_EXIT2), local, remote, reason);
+    DeclareTmpHeapNoproc(ctl_heap,5);
+    int res;
+    Eterm ctl;
+
+    UseTmpHeapNoproc(5);
+    ctl = TUPLE4(&ctl_heap[0],
+		 make_small(DOP_EXIT2), local, remote, reason);
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UnUseTmpHeapNoproc(5);
+    return res;
 }
 
 
 int
 erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
 {
-    Eterm ctl_heap[4];
-    Eterm ctl = TUPLE3(&ctl_heap[0],
-		       make_small(DOP_GROUP_LEADER), leader, remote);
+    DeclareTmpHeapNoproc(ctl_heap,4);
+    int res;
+    Eterm ctl;
+
+    UseTmpHeapNoproc(4);
+    ctl = TUPLE3(&ctl_heap[0],
+		 make_small(DOP_GROUP_LEADER), leader, remote);
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UnUseTmpHeapNoproc(4);
+    return res;
 }
 
 #if defined(PURIFY)
@@ -808,11 +864,15 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
 #include <valgrind/valgrind.h>
 #include <valgrind/memcheck.h>
 
+#ifndef HAVE_VALGRIND_PRINTF_XML
+#define VALGRIND_PRINTF_XML VALGRIND_PRINTF
+#endif
+
 #  define PURIFY_MSG(msg)                                                \
     do {								 \
 	char buf__[1]; size_t bufsz__ = sizeof(buf__);			 \
 	if (erts_sys_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \
-	    VALGRIND_PRINTF("<erlang_error_log>"			 \
+	    VALGRIND_PRINTF_XML("<erlang_error_log>"			 \
 			    "%s, line %d: %s</erlang_error_log>\n",	 \
 			    __FILE__, __LINE__, msg);			 \
 	} else {							 \
@@ -832,6 +892,7 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
 **
 **   assert  hlen == 0 !!!
 */
+
 int erts_net_message(Port *prt,
 		     DistEntry *dep,
 		     byte *hbuf,
@@ -839,10 +900,10 @@ int erts_net_message(Port *prt,
 		     byte *buf,
 		     int len)
 {
+#define DIST_CTL_DEFAULT_SIZE 64
     ErtsDistExternal ede;
     byte *t;
     Sint ctl_len;
-    int orig_ctl_len;
     Eterm arg;
     Eterm from, to;
     Eterm watcher, watched;
@@ -850,7 +911,7 @@ int erts_net_message(Port *prt,
     Eterm *tuple;
     Eterm reason;
     Process* rp;
-    Eterm ctl_default[64];
+    DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE);
     Eterm* ctl = ctl_default;
     ErlOffHeap off_heap;
     Eterm* hp;
@@ -859,29 +920,31 @@ int erts_net_message(Port *prt,
     Eterm token_size;
     ErtsMonitor *mon;
     ErtsLink *lnk;
+    Uint tuple_arity;
     int res;
 #ifdef ERTS_DIST_MSG_DBG
     int orig_len = len;
 #endif
 
+    UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
     /* Thanks to Luke Gorrie */
-    off_heap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    off_heap.funs = NULL;
-#endif
+    off_heap.first = NULL;
     off_heap.overhead = 0;
-    off_heap.externals = NULL;
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
-    if (!erts_is_alive)
+    if (!erts_is_alive) {
+	UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
 	return 0;
+    }
     if (hlen > 0)
 	goto data_error;
-    if (len == 0)  /* HANDLE TICK !!! */
+    if (len == 0) {  /* HANDLE TICK !!! */
+	UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
 	return 0;
+    }
 
 #ifdef ERTS_RAW_DIST_MSG_DBG
     erts_fprintf(stderr, "<< ");
@@ -921,8 +984,8 @@ int erts_net_message(Port *prt,
 	PURIFY_MSG("data error");
 	goto data_error;
     }
-    orig_ctl_len = ctl_len;
-    if (ctl_len > sizeof(ctl_default)/sizeof(ctl_default[0])) {
+
+    if (ctl_len > DIST_CTL_DEFAULT_SIZE) {
 	ctl = erts_alloc(ERTS_ALC_T_DCTRL_BUF, ctl_len * sizeof(Eterm));
     }
     hp = ctl;
@@ -943,29 +1006,23 @@ int erts_net_message(Port *prt,
 #endif
 
     if (is_not_tuple(arg) || 
-	(tuple = tuple_val(arg), arityval(*tuple) < 1) ||
+	(tuple = tuple_val(arg), (tuple_arity = arityval(*tuple)) < 1) ||
 	is_not_small(tuple[1])) {
-	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg);
-	erts_send_error_to_logger_nogl(dsbufp);
-	goto data_error;
+ 	goto invalid_message;
     }
 
     token_size = 0;
 
     switch (type = unsigned_val(tuple[1])) {
     case DOP_LINK:
+	if (tuple_arity != 3) {
+	    goto invalid_message;
+	}
 	from = tuple[2];
 	to   = tuple[3];  /* local proc to link to */
 
 	if (is_not_pid(from) || is_not_pid(to)) {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    PURIFY_MSG("data error");
-	    erts_dsprintf(dsbufp,
-			  "Invalid DOP_LINK distribution message: %.200T",
-			  arg);
-	    erts_send_error_to_logger_nogl(dsbufp);
-	    goto data_error;
+	    goto invalid_message;
 	}
 
 	rp = erts_pid2proc_opt(NULL, 0,
@@ -1004,8 +1061,14 @@ int erts_net_message(Port *prt,
 
     case DOP_UNLINK: {
 	ErtsDistLinkData dld;
+	if (tuple_arity != 3) {
+	    goto invalid_message;
+	}
 	from = tuple[2];
 	to = tuple[3];
+	if (is_not_pid(from) || is_not_pid(to)) {
+	    goto invalid_message;
+	}
 	
 	rp = erts_pid2proc_opt(NULL, 0,
 			       to, ERTS_PROC_LOCK_LINK,
@@ -1032,11 +1095,19 @@ int erts_net_message(Port *prt,
 	/* A remote process wants to monitor us, we get:
 	   {DOP_MONITOR_P, Remote pid, local pid or name, ref} */
 	Eterm name;
+	
+	if (tuple_arity != 4) {
+	    goto invalid_message;
+	}
 
 	watcher = tuple[2];
 	watched = tuple[3];  /* local proc to monitor */
 	ref     = tuple[4];
 
+	if (is_not_ref(ref)) {
+	    goto invalid_message;
+	}
+
 	if (is_atom(watched)) {
 	    name = watched;
 	    rp = erts_whereis_process(NULL, 0,
@@ -1078,10 +1149,17 @@ int erts_net_message(Port *prt,
 	   We get {DOP_DEMONITOR_P, Remote pid, Local pid or name, ref},
 	   We need only the ref of course */
 
+	if (tuple_arity != 4) {
+	    goto invalid_message;
+	}
 	/* watcher = tuple[2]; */
 	/* watched = tuple[3]; May be an atom in case of monitor name */
 	ref = tuple[4];
 
+	if(is_not_ref(ref)) {
+	    goto invalid_message;
+	}
+
 	erts_smp_de_links_lock(dep);
 	mon = erts_remove_monitor(&(dep->monitors),ref);
 	erts_smp_de_links_unlock(dep);
@@ -1106,10 +1184,11 @@ int erts_net_message(Port *prt,
 	erts_destroy_monitor(mon);
 	break;
 
-    case DOP_NODE_LINK: /* XXX never sent ?? */
-	break;
-
     case DOP_REG_SEND_TT:
+	if (tuple_arity != 5) {
+	    goto invalid_message;
+	}
+
 	token_size = size_object(tuple[5]);
 	/* Fall through ... */
     case DOP_REG_SEND:
@@ -1120,12 +1199,19 @@ int erts_net_message(Port *prt,
 	 * There is intentionally no testing of the cookie (it is always '')
 	 * from R9B and onwards.
 	 */
+	if (type != DOP_REG_SEND_TT && tuple_arity != 4) {
+	    goto invalid_message;
+	}
+
 #ifdef ERTS_DIST_MSG_DBG
 	dist_msg_dbg(&ede, "MSG", buf, orig_len);
 #endif
 
 	from = tuple[2];
 	to = tuple[4];
+	if (is_not_pid(from) || is_not_atom(to)){
+	    goto invalid_message;
+	}
 	rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
 	if (rp) {
 	    Uint xsize = (type == DOP_REG_SEND
@@ -1157,6 +1243,10 @@ int erts_net_message(Port *prt,
 	break;
 
     case DOP_SEND_TT:
+	if (tuple_arity != 4) {
+	    goto invalid_message;
+	}
+	
 	token_size = size_object(tuple[4]);
 	/* Fall through ... */
     case DOP_SEND:
@@ -1167,8 +1257,13 @@ int erts_net_message(Port *prt,
 #ifdef ERTS_DIST_MSG_DBG
 	dist_msg_dbg(&ede, "MSG", buf, orig_len);
 #endif
-
+	if (type != DOP_SEND_TT && tuple_arity != 3) {
+	    goto invalid_message;
+	}
 	to = tuple[3];
+	if (is_not_pid(to)) {
+	    goto invalid_message;
+	}
 	rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
 	if (rp) {
 	    Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size);
@@ -1202,15 +1297,23 @@ int erts_net_message(Port *prt,
 	   {DOP_MONITOR_P_EXIT, Remote pid or name, Local pid, ref, reason} */
 	   
 
-	Eterm lhp[3];
+	DeclareTmpHeapNoproc(lhp,3);
 	Eterm sysname;
 	ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_MSG_SEND|ERTS_PROC_LOCK_LINK;
 
+	if (tuple_arity != 5) {
+	    goto invalid_message;
+	}
+
 	/* watched = tuple[2]; */  /* remote proc which died */
 	/* watcher = tuple[3]; */
 	ref     = tuple[4];
 	reason  = tuple[5];
 
+	if(is_not_ref(ref)) {
+	    goto invalid_message;
+	}
+
 	erts_smp_de_links_lock(dep);
 	sysname = dep->sysname;
 	mon = erts_remove_monitor(&(dep->monitors), ref);
@@ -1237,6 +1340,7 @@ int erts_net_message(Port *prt,
 	    erts_smp_proc_unlock(rp, rp_locks);
 	    break;
 	}
+	UseTmpHeapNoproc(3);
 	
 	watched = (is_not_nil(mon->name)
 		   ? TUPLE2(&lhp[0], mon->name, sysname)
@@ -1246,6 +1350,7 @@ int erts_net_message(Port *prt,
 				   ref, am_process, watched, reason);
 	erts_smp_proc_unlock(rp, rp_locks);
 	erts_destroy_monitor(mon);
+	UnUseTmpHeapNoproc(3);
 	break;
     }
 
@@ -1255,24 +1360,25 @@ int erts_net_message(Port *prt,
 	ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND;
 	/* 'from', which 'to' is linked to, died */
 	if (type == DOP_EXIT) {
-	   from = tuple[2];
-	   to = tuple[3];
-	   reason = tuple[4];
-	   token = NIL;
+	    if (tuple_arity != 4) {
+		goto invalid_message;
+	    }
+	    
+	    from = tuple[2];
+	    to = tuple[3];
+	    reason = tuple[4];
+	    token = NIL;
 	} else {
-	   from = tuple[2];
-	   to = tuple[3];
-	   token = tuple[4];
-	   reason = tuple[5];
+	    if (tuple_arity != 5) {
+		goto invalid_message;
+	    }
+	    from = tuple[2];
+	    to = tuple[3];
+	    token = tuple[4];
+	    reason = tuple[5];
 	}
-	if (is_not_internal_pid(to)) {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    PURIFY_MSG("data error");
-	    erts_dsprintf(dsbufp,
-			  "Invalid DOP_EXIT distribution message: %.200T",
-			  arg);
-	    erts_send_error_to_logger_nogl(dsbufp);
-	    goto data_error;
+	if (is_not_pid(from) || is_not_internal_pid(to)) {
+	    goto invalid_message;
 	}
 
 	rp = erts_pid2proc(NULL, 0, to, rp_locks);
@@ -1319,15 +1425,24 @@ int erts_net_message(Port *prt,
 	ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
 	/* 'from' is send an exit signal to 'to' */
 	if (type == DOP_EXIT2) {
-	   from = tuple[2];
-	   to = tuple[3];
-	   reason = tuple[4];
-	   token = NIL;
+	    if (tuple_arity != 4) {
+		goto invalid_message;
+	    }
+	    from = tuple[2];
+	    to = tuple[3];
+	    reason = tuple[4];
+	    token = NIL;
 	} else {
-	   from = tuple[2];
-	   to = tuple[3];
-	   token = tuple[4];
-	   reason = tuple[5];
+	    if (tuple_arity != 5) {
+		goto invalid_message;
+	    }
+	    from = tuple[2];
+	    to = tuple[3];
+	    token = tuple[4];
+	    reason = tuple[5];
+	}
+	if (is_not_pid(from) || is_not_internal_pid(to)) {
+	    goto invalid_message;
 	}
 	rp = erts_pid2proc_opt(NULL, 0, to, rp_locks,
 			       ERTS_P2P_FLG_SMP_INC_REFC);
@@ -1346,10 +1461,14 @@ int erts_net_message(Port *prt,
 	break;
     }
     case DOP_GROUP_LEADER:
+	if (tuple_arity != 3) {
+	    goto invalid_message;
+	}
 	from = tuple[2];   /* Group leader  */
 	to = tuple[3];     /* new member */
-	if (is_not_pid(from))
-	    break;
+	if (is_not_pid(from) || is_not_pid(to)) {
+	    goto invalid_message;
+	}
 
 	rp = erts_pid2proc(NULL, 0, to, ERTS_PROC_LOCK_MAIN);
 	if (!rp)
@@ -1358,57 +1477,39 @@ int erts_net_message(Port *prt,
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
 	break;
 
-    default: {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    erts_dsprintf(dsbufp,
-			  "Illegal value in distribution dispatch switch: "
-			  "%.200T",
-			  arg);
-	    erts_send_error_to_logger_nogl(dsbufp);
-	    PURIFY_MSG("data error");
-	    goto data_error;
-	}
+    default: 
+	goto invalid_message;
     }
 
-    if (off_heap.mso) {
-	erts_cleanup_mso(off_heap.mso);
-    }
-    if (off_heap.externals) {
-	erts_cleanup_externals(off_heap.externals);
-    }
+    erts_cleanup_offheap(&off_heap);
 #ifndef HYBRID /* FIND ME! */
-    if (off_heap.funs) {
-	erts_cleanup_funs(off_heap.funs);
-    }
     if (ctl != ctl_default) {
 	erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl);
     }
 #endif
+    UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     return 0;
-
- data_error:
-    if (off_heap.mso) {
-	erts_cleanup_mso(off_heap.mso);
-    }
-    if (off_heap.externals) {
-	erts_cleanup_externals(off_heap.externals);
+ invalid_message:
+    {
+	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+	erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg);
+	erts_send_error_to_logger_nogl(dsbufp);
     }
+ data_error:
+    PURIFY_MSG("data error");
+    erts_cleanup_offheap(&off_heap);
 #ifndef HYBRID /* FIND ME! */
-    if (off_heap.funs) {
-	erts_cleanup_funs(off_heap.funs);
-    }
     if (ctl != ctl_default) {
 	erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl);
     }
 #endif
+    UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
     erts_do_exit_port(prt, dep->cid, am_killed);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     return -1;
 }
 
-#define ERTS_DE_BUSY_LIMIT (128*1024)
-
 static int
 dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
 {
@@ -1492,18 +1593,18 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
     }
     else {
 	ErtsProcList *plp = NULL;
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	dep->qsize += size_obuf(obuf);
-	if (dep->qsize >= ERTS_DE_BUSY_LIMIT)
+	if (dep->qsize >= erts_dist_buf_busy_limit)
 	    dep->qflgs |= ERTS_DE_QFLG_BUSY;
 	if (!force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) {
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
 
 	    plp = erts_proclist_create(c_p);
 	    plp->next = NULL;
 	    erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
 	    suspended = 1;
-	    erts_smp_spin_lock(&dep->qlock);
+	    erts_smp_mtx_lock(&dep->qlock);
 	}
 
 	/* Enqueue obuf on dist entry */
@@ -1529,7 +1630,7 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
 	    }
 	}
 
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
 	erts_schedule_dist_command(NULL, dep);
 	erts_smp_de_runlock(dep);
 	
@@ -1554,9 +1655,9 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
 	 */
 
 	data_size >>= (10-4);
-#if defined(ARCH_64)
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	data_size &= 0x003fffffffffffff;
-#elif defined(ARCH_32)
+#elif defined(ARCH_32) || HALFWORD_HEAP
 	data_size &= 0x003fffff;
 #else
 #       error "Ohh come on ... !?!"
@@ -1586,7 +1687,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
     if (size > (Uint) INT_MAX)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "Absurdly large distribution output data buffer "
-		 "(%bpu bytes) passed.\n",
+		 "(%beu bytes) passed.\n",
 		 size);
 
     prt->caller = NIL;
@@ -1613,7 +1714,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
     if (size > (Uint) INT_MAX)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "Absurdly large distribution output data buffer "
-		 "(%bpu bytes) passed.\n",
+		 "(%beu bytes) passed.\n",
 		 size);
 
     iov[0].iov_base = NULL;
@@ -1640,9 +1741,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
 }
 
 
-#if defined(ARCH_64)
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define ERTS_PORT_REDS_MASK__ 0x003fffffffffffffL
-#elif defined(ARCH_32)
+#elif defined(ARCH_32) || HALFWORD_HEAP
 #define ERTS_PORT_REDS_MASK__ 0x003fffff
 #else
 #  error "Ohh come on ... !?!"
@@ -1662,10 +1763,8 @@ erts_dist_command(Port *prt, int reds_limit)
 {
     Sint reds = ERTS_PORT_REDS_DIST_CMD_START;
     int prt_busy;
-    int de_busy;
     Uint32 status;
     Uint32 flags;
-    Uint32 qflgs;
     Sint obufsize = 0;
     ErtsDistOutputQueue oq, foq;
     DistEntry *dep = prt->dist_entry;
@@ -1700,13 +1799,12 @@ erts_dist_command(Port *prt, int reds_limit)
      * a mess.
      */
 
-    erts_smp_spin_lock(&dep->qlock);
+    erts_smp_mtx_lock(&dep->qlock);
     oq.first = dep->out_queue.first;
     oq.last = dep->out_queue.last;
     dep->out_queue.first = NULL;
     dep->out_queue.last = NULL;
-    qflgs = dep->qflgs;
-    erts_smp_spin_unlock(&dep->qlock);
+    erts_smp_mtx_unlock(&dep->qlock);
 
     foq.first = dep->finalized_out_queue.first;
     foq.last = dep->finalized_out_queue.last;
@@ -1717,17 +1815,8 @@ erts_dist_command(Port *prt, int reds_limit)
 	goto preempted;
 
     prt_busy = (int) (prt->status & ERTS_PORT_SFLG_PORT_BUSY);
-    de_busy = (int) (qflgs & ERTS_DE_QFLG_BUSY);
 
-    if (prt_busy) {
-	if (!de_busy) {
-	    erts_smp_spin_lock(&dep->qlock);
-	    dep->qflgs |= ERTS_DE_QFLG_BUSY;
-	    erts_smp_spin_unlock(&dep->qlock);
-	    de_busy = 1;
-	}
-    }
-    else if (foq.first) {
+    if (!prt_busy && foq.first) {
 	int preempt = 0;
 	do {
 	    Uint size;
@@ -1745,10 +1834,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	    free_dist_obuf(fob);
 	    preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD);
 	    if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) {
-		erts_smp_spin_lock(&dep->qlock);
-		dep->qflgs |= ERTS_DE_QFLG_BUSY;
-		erts_smp_spin_unlock(&dep->qlock);
-		de_busy = prt_busy = 1;
+		prt_busy = 1;
 		break;
 	    }
 	} while (foq.first && !preempt);
@@ -1831,10 +1917,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	    free_dist_obuf(fob);
 	    preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD);
 	    if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) {
-		erts_smp_spin_lock(&dep->qlock);
-		dep->qflgs |= ERTS_DE_QFLG_BUSY;
-		erts_smp_spin_unlock(&dep->qlock);
-		de_busy = prt_busy = 1;
+		prt_busy = 1;
 		if (oq.first && !preempt)
 		    goto finalize_only;
 	    }
@@ -1861,22 +1944,23 @@ erts_dist_command(Port *prt, int reds_limit)
 	 * dist entry in a non-busy state and resume suspended
 	 * processes.
 	 */
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize >= obufsize);
 	dep->qsize -= obufsize;
 	obufsize = 0;
-	if (de_busy && !prt_busy && dep->qsize < ERTS_DE_BUSY_LIMIT) {
+	if (!prt_busy
+	    && (dep->qflgs & ERTS_DE_QFLG_BUSY)
+	    && dep->qsize < erts_dist_buf_busy_limit) {
 	    ErtsProcList *suspendees;
 	    int resumed;
 	    suspendees = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY);
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
 
 	    resumed = erts_resume_processes(suspendees);
 	    reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED;
-	    de_busy = 0;
 	}
 	else
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
     }
 
     ASSERT(!oq.first && !oq.last);
@@ -1885,10 +1969,10 @@ erts_dist_command(Port *prt, int reds_limit)
 
     if (obufsize != 0) {
 	ASSERT(obufsize > 0);
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize >= obufsize);
 	dep->qsize -= obufsize;
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
     }
 
     ASSERT(foq.first || !foq.last);
@@ -1938,9 +2022,9 @@ erts_dist_command(Port *prt, int reds_limit)
 	foq.last = NULL;
 
 #ifdef DEBUG
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize == obufsize);
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
 #endif
     }
     else {
@@ -1949,14 +2033,14 @@ erts_dist_command(Port *prt, int reds_limit)
 	     * Unhandle buffers need to be put back first
 	     * in out_queue.
 	     */
-	    erts_smp_spin_lock(&dep->qlock);
+	    erts_smp_mtx_lock(&dep->qlock);
 	    dep->qsize -= obufsize;
 	    obufsize = 0;
 	    oq.last->next = dep->out_queue.first;
 	    dep->out_queue.first = oq.first;
 	    if (!dep->out_queue.last)
 		dep->out_queue.last = oq.last;
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
 	}
 
 	erts_schedule_dist_command(prt, NULL);
@@ -1980,10 +2064,10 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id)
 
 	dep->status |= ERTS_DE_SFLG_EXITING;
 
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT));
 	dep->qflgs |= ERTS_DE_QFLG_EXIT;
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
 
 	erts_schedule_dist_command(NULL, dep);
     }
@@ -2354,13 +2438,13 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
 	ErtsProcList *plp = erts_proclist_create(BIF_P);
 	plp->next = NULL;
 	erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	if (dep->suspended.last)
 	    dep->suspended.last->next = plp;
 	else
 	    dep->suspended.first = plp;
 	dep->suspended.last = plp;
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
 	goto yield;
     }
 
@@ -2388,9 +2472,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     ASSERT(dep->send);
 
 #ifdef DEBUG
-    erts_smp_spin_lock(&dep->qlock);
+    erts_smp_mtx_lock(&dep->qlock);
     ASSERT(dep->qsize == 0);
-    erts_smp_spin_unlock(&dep->qlock);
+    erts_smp_mtx_unlock(&dep->qlock);
 #endif
 
     erts_set_dist_entry_connected(dep, BIF_ARG_2, flags);
@@ -2547,12 +2631,15 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
     int visible = 0;
     int hidden = 0;
     int this = 0;
-    Uint  buf[2]; /* For one cons-cell */
+    DeclareTmpHeap(buf,2,BIF_P); /* For one cons-cell */
     DistEntry *dep;
     Eterm arg_list = BIF_ARG_1;
 #ifdef DEBUG
     Eterm* endp;
 #endif
+
+    UseTmpHeap(2,BIF_P);
+
     if (is_atom(BIF_ARG_1))
       arg_list = CONS(buf, BIF_ARG_1, NIL);
 
@@ -2563,13 +2650,14 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
       case am_known:     visible = hidden = not_connected = this = 1; break;
       case am_this:      this = 1;                                    break;
       case am_connected: visible = hidden = 1;                        break;
-      default:           BIF_ERROR(BIF_P, BADARG);                    break;
+      default:           goto error;                                  break;
       }
       arg_list = CDR(list_val(arg_list));
     }
 
-    if (is_not_nil(arg_list))
-      BIF_ERROR(BIF_P, BADARG);
+    if (is_not_nil(arg_list)) {
+	goto error;
+    }
 
     length = 0;
 
@@ -2591,7 +2679,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
 
     if (length == 0) {
 	erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
-	BIF_RET(result);
+	goto done;
     }
 
     hp = HAlloc(BIF_P, 2*length);
@@ -2620,7 +2708,14 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
     }
     ASSERT(endp == hp);
     erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+
+done:
+    UnUseTmpHeap(2,BIF_P);
     BIF_RET(result);
+
+error:
+    UnUseTmpHeap(2,BIF_P);
+    BIF_ERROR(BIF_P,BADARG);
 }
 
 /**********************************************************************/
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index ea1abcaeed..695a4fc3fe 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -38,6 +38,7 @@
 #define DFLAG_UNICODE_IO          0x1000
 #define DFLAG_DIST_HDR_ATOM_CACHE 0x2000
 #define DFLAG_SMALL_ATOM_TAGS     0x4000
+#define DFLAGS_INTERNAL_TAGS      0x8000
 
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
@@ -51,7 +52,7 @@
 #define DOP_SEND		2
 #define DOP_EXIT		3
 #define DOP_UNLINK		4
-#define DOP_NODE_LINK		5
+/* Ancient DOP_NODE_LINK (5) was here, can be reused */
 #define DOP_REG_SEND		6
 #define DOP_GROUP_LEADER	7
 #define DOP_EXIT2		8
@@ -68,7 +69,6 @@
 /* distribution trap functions */
 extern Export* dsend2_trap;
 extern Export* dsend3_trap;
-/*extern Export* dsend_nosuspend_trap;*/
 extern Export* dlink_trap;
 extern Export* dunlink_trap;
 extern Export* dmonitor_node_trap;
@@ -99,7 +99,8 @@ typedef struct {
 #define ERTS_DE_IS_CONNECTED(DEP) \
   (!ERTS_DE_IS_NOT_CONNECTED((DEP)))
 
-
+#define ERTS_DE_BUSY_LIMIT (1024*1024)
+extern int erts_dist_buf_busy_limit;
 extern int erts_is_alive;
 
 /*
@@ -153,10 +154,10 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
     }
     if (no_suspend) {
 	failure = ERTS_DSIG_PREP_CONNECTED;
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	if (dep->qflgs & ERTS_DE_QFLG_BUSY)
 	    failure = ERTS_DSIG_PREP_WOULD_SUSPEND;
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
 	if (failure == ERTS_DSIG_PREP_WOULD_SUSPEND)
 	    goto fail;
     }
@@ -287,4 +288,5 @@ extern void erts_kill_dist_connection(DistEntry *dep, Uint32);
 
 extern Uint erts_dist_cache_size(void);
 
+
 #endif
diff --git a/erts/emulator/beam/elib_malloc.c b/erts/emulator/beam/elib_malloc.c
deleted file mode 100644
index b18c48d8d6..0000000000
--- a/erts/emulator/beam/elib_malloc.c
+++ /dev/null
@@ -1,2334 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-/*
-** Description: Faster malloc().
-*/
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "sys.h"
-
-#ifdef ENABLE_ELIB_MALLOC
-
-#undef THREAD_SAFE_ELIB_MALLOC
-#ifdef USE_THREADS
-#define THREAD_SAFE_ELIB_MALLOC 1
-#else
-#define THREAD_SAFE_ELIB_MALLOC 0
-#endif
-
-#include "erl_driver.h"
-#include "erl_threads.h"
-#include "elib_stat.h"
-#include <stdio.h>
-#include <stdlib.h>
-
-/* To avoid clobbering of names becaure of reclaim on VxWorks,
-   we undefine all possible malloc, calloc etc. */
-#undef malloc
-#undef calloc
-#undef free
-#undef realloc
-
-#define ELIB_INLINE         /* inline all possible functions */
-
-#ifndef ELIB_ALIGN
-#define ELIB_ALIGN             sizeof(double)
-#endif
-
-#ifndef ELIB_HEAP_SIZE
-#define ELIB_HEAP_SIZE         (64*1024)  /* Default 64K */
-#endif
-
-#ifndef ELIB_HEAP_INCREAMENT
-#define ELIB_HEAP_INCREAMENT   (32*1024)  /* Default 32K */
-#endif
-
-#ifndef ELIB_FAILURE
-#define ELIB_FAILURE    abort()
-#endif
-
-#undef ASSERT
-#ifdef DEBUG
-#define ASSERT(B) \
- ((void) ((B) ? 1 : (fprintf(stderr, "%s:%d: Assertion failed: %s\n", \
-			     __FILE__, __LINE__, #B), abort(), 0)))
-#else
-#define ASSERT(B) ((void) 1)
-#endif
-
-#ifndef USE_RECURSIVE_MALLOC_MUTEX
-#define USE_RECURSIVE_MALLOC_MUTEX 0
-#endif
-
-#if USE_RECURSIVE_MALLOC_MUTEX
-static erts_mtx_t malloc_mutex = ERTS_REC_MTX_INITER;
-#else /* #if USE_RECURSIVE_MALLOC_MUTEX */
-static erts_mtx_t malloc_mutex = ERTS_MTX_INITER;
-#if THREAD_SAFE_ELIB_MALLOC
-static erts_cnd_t  malloc_cond  = ERTS_CND_INITER;
-#endif
-#endif  /* #if USE_RECURSIVE_MALLOC_MUTEX */
-
-typedef unsigned long EWord;       /* Assume 32-bit in this implementation */
-typedef unsigned short EHalfWord;  /* Assume 16-bit in this implementation */
-typedef unsigned char EByte;       /* Assume 8-bit byte */
-
-
-#define elib_printf fprintf
-#define elib_putc   fputc
-
-
-#if defined(__STDC__) || defined(__WIN32__)
-#define CONCAT(x,y) x##y
-#else
-#define CONCAT(x,y) x/**/y
-#endif
-
-
-#ifdef ELIB_DEBUG
-#define ELIB_PREFIX(fun, args) CONCAT(elib__,fun) args
-#else
-#define ELIB_PREFIX(fun, args) CONCAT(elib_,fun) args
-#endif
-
-#if defined(__STDC__)
-void *ELIB_PREFIX(malloc, (size_t));
-void *ELIB_PREFIX(calloc, (size_t, size_t));
-void ELIB_PREFIX(cfree, (EWord *));
-void ELIB_PREFIX(free, (EWord *));
-void *ELIB_PREFIX(realloc, (EWord *, size_t));
-void* ELIB_PREFIX(memresize, (EWord *, int));
-void* ELIB_PREFIX(memalign, (int, int));
-void* ELIB_PREFIX(valloc, (int));
-void* ELIB_PREFIX(pvalloc, (int));
-int ELIB_PREFIX(memsize, (EWord *));
-/* Extern interfaces used by VxWorks */
-size_t elib_sizeof(void *);
-void elib_init(EWord *, EWord);
-void elib_force_init(EWord *, EWord);
-#endif
-
-#if defined(__STDC__)
-/* define prototypes for missing */
-void* memalign(size_t a, size_t s);
-void* pvalloc(size_t nb);
-void* memresize(void *p, int nb);
-int memsize(void *p);
-#endif
-
-/* bytes to pages */
-#define PAGES(x)      (((x)+page_size-1) / page_size)
-#define PAGE_ALIGN(p) ((char*)((((EWord)(p))+page_size-1)&~(page_size-1)))
-
-/* bytes to words */
-#define WORDS(x)      (((x)+sizeof(EWord)-1) / sizeof(EWord))
-
-/* Align an address */
-#define ALIGN(p)     ((EWord*)((((EWord)(p)+ELIB_ALIGN-1)&~(ELIB_ALIGN-1))))
-
-/* Calculate the size needed to keep alignment */
-
-#define ALIGN_BSZ(nb)  ((nb+sizeof(EWord)+ELIB_ALIGN-1) & ~(ELIB_ALIGN-1))
-
-#define ALIGN_WSZ(nb)  WORDS(ALIGN_BSZ(nb))
-
-#define ALIGN_SIZE(nb) (ALIGN_WSZ(nb) - 1)
-
-
-/* PARAMETERS */
-
-#if defined(ELIB_HEAP_SBRK)
-
-#undef PAGE_SIZE
-
-/* Get the system page size (NEED MORE DEFINES HERE) */
-#ifdef _SC_PAGESIZE
-#define PAGE_SIZE   sysconf(_SC_PAGESIZE)
-#elif defined(_MSC_VER)
-#  ifdef _M_ALPHA
-#    define PAGE_SIZE      0x2000
-#  else
-#    define PAGE_SIZE      0x1000
-#  endif
-#else
-#define PAGE_SIZE   getpagesize()
-#endif
-
-#define ELIB_EXPAND(need)  expand_sbrk(need)
-static FUNCTION(int, expand_sbrk, (EWord));
-
-#elif defined(ELIB_HEAP_FIXED)
-
-#define PAGE_SIZE 1024
-#define ELIB_EXPAND(need) -1
-static EWord fix_heap[WORDS(ELIB_HEAP_SIZE)];
-
-#elif defined(ELIB_HEAP_USER)
-
-#define PAGE_SIZE 1024
-#define ELIB_EXPAND(need) -1
-
-#else
-
-#error "ELIB HEAP TYPE NOT SET"
-
-#endif
-
-
-#define STAT_ALLOCED_BLOCK(SZ)		\
-do {					\
-    tot_allocated += (SZ);		\
-    if (max_allocated < tot_allocated)	\
-	max_allocated = tot_allocated;	\
-} while (0)
-
-#define STAT_FREED_BLOCK(SZ)		\
-do {					\
-    tot_allocated -= (SZ);		\
-} while (0)
-
-static int max_allocated = 0;
-static int tot_allocated = 0;
-static EWord* eheap;        /* Align heap start */
-static EWord* eheap_top;    /* Point to end of heap */
-EWord page_size = 0;        /* Set by elib_init */
-
-#if defined(ELIB_DEBUG) || defined(DEBUG)
-#define ALIGN_CHECK(a, p)						\
- do {									\
-    if ((EWord)(p) & (a-1)) {						\
-	elib_printf(stderr,						\
-		    "RUNTIME ERROR: bad alignment (0x%lx:%d:%d)\n",	\
-		    (unsigned long) (p), (int) a, __LINE__);			\
-	ELIB_FAILURE;							\
-    }									\
-  } while(0)
-#define ELIB_ALIGN_CHECK(p) ALIGN_CHECK(ELIB_ALIGN, p)
-#else
-#define ALIGN_CHECK(a, p)
-#define ELIB_ALIGN_CHECK(p)
-#endif
-
-#define DYNAMIC 32
-
-/*
-** Free block layout
-**   1 1          30
-**  +--------------------------+
-**  |F|P|        Size          |
-**  +--------------------------+
-**
-** Where F is the free bit
-**       P is the free above bit
-**       Size is messured in words and does not include the hdr word
-**
-** If block is on the free list the size is also stored last in the block.
-** 
-*/
-typedef struct _free_block FreeBlock;
-struct _free_block {
-    EWord hdr;
-    Uint flags;
-    FreeBlock* parent;
-    FreeBlock* left;
-    FreeBlock* right;
-    EWord v[1];
-};
-
-typedef struct _allocated_block {
-    EWord hdr;
-    EWord v[5];
-} AllocatedBlock;
-
-
-/*
- * Interface to tree routines.
- */
-typedef Uint Block_t;
-
-static Block_t*	get_free_block(Uint);
-static void link_free_block(Block_t *);
-static void unlink_free_block(Block_t *del);
-
-#define FREE_BIT       0x80000000
-#define FREE_ABOVE_BIT 0x40000000
-#define SIZE_MASK      0x3fffffff     /* 2^30 words = 2^32 bytes */
-
-/* Work on both FreeBlock and AllocatedBlock */
-#define SIZEOF(p)         ((p)->hdr & SIZE_MASK)
-#define IS_FREE(p)        (((p)->hdr & FREE_BIT) != 0)
-#define IS_FREE_ABOVE(p)  (((p)->hdr & FREE_ABOVE_BIT) != 0)
-
-/* Given that we have a free block above find its size */
-#define SIZEOF_ABOVE(p)    *(((EWord*) (p)) - 1)
-
-#define MIN_BLOCK_SIZE      (sizeof(FreeBlock)/sizeof(EWord))
-#define MIN_WORD_SIZE       (MIN_BLOCK_SIZE-1)
-#define MIN_BYTE_SIZE       (sizeof(FreeBlock)-sizeof(EWord))
-
-#define MIN_ALIGN_SIZE      ALIGN_SIZE(MIN_BYTE_SIZE)
-
-
-static AllocatedBlock* heap_head = 0;
-static AllocatedBlock* heap_tail = 0;
-static EWord eheap_size = 0;
-
-static int heap_locked;
-
-static int elib_need_init = 1;
-#if THREAD_SAFE_ELIB_MALLOC
-static int elib_is_initing = 0;
-#endif
-
-typedef FreeBlock RBTree_t;
-
-static RBTree_t* root = NULL;
-
-
-static FUNCTION(void, deallocate, (AllocatedBlock*, int));
-
-/*
- * Unlink a free block
- */
-
-#define mark_allocated(p, szp) do { \
-      (p)->hdr = ((p)->hdr & FREE_ABOVE_BIT) | (szp); \
-      (p)->v[szp] &= ~FREE_ABOVE_BIT; \
-   } while(0)
-
-#define mark_free(p, szp) do { \
-      (p)->hdr = FREE_BIT | (szp); \
-      ((FreeBlock *)p)->v[szp-sizeof(FreeBlock)/sizeof(EWord)+1] = (szp); \
-   } while(0)
-
-#if 0
-/* Help macros to log2 */
-#define LOG_1(x)  (((x) > 1) ? 1 : 0)
-#define LOG_2(x)  (((x) > 3) ? 2+LOG_1((x) >> 2) : LOG_1(x))
-#define LOG_4(x)  (((x) > 15) ? 4+LOG_2((x) >> 4) : LOG_2(x))
-#define LOG_8(x)  (((x) > 255) ? 8+LOG_4((x)>>8) : LOG_4(x))
-#define LOG_16(x) (((x) > 65535) ? 16+LOG_8((x)>>16) : LOG_8(x))
-
-#define log2(x)   LOG_16(x)
-#endif
-
-/*
- * Split a block to be allocated.
- * Mark block as ALLOCATED and clear
- * FREE_ABOVE_BIT on next block
- *
- * nw is SIZE aligned and szp is SIZE aligned + 1
- */
-static void
-split_block(FreeBlock* p, EWord nw, EWord szp)
-{
-    EWord szq;
-    FreeBlock* q;
-
-    szq = szp - nw;
-    /* Preserve FREE_ABOVE bit in p->hdr !!! */
-
-    if (szq >= MIN_ALIGN_SIZE+1) {
-	szq--;
-	p->hdr = (p->hdr & FREE_ABOVE_BIT) | nw;
-
-	q = (FreeBlock*) (((EWord*) p) + nw + 1);
-	mark_free(q, szq);
-	link_free_block((Block_t *) q);
-
-	q = (FreeBlock*) (((EWord*) q) + szq + 1);
-	q->hdr |= FREE_ABOVE_BIT;
-    }
-    else {
-	mark_allocated((AllocatedBlock*)p, szp);
-    }
-}
-
-/*
- * Find a free block
- */
-static FreeBlock*
-alloc_block(EWord nw)
-{
-    for (;;) {
-	FreeBlock* p = (FreeBlock *) get_free_block(nw);
-
-	if (p != NULL) {
-	    return p;
-	} else if (ELIB_EXPAND(nw+MIN_WORD_SIZE)) {
-	    return 0;
-	}
-    }
-}
-
-
-size_t elib_sizeof(void *p)
-{
-    AllocatedBlock* pp;
-
-    if (p != 0) {
-	pp = (AllocatedBlock*) (((char *)p)-1);
-	return SIZEOF(pp);
-    }
-    return 0;
-}
-
-static void locked_elib_init(EWord*, EWord);
-static void init_elib_malloc(EWord*, EWord);
-
-/*
-** Initialize the elib
-** The addr and sz is only used when compiled with EXPAND_ADDR 
-*/
-/* Not static, this is used by VxWorks */
-void elib_init(EWord* addr, EWord sz)
-{
-    if (!elib_need_init)
-	return;
-    erts_mtx_lock(&malloc_mutex);
-    locked_elib_init(addr, sz);
-    erts_mtx_unlock(&malloc_mutex);
-}
-
-static void locked_elib_init(EWord* addr, EWord sz)
-{
-    if (!elib_need_init)
-	return;
-
-#if THREAD_SAFE_ELIB_MALLOC
-
-#if !USE_RECURSIVE_MALLOC_MUTEX
-    {
-	static erts_tid_t initer_tid;
-
-	if(elib_is_initing) {
-
-	    if(erts_equal_tids(initer_tid, erts_thr_self()))
-		return;
-
-	    /* Wait until initializing thread is done with initialization */
-
-	    while(elib_need_init)
-		erts_cnd_wait(&malloc_cond, &malloc_mutex);
-
-	    return;
-	}
-	else {
-	    initer_tid = erts_thr_self();
-	    elib_is_initing = 1;
-	}
-    }
-#else
-    if(elib_is_initing)
-	return;
-    elib_is_initing = 1;
-#endif
-
-#endif /* #if THREAD_SAFE_ELIB_MALLOC */
-
-    /* Do the actual initialization of the malloc implementation */
-    init_elib_malloc(addr, sz);
-
-#if THREAD_SAFE_ELIB_MALLOC
-
-#if !USE_RECURSIVE_MALLOC_MUTEX
-    erts_mtx_unlock(&malloc_mutex);
-#endif
-
-    /* Recursive calls to malloc are allowed here... */
-    erts_mtx_set_forksafe(&malloc_mutex);
-
-#if !USE_RECURSIVE_MALLOC_MUTEX
-    erts_mtx_lock(&malloc_mutex);
-    elib_is_initing = 0;
-#endif
-
-#endif /* #if THREAD_SAFE_ELIB_MALLOC */
-
-    elib_need_init = 0;
-
-#if THREAD_SAFE_ELIB_MALLOC && !USE_RECURSIVE_MALLOC_MUTEX
-    erts_cnd_broadcast(&malloc_cond);
-#endif
-
-}
-
-static void init_elib_malloc(EWord* addr, EWord sz)
-{
-    int i;
-    FreeBlock* freep;
-    EWord tmp_sz;
-#ifdef ELIB_HEAP_SBRK
-    char* top;
-    EWord n;
-#endif
-
-    max_allocated = 0;
-    tot_allocated = 0;
-    root = NULL;
-
-    /* Get the page size (may involve system call!!!) */
-    page_size = PAGE_SIZE;
-
-#if defined(ELIB_HEAP_SBRK)
-    sz = PAGES(ELIB_HEAP_SIZE)*page_size;
-
-    if ((top = (char*) sbrk(0)) == (char*)-1) {
-	elib_printf(stderr, "could not initialize elib, sbrk(0)");
-	ELIB_FAILURE;
-    }
-    n = PAGE_ALIGN(top) - top;
-    if ((top = (char*) sbrk(n)) == (char*)-1) {
-	elib_printf(stderr, "could not initialize elib, sbrk(n)");
-	ELIB_FAILURE;
-    }
-    if ((eheap = (EWord*) sbrk(sz)) == (EWord*)-1) {
-	elib_printf(stderr, "could not initialize elib, sbrk(SIZE)");
-	ELIB_FAILURE;
-    }
-    sz = WORDS(ELIB_HEAP_SIZE);
-#elif defined(ELIB_HEAP_FIXED)
-    eheap = fix_heap;
-    sz = WORDS(ELIB_HEAP_SIZE);
-#elif defined(ELIB_HEAP_USER)
-    eheap = addr;
-    sz = WORDS(sz);
-#else
-    return -1;
-#endif
-    eheap_size = 0;
-
-    /* Make sure that the first word of the heap_head is aligned */
-    addr = ALIGN(eheap+1);
-    sz -= ((addr - 1) - eheap);      /* Subtract unusable size */
-    eheap_top = eheap = addr - 1;    /* Set new aligned heap start */
-
-    eheap_top[sz-1] = 0;	     /* Heap stop mark */
-
-    addr = eheap;
-    heap_head = (AllocatedBlock*) addr;
-    heap_head->hdr = MIN_ALIGN_SIZE;
-    for (i = 0; i < MIN_ALIGN_SIZE; i++)
-	heap_head->v[i] = 0;
-
-    addr += (MIN_ALIGN_SIZE+1);
-    freep = (FreeBlock*) addr;
-    tmp_sz = sz - (((MIN_ALIGN_SIZE+1) + MIN_BLOCK_SIZE) + 1 + 1);
-    mark_free(freep, tmp_sz);
-    link_free_block((Block_t *) freep);
-
-    /* No need to align heap tail */
-    heap_tail = (AllocatedBlock*) &eheap_top[sz-MIN_BLOCK_SIZE-1];
-    heap_tail->hdr = FREE_ABOVE_BIT | MIN_WORD_SIZE;
-    heap_tail->v[0] = 0;
-    heap_tail->v[1] = 0;
-    heap_tail->v[2] = 0;
-
-    eheap_top += sz;
-    eheap_size += sz;
-
-    heap_locked = 0;
-}
-
-#ifdef ELIB_HEAP_USER
-void elib_force_init(EWord* addr, EWord sz)
-{
-    elib_need_init = 1;
-    elib_init(addr,sz);
-}
-#endif
-
-#ifdef ELIB_HEAP_SBRK
-
-/*
-** need in number of words (should include head and tail words)
-*/
-static int expand_sbrk(EWord sz)
-{
-    EWord* p;
-    EWord  bytes = sz * sizeof(EWord);
-    EWord  size;
-    AllocatedBlock* tail;
-
-    if (bytes < ELIB_HEAP_SIZE)
-	size = PAGES(ELIB_HEAP_INCREAMENT)*page_size;
-    else
-	size = PAGES(bytes)*page_size;
-
-    if ((p = (EWord*) sbrk(size)) == ((EWord*) -1))
-	return -1;
-
-    if (p != eheap_top) {
-	elib_printf(stderr, "panic: sbrk moved\n");
-	ELIB_FAILURE;
-    }
-
-    sz = WORDS(size);
-
-    /* Set new endof heap marker and a new heap tail */
-    eheap_top[sz-1] = 0;
-
-    tail = (AllocatedBlock*) &eheap_top[sz-MIN_BLOCK_SIZE-1];
-    tail->hdr = FREE_ABOVE_BIT | MIN_WORD_SIZE;
-    tail->v[0] = 0;
-    tail->v[1] = 0;
-    tail->v[2] = 0;
-
-    /* Patch old tail with new appended size */
-    heap_tail->hdr = (heap_tail->hdr & FREE_ABOVE_BIT) |
-	(MIN_WORD_SIZE+1+(sz-MIN_BLOCK_SIZE-1));
-    deallocate(heap_tail, 0);
-
-    heap_tail = tail;
-
-    eheap_size += sz;
-    eheap_top += sz;
-
-    return 0;
-}
-
-#endif /* ELIB_HEAP_SBRK */
-
-
-/*
-** Scan heap and check for corrupted heap
-*/
-int elib_check_heap(void)
-{
-    AllocatedBlock* p = heap_head;
-    EWord sz;
-
-    if (heap_locked) {
-	elib_printf(stderr, "heap is locked no info avaiable\n");
-	return 0;
-    }
-
-    while((sz = SIZEOF(p)) != 0) {
-	if (IS_FREE(p)) {
-	    if (p->v[sz-1] != sz) {
-		elib_printf(stderr, "panic: heap corrupted\r\n"); 
-		ELIB_FAILURE;	
-	    }
-	    p = (AllocatedBlock*) (p->v + sz);
-	    if (!IS_FREE_ABOVE(p)) {
-		elib_printf(stderr, "panic: heap corrupted\r\n"); 
-		ELIB_FAILURE;	
-	    }
-	}
-	else
-	    p = (AllocatedBlock*) (p->v + sz);
-    }
-    return 1;
-}
-
-/*
-** Load the byte vector pointed to by v of length vsz
-** with a heap image
-** The scale is defined by vsz and the current heap size
-** free = 0, full = 255
-** 
-** 
-*/
-int elib_heap_map(EByte* v, int vsz)
-{
-    AllocatedBlock* p = heap_head;
-    EWord sz;
-    int gsz = eheap_size / vsz;  /* The granuality used */
-    int fsz = 0;
-    int usz = 0;
-
-    if (gsz == 0)
-	return -1;  /* too good reolution */
-
-    while((sz = SIZEOF(p)) != 0) {
-	if (IS_FREE(p)) {
-	    fsz += sz;
-	    if ((fsz + usz) > gsz) {
-		*v++ = (255*usz)/gsz;
-		fsz -= (gsz - usz);
-		usz = 0;
-		while(fsz >= gsz) {
-		    *v++ = 0;
-		    fsz -= gsz;
-		}
-	    }
-	}
-	else {
-	    usz += sz;
-	    if ((fsz + usz) > gsz) {
-		*v++ = 255 - (255*fsz)/gsz;
-		usz -= (gsz - fsz);
-		fsz = 0;
-		while(usz >= gsz) {
-		    *v++ = 255;
-		    usz -= gsz;
-		}
-	    }
-	}
-	p = (AllocatedBlock*) (p->v + sz);
-    }
-    return 0;
-}
-
-/*
-** Generate a histogram of free/allocated blocks
-** Count granuality of 10 gives
-** (0-10],(10-100],(100-1000],(1000-10000] ...
-** (0-2], (2-4], (4-8], (8-16], ....
-*/
-static int i_logb(EWord size, int base)
-{
-    int lg = 0;
-    while(size >= base) {
-	size /= base;
-	lg++;
-    }
-    return lg;
-}
-
-int elib_histo(EWord* vf, EWord* va, int vsz, int base)
-{
-    AllocatedBlock* p = heap_head;
-    EWord sz;
-    int i;
-    int linear;
-
-    if ((vsz <= 1) || (vf == 0 && va == 0))
-	return -1;
-
-    if (base < 0) {
-	linear = 1;
-	base = -base;
-    }
-    else
-	linear = 0;
-
-    if (base <= 1)
-	return -1;
-
-    if (vf != 0) {
-	for (i = 0; i < vsz; i++)
-	    vf[i] = 0;
-    }
-    if (va != 0) {
-	for (i = 0; i < vsz; i++)
-	    va[i] = 0;
-    }
-
-    while((sz = SIZEOF(p)) != 0) {
-	if (IS_FREE(p)) {
-	    if (vf != 0) {
-		int val;
-		if (linear)
-		    val = sz / base;
-		else
-		    val = i_logb(sz, base);
-		if (val >= vsz)
-		    vf[vsz-1]++;
-		else
-		    vf[val]++;
-	    }
-	}
-	else {
-	    if (va != 0) {
-		int val;
-		if (linear)
-		    val = sz / base;
-		else
-		    val = i_logb(sz, base);
-		if (val >= vsz)
-		    va[vsz-1]++;
-		else
-		    va[val]++;
-	    }
-	}
-	p = (AllocatedBlock*) (p->v + sz);
-    }
-    return 0;
-}
-
-/*
-** Fill the info structure with actual values
-** Total
-** Allocated
-** Free
-** maxMaxFree     
-*/
-void elib_stat(struct elib_stat* info)
-{
-    EWord blks = 0;
-    EWord sz_free = 0;
-    EWord sz_alloc = 0;
-    EWord sz_max_free = 0;
-    EWord sz_min_used = 0x7fffffff;
-    EWord sz;
-    EWord num_free = 0;
-    AllocatedBlock* p = heap_head;
-
-    info->mem_total = eheap_size;
-
-    p = (AllocatedBlock*) (p->v + SIZEOF(p));
-
-    while((sz = SIZEOF(p)) != 0) {
-	blks++;
-	if (IS_FREE(p)) {
-	    if (sz > sz_max_free)
-		sz_max_free = sz;
-	    sz_free += sz;
-	    ++num_free;
-	}
-	else {
-	    if (sz < sz_min_used)
-		sz_min_used = sz;
-	    sz_alloc += sz;
-	}
-	p = (AllocatedBlock*) (p->v + sz);
-    }
-    info->mem_blocks = blks;
-    info->free_blocks = num_free;
-    info->mem_alloc = sz_alloc;
-    info->mem_free = sz_free;
-    info->min_used = sz_min_used;
-    info->max_free = sz_max_free;
-    info->mem_max_alloc = max_allocated;
-    ASSERT(sz_alloc == tot_allocated);
-}
-
-/*
-** Dump the heap
-*/
-void elib_heap_dump(char* label)
-{
-    AllocatedBlock* p = heap_head;
-    EWord sz;
-
-    elib_printf(stderr, "HEAP DUMP (%s)\n", label);
-    if (!elib_check_heap())
-	return;
-    
-    while((sz = SIZEOF(p)) != 0) {
-	if (IS_FREE(p)) {
-	    elib_printf(stderr, "%p: FREE, size = %d\n", p, (int) sz);
-	}
-	else {
-	    elib_printf(stderr, "%p: USED, size = %d %s\n", p, (int) sz,
-		       IS_FREE_ABOVE(p)?"(FREE ABOVE)":"");
-	}
-	p = (AllocatedBlock*) (p->v + sz);
-    }
-}
-
-/*
-**  Scan heaps and count:
-**  free_size, allocated_size, max_free_block
-*/
-void elib_statistics(void* to)
-{
-    struct elib_stat info;
-    EWord frag;
-
-    if (!elib_check_heap())
-	return;
-
-    elib_stat(&info);
-
-    frag = 1000 - ((1000 * info.max_free) / info.mem_free);
-
-    elib_printf(to, "Heap Statistics: total(%d), blocks(%d), frag(%d.%d%%)\n", 
-	       info.mem_total, info.mem_blocks,
-	       (int) frag/10, (int) frag % 10);
-
-    elib_printf(to, "                 allocated(%d), free(%d), "
-		"free_blocks(%d)\n",
-	       info.mem_alloc, info.mem_free,info.free_blocks);
-    elib_printf(to, "                 max_free(%d),  min_used(%d)\n",
-	       info.max_free, info.min_used);
-}
-
-/*
-** Allocate a least nb bytes with alignment a
-** Algorithm:
-**    1) Try locate a block which match exacly among the by direct index.
-**    2) Try using a fix block of greater size
-**    3) Try locate a block by searching in lists where block sizes
-**       X may vary between 2^i < X <= 2^(i+1)
-**
-** Reset memory to zero if clear is true
-*/
-static AllocatedBlock* allocate(EWord nb, EWord a, int clear)
-{
-    FreeBlock* p;
-    EWord nw;
-
-    if (a == ELIB_ALIGN) {
-	/*
-	 * Common case: Called by malloc(), realloc(), calloc().
-	 */
-	nw = nb < MIN_BYTE_SIZE ? MIN_ALIGN_SIZE : ALIGN_SIZE(nb);
-
-	if ((p = alloc_block(nw)) == 0)
-	    return NULL;
-    } else {
-	/*
-	 * Special case: Called by memalign().
-	 */
-	EWord asz, szp, szq, tmpsz;
-	FreeBlock *q;
-
-	if ((p = alloc_block((1+MIN_ALIGN_SIZE)*sizeof(EWord)+a-1+nb)) == 0)
-	    return NULL;
-
-	asz = a - ((EWord) ((AllocatedBlock *)p)->v) % a;
-
-	if (asz != a) {
-	    /* Enforce the alignment requirement by cutting of a free
-	       block at the beginning of the block. */
-
-	    if (asz < (1+MIN_ALIGN_SIZE)*sizeof(EWord) && !IS_FREE_ABOVE(p)) {
-		/* Not enough room to cut of a free block;
-		   increase align size */
-		asz += (((1+MIN_ALIGN_SIZE)*sizeof(EWord) + a - 1)/a)*a;
-	    }
-
-	    szq = ALIGN_SIZE(asz - sizeof(EWord));
-	    szp = SIZEOF(p) - szq - 1;
-
-	    q = p;
-	    p = (FreeBlock*) (((EWord*) q) + szq + 1);
-	    p->hdr = FREE_ABOVE_BIT | FREE_BIT | szp;
-
-	    if (IS_FREE_ABOVE(q)) { /* This should not be possible I think,
-				       but just in case... */
-		tmpsz = SIZEOF_ABOVE(q) + 1;
-		szq += tmpsz;
-		q = (FreeBlock*) (((EWord*) q) - tmpsz);
-		unlink_free_block((Block_t *) q);
-		q->hdr = (q->hdr & FREE_ABOVE_BIT) | FREE_BIT | szq;
-	    }
-	    mark_free(q, szq);
-	    link_free_block((Block_t *) q);
-
-	} /* else already had the correct alignment */
- 
-	nw = nb < MIN_BYTE_SIZE ? MIN_ALIGN_SIZE : ALIGN_SIZE(nb);
-    }
-
-    split_block(p, nw, SIZEOF(p));
-
-    STAT_ALLOCED_BLOCK(SIZEOF(p));
-
-    if (clear) {
-	EWord* pp = ((AllocatedBlock*)p)->v;
-
-	while(nw--)
-	    *pp++ = 0;
-    }
-
-    return (AllocatedBlock*) p;
-}
-
-
-/*
-** Deallocate memory pointed to by p
-** 1. Merge with block above if this block is free
-** 2. Merge with block below if this block is free
-** Link the block to the correct free list
-**
-** p points to the block header!
-**
-*/
-static void deallocate(AllocatedBlock* p, int stat_count)
-{
-    FreeBlock* q;
-    EWord szq;
-    EWord szp;
-
-    szp = SIZEOF(p);
-
-    if (stat_count)
-	STAT_FREED_BLOCK(SIZEOF(p));
-
-    if (IS_FREE_ABOVE(p)) {
-	szq = SIZEOF_ABOVE(p);
-	q = (FreeBlock*) ( ((EWord*) p) - szq - 1);
-	unlink_free_block((Block_t *) q);
-
-	p = (AllocatedBlock*) q;
-	szp += (szq + 1);
-    }
-    q = (FreeBlock*) (p->v + szp);
-    if (IS_FREE(q)) {
-	szq = SIZEOF(q);
-	unlink_free_block((Block_t *) q);
-	szp += (szq + 1);
-    }
-    else
-	q->hdr |= FREE_ABOVE_BIT;
-
-    /* The block above p can NEVER be free !!! */
-    p->hdr = FREE_BIT | szp;
-    p->v[szp-1] = szp;
-
-    link_free_block((Block_t *) p);
-}
-
-/*
-** Reallocate memory
-** If preserve is true then data is moved if neccesary
-*/
-static AllocatedBlock* reallocate(AllocatedBlock* p, EWord nb, int preserve)
-{
-    EWord szp;
-    EWord szq;
-    EWord sz;
-    EWord nw;
-    FreeBlock* q;
-
-    if (nb < MIN_BYTE_SIZE)
-	nw = MIN_ALIGN_SIZE;
-    else
-	nw = ALIGN_SIZE(nb);
-
-    sz = szp = SIZEOF(p);
-
-    STAT_FREED_BLOCK(szp);
-
-    /* Merge with block below */
-    q = (FreeBlock*) (p->v + szp);
-    if (IS_FREE(q)) {
-	szq = SIZEOF(q);
-	unlink_free_block((Block_t *) q);
-	szp += (szq + 1);
-    }
-
-    if (nw <= szp) {
-	split_block((FreeBlock *) p, nw, szp);
-	STAT_ALLOCED_BLOCK(SIZEOF(p));
-	return p;
-    }
-    else {
-	EWord* dp = p->v;
-	AllocatedBlock* npp;
-
-	if (IS_FREE_ABOVE(p)) {
-	    szq = SIZEOF_ABOVE(p);
-	    if (szq + szp + 1 >= nw) {
-		q = (FreeBlock*) (((EWord*) p) - szq - 1);
-		unlink_free_block((Block_t * )q);
-		szp += (szq + 1);
-		p = (AllocatedBlock*) q;
-
-		if (preserve) {
-		    EWord* pp = p->v;
-		    while(sz--)
-			*pp++ = *dp++;
-		}
-		split_block((FreeBlock *) p, nw, szp);
-		STAT_ALLOCED_BLOCK(SIZEOF(p));
-		return p;
-	    }
-	}
-
-	/*
-	 * Update p so that allocate() and deallocate() works.
-	 * (Note that allocate() may call expand_sbrk(), which in
-	 * in turn calls deallocate().)
-	 */
-
-	p->hdr = (p->hdr & FREE_ABOVE_BIT) | szp;
-	p->v[szp] &= ~FREE_ABOVE_BIT;
-
-	npp = allocate(nb, ELIB_ALIGN, 0);
-	if(npp == NULL)
-	    return NULL;
-	if (preserve) {
-	    EWord* pp = npp->v;
-	    while(sz--)
-		*pp++ = *dp++;
-	}
-	deallocate(p, 0);
-	return npp;
-    }
-}
-
-/*
-** What malloc() and friends should do (and return) when the heap is
-** exhausted.  [sverkerw]
-*/
-static void* heap_exhausted(void)
-{
-    /* Choose behaviour */
-#if 0
-    /* Crash-and-burn --- leave a usable corpse (hopefully) */
-    abort();
-#endif    
-    /* The usual ANSI-compliant behaviour */
-    return NULL;
-}
-
-/*
-** Allocate size bytes of memory
-*/
-void* ELIB_PREFIX(malloc, (size_t nb))
-{
-    void *res;
-    AllocatedBlock* p;
-
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if (nb == 0)
-	res = NULL;
-    else if ((p = allocate(nb, ELIB_ALIGN, 0)) != 0) {
-	ELIB_ALIGN_CHECK(p->v);
-	res = p->v;
-    }
-    else
-	res = heap_exhausted();
-
-    erts_mtx_unlock(&malloc_mutex);
-
-    return res;
-}
-
-
-void* ELIB_PREFIX(calloc, (size_t nelem, size_t size))
-{
-    void *res;
-    int nb;
-    AllocatedBlock* p;
-    
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if ((nb = nelem * size) == 0)
-	res = NULL;
-    else if ((p = allocate(nb, ELIB_ALIGN, 1)) != 0) {
-	ELIB_ALIGN_CHECK(p->v);
-	res = p->v;
-    }
-    else
-	res = heap_exhausted();
-
-    erts_mtx_unlock(&malloc_mutex);
-
-    return res;
-}
-
-/*
-** Free memory allocated by malloc
-*/
-
-void ELIB_PREFIX(free, (EWord* p))
-{
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if (p != 0)
-	deallocate((AllocatedBlock*)(p-1), 1);
-
-    erts_mtx_unlock(&malloc_mutex);
-}
-
-void ELIB_PREFIX(cfree, (EWord* p))
-{
-    ELIB_PREFIX(free, (p));
-}
-
-
-/*
-** Realloc the memory allocated in p to nb number of bytes
-**
-*/
-
-void* ELIB_PREFIX(realloc, (EWord* p, size_t nb))
-{
-    void *res = NULL;
-    AllocatedBlock* pp;
-
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if (p != 0) {
-	pp = (AllocatedBlock*) (p-1);
-	if (nb > 0) {
-	    if ((pp = reallocate(pp, nb, 1)) != 0) {
-		ELIB_ALIGN_CHECK(pp->v);
-		res = pp->v;
-	    }
-	}
-	else
-	    deallocate(pp, 1);
-    }
-    else if (nb > 0) {
-	if ((pp = allocate(nb, ELIB_ALIGN, 0)) != 0) {
-	    ELIB_ALIGN_CHECK(pp->v);
-	    res = pp->v;
-	}
-	else
-	    res = heap_exhausted();
-    }
-
-    erts_mtx_unlock(&malloc_mutex);
-
-    return res;
-}
-
-/*
-** Resize the memory area pointed to by p with nb number of bytes
-*/
-void* ELIB_PREFIX(memresize, (EWord* p, int nb))
-{
-    void *res = NULL;
-    AllocatedBlock* pp;
-
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if (p != 0) {
-	pp = (AllocatedBlock*) (p-1);
-	if (nb > 0) {
-	    if ((pp = reallocate(pp, nb, 0)) != 0) {
-		ELIB_ALIGN_CHECK(pp->v);
-		res = pp->v;
-	    }
-	}
-	else
-	    deallocate(pp, 1);
-    }
-    else if (nb > 0) {
-	if ((pp = allocate(nb, ELIB_ALIGN, 0)) != 0) {
-	    ELIB_ALIGN_CHECK(pp->v);
-	    res = pp->v;
-	}
-	else
-	    res = heap_exhausted();
-    }
-
-    erts_mtx_unlock(&malloc_mutex);
-
-    return res;
-}
-
-
-/* Create aligned memory a must be a power of 2 !!! */
-
-void* ELIB_PREFIX(memalign, (int a, int nb))
-{
-    void *res;
-    AllocatedBlock* p;
-
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if (nb == 0 || a <= 0)
-	res = NULL;
-    else if ((p = allocate(nb, a, 0)) != 0) {
-	ALIGN_CHECK(a, p->v);
-	res = p->v;
-    }
-    else
-	res = heap_exhausted();
-
-    erts_mtx_unlock(&malloc_mutex);
-
-    return res;
-}
-
-void* ELIB_PREFIX(valloc, (int nb))
-{
-    return ELIB_PREFIX(memalign, (page_size, nb));
-}
-
-
-void* ELIB_PREFIX(pvalloc, (int nb))
-{
-    return ELIB_PREFIX(memalign, (page_size, PAGES(nb)*page_size));
-}
-/* Return memory size for pointer p in bytes */
-
-int ELIB_PREFIX(memsize, (p))
-EWord* p;
-{
-    return SIZEOF((AllocatedBlock*)(p-1))*4;
-}
-
-
-/*
-** --------------------------------------------------------------------------
-**   DEBUG LIBRARY
-** --------------------------------------------------------------------------
-*/
-
-#ifdef ELIB_DEBUG
-
-#define IN_HEAP(p)  (((p) >= (char*) eheap) && (p) < (char*) eheap_top)
-/*
-** ptr_to_block: return the pointer to heap block pointed into by ptr
-** Returns 0 if not pointing into a block
-*/
-
-static EWord* ptr_to_block(char* ptr)
-{
-    AllocatedBlock* p = heap_head;
-    EWord sz;
-
-    while((sz = SIZEOF(p)) != 0) {
-	if ((ptr >= (char*) p->v) && (ptr < (char*)(p->v+sz)))
-	    return p->v;
-	p = (AllocatedBlock*) (p->v + sz);
-    }
-    return 0;
-}
-
-/*
-** Validate a pointer
-** returns:
-**      0  - if points to start of a block
-**      1  - if points outsize heap
-**     -1  - if points inside block
-**
-*/
-static int check_pointer(char* ptr)
-{
-    if (IN_HEAP(ptr)) {
-	if (ptr_to_block(ptr) == 0)
-	    return 1;
-	return 0;
-    }
-    return -1;
-}
-
-/*
-** Validate a memory area
-** returns:
-**      0 - if area is included in a block
-**     -1 - if area overlap a heap block
-**      1 - if area is outside heap
-*/
-static int check_area(char* ptr, int n)
-{
-    if (IN_HEAP(ptr)) {
-	if (IN_HEAP(ptr+n-1)) {
-	    EWord* p1 = ptr_to_block(ptr);
-	    EWord* p2 = ptr_to_block(ptr+n-1);
-
-	    if (p1 == p2)
-		return (p1 == 0) ? -1 : 0;
-	    return -1;
-	}
-    }
-    else if (IN_HEAP(ptr+n-1))
-	return -1;
-    return 1;
-}
-
-/*
-** Check if a block write will overwrite heap block
-*/
-static void check_write(char* ptr, int n, char* file, int line, char* fun)
-{
-    if (check_area(ptr, n) == -1) {
-	elib_printf(stderr, "RUNTIME ERROR: %s heap overwrite\n", fun);
-	elib_printf(stderr, "File: %s Line: %d\n", file, line);
-	ELIB_FAILURE;
-    }
-}
-
-/*
-** Check if a pointer is an allocated object
-*/
-static void check_allocated_block(char* ptr, char* file, int line, char* fun)
-{
-    EWord* q;
-
-    if (!IN_HEAP(ptr) || ((q=ptr_to_block(ptr)) == 0) || (ptr != (char*) q)) {
-	elib_printf(stderr, "RUNTIME ERROR: %s non heap pointer\n", fun);
-	elib_printf(stderr, "File: %s Line: %d\n", file, line);
-	ELIB_FAILURE;
-    }
-
-    if (IS_FREE((AllocatedBlock*)(q-1))) {
-	elib_printf(stderr, "RUNTIME ERROR: %s free pointer\n", fun);
-	elib_printf(stderr, "File: %s Line: %d\n", file, line);
-	ELIB_FAILURE;
-    }
-
-}
-
-/*
-** --------------------------------------------------------------------------
-**  DEBUG VERSIONS (COMPILED WITH THE ELIB.H)
-** --------------------------------------------------------------------------
-*/
-
-void* elib_dbg_malloc(int n, char* file, int line)
-{
-    return elib__malloc(n);
-}
-
-void* elib_dbg_calloc(int n, int s, char* file, int line)
-{
-    return elib__calloc(n, s);
-}
-
-void* elib_dbg_realloc(EWord* p, int n, char* file, int line)
-{
-    if (p == 0)
-	return elib__malloc(n);
-    check_allocated_block(p, file, line, "elib_realloc");
-    return elib__realloc(p, n);
-}
-
-void elib_dbg_free(EWord* p, char* file, int line)
-{
-    if (p == 0)
-	return;
-    check_allocated_block(p, file, line, "elib_free");
-    elib__free(p);
-}
-
-void elib_dbg_cfree(EWord* p, char* file, int line)
-{
-    if (p == 0)
-	return;
-    check_allocated_block(p, file, line, "elib_free");
-    elib__cfree(p);
-}
-
-void* elib_dbg_memalign(int a, int n, char* file, int line)
-{
-    return elib__memalign(a, n);
-}
-
-void* elib_dbg_valloc(int n, char* file, int line)
-{
-    return elib__valloc(n);
-}
-
-void* elib_dbg_pvalloc(int n, char* file, int line)
-{
-    return elib__pvalloc(n);
-}
-
-void* elib_dbg_memresize(EWord* p, int n, char* file, int line)
-{
-    if (p == 0)
-	return elib__malloc(n);
-    check_allocated_block(p, file, line, "elib_memresize");
-    return elib__memresize(p, n);
-}
-
-int elib_dbg_memsize(void* p, char* file, int line)
-{
-    check_allocated_block(p, file, line, "elib_memsize");
-    return elib__memsize(p);
-}
-
-/*
-** --------------------------------------------------------------------------
-**  LINK TIME FUNCTIONS (NOT COMPILED CALLS)
-** --------------------------------------------------------------------------
-*/
-
-void* elib_malloc(int n)
-{
-    return elib_dbg_malloc(n, "", -1);
-}
-
-void* elib_calloc(int n, int s)
-{
-    return elib_dbg_calloc(n, s, "", -1);
-}
-
-void* elib_realloc(EWord* p, int n)
-{
-    return elib_dbg_realloc(p, n, "", -1);
-}
-
-void elib_free(EWord* p)
-{
-    elib_dbg_free(p, "", -1);
-}
-
-void elib_cfree(EWord* p)
-{
-    elib_dbg_cfree(p, "", -1);
-}
-
-void* elib_memalign(int a, int n)
-{
-    return elib_dbg_memalign(a, n, "", -1);
-}
-
-void* elib_valloc(int n)
-{
-    return elib_dbg_valloc(n, "", -1);
-}
-
-void* elib_pvalloc(int n)
-{
-    return elib_dbg_pvalloc(n, "", -1);
-}
-
-void* elib_memresize(EWord* p, int n)
-{
-    return elib_dbg_memresize(p, n, "", -1);
-}
-
-
-int elib_memsize(EWord* p)
-{
-    return elib_dbg_memsize(p, "", -1);
-}
-
-#endif /* ELIB_DEBUG */
-
-/*
-** --------------------------------------------------------------------------
-** Map c library functions to elib
-** --------------------------------------------------------------------------
-*/
-
-#if defined(ELIB_ALLOC_IS_CLIB)
-void* malloc(size_t nb)
-{
-    return elib_malloc(nb);
-}
-
-void* calloc(size_t nelem, size_t size)
-{
-    return elib_calloc(nelem, size);
-}
-
-
-void free(void *p)
-{
-    elib_free(p);
-}
-
-void cfree(void *p)
-{
-    elib_cfree(p);
-}
-
-void* realloc(void* p, size_t nb)
-{
-    return elib_realloc(p, nb);
-}
-
-
-void* memalign(size_t a, size_t s)
-{
-    return elib_memalign(a, s);
-}
-
-void* valloc(size_t nb)
-{
-    return elib_valloc(nb);
-}
-
-void* pvalloc(size_t nb)
-{
-    return elib_pvalloc(nb);
-}
-
-#if 0
-void* memresize(void* p, int nb)
-{
-    return elib_memresize(p, nb);
-}
-
-int memsize(void* p)
-{
-    return elib_memsize(p);
-}
-#endif
-#endif /* ELIB_ALLOC_IS_CLIB */
-
-#endif /* ENABLE_ELIB_MALLOC */
-
-void elib_ensure_initialized(void)
-{
-#ifdef ENABLE_ELIB_MALLOC
-#ifndef ELIB_DONT_INITIALIZE
-    elib_init(NULL, 0);
-#endif
-#endif
-}
-
-#ifdef ENABLE_ELIB_MALLOC
-/**
- ** A Slightly modified version of the "address order best fit" algorithm
- ** used in erl_bestfit_alloc.c. Comments refer to that implementation.
- **/
-
-/*
- * Description:	A combined "address order best fit"/"best fit" allocator
- *              based on a Red-Black (binary search) Tree. The search,
- *              insert, and delete operations are all O(log n) operations
- *              on a Red-Black Tree. In the "address order best fit" case
- *              n equals number of free blocks, and in the "best fit" case
- *              n equals number of distinct sizes of free blocks. Red-Black
- *              Trees are described in "Introduction to Algorithms", by
- *              Thomas H. Cormen, Charles E. Leiserson, and
- *              Ronald L. Riverest.
- *
- *              This module is a callback-module for erl_alloc_util.c
- *
- * Author: 	Rickard Green
- */
-
-#ifdef DEBUG
-#if 0
-#define HARD_DEBUG
-#endif
-#else
-#undef HARD_DEBUG
-#endif
-
-#define SZ_MASK			SIZE_MASK
-#define FLG_MASK		(~(SZ_MASK))
-
-#define BLK_SZ(B)  (*((Block_t *) (B)) & SZ_MASK)
-
-#define TREE_NODE_FLG		(((Uint) 1) << 0)
-#define RED_FLG			(((Uint) 1) << 1)
-#ifdef HARD_DEBUG
-#  define LEFT_VISITED_FLG	(((Uint) 1) << 2)
-#  define RIGHT_VISITED_FLG	(((Uint) 1) << 3)
-#endif
-
-#define IS_TREE_NODE(N)		(((RBTree_t *) (N))->flags & TREE_NODE_FLG)
-#define IS_LIST_ELEM(N)		(!IS_TREE_NODE(((RBTree_t *) (N))))
-
-#define SET_TREE_NODE(N)	(((RBTree_t *) (N))->flags |= TREE_NODE_FLG)
-#define SET_LIST_ELEM(N)	(((RBTree_t *) (N))->flags &= ~TREE_NODE_FLG)
-
-#define IS_RED(N)		(((RBTree_t *) (N)) \
-				 && ((RBTree_t *) (N))->flags & RED_FLG)
-#define IS_BLACK(N)		(!IS_RED(((RBTree_t *) (N))))
-
-#define SET_RED(N)		(((RBTree_t *) (N))->flags |= RED_FLG)
-#define SET_BLACK(N)		(((RBTree_t *) (N))->flags &= ~RED_FLG)
-
-#undef ASSERT
-#define ASSERT ASSERT_EXPR
-
-#if 1
-#define RBT_ASSERT	ASSERT
-#else
-#define RBT_ASSERT(x)
-#endif
-
-
-#ifdef HARD_DEBUG
-static RBTree_t * check_tree(Uint);
-#endif
-
-#ifdef ERTS_INLINE
-#  ifndef ERTS_CAN_INLINE
-#    define ERTS_CAN_INLINE 1
-#  endif
-#else
-#  if defined(__GNUC__)
-#    define ERTS_CAN_INLINE 1
-#    define ERTS_INLINE __inline__
-#  elif defined(__WIN32__)
-#    define ERTS_CAN_INLINE 1
-#    define ERTS_INLINE __inline
-#  else
-#    define ERTS_CAN_INLINE 0
-#    define ERTS_INLINE
-#  endif
-#endif
-
-/* Types... */
-#if 0
-typedef struct RBTree_t_ RBTree_t;
-
-struct RBTree_t_ {
-    Block_t hdr;
-    Uint flags;
-    RBTree_t *parent;
-    RBTree_t *left;
-    RBTree_t *right;
-};
-#endif
-
-#if 0
-typedef struct {
-    RBTree_t t;
-    RBTree_t *next;
-} RBTreeList_t;
-
-#define LIST_NEXT(N) (((RBTreeList_t *) (N))->next)
-#define LIST_PREV(N) (((RBTreeList_t *) (N))->t.parent)
-#endif
-
-#ifdef DEBUG
-
-/* Destroy all tree fields */
-#define DESTROY_TREE_NODE(N)						\
-  sys_memset((void *) (((Block_t *) (N)) + 1),				\
-	     0xff,							\
-	     (sizeof(RBTree_t) - sizeof(Block_t)))
-
-/* Destroy all tree and list fields */
-#define DESTROY_LIST_ELEM(N)						\
-  sys_memset((void *) (((Block_t *) (N)) + 1),				\
-	     0xff,							\
-	     (sizeof(RBTreeList_t) - sizeof(Block_t)))
-
-#else
-
-#define DESTROY_TREE_NODE(N)
-#define DESTROY_LIST_ELEM(N)
-
-#endif
-
-
-/*
- * Red-Black Tree operations needed
- */
-
-static ERTS_INLINE void
-left_rotate(RBTree_t **root, RBTree_t *x)
-{
-    RBTree_t *y = x->right;
-    x->right = y->left;
-    if (y->left)
-	y->left->parent = x;
-    y->parent = x->parent;
-    if (!y->parent) {
-	RBT_ASSERT(*root == x);
-	*root = y;
-    }
-    else if (x == x->parent->left)
-	x->parent->left = y;
-    else {
-	RBT_ASSERT(x == x->parent->right);
-	x->parent->right = y;
-    }
-    y->left = x;
-    x->parent = y;
-}
-
-static ERTS_INLINE void
-right_rotate(RBTree_t **root, RBTree_t *x)
-{
-    RBTree_t *y = x->left;
-    x->left = y->right;
-    if (y->right)
-	y->right->parent = x;
-    y->parent = x->parent;
-    if (!y->parent) {
-	RBT_ASSERT(*root == x);
-	*root = y;
-    }
-    else if (x == x->parent->right)
-	x->parent->right = y;
-    else {
-	RBT_ASSERT(x == x->parent->left);
-	x->parent->left = y;
-    }
-    y->right = x;
-    x->parent = y;
-}
-
-
-/*
- * Replace node x with node y
- * NOTE: block header of y is not changed
- */
-static ERTS_INLINE void
-replace(RBTree_t **root, RBTree_t *x, RBTree_t *y)
-{
-
-    if (!x->parent) {
-	RBT_ASSERT(*root == x);
-	*root = y;
-    }
-    else if (x == x->parent->left)
-	x->parent->left = y;
-    else {
-	RBT_ASSERT(x == x->parent->right);
-	x->parent->right = y;
-    }
-    if (x->left) {
-	RBT_ASSERT(x->left->parent == x);
-	x->left->parent = y;
-    }
-    if (x->right) {
-	RBT_ASSERT(x->right->parent == x);
-	x->right->parent = y;
-    }
-
-    y->flags	= x->flags;
-    y->parent	= x->parent;
-    y->right	= x->right;
-    y->left	= x->left;
-
-    DESTROY_TREE_NODE(x);
-
-}
-
-static void
-tree_insert_fixup(RBTree_t *blk)
-{
-    RBTree_t *x = blk, *y;
-
-    /*
-     * Rearrange the tree so that it satisfies the Red-Black Tree properties
-     */
-
-    RBT_ASSERT(x != root && IS_RED(x->parent));
-    do {
-
-	/*
-	 * x and its parent are both red. Move the red pair up the tree
-	 * until we get to the root or until we can separate them.
-	 */
-
-	RBT_ASSERT(IS_RED(x));
-	RBT_ASSERT(IS_BLACK(x->parent->parent));
-	RBT_ASSERT(x->parent->parent);
-
-	if (x->parent == x->parent->parent->left) {
-	    y = x->parent->parent->right;
-	    if (IS_RED(y)) {
-		SET_BLACK(y);
-		x = x->parent;
-		SET_BLACK(x);
-		x = x->parent;
-		SET_RED(x);
-	    }
-	    else {
-
-		if (x == x->parent->right) {
-		    x = x->parent;
-		    left_rotate(&root, x);
-		}
-
-		RBT_ASSERT(x == x->parent->parent->left->left);
-		RBT_ASSERT(IS_RED(x));
-		RBT_ASSERT(IS_RED(x->parent));
-		RBT_ASSERT(IS_BLACK(x->parent->parent));
-		RBT_ASSERT(IS_BLACK(y));
-
-		SET_BLACK(x->parent);
-		SET_RED(x->parent->parent);
-		right_rotate(&root, x->parent->parent);
-
-		RBT_ASSERT(x == x->parent->left);
-		RBT_ASSERT(IS_RED(x));
-		RBT_ASSERT(IS_RED(x->parent->right));
-		RBT_ASSERT(IS_BLACK(x->parent));
-		break;
-	    }
-	}
-	else {
-	    RBT_ASSERT(x->parent == x->parent->parent->right);
-	    y = x->parent->parent->left;
-	    if (IS_RED(y)) {
-		SET_BLACK(y);
-		x = x->parent;
-		SET_BLACK(x);
-		x = x->parent;
-		SET_RED(x);
-	    }
-	    else {
-
-		if (x == x->parent->left) {
-		    x = x->parent;
-		    right_rotate(&root, x);
-		}
-
-		RBT_ASSERT(x == x->parent->parent->right->right);
-		RBT_ASSERT(IS_RED(x));
-		RBT_ASSERT(IS_RED(x->parent));
-		RBT_ASSERT(IS_BLACK(x->parent->parent));
-		RBT_ASSERT(IS_BLACK(y));
-
-		SET_BLACK(x->parent);
-		SET_RED(x->parent->parent);
-		left_rotate(&root, x->parent->parent);
-
-		RBT_ASSERT(x == x->parent->right);
-		RBT_ASSERT(IS_RED(x));
-		RBT_ASSERT(IS_RED(x->parent->left));
-		RBT_ASSERT(IS_BLACK(x->parent));
-		break;
-	    }
-	}
-    } while (x != root && IS_RED(x->parent));
-
-    SET_BLACK(root);
-}
-
-static void
-unlink_free_block(Block_t *del)
-{
-    Uint spliced_is_black;
-    RBTree_t *x, *y, *z = (RBTree_t *) del;
-    RBTree_t null_x; /* null_x is used to get the fixup started when we
-			splice out a node without children. */
-
-    null_x.parent = NULL;
-
-#ifdef HARD_DEBUG
-    check_tree(0);
-#endif
-
-    /* Remove node from tree... */
-
-    /* Find node to splice out */
-    if (!z->left || !z->right)
-	y = z;
-    else
-	/* Set y to z:s successor */
-	for(y = z->right; y->left; y = y->left);
-    /* splice out y */
-    x = y->left ? y->left : y->right;
-    spliced_is_black = IS_BLACK(y);
-    if (x) {
-	x->parent = y->parent;
-    }
-    else if (!x && spliced_is_black) {
-	x = &null_x;
-	x->flags = 0;
-	SET_BLACK(x);
-	x->right = x->left = NULL;
-	x->parent = y->parent;
-	y->left = x;
-    }
-
-    if (!y->parent) {
-	RBT_ASSERT(root == y);
-	root = x;
-    }
-    else if (y == y->parent->left)
-	y->parent->left = x;
-    else {
-	RBT_ASSERT(y == y->parent->right);
-	y->parent->right = x;
-    }
-    if (y != z) {
-	/* We spliced out the successor of z; replace z by the successor */
-	replace(&root, z, y);
-    }
-
-    if (spliced_is_black) {
-	/* We removed a black node which makes the resulting tree
-	   violate the Red-Black Tree properties. Fixup tree... */
-
-	while (IS_BLACK(x) && x->parent) {
-
-	    /*
-	     * x has an "extra black" which we move up the tree
-	     * until we reach the root or until we can get rid of it.
-	     *
-	     * y is the sibbling of x
-	     */
-
-	    if (x == x->parent->left) {
-		y = x->parent->right;
-		RBT_ASSERT(y);
-		if (IS_RED(y)) {
-		    RBT_ASSERT(y->right);
-		    RBT_ASSERT(y->left);
-		    SET_BLACK(y);
-		    RBT_ASSERT(IS_BLACK(x->parent));
-		    SET_RED(x->parent);
-		    left_rotate(&root, x->parent);
-		    y = x->parent->right;
-		}
-		RBT_ASSERT(y);
-		RBT_ASSERT(IS_BLACK(y));
-		if (IS_BLACK(y->left) && IS_BLACK(y->right)) {
-		    SET_RED(y);
-		    x = x->parent;
-		}
-		else {
-		    if (IS_BLACK(y->right)) {
-			SET_BLACK(y->left);
-			SET_RED(y);
-			right_rotate(&root, y);
-			y = x->parent->right;
-		    }
-		    RBT_ASSERT(y);
-		    if (IS_RED(x->parent)) {
-
-			SET_BLACK(x->parent);
-			SET_RED(y);
-		    }
-		    RBT_ASSERT(y->right);
-		    SET_BLACK(y->right);
-		    left_rotate(&root, x->parent);
-		    x = root;
-		    break;
-		}
-	    }
-	    else {
-		RBT_ASSERT(x == x->parent->right);
-		y = x->parent->left;
-		RBT_ASSERT(y);
-		if (IS_RED(y)) {
-		    RBT_ASSERT(y->right);
-		    RBT_ASSERT(y->left);
-		    SET_BLACK(y);
-		    RBT_ASSERT(IS_BLACK(x->parent));
-		    SET_RED(x->parent);
-		    right_rotate(&root, x->parent);
-		    y = x->parent->left;
-		}
-		RBT_ASSERT(y);
-		RBT_ASSERT(IS_BLACK(y));
-		if (IS_BLACK(y->right) && IS_BLACK(y->left)) {
-		    SET_RED(y);
-		    x = x->parent;
-		}
-		else {
-		    if (IS_BLACK(y->left)) {
-			SET_BLACK(y->right);
-			SET_RED(y);
-			left_rotate(&root, y);
-			y = x->parent->left;
-		    }
-		    RBT_ASSERT(y);
-		    if (IS_RED(x->parent)) {
-			SET_BLACK(x->parent);
-			SET_RED(y);
-		    }
-		    RBT_ASSERT(y->left);
-		    SET_BLACK(y->left);
-		    right_rotate(&root, x->parent);
-		    x = root;
-		    break;
-		}
-	    }
-	}
-	SET_BLACK(x);
-
-	if (null_x.parent) {
-	    if (null_x.parent->left == &null_x)
-		null_x.parent->left = NULL;
-	    else {
-		RBT_ASSERT(null_x.parent->right == &null_x);
-		null_x.parent->right = NULL;
-	    }
-	    RBT_ASSERT(!null_x.left);
-	    RBT_ASSERT(!null_x.right);
-	}
-	else if (root == &null_x) {
-	    root = NULL;
-	    RBT_ASSERT(!null_x.left);
-	    RBT_ASSERT(!null_x.right);
-	}
-    }
-
-
-    DESTROY_TREE_NODE(del);
-
-#ifdef HARD_DEBUG
-    check_tree(0);
-#endif
-
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * "Address order best fit" specific callbacks.                              *
-\*                                                                           */
-
-static void
-link_free_block(Block_t *block)
-{
-    RBTree_t *blk = (RBTree_t *) block;
-    Uint blk_sz = BLK_SZ(blk);
-
-    blk->flags	= 0;
-    blk->left	= NULL;
-    blk->right	= NULL;
-
-    if (!root) {
-	blk->parent = NULL;
-	SET_BLACK(blk);
-	root = blk;
-    } else {
-	RBTree_t *x = root;
-	while (1) {
-	    Uint size;
-
-	    size = BLK_SZ(x);
-
-	    if (blk_sz < size || (blk_sz == size && blk < x)) {
-		if (!x->left) {
-		    blk->parent = x;
-		    x->left = blk;
-		    break;
-		}
-		x = x->left;
-	    }
-	    else {
-		if (!x->right) {
-		    blk->parent = x;
-		    x->right = blk;
-		    break;
-		}
-		x = x->right;
-	    }
-
-	}
-
-	/* Insert block into size tree */
-	RBT_ASSERT(blk->parent);
-
-	SET_RED(blk);
-	if (IS_RED(blk->parent)) {
-	    tree_insert_fixup(blk);
-	}
-    }
-
-#ifdef HARD_DEBUG
-    check_tree(0);
-#endif
-}
-
-
-static Block_t *
-get_free_block(Uint size)
-{
-    RBTree_t *x = root;
-    RBTree_t *blk = NULL;
-    Uint blk_sz;
-
-    while (x) {
-	blk_sz = BLK_SZ(x);
-	if (blk_sz < size) {
-	    x = x->right;
-	}
-	else {
-	    blk = x;
-	    x = x->left;
-	}
-    }
-    
-    if (!blk)
-	return NULL;
-
-#ifdef HARD_DEBUG
-    ASSERT(blk == check_tree(size));
-#endif
-
-    unlink_free_block((Block_t *) blk);
-
-    return (Block_t *) blk;
-}
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * Debug functions                                                           *
-\*                                                                           */
-
-
-#ifdef HARD_DEBUG
-
-#define IS_LEFT_VISITED(FB)	((FB)->flags & LEFT_VISITED_FLG)
-#define IS_RIGHT_VISITED(FB)	((FB)->flags & RIGHT_VISITED_FLG)
-
-#define SET_LEFT_VISITED(FB)	((FB)->flags |= LEFT_VISITED_FLG)
-#define SET_RIGHT_VISITED(FB)	((FB)->flags |= RIGHT_VISITED_FLG)
-
-#define UNSET_LEFT_VISITED(FB)	((FB)->flags &= ~LEFT_VISITED_FLG)
-#define UNSET_RIGHT_VISITED(FB)	((FB)->flags &= ~RIGHT_VISITED_FLG)
-
-
-#if 0
-#  define PRINT_TREE
-#else
-#  undef PRINT_TREE
-#endif
-
-#ifdef PRINT_TREE
-static void print_tree(void);
-#endif
-
-/*
- * Checks that the order between parent and children are correct,
- * and that the Red-Black Tree properies are satisfied. if size > 0,
- * check_tree() returns a node that satisfies "best fit" resp.
- * "address order best fit".
- *
- * The Red-Black Tree properies are:
- *   1. Every node is either red or black.
- *   2. Every leaf (NIL) is black.
- *   3. If a node is red, then both its children are black.
- *   4. Every simple path from a node to a descendant leaf
- *      contains the same number of black nodes.
- */
-
-static RBTree_t *
-check_tree(Uint size)
-{
-    RBTree_t *res = NULL;
-    Sint blacks;
-    Sint curr_blacks;
-    RBTree_t *x;
-
-#ifdef PRINT_TREE
-    print_tree();
-#endif
-
-    if (!root)
-	return res;
-
-    x = root;
-    ASSERT(IS_BLACK(x));
-    ASSERT(!x->parent);
-    curr_blacks = 1;
-    blacks = -1;
-
-    while (x) {
-	if (!IS_LEFT_VISITED(x)) {
-	    SET_LEFT_VISITED(x);
-	    if (x->left) {
-		x = x->left;
-		if (IS_BLACK(x))
-		    curr_blacks++;
-		continue;
-	    }
-	    else {
-		if (blacks < 0)
-		    blacks = curr_blacks;
-		ASSERT(blacks == curr_blacks);
-	    }
-	}
-
-	if (!IS_RIGHT_VISITED(x)) {
-	    SET_RIGHT_VISITED(x);
-	    if (x->right) {
-		x = x->right;
-		if (IS_BLACK(x))
-		    curr_blacks++;
-		continue;
-	    }
-	    else {
-		if (blacks < 0)
-		    blacks = curr_blacks;
-		ASSERT(blacks == curr_blacks);
-	    }
-	}
-
-
-	if (IS_RED(x)) {
-	    ASSERT(IS_BLACK(x->right));
-	    ASSERT(IS_BLACK(x->left));
-	}
-
-	ASSERT(x->parent || x == root);
-
-	if (x->left) {
-	    ASSERT(x->left->parent == x);
-	    ASSERT(BLK_SZ(x->left) < BLK_SZ(x)
-		   || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x));
-	}
-
-	if (x->right) {
-	    ASSERT(x->right->parent == x);
-	    ASSERT(BLK_SZ(x->right) > BLK_SZ(x)
-		   || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x));
-	}
-
-	if (size && BLK_SZ(x) >= size) {
-	    if (!res
-		|| BLK_SZ(x) < BLK_SZ(res)
-		|| (BLK_SZ(x) == BLK_SZ(res) && x < res))
-		res = x;
-	}
-
-	UNSET_LEFT_VISITED(x);
-	UNSET_RIGHT_VISITED(x);
-	if (IS_BLACK(x))
-	    curr_blacks--;
-	x = x->parent;
-
-    }
-
-    ASSERT(curr_blacks == 0);
-
-    UNSET_LEFT_VISITED(root);
-    UNSET_RIGHT_VISITED(root);
-
-    return res;
-
-}
-
-
-#ifdef PRINT_TREE
-#define INDENT_STEP 2
-
-#include <stdio.h>
-
-static void
-print_tree_aux(RBTree_t *x, int indent)
-{
-    int i;
-
-    if (!x) {
-	for (i = 0; i < indent; i++) {
-	    putc(' ', stderr);
-	}
-	fprintf(stderr, "BLACK: nil\r\n");
-    }
-    else {
-	print_tree_aux(x->right, indent + INDENT_STEP);
-	for (i = 0; i < indent; i++) {
-	    putc(' ', stderr);
-	}
-	fprintf(stderr, "%s: sz=%lu addr=0x%lx\r\n",
-		IS_BLACK(x) ? "BLACK" : "RED",
-		BLK_SZ(x),
-		(Uint) x);
-	print_tree_aux(x->left,  indent + INDENT_STEP);
-    }
-}
-
-
-static void
-print_tree(void)
-{
-    fprintf(stderr, " --- Size-Adress tree begin ---\r\n");
-    print_tree_aux(root, 0);
-    fprintf(stderr, " --- Size-Adress tree end ---\r\n");
-}
-
-#endif
-
-#endif
-
-#endif /* ENABLE_ELIB_MALLOC */
diff --git a/erts/emulator/beam/elib_stat.h b/erts/emulator/beam/elib_stat.h
deleted file mode 100644
index d8c7f31737..0000000000
--- a/erts/emulator/beam/elib_stat.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-/*
-** Interface to elib statistics
-**
-*/
-#ifndef __ELIB_STAT_H__
-#define __ELIB_STAT_H__
-
-struct elib_stat {
-    int mem_total;    /* Number of heap words  */
-    int mem_blocks;   /* Number of block */
-    int mem_alloc;    /* Number of words in use */
-    int mem_free;     /* Number of words free */
-    int min_used;     /* Size of the smallest block used */
-    int max_free;     /* Size of the largest free block */
-    int free_blocks;  /* Number of fragments in free list */
-    int mem_max_alloc;/* Max number of words in use */
-};
-
-EXTERN_FUNCTION(void, elib_statistics, (void*));
-EXTERN_FUNCTION(int,  elib_check_heap, (_VOID_));
-EXTERN_FUNCTION(void, elib_heap_dump, (char*));
-EXTERN_FUNCTION(void, elib_stat, (struct elib_stat*));
-EXTERN_FUNCTION(int,  elib_heap_map, (unsigned char*, int));
-EXTERN_FUNCTION(int,  elib_histo, (unsigned long*, unsigned long*, int, int));
-
-#endif
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index b853ec0f01..cda404af5e 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -38,9 +38,6 @@
 #include "erl_bits.h"
 #include "erl_instrument.h"
 #include "erl_mseg.h"
-#ifdef ELIB_ALLOC_IS_CLIB
-#include "erl_version.h"
-#endif
 #include "erl_monitors.h"
 #include "erl_bif_timer.h"
 #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
@@ -64,8 +61,15 @@
 
 #ifdef DEBUG
 static Uint install_debug_functions(void);
+#if 0
+#define HARD_DEBUG
+#ifdef __GNUC__
+#warning "* * * * * * * * * * * * * *"
+#warning "* HARD DEBUG IS ENABLED!  *"
+#warning "* * * * * * * * * * * * * *"
+#endif
+#endif
 #endif
-extern void elib_ensure_initialized(void);
 
 ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1];
 ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
@@ -86,6 +90,10 @@ typedef union {
 static ErtsAllocatorState_t sl_alloc_state;
 static ErtsAllocatorState_t std_alloc_state;
 static ErtsAllocatorState_t ll_alloc_state;
+#if HALFWORD_HEAP
+static ErtsAllocatorState_t std_alloc_low_state;
+static ErtsAllocatorState_t ll_alloc_low_state;
+#endif
 static ErtsAllocatorState_t temp_alloc_state;
 static ErtsAllocatorState_t eheap_alloc_state;
 static ErtsAllocatorState_t binary_alloc_state;
@@ -162,6 +170,10 @@ typedef struct {
     struct au_init binary_alloc;
     struct au_init ets_alloc;
     struct au_init driver_alloc;
+#if HALFWORD_HEAP
+    struct au_init std_alloc_low;
+    struct au_init ll_alloc_low;
+#endif
 } erts_alc_hndl_args_init_t;
 
 #define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
@@ -189,6 +201,10 @@ set_default_sl_alloc_opts(struct au_init *ip)
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_SHORT_LIVED;
     ip->init.util.rsbcst	= 80;
+#if HALFWORD_HEAP
+    ip->init.util.low_mem       = 1;
+#endif
+
 }
 
 static void
@@ -220,7 +236,7 @@ set_default_ll_alloc_opts(struct au_init *ip)
     ip->init.util.ramv		= 0;
     ip->init.util.mmsbc		= 0;
     ip->init.util.mmmbc		= 0;
-    ip->init.util.sbct		= ~((Uint) 0);
+    ip->init.util.sbct		= ~((UWord) 0);
     ip->init.util.name_prefix	= "ll_";
     ip->init.util.alloc_no	= ERTS_ALC_A_LONG_LIVED;
 #ifndef SMALL_MEMORY
@@ -252,6 +268,9 @@ set_default_temp_alloc_opts(struct au_init *ip)
     ip->init.util.ts 		= ERTS_ALC_MTA_TEMPORARY;
     ip->init.util.rsbcst	= 90;
     ip->init.util.rmbcmt	= 100;
+#if HALFWORD_HEAP
+    ip->init.util.low_mem       = 1;
+#endif
 }
 
 static void
@@ -271,6 +290,9 @@ set_default_eheap_alloc_opts(struct au_init *ip)
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_EHEAP;
     ip->init.util.rsbcst	= 50;
+#if HALFWORD_HEAP
+    ip->init.util.low_mem       = 1;
+#endif
 }
 
 static void
@@ -391,10 +413,14 @@ refuse_af_strategy(struct au_init *init)
 
 static void init_thr_ix(int static_ixs);
 
+#ifdef HARD_DEBUG
+static void hdbg_init(void);
+#endif
+
 void
 erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 {
-    Uint extra_block_size = 0;
+    UWord extra_block_size = 0;
     int i;
     erts_alc_hndl_args_init_t init = {
 	0,
@@ -406,6 +432,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	ERTS_DEFAULT_ALCU_INIT
     };
 
+#ifdef HARD_DEBUG
+    hdbg_init();
+#endif
+
     erts_sys_alloc_init();
     init_thr_ix(erts_no_schedulers);
     erts_init_utils_mem();
@@ -519,6 +549,20 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     erts_allctrs[ERTS_ALC_A_SYSTEM].free		= erts_sys_free;
     erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled	= 1;
 
+#if HALFWORD_HEAP
+    /* Init low memory variants by cloning */
+    init.std_alloc_low = init.std_alloc;
+    init.std_alloc_low.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW;
+    init.std_alloc_low.init.util.low_mem  = 1;
+
+    init.ll_alloc_low = init.ll_alloc;
+    init.ll_alloc_low.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW;
+    init.ll_alloc_low.init.util.low_mem = 1;
+
+    set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_alloc_low);
+    set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_alloc_low);
+#endif /* HALFWORD */
+
     set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc);
     set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc);
     set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc);
@@ -542,7 +586,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 
     sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold);
     sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad);
-
     if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled)
 	erts_fix_core_allocator_ix = ERTS_FIX_CORE_ALLOCATOR;
     else
@@ -565,7 +608,14 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     start_au_allocator(ERTS_ALC_A_LONG_LIVED,
 		       &init.ll_alloc,
 		       &ll_alloc_state);
-
+#if HALFWORD_HEAP
+    start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW,
+		       &init.ll_alloc_low,
+		       &ll_alloc_low_state);
+    start_au_allocator(ERTS_ALC_A_STANDARD_LOW,
+		       &init.std_alloc_low,
+		       &std_alloc_low_state);
+#endif
     start_au_allocator(ERTS_ALC_A_EHEAP,
 		       &init.eheap_alloc,
 		       &eheap_alloc_state);
@@ -601,11 +651,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     erts_set_fix_size(ERTS_ALC_T_PROC,		sizeof(Process));
     erts_set_fix_size(ERTS_ALC_T_DB_TABLE,	sizeof(DbTable));
     erts_set_fix_size(ERTS_ALC_T_ATOM,		sizeof(Atom));
-    erts_set_fix_size(ERTS_ALC_T_EXPORT,	sizeof(Export));
+
     erts_set_fix_size(ERTS_ALC_T_MODULE,	sizeof(Module));
     erts_set_fix_size(ERTS_ALC_T_REG_PROC,	sizeof(RegProc));
-    erts_set_fix_size(ERTS_ALC_T_MONITOR_SH,	ERTS_MONITOR_SH_SIZE*sizeof(Uint));
-    erts_set_fix_size(ERTS_ALC_T_NLINK_SH,	ERTS_LINK_SH_SIZE*sizeof(Uint));
     erts_set_fix_size(ERTS_ALC_T_FUN_ENTRY,	sizeof(ErlFunEntry));
 #ifdef ERTS_ALC_T_DRV_EV_D_STATE
     erts_set_fix_size(ERTS_ALC_T_DRV_EV_D_STATE,
@@ -615,6 +663,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     erts_set_fix_size(ERTS_ALC_T_DRV_SEL_D_STATE,
 		      sizeof(ErtsDrvSelectDataState));
 #endif
+#if !HALFWORD_HEAP
+    erts_set_fix_size(ERTS_ALC_T_EXPORT,	sizeof(Export));
+    erts_set_fix_size(ERTS_ALC_T_MONITOR_SH,	ERTS_MONITOR_SH_SIZE*sizeof(Uint));
+    erts_set_fix_size(ERTS_ALC_T_NLINK_SH,	ERTS_LINK_SH_SIZE*sizeof(Uint));
+#endif
 #endif
 #endif
 
@@ -627,6 +680,15 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
     ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
     ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
 
+#if HALFWORD_HEAP
+    /* If halfword heap, silently ignore any disabling of internal
+     * allocators for low memory
+     */
+    if (init->init.util.low_mem) {
+	init->enable = 1;
+    }
+#endif
+
     if (!init->enable) {
 	af->alloc = erts_sys_alloc;
 	af->realloc = erts_sys_realloc;
@@ -719,8 +781,8 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 		     init->init.util.name_prefix);
 	tspec->allctr = (Allctr_t **) states;
 	states = ((char *) states) + sizeof(Allctr_t *) * (tspec->size + 1);
-	states = ((((Uint) states) & ERTS_CACHE_LINE_MASK)
-		  ? (void *) ((((Uint) states) & ~ERTS_CACHE_LINE_MASK)
+	states = ((((UWord) states) & ERTS_CACHE_LINE_MASK)
+		  ? (void *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK)
 			      + ERTS_CACHE_LINE_SIZE)
 		  : (void *) states);
 	tspec->allctr[0] = init->thr_spec > 0 ? (Allctr_t *) state : (Allctr_t *) NULL;
@@ -1337,7 +1399,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 	    argv[j++] = argv[i];
     }
     *argc = j;
-    
 }
 
 static char *type_no_str(ErtsAlcType_t n)
@@ -1393,6 +1454,33 @@ void erts_alloc_reg_scheduler_id(Uint id)
     erts_tsd_set(thr_ix_key, (void *)(long) ix);
 }
 
+static void
+no_verify(Allctr_t *allctr)
+{
+
+}
+
+erts_alloc_verify_func_t
+erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr)
+{
+    if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util
+	&& erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) {
+	ErtsAllocatorThrSpec_t *tspec;
+	tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY];
+	if (!tspec->all_thr_safe) {
+	    int ix = erts_alc_get_thr_ix();
+
+	    if (ix < tspec->size) {
+		*allctr = tspec->allctr[ix];
+		return erts_alcu_verify_unused;
+	    }
+	}
+    }
+
+    *allctr = NULL;
+    return no_verify;
+}
+
 __decl_noreturn void
 erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
 {
@@ -1483,10 +1571,10 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size)
     erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size);
 }
 
-static ERTS_INLINE Uint
+static ERTS_INLINE UWord
 alcu_size(ErtsAlcType_t ai)
 {
-    Uint res = 0;
+    UWord res = 0;
 
     ASSERT(erts_allctrs_info[ai].enabled);
     ASSERT(erts_allctrs_info[ai].alloc_util);
@@ -1518,6 +1606,49 @@ alcu_size(ErtsAlcType_t ai)
     return res;
 }
 
+#if HALFWORD_HEAP
+static ERTS_INLINE int
+alcu_is_low(ErtsAlcType_t ai)
+{
+    int is_low = 0;
+    ASSERT(erts_allctrs_info[ai].enabled);
+    ASSERT(erts_allctrs_info[ai].alloc_util);
+
+    if (!erts_allctrs_info[ai].thr_spec) {
+	Allctr_t *allctr = erts_allctrs_info[ai].extra;
+	is_low = allctr->mseg_opt.low_mem;
+    }
+    else {
+	ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai];
+	int i;
+# ifdef DEBUG
+	int found_one = 0;
+# endif
+
+	ASSERT(tspec->all_thr_safe);
+	ASSERT(tspec->enabled);
+
+	for (i = tspec->size - 1; i >= 0; i--) {
+	    Allctr_t *allctr = tspec->allctr[i];
+	    if (allctr) {
+# ifdef DEBUG
+		if (!found_one) {
+		    is_low = allctr->mseg_opt.low_mem;
+		    found_one = 1;
+		}
+		else ASSERT(is_low == allctr->mseg_opt.low_mem);
+# else
+		is_low = allctr->mseg_opt.low_mem;
+		break;
+# endif
+	    }
+	}
+	ASSERT(found_one);
+    }
+    return is_low;
+}
+#endif /* HALFWORD */
+
 Eterm
 erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 {
@@ -1534,23 +1665,28 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	int code;
 	int ets;
 	int maximum;
+#if HALFWORD_HEAP
+	int low;
+#endif
     } want = {0};
     struct {
-	Uint total;
-	Uint processes;
-	Uint processes_used;
-	Uint system;
-	Uint atom;
-	Uint atom_used;
-	Uint binary;
-	Uint code;
-	Uint ets;
-	Uint maximum;
+	UWord total;
+	UWord processes;
+	UWord processes_used;
+	UWord system;
+	UWord atom;
+	UWord atom_used;
+	UWord binary;
+	UWord code;
+	UWord ets;
+	UWord maximum;
+#if HALFWORD_HEAP
+	UWord low;
+#endif
     } size = {0};
-    Eterm atoms[sizeof(size)/sizeof(Uint)];
-    Uint *uintps[sizeof(size)/sizeof(Uint)];
-    Eterm euints[sizeof(size)/sizeof(Uint)];
-    int need_atom;
+    Eterm atoms[sizeof(size)/sizeof(UWord)];
+    UWord *uintps[sizeof(size)/sizeof(UWord)];
+    Eterm euints[sizeof(size)/sizeof(UWord)];
     int want_tot_or_sys;
     int length;
     Eterm res = THE_NON_VALUE;
@@ -1602,15 +1738,20 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	    atoms[length] = am_maximum;
 	    uintps[length++] = &size.maximum;
 	}
-
+#if HALFWORD_HEAP
+	want.low = 1;
+	atoms[length] = am_low;
+	uintps[length++] = &size.low;
+#endif
     }
     else {
-	Eterm tmp_heap[2];
+	DeclareTmpHeapNoproc(tmp_heap,2);
 	Eterm wanted_list;
 
 	if (is_nil(earg))
 	    return NIL;
 
+	UseTmpHeapNoproc(2);
 	if (is_not_atom(earg))
 	    wanted_list = earg;
 	else {
@@ -1690,15 +1831,27 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 			atoms[length] = am_maximum;
 			uintps[length++] = &size.maximum;
 		    }
-		}
-		else
+		} else {
+		    UnUseTmpHeapNoproc(2);
 		    return am_badarg;
+		}
+		break;
+#if HALFWORD_HEAP
+	    case am_low:
+		if (!want.low) {
+		    want.low = 1;
+		    atoms[length] = am_low;
+		    uintps[length++] = &size.low;
+		}
 		break;
+#endif
 	    default:
+		UnUseTmpHeapNoproc(2);
 		return am_badarg;
 	    }
 	    wanted_list = CDR(list_val(wanted_list));
 	}
+	UnUseTmpHeapNoproc(2);
 	if (is_not_nil(wanted_list))
 	    return am_badarg;
     }
@@ -1721,7 +1874,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 
     ASSERT(length <= sizeof(atoms)/sizeof(Eterm));
     ASSERT(length <= sizeof(euints)/sizeof(Eterm));
-    ASSERT(length <= sizeof(uintps)/sizeof(Uint));
+    ASSERT(length <= sizeof(uintps)/sizeof(UWord));
 
 
     if (proc) {
@@ -1734,15 +1887,14 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
     /* Calculate values needed... */
 
     want_tot_or_sys = want.total || want.system;
-    need_atom = ERTS_MEM_NEED_ALL_ALCU || want.atom;
 
     if (ERTS_MEM_NEED_ALL_ALCU) {
 	size.total = 0;
 
 	for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
 	    if (erts_allctrs_info[ai].alloc_util) {
-		Uint *save;
-		Uint asz;
+		UWord *save;
+		UWord asz;
 		switch (ai) {
 		case ERTS_ALC_A_TEMPORARY:
 		     /*
@@ -1767,6 +1919,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		if (save)
 		    *save = asz;
 		size.total += asz;
+#if HALFWORD_HEAP
+		if (alcu_is_low(ai)) {
+		    size.low += asz;
+		}
+#endif
 	    }
 	}
     }
@@ -1774,7 +1931,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 
 
     if (want_tot_or_sys || want.processes || want.processes_used) {
-	Uint tmp;
+	UWord tmp;
 
 	if (ERTS_MEM_NEED_ALL_ALCU)
 	    tmp = size.processes;
@@ -1789,6 +1946,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 
 	size.processes = size.processes_used = tmp;
 
+#if HALFWORD_HEAP
+	/* BUG: We ignore link and monitor memory */
+#else
 	erts_fix_info(ERTS_ALC_T_NLINK_SH, &efi);
 	size.processes += efi.total;
 	size.processes_used += efi.used;
@@ -1796,6 +1956,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	erts_fix_info(ERTS_ALC_T_MONITOR_SH, &efi);
 	size.processes += efi.total;
 	size.processes_used += efi.used;
+#endif
 
 	erts_fix_info(ERTS_ALC_T_PROC, &efi);
 	size.processes += efi.total;
@@ -1832,8 +1993,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	erts_fix_info(ERTS_ALC_T_MODULE, &efi);
 	size.code += efi.used;
 	size.code += export_table_sz();
+#if HALFWORD_HEAP
+	size.code += export_list_size() * sizeof(Export);
+#else
 	erts_fix_info(ERTS_ALC_T_EXPORT, &efi);
 	size.code += efi.used;
+#endif
 	size.code += erts_fun_table_sz();
 	erts_fix_info(ERTS_ALC_T_FUN_ENTRY, &efi);
 	size.code += efi.used;
@@ -1879,9 +2044,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	if (only_one_value) {
 	    ASSERT(length == 1);
 	    hsz = 0;
-	    erts_bld_uint(NULL, &hsz, *uintps[0]);
+	    erts_bld_uword(NULL, &hsz, *uintps[0]);
 	    hp = hsz ? HAlloc((Process *) proc, hsz) : NULL;
-	    res = erts_bld_uint(&hp, NULL, *uintps[0]);
+	    res = erts_bld_uword(&hp, NULL, *uintps[0]);
 	}
 	else {
 	    Uint **hpp = NULL;
@@ -1891,7 +2056,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	    while (1) {
 		int i;
 		for (i = 0; i < length; i++)
-		    euints[i] = erts_bld_uint(hpp, hszp, *uintps[i]);
+		    euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]);
 		res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints);
 		if (hpp)
 		    break;
@@ -2060,11 +2225,11 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
 	for (i = 0; i < length; i++) {
 	    switch (values[i].arity) {
 	    case 2:
-		erts_print(to, arg, "%s: %bpu\n",
+		erts_print(to, arg, "%s: %beu\n",
 			   values[i].name, values[i].ui[0]);
 		break;
 	    case 3:
-		erts_print(to, arg, "%s: %bpu %bpu\n",
+		erts_print(to, arg, "%s: %beu %beu\n",
 			   values[i].name, values[i].ui[0], values[i].ui[1]);
 		break;
 	    default:
@@ -2285,8 +2450,8 @@ erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz)
 			SysAllocStat sas;
 			Eterm opts_am;
 			Eterm opts;
-			Eterm as[4];
-			Eterm ts[4];
+			Eterm as[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */
+			Eterm ts[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */
 			int l;
 
 			if (only_sz)
@@ -2302,13 +2467,8 @@ erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz)
 			l = 0;
 			as[l] = am_atom_put("e", 1);
 			ts[l++] = am_true;
-#ifdef ELIB_ALLOC_IS_CLIB
-			as[l] = am_atom_put("m", 1);
-			ts[l++] = am_atom_put("elib", 4);
-#else
 			as[l] = am_atom_put("m", 1);
 			ts[l++] = am_atom_put("libc", 4);
-#endif
 			if(sas.trim_threshold >= 0) {
 			    as[l] = am_atom_put("tt", 2);
 			    ts[l++] = erts_bld_uint(hpp, szp,
@@ -2462,11 +2622,7 @@ erts_allocator_info(int to, void *arg)
 		    case ERTS_ALC_A_SYSTEM: {
 			SysAllocStat sas;
 			erts_print(to, arg, "option e: true\n");
-#ifdef ELIB_ALLOC_IS_CLIB
-			erts_print(to, arg, "option m: elib\n");
-#else
 			erts_print(to, arg, "option m: libc\n");
-#endif
 			sys_alloc_stat(&sas);
 			if(sas.trim_threshold >= 0)
 			    erts_print(to, arg, "option tt: %d\n", sas.trim_threshold);
@@ -2570,13 +2726,8 @@ erts_allocator_options(void *proc)
 
 		switch (a) {
 		case ERTS_ALC_A_SYSTEM:
-#ifdef ELIB_ALLOC_IS_CLIB
-		    as[l] = am_atom_put("m", 1);
-		    ts[l++] = am_atom_put("elib", 4);
-#else
 		    as[l] = am_atom_put("m", 1);
 		    ts[l++] = am_atom_put("libc", 4);
-#endif
 		    if(sas.trim_threshold >= 0) {
 			as[l] = am_atom_put("tt", 2);
 			ts[l++] = erts_bld_uint(hpp, szp,
@@ -2647,23 +2798,7 @@ erts_allocator_options(void *proc)
 
     features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
 
-#if defined(ELIB_ALLOC_IS_CLIB)
-    {
-	Eterm version;
-	int i;
-	int ver[5];
-	i = sscanf(ERLANG_VERSION,
-		   "%d.%d.%d.%d.%d",
-		   &ver[0], &ver[1], &ver[2], &ver[3], &ver[4]);
-
-	version = NIL;
-	for(i--; i >= 0; i--)
-	  version = erts_bld_cons(hpp, szp, make_small(ver[i]), version);
-
-	res = erts_bld_tuple(hpp, szp, 4,
-			     am_elib_malloc, version, features, settings);
-    }
-#elif defined(__GLIBC__)
+#if defined(__GLIBC__)
     {
 	Eterm AM_glibc = am_atom_put("glibc", 5);
 	Eterm version;
@@ -2859,12 +2994,10 @@ unsigned long erts_alc_test(unsigned long op,
 	    break;
 	}
 	case 0xf0a:
-	    if (ethr_mutex_lock((ethr_mutex *) a1) != 0)
-		ERTS_ALC_TEST_ABORT;
+	    ethr_mutex_lock((ethr_mutex *) a1);
 	    break;
 	case 0xf0b:
-	    if (ethr_mutex_unlock((ethr_mutex *) a1) != 0)
-		ERTS_ALC_TEST_ABORT;
+	    ethr_mutex_unlock((ethr_mutex *) a1);
 	    break;
 	case 0xf0c: {
 	    ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond));
@@ -2880,31 +3013,21 @@ unsigned long erts_alc_test(unsigned long op,
 	    break;
 	}
 	case 0xf0e:
-	    if (ethr_cond_broadcast((ethr_cond *) a1) != 0)
-		ERTS_ALC_TEST_ABORT;
+	    ethr_cond_broadcast((ethr_cond *) a1);
 	    break;
 	case 0xf0f: {
 	    int res;
 	    do {
 		res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2);
 	    } while (res == EINTR);
-	    if (res != 0)
-		ERTS_ALC_TEST_ABORT;
 	    break;
 	}
 	case 0xf10: {
 	    ethr_tid *tid = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_tid));
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	    if (erts_lcnt_thr_create(tid,
-				(void * (*)(void *)) a1,
-				(void *) a2,
-				NULL) != 0)
-#else
 	    if (ethr_thr_create(tid,
 				(void * (*)(void *)) a1,
 				(void *) a2,
 				NULL) != 0)
-#endif
 		ERTS_ALC_TEST_ABORT;
 	    return (unsigned long) tid;
 	}
@@ -2943,10 +3066,13 @@ unsigned long erts_alc_test(unsigned long op,
 #undef PRINT_OPS
 #endif
 
+#ifdef HARD_DEBUG
+#define FENCE_SZ		(4*sizeof(UWord))
+#else
+#define FENCE_SZ		(3*sizeof(UWord))
+#endif
 
-#define FENCE_SZ		(3*sizeof(Uint))
-
-#ifdef ARCH_64
+#if defined(ARCH_64)
 #define FENCE_PATTERN 0xABCDEF97ABCDEF97
 #else
 #define FENCE_PATTERN 0xABCDEF97
@@ -2956,7 +3082,7 @@ unsigned long erts_alc_test(unsigned long op,
 #define TYPE_PATTERN_SHIFT 16
 
 #define FIXED_FENCE_PATTERN_MASK \
-  (~((Uint) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT)))
+  (~((UWord) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT)))
 #define FIXED_FENCE_PATTERN \
   (FENCE_PATTERN & FIXED_FENCE_PATTERN_MASK)
 
@@ -2966,22 +3092,170 @@ unsigned long erts_alc_test(unsigned long op,
 #define GET_TYPE_OF_PATTERN(P) \
   (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK)
 
+#ifdef HARD_DEBUG
+
+#define ERL_ALC_HDBG_MAX_MBLK 100000
+#define ERTS_ALC_O_CHECK -1
+
+typedef struct hdbg_mblk_ hdbg_mblk;
+struct hdbg_mblk_ {
+    hdbg_mblk *next;
+    hdbg_mblk *prev;
+    void *p;
+    Uint s;
+    ErtsAlcType_t n;
+};
+
+static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK];
+
+static hdbg_mblk *free_hdbg_mblks;
+static hdbg_mblk *used_hdbg_mblks;
+static erts_mtx_t hdbg_mblk_mtx;
+
+static void
+hdbg_init(void)
+{
+    int i;
+    for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++)
+	hdbg_mblks[i].next = &hdbg_mblks[i+1];
+    hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL;
+    free_hdbg_mblks = &hdbg_mblks[0];
+    used_hdbg_mblks = NULL;
+    erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug");
+}
+
+static void *check_memory_fence(void *ptr,
+				Uint *size,
+				ErtsAlcType_t n,
+				int func);
+void erts_hdbg_chk_blks(void);
+
+void
+erts_hdbg_chk_blks(void)
+{
+    hdbg_mblk *mblk;
+
+    erts_mtx_lock(&hdbg_mblk_mtx);
+    for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) {
+	Uint sz;
+	check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK);
+	ASSERT(sz == mblk->s);
+    }
+    erts_mtx_unlock(&hdbg_mblk_mtx);
+}
+
+static hdbg_mblk *
+hdbg_alloc(void *p, Uint s, ErtsAlcType_t n)
+{
+    hdbg_mblk *mblk;
+
+    erts_mtx_lock(&hdbg_mblk_mtx);
+    mblk = free_hdbg_mblks;
+    if (!mblk) {
+	erts_fprintf(stderr,
+		     "Ran out of debug blocks; please increase "
+		     "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n",
+		     ERL_ALC_HDBG_MAX_MBLK);
+	abort();
+    }
+    free_hdbg_mblks = mblk->next;
+
+    mblk->p = p;
+    mblk->s = s;
+    mblk->n = n;
+
+    mblk->next = used_hdbg_mblks;
+    mblk->prev = NULL;
+    if (used_hdbg_mblks)
+	used_hdbg_mblks->prev = mblk;
+    used_hdbg_mblks = mblk;
+    erts_mtx_unlock(&hdbg_mblk_mtx);
+    return (void *) mblk;
+}
+
+static void
+hdbg_free(hdbg_mblk *mblk)
+{
+    erts_mtx_lock(&hdbg_mblk_mtx);
+    if (mblk->next)
+	mblk->next->prev = mblk->prev;
+    if (mblk->prev)
+	mblk->prev->next = mblk->next;
+    else
+	used_hdbg_mblks = mblk->next;
+
+    mblk->next = free_hdbg_mblks;
+    free_hdbg_mblks = mblk;
+    erts_mtx_unlock(&hdbg_mblk_mtx);
+}
+
+#endif
+
+#ifdef  ERTS_ALLOC_UTIL_HARD_DEBUG
+static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func);
+
+void check_allocated_block( Uint type, void *blk)
+{
+    Uint dummy;
+    check_memory_fence(blk, &dummy, ERTS_ALC_T2N(type), ERTS_ALC_O_FREE);
+}
+
+void check_allocators(void)
+{
+    int i;
+    if (!erts_initialized)
+	return;
+    for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; ++i) {
+	if (erts_allctrs_info[i].alloc_util) {
+	    ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra;
+	    Allctr_t *allctr = real_af->extra;
+	    Carrier_t *ct;
+#ifdef USE_THREADS
+	if (allctr->thread_safe)
+	    erts_mtx_lock(&allctr->mutex);
+#endif
+
+	    if (allctr->check_mbc) {
+		for (ct = allctr->mbc_list.first; ct; ct = ct->next) {
+		    fprintf(stderr,"Checking allocator %d\r\n",i);
+		    allctr->check_mbc(allctr,ct);
+		}
+	    }
+#ifdef USE_THREADS
+	if (allctr->thread_safe)
+	    erts_mtx_unlock(&allctr->mutex);
+#endif
+	}
+    }
+}
+#endif
 
 static void *
 set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n)
 {
-    Uint *ui_ptr;
-    Uint pattern;
+    UWord *ui_ptr;
+    UWord pattern;
+#ifdef HARD_DEBUG
+    hdbg_mblk **mblkpp;
+#endif
 
     if (!ptr)
 	return NULL;
 
-    ui_ptr = (Uint *) ptr;
+    ui_ptr = (UWord *) ptr;
     pattern = MK_PATTERN(n);
-    
+
+#ifdef HARD_DEBUG
+    mblkpp = (hdbg_mblk **) ui_ptr++;
+#endif
+
     *(ui_ptr++) = sz;
     *(ui_ptr++) = pattern;
-    memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(Uint));
+    memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord));
+
+#ifdef HARD_DEBUG
+    *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n);
+#endif
 
     return (void *) ui_ptr;
 }
@@ -2991,16 +3265,22 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
 {
     Uint sz;
     Uint found_type;
-    Uint pre_pattern;
-    Uint post_pattern;
-    Uint *ui_ptr;
+    UWord pre_pattern;
+    UWord post_pattern;
+    UWord *ui_ptr;
+#ifdef HARD_DEBUG
+    hdbg_mblk *mblk;
+#endif
 
     if (!ptr)
 	return NULL;
 
-    ui_ptr = (Uint *) ptr;
+    ui_ptr = (UWord *) ptr;
     pre_pattern = *(--ui_ptr);
     *size = sz = *(--ui_ptr);
+#ifdef HARD_DEBUG
+    mblk = (hdbg_mblk *) *(--ui_ptr);
+#endif
 
     found_type = GET_TYPE_OF_PATTERN(pre_pattern);
     if (pre_pattern != MK_PATTERN(n)) {
@@ -3011,7 +3291,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
 		     (unsigned long) ptr);
     }
 
-    memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(Uint));
+    memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord));
 
     if (post_pattern != MK_PATTERN(n)
 	|| pre_pattern != post_pattern) {
@@ -3056,6 +3336,17 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
 		 (unsigned long) ptr, (unsigned long) sz, ftype, op_str, otype);
     }
 
+#ifdef HARD_DEBUG
+    switch (func) {
+    case ERTS_ALC_O_REALLOC:
+    case ERTS_ALC_O_FREE:
+	hdbg_free(mblk);
+	break;
+    default:
+	break;
+    }
+#endif
+
     return (void *) ui_ptr;
 }
 
@@ -3068,6 +3359,10 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size)
     Uint dsize;
     void *res;
 
+#ifdef HARD_DEBUG
+    erts_hdbg_chk_blks();
+#endif
+
     ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
     dsize = size + FENCE_SZ;
     res = (*real_af->alloc)(n, real_af->extra, dsize);
@@ -3097,13 +3392,17 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size)
     dsize = size + FENCE_SZ;
     dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC);
 
+#ifdef HARD_DEBUG
+    erts_hdbg_chk_blks();
+#endif
+
     if (old_size > size)
 	sys_memset((void *) (((char *) ptr) + size),
 		   0xf,
 		   sizeof(Uint) + old_size - size);
 
     res = (*real_af->realloc)(n, real_af->extra, dptr, dsize);
-    
+
     res = set_memory_fence(res, size, n);
 
 #ifdef PRINT_OPS
@@ -3133,6 +3432,10 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr)
     fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr);
 #endif
 
+#ifdef HARD_DEBUG
+    erts_hdbg_chk_blks();
+#endif
+
 }
 
 static Uint
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index e7a203002f..ce792d4d17 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -172,9 +172,17 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size);
 void erts_free(ErtsAlcType_t type, void *ptr);
 void *erts_alloc_fnf(ErtsAlcType_t type, Uint size);
 void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size);
+void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size);
+
 
 #endif /* #if !ERTS_ALC_DO_INLINE */
 
+#ifndef ERTS_CACHE_LINE_SIZE
+/* Assume a cache line size of 64 bytes */
+#  define ERTS_CACHE_LINE_SIZE ((UWord) 64)
+#  define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1)
+#endif
+
 #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__)
 
 ERTS_ALC_INLINE
@@ -234,13 +242,25 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size)
 	size);
 }
 
+ERTS_ALC_INLINE
+void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size)
+{
+    UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1));
+
+    if (v & ERTS_CACHE_LINE_MASK) {
+	v = (v & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
+    }
+    ASSERT((v & ERTS_CACHE_LINE_MASK) == 0);
+    return (void*)v;
+}
+
 #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */
 
-#ifndef ERTS_CACHE_LINE_SIZE
-/* Assume a cache line size of 64 bytes */
-#  define ERTS_CACHE_LINE_SIZE ((Uint) 64)
-#  define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1)
-#endif
+typedef void (*erts_alloc_verify_func_t)(Allctr_t *);
+
+erts_alloc_verify_func_t
+erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr);
+
 
 #define ERTS_ALC_CACHE_LINE_ALIGN_SIZE(SZ) \
   (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE)
@@ -486,9 +506,9 @@ init_##NAME##_alloc(void)						\
     qa_data_##NAME##__ = erts_alloc(ERTS_ALC_T_PRE_ALLOC_DATA,tot_size);\
     chunk_start = (((char *) qa_data_##NAME##__)			\
 		   + sizeof(erts_sched_pref_quick_alloc_data_t));	\
-    if ((((Uint) chunk_start) & ERTS_CACHE_LINE_MASK) != ((Uint) 0))	\
+    if ((((UWord) chunk_start) & ERTS_CACHE_LINE_MASK) != ((UWord) 0))	\
 	chunk_start = ((char *)						\
-		       ((((Uint) chunk_start) & ~ERTS_CACHE_LINE_MASK)	\
+		       ((((UWord) chunk_start) & ~ERTS_CACHE_LINE_MASK)	\
 		       + ERTS_CACHE_LINE_SIZE));			\
     qa_data_##NAME##__->chunks_mem_size = chunk_mem_size;		\
     qa_data_##NAME##__->start = (void *) chunk_start;			\
@@ -553,7 +573,7 @@ NAME##_free(TYPE *p)							\
 }
 
 #ifdef DEBUG
-#define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((Uint *) (PTR)) - 2))
+#define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((UWord *) (PTR)) - 2))
 #endif /* #ifdef DEBUG */
 
 #undef ERTS_ALC_INLINE
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index f701f71c7d..c6cc0e1fac 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -1,19 +1,19 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2003-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2003-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 
@@ -75,6 +75,11 @@ allocator	EHEAP		true		eheap_alloc
 allocator	ETS		true		ets_alloc
 allocator	FIXED_SIZE	true		fix_alloc
 
++if halfword
+allocator	LONG_LIVED_LOW	true		ll_alloc_low
+allocator	STANDARD_LOW	true		std_alloc_low
++endif
+
 +else # Non smp build
 
 allocator	TEMPORARY	false		temp_alloc
@@ -85,12 +90,18 @@ allocator	EHEAP		false		eheap_alloc
 allocator	ETS		false		ets_alloc
 allocator	FIXED_SIZE	false		fix_alloc
 
++if halfword
+allocator	LONG_LIVED_LOW	false		ll_alloc_low
+allocator	STANDARD_LOW	false		std_alloc_low
++endif
+
 +endif
 
 allocator	BINARY		true		binary_alloc
 allocator	DRIVER		true		driver_alloc
 
 
+
 # --- Class declarations -----------------------------------------------------
 #
 # Syntax: class <CLASS> <DESCRIPTION>
@@ -125,19 +136,15 @@ class	SYSTEM		system_data
 
 type	PROC		FIXED_SIZE	PROCESSES	proc
 type	ATOM		FIXED_SIZE	ATOM		atom_entry
-type	EXPORT		FIXED_SIZE	CODE		export_entry
 type	MODULE		FIXED_SIZE	CODE		module_entry
 type	REG_PROC	FIXED_SIZE	PROCESSES	reg_proc
 type	LINK_LH		STANDARD	PROCESSES	link_lh
-type	MONITOR_SH	FIXED_SIZE	PROCESSES	monitor_sh
-type	MONITOR_LH	STANDARD	PROCESSES	monitor_lh
-type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
-type	NLINK_LH	STANDARD	PROCESSES	nlink_lh
 type	SUSPEND_MON	STANDARD	PROCESSES	suspend_monitor
 type	PEND_SUSPEND	SHORT_LIVED	PROCESSES	pending_suspend
 type	PROC_LIST	SHORT_LIVED	PROCESSES	proc_list
 type	FUN_ENTRY	FIXED_SIZE	CODE		fun_entry
 type	ATOM_TXT	LONG_LIVED	ATOM		atom_text
+type 	BEAM_REGISTER	EHEAP		PROCESSES	beam_register
 type	HEAP		EHEAP		PROCESSES	heap
 type	OLD_HEAP	EHEAP		PROCESSES	old_heap
 type	HEAP_FRAG	EHEAP		PROCESSES	heap_frag
@@ -174,7 +181,6 @@ type	DRIVER		STANDARD	SYSTEM		driver
 type	NIF		DRIVER		SYSTEM		nif_internal
 type	BINARY		BINARY		BINARIES	binary
 type	NBIF_TABLE	SYSTEM		SYSTEM		nbif_tab
-type	CODE		LONG_LIVED	CODE		code
 type	ARG_REG		STANDARD	PROCESSES	arg_reg
 type	PROC_DICT	STANDARD	PROCESSES	proc_dict
 type	CALLS_BUF	STANDARD	PROCESSES	calls_buf
@@ -194,7 +200,6 @@ type	DB_TABLES	LONG_LIVED	ETS		db_tabs
 type    DB_NTAB_ENT	STANDARD	ETS		db_named_table_entry
 type	DB_TMP		TEMPORARY	ETS		db_tmp
 type	DB_MC_STK	TEMPORARY	ETS		db_mc_stack
-type	DB_MS_PSDO_PROC	LONG_LIVED	ETS		db_match_pseudo_proc
 type	DB_MS_RUN_HEAP	SHORT_LIVED	ETS		db_match_spec_run_heap
 type	DB_MS_CMPL_HEAP	TEMPORARY	ETS		db_match_spec_cmpl_heap
 type	DB_SEG		ETS		ETS		db_segment
@@ -211,9 +216,8 @@ type	LOGGER_DSBUF	TEMPORARY	SYSTEM		logger_dsbuf
 type	TMP_DSBUF	TEMPORARY	SYSTEM		tmp_dsbuf
 type	INFO_DSBUF	SYSTEM		SYSTEM		info_dsbuf
 # INFO_DSBUF have to use the SYSTEM allocator; otherwise, a deadlock might occur
-type	SCHDLR_DATA	LONG_LIVED	PROCESSES	scheduler_data
+type	SCHDLR_SLP_INFO	LONG_LIVED	SYSTEM		scheduler_sleep_info
 type	RUNQS		LONG_LIVED	SYSTEM		run_queues
-type	DDLL_PROCESS	STANDARD	SYSTEM		ddll_processes
 type	DDLL_HANDLE	STANDARD	SYSTEM		ddll_handle
 type	DDLL_ERRCODES	LONG_LIVED	SYSTEM		ddll_errcodes
 type	DDLL_TMP_BUF	TEMPORARY	SYSTEM		ddll_tmp_buf
@@ -231,6 +235,7 @@ type    RE_SUBJECT      SHORT_LIVED     SYSTEM          re_subject
 type  	RE_HEAP 	STANDARD	SYSTEM		re_heap
 type	RE_STACK 	SHORT_LIVED	SYSTEM		re_stack
 type	UNICODE_BUFFER 	SHORT_LIVED	SYSTEM		unicode_buffer
+type	BINARY_BUFFER 	SHORT_LIVED	SYSTEM		binary_buffer
 type	PRE_ALLOC_DATA	LONG_LIVED	SYSTEM		pre_alloc_data
 type	DRV_THR_OPTS	DRIVER		SYSTEM		driver_thread_opts
 type	DRV_TID		DRIVER		SYSTEM		driver_tid
@@ -244,6 +249,7 @@ type	CPUDATA		LONG_LIVED	SYSTEM		cpu_data
 type	TMP_CPU_IDS	SHORT_LIVED	SYSTEM		tmp_cpu_ids
 type	EXT_TERM_DATA	SHORT_LIVED	PROCESSES	external_term_data
 type	ZLIB		STANDARD	SYSTEM		zlib
+type	CPU_GRPS_MAP	LONG_LIVED	SYSTEM		cpu_groups_map
 
 +if smp
 type	ASYNC		SHORT_LIVED	SYSTEM		async
@@ -259,6 +265,8 @@ type	XPORTS_LIST	SHORT_LIVED	SYSTEM		extra_port_list
 type	PROC_LCK_WTR	LONG_LIVED	SYSTEM		proc_lock_waiter
 type	PROC_LCK_QS	LONG_LIVED	SYSTEM		proc_lock_queues
 type	RUNQ_BLNS	LONG_LIVED	SYSTEM		run_queue_balancing
+type	MISC_AUX_WORK_Q	LONG_LIVED	SYSTEM		misc_aux_work_q
+type	MISC_AUX_WORK	SHORT_LIVED	SYSTEM		misc_aux_work
 +endif
 
 #
@@ -267,7 +275,9 @@ type	RUNQ_BLNS	LONG_LIVED	SYSTEM		run_queue_balancing
 
 +if threads
 
-type	ETHR_INTERNAL	SYSTEM		SYSTEM		ethread_internal
+type	ETHR_STD	STANDARD	SYSTEM		ethread_standard
+type	ETHR_SL		SHORT_LIVED	SYSTEM		ethread_short_lived
+type	ETHR_LL		LONG_LIVED	SYSTEM		ethread_long_lived
 
 +ifnot smp
 
@@ -318,10 +328,45 @@ type	SSB		SHORT_LIVED	PROCESSES	ssb
 +endif
 
 
++if halfword
+
+type	DDLL_PROCESS	STANDARD_LOW	SYSTEM		ddll_processes
+type	MONITOR_LH	STANDARD_LOW	PROCESSES	monitor_lh
+type	NLINK_LH	STANDARD_LOW	PROCESSES	nlink_lh
+type	CODE		LONG_LIVED_LOW	CODE		code
+type	DB_HEIR_DATA	STANDARD_LOW	ETS		db_heir_data
+type	DB_MS_PSDO_PROC	LONG_LIVED_LOW	ETS		db_match_pseudo_proc
+type	SCHDLR_DATA	LONG_LIVED_LOW	SYSTEM		scheduler_data
+type  	LL_TEMP_TERM 	LONG_LIVED_LOW	SYSTEM		ll_temp_term
+
+# no FIXED_SIZE for low memory
+type	EXPORT		STANDARD_LOW	CODE		export_entry
+type	MONITOR_SH	STANDARD_LOW	PROCESSES	monitor_sh
+type	NLINK_SH	STANDARD_LOW	PROCESSES	nlink_sh
+
++else # "fullword"
+
+type	DDLL_PROCESS	STANDARD	SYSTEM		ddll_processes
+type	MONITOR_LH	STANDARD	PROCESSES	monitor_lh
+type	NLINK_LH	STANDARD	PROCESSES	nlink_lh
+type	CODE		LONG_LIVED	CODE		code
+type	DB_HEIR_DATA	STANDARD	ETS		db_heir_data
+type	DB_MS_PSDO_PROC	LONG_LIVED	ETS		db_match_pseudo_proc
+type	SCHDLR_DATA	LONG_LIVED	SYSTEM		scheduler_data
+type  	LL_TEMP_TERM 	LONG_LIVED	SYSTEM		ll_temp_term
+
+type	EXPORT		FIXED_SIZE	CODE		export_entry
+type	MONITOR_SH	FIXED_SIZE	PROCESSES	monitor_sh
+type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
+
++endif
+
+
 #
 # Types used by system specific code
 #
 
+type  	TEMP_TERM 	TEMPORARY	SYSTEM		temp_term
 type	DRV_TAB		LONG_LIVED	SYSTEM		drv_tab
 type	DRV_EV_STATE	LONG_LIVED	SYSTEM		driver_event_state
 type	DRV_EV_D_STATE	FIXED_SIZE	SYSTEM		driver_event_data_state
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 9b7bc24c1c..cc04ef65bf 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -69,14 +69,14 @@ static int initialized = 0;
 
 #if HAVE_ERTS_MSEG
 
-#define INV_MSEG_UNIT_MASK	((Uint) (mseg_unit_size - 1))
+#define INV_MSEG_UNIT_MASK	((UWord) (mseg_unit_size - 1))
 #define MSEG_UNIT_MASK		(~INV_MSEG_UNIT_MASK)
 #define MSEG_UNIT_FLOOR(X)	((X) & MSEG_UNIT_MASK)
 #define MSEG_UNIT_CEILING(X)	MSEG_UNIT_FLOOR((X) + INV_MSEG_UNIT_MASK)
 
 #endif
 
-#define INV_SYS_ALLOC_CARRIER_MASK	((Uint) (sys_alloc_carrier_size - 1))
+#define INV_SYS_ALLOC_CARRIER_MASK	((UWord) (sys_alloc_carrier_size - 1))
 #define SYS_ALLOC_CARRIER_MASK		(~INV_SYS_ALLOC_CARRIER_MASK)
 #define SYS_ALLOC_CARRIER_FLOOR(X)	((X) & SYS_ALLOC_CARRIER_MASK)
 #define SYS_ALLOC_CARRIER_CEILING(X) \
@@ -85,7 +85,7 @@ static int initialized = 0;
 #undef ASSERT
 #define ASSERT ASSERT_EXPR
 
-#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE ((Uint) 1)
+#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE ((UWord) 1)
 
 #if 0
 /* Can be useful for debugging */
@@ -114,12 +114,12 @@ static Uint mseg_unit_size;
 
 /* Blocks ... */
 
-#define SBC_BLK_FTR_FLG		(((Uint) 1) << 0)
-#define UNUSED1_BLK_FTR_FLG	(((Uint) 1) << 1)
-#define UNUSED2_BLK_FTR_FLG	(((Uint) 1) << 2)
+#define SBC_BLK_FTR_FLG		(((UWord) 1) << 0)
+#define UNUSED1_BLK_FTR_FLG	(((UWord) 1) << 1)
+#define UNUSED2_BLK_FTR_FLG	(((UWord) 1) << 2)
 
 #define ABLK_HDR_SZ (sizeof(Block_t))
-#define FBLK_FTR_SZ (sizeof(Uint))
+#define FBLK_FTR_SZ (sizeof(UWord))
 
 #define UMEMSZ2BLKSZ(AP, SZ)						\
   (ABLK_HDR_SZ + (SZ) <= (AP)->min_block_size				\
@@ -130,14 +130,14 @@ static Uint mseg_unit_size;
 #define BLK2UMEM(P) ((void *)    (((char *) (P)) + ABLK_HDR_SZ))
 
 #define PREV_BLK_SZ(B) \
-  ((Uint) (*(((Uint *) (B)) - 1) & SZ_MASK))
+  ((UWord) (*(((UWord *) (B)) - 1) & SZ_MASK))
 
 #define SET_BLK_SZ_FTR(B, SZ) \
-  (*((Uint *) (((char *) (B)) + (SZ) - sizeof(Uint))) = (SZ))
+  (*((UWord *) (((char *) (B)) + (SZ) - sizeof(UWord))) = (SZ))
 
-#define THIS_FREE_BLK_HDR_FLG 	(((Uint) 1) << 0)
-#define PREV_FREE_BLK_HDR_FLG 	(((Uint) 1) << 1)
-#define LAST_BLK_HDR_FLG 	(((Uint) 1) << 2)
+#define THIS_FREE_BLK_HDR_FLG 	(((UWord) 1) << 0)
+#define PREV_FREE_BLK_HDR_FLG 	(((UWord) 1) << 1)
+#define LAST_BLK_HDR_FLG 	(((UWord) 1) << 2)
 
 #define SET_BLK_SZ(B, SZ) \
   (ASSERT(((SZ) & FLG_MASK) == 0), \
@@ -156,11 +156,11 @@ static Uint mseg_unit_size;
   (*((Block_t *) (B)) &= ~LAST_BLK_HDR_FLG)
 
 #define SBH_THIS_FREE		THIS_FREE_BLK_HDR_FLG
-#define SBH_THIS_ALLOCED	((Uint) 0)
+#define SBH_THIS_ALLOCED	((UWord) 0)
 #define SBH_PREV_FREE		PREV_FREE_BLK_HDR_FLG
-#define SBH_PREV_ALLOCED	((Uint) 0)
+#define SBH_PREV_ALLOCED	((UWord) 0)
 #define SBH_LAST_BLK		LAST_BLK_HDR_FLG
-#define SBH_NOT_LAST_BLK	((Uint) 0)
+#define SBH_NOT_LAST_BLK	((UWord) 0)
 
 #define SET_BLK_HDR(B, Sz, F) \
   (ASSERT(((Sz) & FLG_MASK) == 0), *((Block_t *) (B)) = ((Sz) | (F)))
@@ -200,7 +200,7 @@ static Uint mseg_unit_size;
   ((FTR) = 0)
 
 #define IS_SBC_BLK(B) \
-  (IS_PREV_BLK_FREE((B)) && (((Uint *) (B))[-1] & SBC_BLK_FTR_FLG))
+  (IS_PREV_BLK_FREE((B)) && (((UWord *) (B))[-1] & SBC_BLK_FTR_FLG))
 #define IS_MBC_BLK(B) \
   (!IS_SBC_BLK((B)))
 
@@ -211,8 +211,8 @@ static Uint mseg_unit_size;
 
 /* Carriers ... */
 
-#define MSEG_CARRIER_HDR_FLAG		(((Uint) 1) << 0)
-#define SBC_CARRIER_HDR_FLAG		(((Uint) 1) << 1)
+#define MSEG_CARRIER_HDR_FLAG		(((UWord) 1) << 0)
+#define SBC_CARRIER_HDR_FLAG		(((UWord) 1) << 1)
 
 #define SCH_SYS_ALLOC			0
 #define SCH_MSEG			MSEG_CARRIER_HDR_FLAG
@@ -407,18 +407,18 @@ do {									\
 
 /* Debug stuff... */
 #ifdef DEBUG
-static Uint carrier_alignment;
+static UWord carrier_alignment;
 #define DEBUG_SAVE_ALIGNMENT(C)						\
 do {									\
-    Uint algnmnt__ = sizeof(Unit_t) - (((Uint) (C)) % sizeof(Unit_t));	\
+    UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t));	\
     carrier_alignment = MIN(carrier_alignment, algnmnt__);		\
-    ASSERT(((Uint) (C)) % sizeof(Uint) == 0);				\
+    ASSERT(((UWord) (C)) % sizeof(UWord) == 0);				\
 } while (0)
 #define DEBUG_CHECK_ALIGNMENT(P)					\
 do {									\
-    ASSERT(sizeof(Unit_t) - (((Uint) (P)) % sizeof(Unit_t))		\
+    ASSERT(sizeof(Unit_t) - (((UWord) (P)) % sizeof(Unit_t))		\
 	   >= carrier_alignment);					\
-    ASSERT(((Uint) (P)) % sizeof(Uint) == 0);				\
+    ASSERT(((UWord) (P)) % sizeof(UWord) == 0);				\
 } while (0)
 
 #else
@@ -610,7 +610,7 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr)
 }
 
 
-static Block_t *create_carrier(Allctr_t *, Uint, Uint);
+static Block_t *create_carrier(Allctr_t *, Uint, UWord);
 static void destroy_carrier(Allctr_t *, Block_t *);
 
 /* Multi block carrier alloc/realloc/free ... */
@@ -630,6 +630,11 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp)
 
     blk = (*allctr->get_free_block)(allctr, *blk_szp, NULL, 0);
 
+#if HALFWORD_HEAP
+    if (!blk) {
+	blk = create_carrier(allctr, *blk_szp, CFLG_MBC|CFLG_FORCE_MSEG);
+    }
+#else
     if (!blk) {
 	blk = create_carrier(allctr, *blk_szp, CFLG_MBC);
 	if (!blk) {
@@ -640,6 +645,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp)
 				 CFLG_SBC|CFLG_FORCE_SIZE|CFLG_FORCE_SYS_ALLOC);
 	}
     }
+#endif
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
     if (IS_MBC_BLK(blk)) {
@@ -656,14 +662,14 @@ static ERTS_INLINE void
 mbc_alloc_finalize(Allctr_t *allctr,
 		   Block_t *blk,
 		   Uint org_blk_sz,
-		   Uint flags,
+		   UWord flags,
 		   Uint want_blk_sz,
 		   int valid_blk_info)
 {
     Uint blk_sz;
     Uint nxt_blk_sz;
     Block_t *nxt_blk;
-    Uint prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG;
+    UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG;
 
     ASSERT(org_blk_sz >= want_blk_sz);
     ASSERT(blk);
@@ -853,7 +859,7 @@ mbc_free(Allctr_t *allctr, void *p)
 }
 
 static void *
-mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
+mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
 {
     void *new_p;
     Uint old_blk_sz;
@@ -1144,7 +1150,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 	}
 	else {
 	    Uint new_blk_sz;
-	    Uint new_blk_flgs;
+	    UWord new_blk_flgs;
 	    Uint prev_blk_sz;
 	    Uint blk_cpy_sz;
 
@@ -1239,7 +1245,7 @@ do {									\
 
 
 static Block_t *
-create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
+create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 {
     Block_t *blk;
     Carrier_t *crr;
@@ -1283,8 +1289,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 	if (crr_sz < allctr->mbc_header_size + blk_sz)
 	    crr_sz = allctr->mbc_header_size + blk_sz;
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	if (sizeof(Unit_t) == sizeof(Uint))
-	    crr_sz += sizeof(Uint);
+	if (sizeof(Unit_t) == sizeof(UWord))
+	    crr_sz += sizeof(UWord);
 #endif
     }
     crr_sz = MSEG_UNIT_CEILING(crr_sz);
@@ -1324,8 +1330,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 	    && bcrr_sz < allctr->smallest_mbc_size)
 	    bcrr_sz = allctr->smallest_mbc_size;
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	if (sizeof(Unit_t) == sizeof(Uint))
-	    bcrr_sz += sizeof(Uint);
+	if (sizeof(Unit_t) == sizeof(UWord))
+	    bcrr_sz += sizeof(UWord);
 #endif
 
     }
@@ -1360,7 +1366,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 
 	blk = SBC2BLK(allctr, crr);
 
-	SET_SBC_BLK_FTR(((Uint *) blk)[-1]);
+	SET_SBC_BLK_FTR(((UWord *) blk)[-1]);
 	SET_BLK_HDR(blk, blk_sz, SBH_THIS_ALLOCED|SBH_PREV_FREE|SBH_LAST_BLK);
 
 	link_carrier(&allctr->sbc_list, crr);
@@ -1379,13 +1385,13 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 	blk = MBC2FBLK(allctr, crr);
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	if (sizeof(Unit_t) == sizeof(Uint))
-	    crr_sz -= sizeof(Uint);
+	if (sizeof(Unit_t) == sizeof(UWord))
+	    crr_sz -= sizeof(UWord);
 #endif
 
 	blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size);
 
-	SET_MBC_BLK_FTR(((Uint *) blk)[-1]);
+	SET_MBC_BLK_FTR(((UWord *) blk)[-1]);
 	SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK);
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
@@ -1400,13 +1406,13 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 	link_carrier(&allctr->mbc_list, crr);
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	if (sizeof(Unit_t) == sizeof(Uint))
-	    crr_sz += sizeof(Uint);
+	if (sizeof(Unit_t) == sizeof(UWord))
+	    crr_sz += sizeof(UWord);
 #endif
 	CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz);
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	if (sizeof(Unit_t) == sizeof(Uint))
-	    crr_sz -= sizeof(Uint);
+	if (sizeof(Unit_t) == sizeof(UWord))
+	    crr_sz -= sizeof(UWord);
 #endif
 	if (allctr->creating_mbc)
 	    (*allctr->creating_mbc)(allctr, crr);
@@ -1418,11 +1424,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 }
 
 static Block_t *
-resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, Uint flags)
+resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 {
     Block_t *new_blk;
     Carrier_t *new_crr, *old_crr;
-    Uint create_flags, old_crr_sz, old_blk_sz, new_blk_sz, new_crr_sz;
+    UWord create_flags;
+    Uint old_crr_sz, old_blk_sz, new_blk_sz, new_crr_sz;
     Uint new_bcrr_sz;
 
     if (flags & CFLG_MBC) {
@@ -1632,6 +1639,9 @@ static struct {
     Eterm e;
     Eterm t;
     Eterm ramv;
+#if HALFWORD_HEAP
+    Eterm low;
+#endif
     Eterm sbct;
 #if HAVE_ERTS_MSEG
     Eterm asbcst;
@@ -1717,6 +1727,9 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(e);
 	AM_INIT(t);
 	AM_INIT(ramv);
+#if HALFWORD_HEAP
+	AM_INIT(low);
+#endif
 	AM_INIT(sbct);
 #if HAVE_ERTS_MSEG
 	AM_INIT(asbcst);
@@ -1870,7 +1883,7 @@ sz_info_carriers(Allctr_t *allctr,
 		   cs->blocks.max_ever.size);
 	erts_print(to,
 		   arg,
-		   "%scarriers size: %bpu %bpu %bpu\n",
+		   "%scarriers size: %beu %bpu %bpu\n",
 		   prefix,
 		   curr_size,
 		   cs->max.size,
@@ -1926,7 +1939,7 @@ info_carriers(Allctr_t *allctr,
 		   cs->blocks.max_ever.size);
 	erts_print(to,
 		   arg,
-		   "%scarriers: %bpu %bpu %bpu\n",
+		   "%scarriers: %beu %bpu %bpu\n",
 		   prefix,
 		   curr_no,
 		   cs->max.no,
@@ -1945,7 +1958,7 @@ info_carriers(Allctr_t *allctr,
 		   cs->curr_sys_alloc.no);
 	erts_print(to,
 		   arg,
-		   "%scarriers size: %bpu %bpu %bpu\n",
+		   "%scarriers size: %beu %bpu %bpu\n",
 		   prefix,
 		   curr_size,
 		   cs->max.size,
@@ -2046,15 +2059,15 @@ info_calls(Allctr_t *allctr,
 
 #define PRINT_CC_4(TO, TOA, NAME, CC)					\
     if ((CC).giga_no == 0)						\
-	erts_print(TO, TOA, "%s calls: %bpu\n", NAME, CC.no);		\
+	erts_print(TO, TOA, "%s calls: %b32u\n", NAME, CC.no);		\
     else								\
-	erts_print(TO, TOA, "%s calls: %bpu%09lu\n", NAME, CC.giga_no, CC.no)
+	erts_print(TO, TOA, "%s calls: %b32u%09lu\n", NAME, CC.giga_no, CC.no)
 
 #define PRINT_CC_5(TO, TOA, PRFX, NAME, CC)				\
     if ((CC).giga_no == 0)						\
-	erts_print(TO, TOA, "%s%s calls: %bpu\n",PRFX,NAME,CC.no);	\
+	erts_print(TO, TOA, "%s%s calls: %b32u\n",PRFX,NAME,CC.no);	\
     else								\
-	erts_print(TO, TOA, "%s%s calls: %bpu%09lu\n",PRFX,NAME,CC.giga_no,CC.no)
+	erts_print(TO, TOA, "%s%s calls: %b32u%09lu\n",PRFX,NAME,CC.giga_no,CC.no)
 
 	char *prefix = allctr->name_prefix;
 	int to = *print_to_p;
@@ -2161,23 +2174,29 @@ info_options(Allctr_t *allctr,
 		   "option e: true\n"
 		   "option t: %s\n"
 		   "option ramv: %s\n"
-		   "option sbct: %bpu\n"
+#if HALFWORD_HEAP
+		   "option low: %s\n"
+#endif
+		   "option sbct: %beu\n"
 #if HAVE_ERTS_MSEG
 		   "option asbcst: %bpu\n"
 		   "option rsbcst: %bpu\n"
 #endif
-		   "option rsbcmt: %bpu\n"
-		   "option rmbcmt: %bpu\n"
-		   "option mmbcs: %bpu\n"
+		   "option rsbcmt: %beu\n"
+		   "option rmbcmt: %beu\n"
+		   "option mmbcs: %beu\n"
 #if HAVE_ERTS_MSEG
-		   "option mmsbc: %bpu\n"
-		   "option mmmbc: %bpu\n"
+		   "option mmsbc: %beu\n"
+		   "option mmmbc: %beu\n"
 #endif
-		   "option lmbcs: %bpu\n"
-		   "option smbcs: %bpu\n"
-		   "option mbcgs: %bpu\n",
+		   "option lmbcs: %beu\n"
+		   "option smbcs: %beu\n"
+		   "option mbcgs: %beu\n",
 		   topt,
 		   allctr->ramv ? "true" : "false",
+#if HALFWORD_HEAP
+		   allctr->mseg_opt.low_mem ? "true" : "false",
+#endif
 		   allctr->sbc_threshold,
 #if HAVE_ERTS_MSEG
 		   allctr->mseg_opt.abs_shrink_th,
@@ -2236,6 +2255,9 @@ info_options(Allctr_t *allctr,
 	add_2tup(hpp, szp, &res,
 		 am.sbct,
 		 bld_uint(hpp, szp, allctr->sbc_threshold));
+#if HALFWORD_HEAP
+	add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false);
+#endif
 	add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false);
 	add_2tup(hpp, szp, &res, am.t, (allctr->t
 					? bld_uint(hpp, szp, (Uint) allctr->t)
@@ -2285,9 +2307,9 @@ erts_alcu_au_info_options(int *print_to_p, void *print_to_arg,
 	erts_print(*print_to_p,
 		   print_to_arg,
 #if HAVE_ERTS_MSEG
-		   "option mmc: %bpu\n"
+		   "option mmc: %beu\n"
 #endif
-		   "option ycs: %bpu\n",
+		   "option ycs: %beu\n",
 #if HAVE_ERTS_MSEG
 		   max_mseg_carriers,
 #endif
@@ -2522,7 +2544,12 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
     INC_CC(allctr->calls.this_alloc);
 
     if (size >= allctr->sbc_threshold) {
+#if HALFWORD_HEAP
+	Block_t *blk = create_carrier(allctr, size,
+				      CFLG_SBC | CFLG_FORCE_MSEG);
+#else
 	Block_t *blk = create_carrier(allctr, size, CFLG_SBC);
+#endif
 	res = blk ? BLK2UMEM(blk) : NULL;
     }
     else
@@ -2594,16 +2621,16 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
     Allctr_t *allctr;
     void *res;
 
-    ASSERT(sizeof(Uint) == sizeof(Allctr_t *));
+    ASSERT(sizeof(UWord) == sizeof(Allctr_t *));
     ASSERT(ix > 0);
     if (ix >= tspec->size)
 	ix = (ix % (tspec->size - 1)) + 1;
     allctr = tspec->allctr[ix];
     erts_mtx_lock(&allctr->mutex);
-    res = do_erts_alcu_alloc(type, allctr, size + sizeof(Uint));
+    res = do_erts_alcu_alloc(type, allctr, size + sizeof(UWord));
     if (res) {
 	*((Allctr_t **) res) = allctr;
-	res = (void *) (((char *) res) + sizeof(Uint));
+	res = (void *) (((char *) res) + sizeof(UWord));
     }
     erts_mtx_unlock(&allctr->mutex);
     DEBUG_CHECK_ALIGNMENT(res);
@@ -2681,7 +2708,7 @@ void
 erts_alcu_free_thr_pref(ErtsAlcType_t type, void *unused, void *p)
 {
     if (p) {
-	void *ptr = (void *) (((char *) p) - sizeof(Uint));
+	void *ptr = (void *) (((char *) p) - sizeof(UWord));
 	Allctr_t *allctr = *((Allctr_t **) ptr);
 	erts_mtx_lock(&allctr->mutex);
 	do_erts_alcu_free(type, allctr, ptr);
@@ -2698,7 +2725,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 		     void *extra,
 		     void *p,
 		     Uint size,
-		     Uint flgs)
+		     UWord flgs)
 {
     Allctr_t *allctr = (Allctr_t *) extra; 
     Block_t *blk;
@@ -2780,13 +2807,21 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 	Block_t *new_blk;
 	if(IS_SBC_BLK(blk)) {
 	do_carrier_resize:
+#if HALFWORD_HEAP
+	    new_blk = resize_carrier(allctr, blk, size, CFLG_SBC | CFLG_FORCE_MSEG);
+#else
 	    new_blk = resize_carrier(allctr, blk, size, CFLG_SBC);
+#endif
 	    res = new_blk ? BLK2UMEM(new_blk) : NULL;
 	}
 	else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
 	    return NULL;
 	else {
+#if HALFWORD_HEAP
+	    new_blk = create_carrier(allctr, size, CFLG_SBC | CFLG_FORCE_MSEG);
+#else
 	    new_blk = create_carrier(allctr, size, CFLG_SBC);
+#endif
 	    if (new_blk) {
 		res = BLK2UMEM(new_blk);
 		sys_memcpy((void *) res,
@@ -2962,7 +2997,7 @@ erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size)
     if (!p)
 	return erts_alcu_alloc_thr_pref(type, extra, size);
 
-    ptr = (void *) (((char *) p) - sizeof(Uint));
+    ptr = (void *) (((char *) p) - sizeof(UWord));
     used_allctr = *((Allctr_t **) ptr);
 
     ix = erts_alc_get_thr_ix();
@@ -2976,32 +3011,32 @@ erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size)
     res = do_erts_alcu_realloc(type,
 			       used_allctr,
 			       ptr,
-			       size + sizeof(Uint),
+			       size + sizeof(UWord),
 			       (pref_allctr != used_allctr
 				? ERTS_ALCU_FLG_FAIL_REALLOC_MOVE
 				: 0));
     erts_mtx_unlock(&used_allctr->mutex);
     if (res) {
 	ASSERT(used_allctr == *((Allctr_t **) res));
-	res = (void *) (((char *) res) + sizeof(Uint));
+	res = (void *) (((char *) res) + sizeof(UWord));
 	DEBUG_CHECK_ALIGNMENT(res);
     }
     else {
 	erts_mtx_lock(&pref_allctr->mutex);
-	res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(Uint));
+	res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord));
 	erts_mtx_unlock(&pref_allctr->mutex);
 	if (res) {
 	    Block_t *blk;
 	    size_t cpy_size;
 
 	    *((Allctr_t **) res) = pref_allctr;
-	    res = (void *) (((char *) res) + sizeof(Uint));
+	    res = (void *) (((char *) res) + sizeof(UWord));
 
 	    DEBUG_CHECK_ALIGNMENT(res);
 
 	    erts_mtx_lock(&used_allctr->mutex);
 	    blk = UMEM2BLK(ptr);
-	    cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(Uint);
+	    cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(UWord);
 	    if (cpy_size > size)
 		cpy_size = size;
 	    sys_memcpy(res, p, cpy_size);
@@ -3026,7 +3061,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra,
     if (!p)
 	return erts_alcu_alloc_thr_pref(type, extra, size);
 
-    ptr = (void *) (((char *) p) - sizeof(Uint));
+    ptr = (void *) (((char *) p) - sizeof(UWord));
     used_allctr = *((Allctr_t **) ptr);
 
     ix = erts_alc_get_thr_ix();
@@ -3037,7 +3072,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra,
     ASSERT(used_allctr && pref_allctr);
 
     erts_mtx_lock(&pref_allctr->mutex);
-    res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(Uint));
+    res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord));
     if (!res) {
 	erts_mtx_unlock(&pref_allctr->mutex);
 	res = erts_alcu_realloc_thr_pref(type, extra, p, size);
@@ -3048,7 +3083,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra,
 	Allctr_t *allctr;
 
 	*((Allctr_t **) res) = pref_allctr;
-	res = (void *) (((char *) res) + sizeof(Uint));
+	res = (void *) (((char *) res) + sizeof(UWord));
 
 	DEBUG_CHECK_ALIGNMENT(res);
 
@@ -3061,7 +3096,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra,
 	}
 
 	blk = UMEM2BLK(ptr);
-	cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(Uint);
+	cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(UWord);
 	if (cpy_size > size)
 	    cpy_size = size;
 	sys_memcpy(res, p, cpy_size);
@@ -3085,13 +3120,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	goto error;
 
 #if HAVE_ERTS_MSEG
-    {
-	ErtsMsegOpt_t mseg_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER;
-    
-	sys_memcpy((void *) &allctr->mseg_opt,
-		   (void *) &mseg_opt,
-		   sizeof(ErtsMsegOpt_t));
-    }
+    sys_memcpy((void *) &allctr->mseg_opt,
+	       (void *) &erts_mseg_default_opt,
+	       sizeof(ErtsMsegOpt_t));
+# if HALFWORD_HEAP
+    allctr->mseg_opt.low_mem = init->low_mem;
+# endif
 #endif
 
     allctr->name_prefix			= init->name_prefix;
@@ -3138,11 +3172,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     if (allctr->min_block_size < ABLK_HDR_SZ)
 	goto error;
     allctr->min_block_size		= UNIT_CEILING(allctr->min_block_size
-						       + sizeof(Uint));
+						       + sizeof(UWord));
 
 #if HAVE_ERTS_MSEG
-    if (allctr->mseg_opt.abs_shrink_th > ~((Uint) 0) / 100)
-	allctr->mseg_opt.abs_shrink_th = ~((Uint) 0) / 100;
+    if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100)
+	allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100;
 #endif
 
 #ifdef USE_THREADS
@@ -3182,15 +3216,15 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size
 						+ FBLK_FTR_SZ
 						+ ABLK_HDR_SZ
-						+ sizeof(Uint))
+						+ sizeof(UWord))
 				   - ABLK_HDR_SZ
-				   - sizeof(Uint));
+				   - sizeof(UWord));
 	allctr->sbc_header_size = (UNIT_CEILING(sizeof(Carrier_t)
 						+ FBLK_FTR_SZ
 						+ ABLK_HDR_SZ
-						+ sizeof(Uint))
+						+ sizeof(UWord))
 				   - ABLK_HDR_SZ
-				   - sizeof(Uint));
+				   - sizeof(UWord));
     }
     else
 #endif
@@ -3208,12 +3242,21 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     if (allctr->main_carrier_size) {
 	Block_t *blk;
 
+#if HALFWORD_HEAP
+	blk = create_carrier(allctr,
+			     allctr->main_carrier_size,
+			     CFLG_MBC
+			     | CFLG_FORCE_SIZE
+			     | CFLG_FORCE_MSEG
+			     | CFLG_MAIN_CARRIER);
+#else
 	blk = create_carrier(allctr,
 			     allctr->main_carrier_size,
 			     CFLG_MBC
 			     | CFLG_FORCE_SIZE
 			     | CFLG_FORCE_SYS_ALLOC
 			     | CFLG_MAIN_CARRIER);
+#endif
 	if (!blk)
 	    goto error;
 
@@ -3339,6 +3382,38 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2)
  * Debug functions                                                           *
 \*                                                                           */
 
+void
+erts_alcu_verify_unused(Allctr_t *allctr)
+{
+    UWord no;
+
+    no = allctr->sbcs.curr_mseg.no;
+    no += allctr->sbcs.curr_sys_alloc.no;
+    no += allctr->mbcs.blocks.curr.no;
+
+    if (no) {
+	UWord sz = allctr->sbcs.blocks.curr.size;
+	sz += allctr->mbcs.blocks.curr.size;
+	erl_exit(ERTS_ABORT_EXIT,
+		 "%salloc() used when expected to be unused!\n"
+		 "Total amount of blocks allocated: %bpu\n"
+		 "Total amount of bytes allocated: %bpu\n",
+		 allctr->name_prefix, no, sz);
+    }
+}
+
+void
+erts_alcu_verify_unused_ts(Allctr_t *allctr)
+{
+#ifdef USE_THREADS
+    erts_mtx_lock(&allctr->mutex);
+#endif
+    erts_alcu_verify_unused(allctr);
+#ifdef USE_THREADS
+    erts_mtx_unlock(&allctr->mutex);
+#endif
+}
+
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
 
 static void
@@ -3409,7 +3484,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 	    is_free_blk = (int) IS_FREE_BLK(blk);
 	    if(is_free_blk) {
 		if (IS_NOT_LAST_BLK(blk))
-		    ASSERT(*((Uint *) (((char *) blk)+blk_sz-sizeof(Uint)))
+		    ASSERT(*((UWord *) (((char *) blk)+blk_sz-sizeof(UWord)))
 			   == blk_sz);
 	    }
 
@@ -3417,7 +3492,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 		(*allctr->check_block)(allctr, blk, (int) is_free_blk);
 
 	    if (IS_LAST_BLK(blk)) {
-		carrier_end = ((char *) NXT_BLK(blk)) + sizeof(Uint);
+		carrier_end = ((char *) NXT_BLK(blk)) + sizeof(UWord);
 		mbc = *((Carrier_t **) NXT_BLK(blk));
 		prev_blk = NULL;
 		blk = MBC2FBLK(allctr, mbc);
@@ -3433,7 +3508,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 	ASSERT((((char *) mbc)
 		+ allctr->mbc_header_size
 		+ tot_blk_sz
-		+ sizeof(Uint)) == carrier_end);
+		+ sizeof(UWord)) == carrier_end);
 	ASSERT(((char *) mbc) + CARRIER_SZ(mbc) == carrier_end);
 
 	if (allctr->check_mbc)
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 10b11661e6..ddf84c086c 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -27,8 +27,8 @@
 typedef struct Allctr_t_ Allctr_t;
 
 typedef struct {
-    Uint ycs;
-    Uint mmc;
+    UWord ycs;
+    UWord mmc;
 } AlcUInit_t;
 
 typedef struct {
@@ -38,22 +38,23 @@ typedef struct {
     int tspec;
     int tpref;
     int ramv;
-    Uint sbct;
-    Uint asbcst;
-    Uint rsbcst;
-    Uint rsbcmt;
-    Uint rmbcmt;
-    Uint mmbcs;
-    Uint mmsbc;
-    Uint mmmbc;
-    Uint lmbcs;
-    Uint smbcs;
-    Uint mbcgs;
+    int low_mem;   /* HALFWORD only */
+    UWord sbct;
+    UWord asbcst;
+    UWord rsbcst;
+    UWord rsbcmt;
+    UWord rmbcmt;
+    UWord mmbcs;
+    UWord mmsbc;
+    UWord mmmbc;
+    UWord lmbcs;
+    UWord smbcs;
+    UWord mbcgs;
 } AllctrInit_t;
 
 typedef struct {
-    Uint blocks;
-    Uint carriers;
+    UWord blocks;
+    UWord carriers;
 } AllctrSize_t;
 
 #ifndef SMALL_MEMORY
@@ -70,6 +71,7 @@ typedef struct {
     0,			/* (bool)   tspec:  thread specific              */\
     0,			/* (bool)   tpref:  thread preferred             */\
     0,			/* (bool)   ramv:   realloc always moves         */\
+    0,			/* (bool)   low_mem: HALFWORD only               */\
     512*1024,		/* (bytes)  sbct:   sbc threshold                */\
     2*1024*2024,	/* (amount) asbcst: abs sbc shrink threshold     */\
     20,			/* (%)      rsbcst: rel sbc shrink threshold     */\
@@ -97,6 +99,7 @@ typedef struct {
     0,			/* (bool)   tspec:  thread specific              */\
     0,			/* (bool)   tpref:  thread preferred             */\
     0,			/* (bool)   ramv:   realloc always moves         */\
+    0,			/* (bool)   low_mem: HALFWORD only               */\
     64*1024,		/* (bytes)  sbct:   sbc threshold                */\
     2*1024*2024,	/* (amount) asbcst: abs sbc shrink threshold     */\
     20,			/* (%)      rsbcst: rel sbc shrink threshold     */\
@@ -163,19 +166,19 @@ void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *);
 #define CEILING(X, I)  ((((X) - 1)/(I) + 1)*(I))
 
 #undef  WORD_MASK
-#define INV_WORD_MASK	((Uint) (sizeof(Uint) - 1))
+#define INV_WORD_MASK	((UWord) (sizeof(UWord) - 1))
 #define WORD_MASK	(~INV_WORD_MASK)
 #define WORD_FLOOR(X)	((X) & WORD_MASK)
 #define WORD_CEILING(X)	WORD_FLOOR((X) + INV_WORD_MASK)
 
 #undef  UNIT_MASK
-#define INV_UNIT_MASK	((Uint) (sizeof(Unit_t) - 1))
+#define INV_UNIT_MASK	((UWord) (sizeof(Unit_t) - 1))
 #define UNIT_MASK	(~INV_UNIT_MASK)
 #define UNIT_FLOOR(X)	((X) & UNIT_MASK)
 #define UNIT_CEILING(X)	UNIT_FLOOR((X) + INV_UNIT_MASK)
 
 
-#define SZ_MASK			(~((Uint) 0) << 3)
+#define SZ_MASK			(~((UWord) 0) << 3)
 #define FLG_MASK		(~(SZ_MASK))
 
 
@@ -189,7 +192,7 @@ typedef union {char c[8]; long l; double d;} Unit_t;
 
 typedef struct Carrier_t_ Carrier_t;
 struct Carrier_t_ {
-    Uint chdr;
+    UWord chdr;
     Carrier_t *next;
     Carrier_t *prev;
 };
@@ -199,17 +202,17 @@ typedef struct {
     Carrier_t *last;
 } CarrierList_t;
 
-typedef Uint Block_t;
-typedef Uint FreeBlkFtr_t;
+typedef UWord Block_t;
+typedef UWord FreeBlkFtr_t;
 
 typedef struct {
-    Uint giga_no;
-    Uint no;
+    UWord giga_no;
+    UWord no;
 } CallCounter_t;
 
 typedef struct {
-    Uint		no;
-    Uint		size;
+    UWord		no;
+    UWord		size;
 } StatValues_t;
 
 typedef struct {
@@ -333,6 +336,9 @@ struct Allctr_t_ {
 int	erts_alcu_start(Allctr_t *, AllctrInit_t *);
 void	erts_alcu_stop(Allctr_t *);
 
+void	erts_alcu_verify_unused(Allctr_t *);
+void	erts_alcu_verify_unused_ts(Allctr_t *allctr);
+
 unsigned long	erts_alcu_test(unsigned long, unsigned long, unsigned long);
 
 
diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index 126ec7cc73..64fad9fe0e 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -41,6 +41,16 @@
 #  define MAX(x, y) (((x) > (y)) ? (x) : (y))
 #endif
 
+#if !HEAP_ON_C_STACK
+#  define DECLARE_TMP(VariableName,N,P)  \
+     Eterm *VariableName = ((ERTS_PROC_GET_SCHDATA(P)->erl_arith_tmp_heap) + (2 * N))
+#else
+#  define DECLARE_TMP(VariableName,N,P) \
+     Eterm VariableName[2]
+#endif
+#  define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp)))
+
+
 static Eterm shift(Process* p, Eterm arg1, Eterm arg2, int right);
 
 static ERTS_INLINE void maybe_shrink(Process* p, Eterm* hp, Eterm res, Uint alloc)
@@ -169,7 +179,7 @@ shift(Process* p, Eterm arg1, Eterm arg2, int right)
 {
     Sint i;
     Sint ires;
-    Eterm tmp_big1[2];
+    DECLARE_TMP(tmp_big1,0,p);
     Eterm* bigp;
     Uint need;
 
@@ -312,8 +322,8 @@ BIF_RETTYPE bnot_1(BIF_ALIST_1)
 Eterm
 erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm res;
     Eterm hdr;
     FloatDef f1, f2;
@@ -458,8 +468,8 @@ erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2)
 Eterm
 erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm hdr;
     Eterm res;
     FloatDef f1, f2;
@@ -602,8 +612,8 @@ erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2)
 Eterm
 erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm hdr;
     Eterm res;
     FloatDef f1, f2;
@@ -627,8 +637,8 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
 		    } else if (arg2 == SMALL_ONE) {
 			return(arg1);
 		    } else {
-			Eterm big_res[3];
-
+			DeclareTmpHeap(big_res,3,p);
+			UseTmpHeap(3,p);
 			/*
 			 * The following code is optimized for the case that
 			 * result is small (which should be the most common case
@@ -636,6 +646,7 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
 			 */
 			res = small_times(signed_val(arg1), signed_val(arg2), big_res);
 			if (is_small(res)) {
+			    UnUseTmpHeap(3,p);
 			    return res;
 			} else {
 			    /*
@@ -657,6 +668,7 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
 			    if (arity > 1) {
 				*hp = big_res[2];
 			    }
+			    UnUseTmpHeap(3,p);
 			    return res;
 			}
 		    }
@@ -915,8 +927,8 @@ erts_mixed_div(Process* p, Eterm arg1, Eterm arg2)
 Eterm
 erts_int_div(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     int ires;
 
     switch (NUMBER_CODE(arg1, arg2)) {
@@ -967,8 +979,8 @@ erts_int_div(Process* p, Eterm arg1, Eterm arg2)
 Eterm
 erts_int_rem(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     int ires;
 
     switch (NUMBER_CODE(arg1, arg2)) {
@@ -979,7 +991,8 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2)
 	if (arg1 != make_small(MIN_SMALL)) {
 	    return arg1;
 	} else {
-	    Eterm tmp = small_to_big(signed_val(arg1), tmp_big1);
+	    Eterm tmp;
+	    tmp = small_to_big(signed_val(arg1), tmp_big1);
 	    if ((ires = big_ucomp(tmp, arg2)) == 0) {
 		return SMALL_ZERO;
 	    } else {
@@ -1013,8 +1026,8 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2)
 
 Eterm erts_band(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm* hp;
     int need;
 
@@ -1041,8 +1054,8 @@ Eterm erts_band(Process* p, Eterm arg1, Eterm arg2)
 
 Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm* hp;
     int need;
 
@@ -1069,8 +1082,8 @@ Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2)
 
 Eterm erts_bxor(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm* hp;
     int need;
 
@@ -1148,8 +1161,8 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg1;
     Eterm arg2;
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm res;
     Eterm hdr;
     FloatDef f1, f2;
@@ -1237,10 +1250,10 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live)
 		    need_heap = BIG_NEED_SIZE(sz);
 		    if (ERTS_NEED_GC(p, need_heap)) {
 			erts_garbage_collect(p, need_heap, reg, live+2);
-			if (arg1 != make_big(tmp_big1)) {
+			if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
 			    arg1 = reg[live];
 			}
-			if (arg2 != make_big(tmp_big2)) {
+			if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
 			    arg2 = reg[live+1];
 			}
 		    }
@@ -1316,8 +1329,8 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg1;
     Eterm arg2;
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm hdr;
     Eterm res;
     FloatDef f1, f2;
@@ -1394,10 +1407,10 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live)
 		    need_heap = BIG_NEED_SIZE(sz);
 		    if (ERTS_NEED_GC(p, need_heap)) {
 			erts_garbage_collect(p, need_heap, reg, live+2);
-			if (arg1 != make_big(tmp_big1)) {
+			if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
 			    arg1 = reg[live];
 			}
-			if (arg2 != make_big(tmp_big2)) {
+			if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
 			    arg2 = reg[live+1];
 			}
 		    }
@@ -1482,8 +1495,8 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg1;
     Eterm arg2;
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm hdr;
     Eterm res;
     FloatDef f1, f2;
@@ -1509,7 +1522,8 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
 		    } else if (arg2 == SMALL_ONE) {
 			return(arg1);
 		    } else {
-			Eterm big_res[3];
+			DeclareTmpHeap(big_res,3,p);
+			UseTmpHeap(3,p);
 
 			/*
 			 * The following code is optimized for the case that
@@ -1519,6 +1533,7 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
 			res = small_times(signed_val(arg1), signed_val(arg2),
 					  big_res);
 			if (is_small(res)) {
+			    UnUseTmpHeap(3,p);
 			    return res;
 			} else {
 			    /*
@@ -1546,6 +1561,7 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
 			    if (arity > 1) {
 				*hp = big_res[2];
 			    }
+			    UnUseTmpHeap(3,p);
 			    return res;
 			}
 		    }
@@ -1609,17 +1625,17 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
 		    need_heap = BIG_NEED_SIZE(sz);
 		    if (ERTS_NEED_GC(p, need_heap)) {
 			erts_garbage_collect(p, need_heap, reg, live+2);
-			if (arg1 != make_big(tmp_big1)) {
+			if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
 			    arg1 = reg[live];
 			}
-			if (arg2 != make_big(tmp_big2)) {
+			if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
 			    arg2 = reg[live+1];
 			}
 		    }
 		    hp = p->htop;
 		    p->htop += need_heap;
 		    res = big_times(arg1, arg2, hp);
-                    trim_heap(p, hp, res);
+		    trim_heap(p, hp, res);
 
 		    /*
 		     * Note that the result must be big in this case, since
@@ -1828,8 +1844,8 @@ erts_gc_int_div(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg1;
     Eterm arg2;
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     int ires;
 
     arg1 = reg[live];
@@ -1866,10 +1882,10 @@ erts_gc_int_div(Process* p, Eterm* reg, Uint live)
 	    need = BIG_NEED_SIZE(i-ires+1) + BIG_NEED_SIZE(i);
 	    if (ERTS_NEED_GC(p, need)) {
 		erts_garbage_collect(p, need, reg, live+2);
-		if (arg1 != make_big(tmp_big1)) {
+		if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
 		    arg1 = reg[live];
 		}
-		if (arg2 != make_big(tmp_big2)) {
+		if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
 		    arg2 = reg[live+1];
 		}
 	    }
@@ -1894,8 +1910,8 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg1;
     Eterm arg2;
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     int ires;
 
     arg1 = reg[live];
@@ -1908,7 +1924,8 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live)
 	if (arg1 != make_small(MIN_SMALL)) {
 	    return arg1;
 	} else {
-	    Eterm tmp = small_to_big(signed_val(arg1), tmp_big1);
+	    Eterm tmp;
+	    tmp = small_to_big(signed_val(arg1), tmp_big1);
 	    if ((ires = big_ucomp(tmp, arg2)) == 0) {
 		return SMALL_ZERO;
 	    } else {
@@ -1928,10 +1945,10 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live)
 
 	    if (ERTS_NEED_GC(p, need)) {
 		erts_garbage_collect(p, need, reg, live+2);
-		if (arg1 != make_big(tmp_big1)) {
+		if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
 		    arg1 = reg[live];
 		}
-		if (arg2 != make_big(tmp_big2)) {
+		if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
 		    arg2 = reg[live+1];
 		}
 	    }
@@ -1956,8 +1973,8 @@ Eterm erts_gc_##func(Process* p, Eterm* reg, Uint live)				\
 {										\
     Eterm arg1;									\
     Eterm arg2;									\
-    Eterm tmp_big1[2];								\
-    Eterm tmp_big2[2];								\
+    DECLARE_TMP(tmp_big1,0,p);							\
+    DECLARE_TMP(tmp_big2,1,p);							\
     Eterm* hp;									\
     int need;									\
 										\
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index b090564649..91b64411d4 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 #ifdef HAVE_CONFIG_H
@@ -70,7 +70,6 @@ static ErlAsync* async_ready_list = NULL;
 /* Detach from driver */
 static void async_detach(DE_Handle* dh)
 {
-    /* XXX:PaN what should happen here? we want to unload the driver or??? */
     return;
 }
 
@@ -176,7 +175,6 @@ int exit_async()
 
 static void async_add(ErlAsync* a, AsyncQueue* q)
 {
-    /* XXX:PaN Is this still necessary when ports lock drivers? */
     if (is_internal_port(a->port)) {
 	ERTS_LC_ASSERT(erts_drvportid2port(a->port));
 	/* make sure the driver will stay around */
@@ -253,6 +251,7 @@ static int async_del(long id)
 		erts_free(ERTS_ALC_T_ASYNC, a);
 		return 1;
 	    }
+	    a = a->next;
 	}
 	erts_mtx_unlock(&async_q[i].mtx);
     }
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
new file mode 100644
index 0000000000..684fa5d12f
--- /dev/null
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -0,0 +1,2939 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * NOTE: This file contains the BIF's for the *module* binary in stdlib.
+ * other BIF's concerning binaries are in binary.c.
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_vm.h"
+#include "global.h"
+#include "erl_process.h"
+#include "error.h"
+#include "bif.h"
+#include "big.h"
+#include "erl_binary.h"
+#include "erl_bits.h"
+
+
+/*
+ * The native implementation functions for the module binary.
+ * Searching is implemented using either Boyer-Moore or Aho-Corasick
+ * depending on number of searchstrings (BM if one, AC if more than one).
+ * Native implementation is mostly for efficiency, nothing
+ * (except binary:referenced_byte_size) really *needs* to be implemented
+ * in native code.
+ */
+
+/* #define HARDDEBUG */
+
+/* Init and local variables */
+
+static Export binary_match_trap_export;
+static BIF_RETTYPE binary_match_trap(BIF_ALIST_3);
+static Export binary_matches_trap_export;
+static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3);
+static Export binary_longest_prefix_trap_export;
+static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3);
+static Export binary_longest_suffix_trap_export;
+static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3);
+static Export binary_bin_to_list_trap_export;
+static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3);
+static Export binary_copy_trap_export;
+static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2);
+static Uint max_loop_limit;
+
+
+void erts_init_bif_binary(void)
+{
+    sys_memset((void *) &binary_match_trap_export, 0, sizeof(Export));
+    binary_match_trap_export.address = &binary_match_trap_export.code[3];
+    binary_match_trap_export.code[0] = am_erlang;
+    binary_match_trap_export.code[1] = am_binary_match_trap;
+    binary_match_trap_export.code[2] = 3;
+    binary_match_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_match_trap_export.code[4] = (BeamInstr) &binary_match_trap;
+
+    sys_memset((void *) &binary_matches_trap_export, 0, sizeof(Export));
+    binary_matches_trap_export.address = &binary_matches_trap_export.code[3];
+    binary_matches_trap_export.code[0] = am_erlang;
+    binary_matches_trap_export.code[1] = am_binary_matches_trap;
+    binary_matches_trap_export.code[2] = 3;
+    binary_matches_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_matches_trap_export.code[4] = (BeamInstr) &binary_matches_trap;
+
+    sys_memset((void *) &binary_longest_prefix_trap_export, 0, sizeof(Export));
+    binary_longest_prefix_trap_export.address = &binary_longest_prefix_trap_export.code[3];
+    binary_longest_prefix_trap_export.code[0] = am_erlang;
+    binary_longest_prefix_trap_export.code[1] = am_binary_longest_prefix_trap;
+    binary_longest_prefix_trap_export.code[2] = 3;
+    binary_longest_prefix_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_longest_prefix_trap_export.code[4] = (BeamInstr) &binary_longest_prefix_trap;
+
+    sys_memset((void *) &binary_longest_suffix_trap_export, 0, sizeof(Export));
+    binary_longest_suffix_trap_export.address = &binary_longest_suffix_trap_export.code[3];
+    binary_longest_suffix_trap_export.code[0] = am_erlang;
+    binary_longest_suffix_trap_export.code[1] = am_binary_longest_suffix_trap;
+    binary_longest_suffix_trap_export.code[2] = 3;
+    binary_longest_suffix_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_longest_suffix_trap_export.code[4] = (BeamInstr) &binary_longest_suffix_trap;
+
+    sys_memset((void *) &binary_bin_to_list_trap_export, 0, sizeof(Export));
+    binary_bin_to_list_trap_export.address = &binary_bin_to_list_trap_export.code[3];
+    binary_bin_to_list_trap_export.code[0] = am_erlang;
+    binary_bin_to_list_trap_export.code[1] = am_binary_bin_to_list_trap;
+    binary_bin_to_list_trap_export.code[2] = 3;
+    binary_bin_to_list_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_bin_to_list_trap_export.code[4] = (BeamInstr) &binary_bin_to_list_trap;
+    sys_memset((void *) &binary_copy_trap_export, 0, sizeof(Export));
+    binary_copy_trap_export.address = &binary_copy_trap_export.code[3];
+    binary_copy_trap_export.code[0] = am_erlang;
+    binary_copy_trap_export.code[1] = am_binary_copy_trap;
+    binary_copy_trap_export.code[2] = 2;
+    binary_copy_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_copy_trap_export.code[4] = (BeamInstr) &binary_copy_trap;
+
+    max_loop_limit = 0;
+    return;
+}
+
+/*
+ * Setting the loop_limit for searches for debugging
+ */
+Sint erts_binary_set_loop_limit(Sint limit)
+{
+    Sint save = (Sint) max_loop_limit;
+    if (limit <= 0) {
+	max_loop_limit = 0;
+    } else {
+	max_loop_limit = (Uint) limit;
+    }
+
+    return save;
+}
+
+static Uint get_reds(Process *p, int loop_factor)
+{
+    Uint reds = ERTS_BIF_REDS_LEFT(p) * loop_factor;
+    Uint tmp = max_loop_limit;
+    if (tmp != 0 && tmp < reds) {
+	return tmp;
+    }
+    if (!reds) {
+	reds = 1;
+    }
+    return reds;
+}
+
+/*
+ * A micro allocator used when building search structures, just a convenience
+ * for building structures inside a pre-allocated magic binary using
+ * conventional malloc-like interface.
+ */
+
+#define MYALIGN(Size) (SIZEOF_VOID_P * (((Size) / SIZEOF_VOID_P) + \
+                       !!(((Size) % SIZEOF_VOID_P))))
+
+#ifdef DEBUG
+#define CHECK_ALLOCATOR(My) ASSERT((My).current <= ((My).mem + (My).size))
+#else
+#define CHECK_ALLOCATOR(My) /* nothing */
+#endif
+
+typedef struct _my_allocator {
+    Uint size;
+    byte *current;
+    byte *mem;
+} MyAllocator;
+
+static void init_my_allocator(MyAllocator *my, Uint siz, byte *array)
+{
+    ASSERT((siz % SIZEOF_VOID_P) == 0);
+    my->size = siz;
+    my->mem = array;
+    my->current = my->mem;
+}
+
+static void *my_alloc(MyAllocator *my, Uint size)
+{
+    void *ptr = my->current;
+    my->current += MYALIGN(size);
+    return ptr;
+}
+
+/*
+ * The search functionality.
+ *
+ * The search is byte oriented, which works nicely for UTF-8 as well as
+ * latin1 data
+ */
+
+#define ALPHABET_SIZE 256
+
+typedef struct _ac_node {
+#ifdef HARDDEBUG
+    Uint32 id;                        /* To identify h pointer targets when
+					 dumping */
+#endif
+    Uint32 d;                         /* Depth in trie, also represents the
+					 length (-1) of the matched string if
+					 in final set */
+    Sint32 final;                     /* Members in final set represent
+				       * matches.
+				       * The set representation is scattered
+				       * among the nodes in this way:
+				       * >0 -> this represents a member of
+				       * the final set, <0 -> member of
+				       * final set somewhere in the failure
+				       * chain,
+				       * 0 -> not member of the final set */
+    struct _ac_node *h;                /* h(Hode) is the failure function */
+    struct _ac_node *g[ALPHABET_SIZE]; /* g(Node,Character) is the
+					  transition function */
+} ACNode;
+
+typedef struct _ac_trie {
+#ifdef HARDDEBUG
+    Uint32 idc;
+#endif
+    Uint32 counter;                    /* Number of added patterns */
+    ACNode *root;                      /* pointer to the root state */
+} ACTrie;
+
+typedef struct _bm_data {
+    byte *x;
+    Sint len;
+    Sint *goodshift;
+    Sint badshift[ALPHABET_SIZE];
+} BMData;
+
+#ifdef HARDDEBUG
+static void dump_bm_data(BMData *bm);
+static void dump_ac_trie(ACTrie *act);
+static void dump_ac_node(ACNode *node, int indent, int ch);
+#endif
+
+/*
+ * The needed size of binary data for a search structure - given the
+ * accumulated string lengths.
+ */
+#define BM_SIZE(StrLen) 	      /* StrLen: length of searchstring */ \
+((MYALIGN(sizeof(Sint) * (StrLen))) + /* goodshift array */                \
+ MYALIGN(StrLen) +                    /* searchstring saved */             \
+ (MYALIGN(sizeof(BMData))))           /* Structure */
+
+#define AC_SIZE(StrLens)       /* StrLens: sum of all searchstring lengths */ \
+((MYALIGN(sizeof(ACNode)) *                                                   \
+((StrLens)+1)) + 	       /* The actual nodes (including rootnode) */    \
+ MYALIGN(sizeof(ACTrie)))      /* Structure */
+
+
+#ifndef MAX
+#define MAX(A,B) (((A) > (B)) ? (A) : (B))
+#endif
+
+#ifndef MIN
+#define MIN(A,B) (((A) > (B)) ? (B) : (A))
+#endif
+/*
+ * Callback for the magic binary
+ */
+static void cleanup_my_data_ac(Binary *bp)
+{
+    return;
+}
+static void cleanup_my_data_bm(Binary *bp)
+{
+    return;
+}
+
+/*
+ * Initiate a (allocated) micro allocator and fill in the base
+ * for an Aho-Corasick search trie, given the accumulated length of the search
+ * strings.
+ */
+static ACTrie *create_acdata(MyAllocator *my, Uint len,
+			     ACNode ***qbuff /* out */,
+			     Binary **the_bin /* out */)
+{
+    Uint datasize = AC_SIZE(len);
+    ACTrie *act;
+    ACNode *acn;
+    Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_ac);
+    byte *data = ERTS_MAGIC_BIN_DATA(mb);
+
+    init_my_allocator(my, datasize, data);
+    act = my_alloc(my, sizeof(ACTrie)); /* Important that this is the first
+					   allocation */
+    act->counter = 0;
+    act->root = acn = my_alloc(my, sizeof(ACNode));
+    acn->d = 0;
+    acn->final = 0;
+    acn->h = NULL;
+    memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE);
+#ifdef HARDDEBUG
+    act->idc = 0;
+    acn->id = 0;
+#endif
+    *qbuff = erts_alloc(ERTS_ALC_T_TMP, sizeof(ACNode *) * len);
+    *the_bin = mb;
+    return act;
+}
+
+/*
+ * The same initialization of allocator and basic data for Boyer-Moore.
+ */
+static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len,
+			     Binary **the_bin /* out */)
+{
+    Uint datasize = BM_SIZE(len);
+    BMData *bmd;
+    Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_bm);
+    byte *data = ERTS_MAGIC_BIN_DATA(mb);
+    init_my_allocator(my, datasize, data);
+    bmd = my_alloc(my, sizeof(BMData));
+    bmd->x = my_alloc(my,len);
+    memcpy(bmd->x,x,len);
+    bmd->len = len;
+    bmd->goodshift = my_alloc(my,sizeof(Uint) * len);
+    *the_bin = mb;
+    return bmd;
+}
+
+/*
+ * Compilation of search structures
+ */
+
+/*
+ * Aho Corasick - Build a Trie and fill in the failure functions
+ * when all strings are added.
+ * The algorithm is nicely described by Dieter B�hler of University of
+ * T�bingen:
+ * http://www-sr.informatik.uni-tuebingen.de/~buehler/AC/AC.html
+ */
+
+/*
+ * Helper called once for each search pattern
+ */
+static void ac_add_one_pattern(MyAllocator *my, ACTrie *act, byte *x, Uint len)
+{
+    ACNode *acn = act->root;
+    Uint32 n = ++act->counter; /* Always increase counter, even if it's a
+				  duplicate as this may identify the pattern
+				  in the final set (not in current interface
+				  though) */
+    Uint i = 0;
+
+    while(i < len) {
+	if (acn->g[x[i]] != NULL) {
+	    /* node exists, continue */
+	    acn = acn->g[x[i]];
+	    ++i;
+	} else {
+	    /* allocate a new node */
+	    ACNode *nn = my_alloc(my,sizeof(ACNode));
+#ifdef HARDDEBUG
+	    nn->id = ++(act->idc);
+#endif
+	    nn->d = i+1;
+	    nn->h = act->root;
+	    nn->final = 0;
+	    memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE);
+	    acn->g[x[i]] = nn;
+	    ++i;
+	    acn = nn;
+	}
+    }
+    if (acn->final == 0) { /* New pattern, add to final set */
+	acn->final = n;
+    }
+}
+
+/*
+ * Called when all search patterns are added.
+ */
+static void ac_compute_failure_functions(ACTrie *act, ACNode **qbuff)
+{
+    ACNode *root = act->root;
+    ACNode *parent;
+    int i;
+    int qh = 0,qt = 0;
+    ACNode *child, *r;
+
+    /* Set all children of the root to have the root as failure function */
+    for (i = 0; i < ALPHABET_SIZE; ++i) {
+	if (root->g[i] != NULL) {
+	    root->g[i]->h = root;
+	    /* Add to que for later traversal */
+	    qbuff[qt++] = root->g[i];
+	}
+    }
+
+    /* So, now we've handled children of the root state, traverse the
+       rest of the trie BF... */
+    while (qh < qt) {
+	parent = qbuff[qh++];
+	for (i = 0; i < ALPHABET_SIZE; ++ i) {
+	    if ((child = parent->g[i]) != NULL) {
+		/* Visit this node to */
+		qbuff[qt++] = child;
+		/* Search for correct failure function, follow the parent's
+		   failure function until you find a similar transition
+		   funtion to this child's */
+		r =  parent->h;
+		while (r != NULL && r->g[i] == NULL) {
+		    r = r->h;
+		}
+		if (r == NULL) {
+		    /* Replace NULL failures with the root as we go */
+		    child->h = (root->g[i] == NULL) ? root : root->g[i];
+		} else {
+		    child->h = r->g[i];
+		    /*
+		     * The "final" set is scattered among the nodes. When
+		     * the failure function points to a member of the final
+		     * set, we have a match, but we might not see it in the
+		     * current node if we dont mark it as a special type of
+		     * final, i.e. foolow the failure function and you will
+		     * find a real member of final set. This is marked with
+		     * a negative string id and only done if this node does
+		     * not represent a member in the final set.
+		     */
+		    if (!(child->final) && (child->h->final)) {
+			child->final = -1;
+		    }
+		}
+	    }
+	}
+    }
+    /* Finally the failure function of the root should point to itself */
+    root->h = root;
+}
+
+
+/*
+ * The actual searching for needles in the haystack...
+ * Find first match using Aho-Coracick Trie
+ * return pattern number and fill in mpos + mlen if found, otherwise return 0
+ * Return the matching pattern that *starts* first, and ends
+ * last (difference when overlapping), hence the candidate thing.
+ * Basic AC finds the first end before the first start...
+ *
+ */
+typedef struct {
+    ACNode *q;
+    Uint pos;
+    Uint len;
+    ACNode *candidate;
+    Uint candidate_start;
+} ACFindFirstState;
+
+
+static void ac_init_find_first_match(ACFindFirstState *state, ACTrie *act, Sint startpos, Uint len)
+{
+    state->q = act->root;
+    state->pos = startpos;
+    state->len = len;
+    state->candidate = NULL;
+    state->candidate_start = 0;
+}
+#define AC_OK 0
+#define AC_NOT_FOUND -1
+#define AC_RESTART -2
+
+#define AC_LOOP_FACTOR 10
+
+static int ac_find_first_match(ACFindFirstState *state, byte *haystack,
+				Uint *mpos, Uint *mlen, Uint *reductions)
+{
+    ACNode *q = state->q;
+    Uint i = state->pos;
+    ACNode *candidate = state->candidate, *r;
+    Uint len = state->len;
+    Uint candidate_start = state->candidate_start;
+    Uint rstart;
+    register Uint reds = *reductions;
+
+    while (i < len) {
+	if (--reds == 0) {
+	    state->q = q;
+	    state->pos = i;
+	    state->len = len;
+	    state->candidate = candidate;
+	    state->candidate_start = candidate_start;
+	    return AC_RESTART;
+	}
+
+	while (q->g[haystack[i]] == NULL && q->h != q) {
+	    q = q->h;
+	}
+	if (q->g[haystack[i]] != NULL) {
+	    q = q->g[haystack[i]];
+	}
+#ifdef HARDDEBUG
+	erts_printf("ch = %c, Current: %u\n", (int) haystack[i], (unsigned) q->id);
+#endif
+	++i;
+	if (candidate != NULL && (i - q->d) > candidate_start) {
+	    break;
+	}
+	if (q->final) {
+	    r = q;
+	    while (r->final < 0)
+		r = r->h;
+	    rstart = i - r->d;
+	    if (candidate == NULL || rstart < candidate_start ||
+		(rstart == candidate_start && candidate->d < q->d)) {
+		candidate_start = rstart;
+		candidate = r;
+	    }
+	}
+    }
+    *reductions = reds;
+    if (!candidate) {
+	return AC_NOT_FOUND;
+    }
+#ifdef HARDDEBUG
+    dump_ac_node(candidate,0,'?');
+#endif
+    *mpos = candidate_start;
+    *mlen = candidate->d;
+    return AC_OK;
+}
+
+typedef struct _findall_data {
+    Uint pos;
+    Uint len;
+#ifdef HARDDEBUG
+    Uint id;
+#endif
+    Eterm epos;
+    Eterm elen;
+} FindallData;
+
+typedef struct {
+    ACNode *q;
+    Uint pos;
+    Uint len;
+    Uint m;
+    Uint allocated;
+    FindallData *out;
+} ACFindAllState;
+
+static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos, Uint len)
+{
+    state->q = act->root;
+    state->pos = startpos;
+    state->len = len;
+    state->m = 0;
+    state->allocated = 0;
+    state->out = NULL;
+}
+
+static void ac_restore_find_all(ACFindAllState *state, char *buff)
+{
+    memcpy(state,buff,sizeof(ACFindAllState));
+    if (state->allocated > 0) {
+	state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated));
+	memcpy(state->out,buff+sizeof(ACFindAllState),sizeof(FindallData)*state->m);
+    } else {
+	state->out = NULL;
+    }
+}
+
+static void ac_serialize_find_all(ACFindAllState *state, char *buff)
+{
+    memcpy(buff,state,sizeof(ACFindAllState));
+    memcpy(buff+sizeof(ACFindAllState),state->out,sizeof(FindallData)*state->m);
+}
+
+static void ac_clean_find_all(ACFindAllState *state)
+{
+    if (state->out != NULL) {
+	erts_free(ERTS_ALC_T_TMP, state->out);
+    }
+#ifdef HARDDEBUG
+    state->out = NULL;
+    state->allocated = 0;
+#endif
+}
+
+#define SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(S) \
+          (sizeof(ACFindAllState)+(sizeof(FindallData)*(S).m))
+
+/*
+ * Differs to the find_first function in that it stores all matches and the values
+ * arte returned only in the state.
+ */
+static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack,
+				       Uint *reductions)
+{
+    ACNode *q = state->q;
+    Uint i = state->pos;
+    Uint rstart;
+    ACNode *r;
+    Uint len = state->len;
+    Uint m = state->m, save_m;
+    Uint allocated = state->allocated;
+    FindallData *out = state->out;
+    register Uint reds = *reductions;
+
+
+    while (i < len) {
+	if (--reds == 0) {
+	    state->q = q;
+	    state->pos = i;
+	    state->len = len;
+	    state->m = m;
+	    state->allocated = allocated;
+	    state->out = out;
+	    return AC_RESTART;
+	}
+	while (q->g[haystack[i]] == NULL && q->h != q) {
+	    q = q->h;
+	}
+	if (q->g[haystack[i]] != NULL) {
+	    q = q->g[haystack[i]];
+	}
+	++i;
+	if (q->final) {
+	    r = q;
+	    while (r->final) {
+		while (r->final < 0)
+		    r = r->h;
+#ifdef HARDDEBUG
+		erts_printf("Trying to add %u\n",(unsigned) r->final);
+#endif
+		rstart = i - r->d;
+		save_m = m;
+		while (m > 0 && (out[m-1].pos > rstart ||
+				 (out[m-1].pos == rstart &&
+				  out[m-1].len < r->d))) {
+#ifdef HARDDEBUG
+		    erts_printf("Popping %u\n",(unsigned) out[m-1].id);
+#endif
+		    --m;
+		}
+#ifdef HARDDEBUG
+		if (m > 0) {
+		    erts_printf("Pos %u\n",out[m-1].pos);
+		    erts_printf("Len %u\n",out[m-1].len);
+		}
+		erts_printf("Rstart %u\n",rstart);
+#endif
+		if (m == 0 ||  out[m-1].pos + out[m-1].len <= rstart) {
+		    if (m >= allocated) {
+			if (!allocated) {
+			    allocated = 10;
+			    out = erts_alloc(ERTS_ALC_T_TMP,
+					     sizeof(FindallData) * allocated);
+			} else {
+			    allocated *= 2;
+			    out = erts_realloc(ERTS_ALC_T_TMP, out,
+					       sizeof(FindallData) *
+					       allocated);
+			}
+		    }
+		    out[m].pos = rstart;
+		    out[m].len = r->d;
+#ifdef HARDDEBUG
+		    out[m].id = r->final;
+#endif
+		    ++m;
+#ifdef HARDDEBUG
+		    erts_printf("Pushing %u\n",(unsigned) out[m-1].id);
+#endif
+		} else {
+#ifdef HARDDEBUG
+		    erts_printf("Backtracking %d steps\n",save_m - m);
+#endif
+		    m = save_m;
+		}
+		r = r->h;
+	    }
+	}
+    }
+    *reductions = reds;
+    state->m = m;
+    state->out = out;
+    return (m == 0) ? AC_NOT_FOUND : AC_OK;
+}
+
+/*
+ * Boyer Moore - most obviously implemented more or less exactly as
+ * Christian Charras and Thierry Lecroq describe it in "Handbook of
+ * Exact String-Matching Algorithms"
+ * http://www-igm.univ-mlv.fr/~lecroq/string/
+ */
+
+/*
+ * Call this to compute badshifts array
+ */
+static void compute_badshifts(BMData *bmd)
+{
+    Sint i;
+    Sint m = bmd->len;
+
+    for (i = 0; i < ALPHABET_SIZE; ++i) {
+	bmd->badshift[i] = m;
+    }
+    for (i = 0; i < m - 1; ++i) {
+	bmd->badshift[bmd->x[i]] = m - i - 1;
+    }
+}
+
+/* Helper for "compute_goodshifts" */
+static void compute_suffixes(byte *x, Sint m, Sint *suffixes)
+{
+    int f,g,i;
+
+    suffixes[m - 1] = m;
+
+    f = 0; /* To avoid use before set warning */
+
+    g = m - 1;
+
+    for (i = m - 2; i >= 0; --i) {
+	if (i > g && suffixes[i + m - 1 - f] < i - g) {
+	    suffixes[i] = suffixes[i + m - 1 - f];
+	} else {
+	    if (i < g) {
+		g = i;
+	    }
+	    f = i;
+	    while ( g >= 0 && x[g] == x[g + m - 1 - f] ) {
+		--g;
+	    }
+	    suffixes[i] = f - g;
+	}
+    }
+}
+
+/*
+ * Call this to compute goodshift array
+ */
+static void compute_goodshifts(BMData *bmd)
+{
+    Sint m = bmd->len;
+    byte *x = bmd->x;
+    Sint i, j;
+    Sint *suffixes = erts_alloc(ERTS_ALC_T_TMP, m * sizeof(Sint));
+
+    compute_suffixes(x, m, suffixes);
+
+    for (i = 0; i < m; ++i) {
+	bmd->goodshift[i] = m;
+    }
+
+    j = 0;
+
+    for (i = m - 1; i >= -1; --i) {
+	if (i == -1 || suffixes[i] == i + 1) {
+	    while (j < m - 1 - i) {
+		if (bmd->goodshift[j] == m) {
+		    bmd->goodshift[j] = m - 1 - i;
+		}
+		++j;
+	    }
+	}
+    }
+    for (i = 0; i <= m - 2; ++i) {
+	bmd->goodshift[m - 1 - suffixes[i]] = m - 1 - i;
+    }
+    erts_free(ERTS_ALC_T_TMP, suffixes);
+}
+
+typedef struct {
+    Sint pos;
+    Sint len;
+} BMFindFirstState;
+
+#define BM_OK 0 /* used only for find_all */
+#define BM_NOT_FOUND -1
+#define BM_RESTART -2
+#define BM_LOOP_FACTOR 10 /* Should we have a higher value? */
+
+static void bm_init_find_first_match(BMFindFirstState *state, Sint startpos,
+				     Uint len)
+{
+    state->pos = startpos;
+    state->len = (Sint) len;
+}
+
+
+static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd,
+				byte *haystack, Uint *reductions)
+{
+    Sint blen = bmd->len;
+    Sint len = state->len;
+    Sint *gs = bmd->goodshift;
+    Sint *bs = bmd->badshift;
+    byte *needle = bmd->x;
+    Sint i;
+    Sint j = state->pos;
+    register Uint reds = *reductions;
+
+    while (j <= len - blen) {
+	if (--reds == 0) {
+	    state->pos = j;
+	    return BM_RESTART;
+	}
+	for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i)
+	    ;
+	if (i < 0) { /* found */
+	    *reductions = reds;
+	    return j;
+	}
+	j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i);
+    }
+    *reductions = reds;
+    return BM_NOT_FOUND;
+}
+
+typedef struct {
+    Sint pos;
+    Sint len;
+    Uint m;
+    Uint allocated;
+    FindallData *out;
+} BMFindAllState;
+
+static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len)
+{
+    state->pos = startpos;
+    state->len = (Sint) len;
+    state->m = 0;
+    state->allocated = 0;
+    state->out = NULL;
+}
+
+static void bm_restore_find_all(BMFindAllState *state, char *buff)
+{
+    memcpy(state,buff,sizeof(BMFindAllState));
+    if (state->allocated > 0) {
+	state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) *
+				(state->allocated));
+	memcpy(state->out,buff+sizeof(BMFindAllState),
+	       sizeof(FindallData)*state->m);
+    } else {
+	state->out = NULL;
+    }
+}
+
+static void bm_serialize_find_all(BMFindAllState *state, char *buff)
+{
+    memcpy(buff,state,sizeof(BMFindAllState));
+    memcpy(buff+sizeof(BMFindAllState),state->out,
+	   sizeof(FindallData)*state->m);
+}
+
+static void bm_clean_find_all(BMFindAllState *state)
+{
+    if (state->out != NULL) {
+	erts_free(ERTS_ALC_T_TMP, state->out);
+    }
+#ifdef HARDDEBUG
+    state->out = NULL;
+    state->allocated = 0;
+#endif
+}
+
+#define SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(S) \
+          (sizeof(BMFindAllState)+(sizeof(FindallData)*(S).m))
+
+/*
+ * Differs to the find_first function in that it stores all matches and the
+ * values are returned only in the state.
+ */
+static Sint bm_find_all_non_overlapping(BMFindAllState *state,
+					BMData *bmd, byte *haystack,
+					Uint *reductions)
+{
+    Sint blen = bmd->len;
+    Sint len = state->len;
+    Sint *gs = bmd->goodshift;
+    Sint *bs = bmd->badshift;
+    byte *needle = bmd->x;
+    Sint i;
+    Sint j = state->pos;
+    Uint m = state->m;
+    Uint allocated = state->allocated;
+    FindallData *out = state->out;
+    register Uint reds = *reductions;
+
+    while (j <= len - blen) {
+	if (--reds == 0) {
+	    state->pos = j;
+	    state->m = m;
+	    state->allocated = allocated;
+	    state->out = out;
+	    return BM_RESTART;
+	}
+	for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i)
+	    ;
+	if (i < 0) { /* found */
+	    if (m >= allocated) {
+		if (!allocated) {
+		    allocated = 10;
+		    out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * allocated);
+		} else {
+		    allocated *= 2;
+		    out = erts_realloc(ERTS_ALC_T_TMP, out,
+				       sizeof(FindallData) * allocated);
+		}
+	    }
+	    out[m].pos = j;
+	    out[m].len = blen;
+	    ++m;
+	    j += blen;
+	} else {
+	    j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i);
+	}
+    }
+    state->m = m;
+    state->out = out;
+    *reductions = reds;
+    return (m == 0) ? BM_NOT_FOUND : BM_OK;
+}
+
+/*
+ * Interface functions (i.e. "bif's")
+ */
+
+/*
+ * Search functionality interfaces
+ */
+
+static int do_binary_match_compile(Eterm argument, Eterm *tag, Binary **binp)
+{
+    Eterm t, b, comp_term = NIL;
+    Uint characters;
+    Uint words;
+
+    characters = 0;
+    words = 0;
+
+    if (is_list(argument)) {
+	t = argument;
+	while (is_list(t)) {
+	    b = CAR(list_val(t));
+	    t = CDR(list_val(t));
+	    if (!is_binary(b)) {
+		goto badarg;
+	    }
+	    if (binary_bitsize(b) != 0) {
+		goto badarg;
+	    }
+	    ++words;
+	    characters += binary_size(b);
+	}
+	if (is_not_nil(t)) {
+	    goto badarg;
+	}
+	if (words > 1) {
+	    comp_term = argument;
+	} else {
+	    comp_term = CAR(list_val(argument));
+	}
+    } else if (is_binary(argument)) {
+	if (binary_bitsize(argument) != 0) {
+	    goto badarg;
+	}
+	words = 1;
+	comp_term = argument;
+	characters = binary_size(argument);
+    }
+
+    if (characters == 0) {
+	goto badarg;
+    }
+    ASSERT(words > 0);
+
+    if (words == 1) {
+	byte *bytes;
+	Uint bitoffs, bitsize;
+	byte *temp_alloc = NULL;
+	MyAllocator my;
+	BMData *bmd;
+	Binary *bin;
+
+	ERTS_GET_BINARY_BYTES(comp_term, bytes, bitoffs, bitsize);
+	if (bitoffs != 0) {
+	    bytes = erts_get_aligned_binary_bytes(comp_term, &temp_alloc);
+	}
+	bmd = create_bmdata(&my, bytes, characters, &bin);
+	compute_badshifts(bmd);
+	compute_goodshifts(bmd);
+	erts_free_aligned_binary_bytes(temp_alloc);
+	CHECK_ALLOCATOR(my);
+	*tag = am_bm;
+	*binp = bin;
+	return 0;
+    } else {
+	ACTrie *act;
+	MyAllocator my;
+	ACNode **qbuff;
+	Binary *bin;
+
+	act = create_acdata(&my, characters, &qbuff, &bin);
+	t = comp_term;
+	while (is_list(t)) {
+	    byte *bytes;
+	    Uint bitoffs, bitsize;
+	    byte *temp_alloc = NULL;
+	    b = CAR(list_val(t));
+	    t = CDR(list_val(t));
+	    ERTS_GET_BINARY_BYTES(b, bytes, bitoffs, bitsize);
+	    if (bitoffs != 0) {
+		bytes = erts_get_aligned_binary_bytes(b, &temp_alloc);
+	    }
+	    ac_add_one_pattern(&my,act,bytes,binary_size(b));
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	}
+	ac_compute_failure_functions(act,qbuff);
+	CHECK_ALLOCATOR(my);
+	erts_free(ERTS_ALC_T_TMP,qbuff);
+	*tag = am_ac;
+	*binp = bin;
+	return 0;
+    }
+ badarg:
+    return -1;
+}
+
+BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1)
+{
+    Binary *bin;
+    Eterm tag, ret;
+    Eterm *hp;
+
+    if (do_binary_match_compile(BIF_ARG_1,&tag,&bin)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    hp = HAlloc(BIF_P, PROC_BIN_SIZE+3);
+    ret = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin);
+    ret = TUPLE2(hp, tag, ret);
+    BIF_RET(ret);
+}
+
+#define DO_BIN_MATCH_OK 0
+#define DO_BIN_MATCH_BADARG -1
+#define DO_BIN_MATCH_RESTART -2
+
+static int do_binary_match(Process *p, Eterm subject, Uint hsstart, Uint hsend,
+			   Eterm type, Binary *bin, Eterm state_term,
+			   Eterm *res_term)
+{
+    byte *bytes;
+    Uint bitoffs, bitsize;
+    byte *temp_alloc = NULL;
+
+    ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	goto badarg;
+    }
+    if (bitoffs != 0) {
+	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
+    }
+    if (state_term != NIL) {
+	Eterm *ptr = big_val(state_term);
+	type = ptr[1];
+    }
+
+    if (type == am_bm) {
+	BMData *bm;
+	Sint pos;
+	Eterm ret;
+	Eterm *hp;
+	BMFindFirstState state;
+	Uint reds = get_reds(p, BM_LOOP_FACTOR);
+	Uint save_reds = reds;
+
+	bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	dump_bm_data(bm);
+#endif
+	if (state_term == NIL) {
+	    bm_init_find_first_match(&state, hsstart, hsend);
+	} else {
+	    Eterm *ptr = big_val(state_term);
+	    memcpy(&state,ptr+2,sizeof(state));
+	}
+#ifdef HARDDEBUG
+	erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos,
+		    state.len);
+#endif
+	pos = bm_find_first_match(&state, bm, bytes, &reds);
+	if (pos == BM_NOT_FOUND) {
+	    ret = am_nomatch;
+	} else if (pos == BM_RESTART) {
+	    int x = (sizeof(BMFindFirstState) / sizeof(Eterm)) +
+		!!(sizeof(BMFindFirstState) % sizeof(Eterm));
+#ifdef HARDDEBUG
+	    erts_printf("Trap bm!\n");
+#endif
+	    hp = HAlloc(p,x+2);
+	    hp[0] = make_pos_bignum_header(x+1);
+	    hp[1] = type;
+	    memcpy(hp+2,&state,sizeof(state));
+	    *res_term = make_big(hp);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    return DO_BIN_MATCH_RESTART;
+	} else {
+	    Eterm erlen = erts_make_integer((Uint) bm->len, p);
+	    ret = erts_make_integer(pos,p);
+	    hp = HAlloc(p,3);
+	    ret = TUPLE2(hp, ret, erlen);
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
+	*res_term = ret;
+	return DO_BIN_MATCH_OK;
+    } else if (type == am_ac) {
+	ACTrie *act;
+	Uint pos, rlen;
+	int acr;
+	ACFindFirstState state;
+	Eterm ret;
+	Eterm *hp;
+	Uint reds = get_reds(p, AC_LOOP_FACTOR);
+	Uint save_reds = reds;
+
+	act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	dump_ac_trie(act);
+#endif
+	if (state_term == NIL) {
+	    ac_init_find_first_match(&state, act, hsstart, hsend);
+	} else {
+	    Eterm *ptr = big_val(state_term);
+	    memcpy(&state,ptr+2,sizeof(state));
+	}
+	acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds);
+	if (acr == AC_NOT_FOUND) {
+	    ret = am_nomatch;
+	} else if (acr == AC_RESTART) {
+	    int x = (sizeof(state) / sizeof(Eterm)) +
+		!!(sizeof(ACFindFirstState) % sizeof(Eterm));
+#ifdef HARDDEBUG
+	    erts_printf("Trap ac!\n");
+#endif
+	    hp = HAlloc(p,x+2);
+	    hp[0] = make_pos_bignum_header(x+1);
+	    hp[1] = type;
+	    memcpy(hp+2,&state,sizeof(state));
+	    *res_term = make_big(hp);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    return DO_BIN_MATCH_RESTART;
+	} else {
+	    Eterm epos = erts_make_integer(pos+hsstart,p);
+	    Eterm erlen = erts_make_integer(rlen,p);
+	    hp = HAlloc(p,3);
+	    ret = TUPLE2(hp, epos, erlen);
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
+	*res_term = ret;
+	return DO_BIN_MATCH_OK;
+    }
+ badarg:
+    return DO_BIN_MATCH_BADARG;
+}
+
+static int do_binary_matches(Process *p, Eterm subject, Uint hsstart,
+			     Uint hsend, Eterm type, Binary *bin,
+			     Eterm state_term, Eterm *res_term)
+{
+    byte *bytes;
+    Uint bitoffs, bitsize;
+    byte *temp_alloc = NULL;
+
+    ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	goto badarg;
+    }
+    if (bitoffs != 0) {
+	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
+    }
+    if (state_term != NIL) {
+	Eterm *ptr = big_val(state_term);
+	type = ptr[1];
+    }
+
+    if (type == am_bm) {
+	BMData *bm;
+	Sint pos;
+	Eterm ret,tpl;
+	Eterm *hp;
+	BMFindAllState state;
+	Uint reds = get_reds(p, BM_LOOP_FACTOR);
+	Uint save_reds = reds;
+
+	bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	dump_bm_data(bm);
+#endif
+	if (state_term == NIL) {
+	    bm_init_find_all(&state, hsstart, hsend);
+	} else {
+	    Eterm *ptr = big_val(state_term);
+	    bm_restore_find_all(&state,(char *) (ptr+2));
+	}
+
+	pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds);
+	if (pos == BM_NOT_FOUND) {
+	    ret = NIL;
+	} else if (pos == BM_RESTART) {
+	    int x =
+		(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
+		!!(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+	    erts_printf("Trap bm!\n");
+#endif
+	    hp = HAlloc(p,x+2);
+	    hp[0] = make_pos_bignum_header(x+1);
+	    hp[1] = type;
+	    bm_serialize_find_all(&state, (char *) (hp+2));
+	    *res_term = make_big(hp);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    bm_clean_find_all(&state);
+	    return DO_BIN_MATCH_RESTART;
+	} else {
+	    FindallData *fad = state.out;
+	    int i;
+	    for (i = 0; i < state.m; ++i) {
+		fad[i].epos = erts_make_integer(fad[i].pos,p);
+		fad[i].elen = erts_make_integer(fad[i].len,p);
+	    }
+	    hp = HAlloc(p,state.m * (3 + 2));
+	    ret = NIL;
+	    for (i = state.m - 1; i >= 0; --i) {
+		tpl = TUPLE2(hp, fad[i].epos, fad[i].elen);
+		hp +=3;
+		ret = CONS(hp,tpl,ret);
+		hp += 2;
+	    }
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	bm_clean_find_all(&state);
+	BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
+	*res_term = ret;
+	return DO_BIN_MATCH_OK;
+    } else if (type == am_ac) {
+	ACTrie *act;
+	int acr;
+	ACFindAllState state;
+	Eterm ret,tpl;
+	Eterm *hp;
+	Uint reds = get_reds(p, AC_LOOP_FACTOR);
+	Uint save_reds = reds;
+
+	act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	dump_ac_trie(act);
+#endif
+	if (state_term == NIL) {
+	    ac_init_find_all(&state, act, hsstart, hsend);
+	} else {
+	    Eterm *ptr = big_val(state_term);
+	    ac_restore_find_all(&state,(char *) (ptr+2));
+	}
+	acr = ac_find_all_non_overlapping(&state, bytes, &reds);
+	if (acr == AC_NOT_FOUND) {
+	    ret = NIL;
+	} else if (acr == AC_RESTART) {
+	    int x =
+		(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
+		!!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+	    erts_printf("Trap ac!\n");
+#endif
+	    hp = HAlloc(p,x+2);
+	    hp[0] = make_pos_bignum_header(x+1);
+	    hp[1] = type;
+	    ac_serialize_find_all(&state, (char *) (hp+2));
+	    *res_term = make_big(hp);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    ac_clean_find_all(&state);
+	    return DO_BIN_MATCH_RESTART;
+	} else {
+	    FindallData *fad = state.out;
+	    int i;
+	    for (i = 0; i < state.m; ++i) {
+		fad[i].epos = erts_make_integer(fad[i].pos,p);
+		fad[i].elen = erts_make_integer(fad[i].len,p);
+	    }
+	    hp = HAlloc(p,state.m * (3 + 2));
+	    ret = NIL;
+	    for (i = state.m - 1; i >= 0; --i) {
+		tpl = TUPLE2(hp, fad[i].epos, fad[i].elen);
+		hp +=3;
+		ret = CONS(hp,tpl,ret);
+		hp += 2;
+	    }
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	ac_clean_find_all(&state);
+	BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
+	*res_term = ret;
+	return DO_BIN_MATCH_OK;
+    }
+ badarg:
+    return DO_BIN_MATCH_BADARG;
+}
+
+static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp)
+{
+    Eterm *tp;
+    Uint pos;
+    Sint len;
+    if (l == ((Eterm) 0) || l == NIL) {
+	/* Invalid term or NIL, we're called from binary_match(es)_2 or
+	   have no options*/
+	*posp = 0;
+	*endp = binary_size(bin);
+	return 0;
+    } else if (is_list(l)) {
+	while(is_list(l)) {
+	    Eterm t = CAR(list_val(l));
+	    Uint orig_size;
+	    if (!is_tuple(t)) {
+		goto badarg;
+	    }
+	    tp = tuple_val(t);
+	    if (arityval(*tp) != 2) {
+		goto badarg;
+	    }
+	    if (tp[1] != am_scope || is_not_tuple(tp[2])) {
+		goto badarg;
+	    }
+	    tp = tuple_val(tp[2]);
+	    if (arityval(*tp) != 2) {
+		goto badarg;
+	    }
+	    if (!term_to_Uint(tp[1], &pos)) {
+		goto badarg;
+	    }
+	    if (!term_to_Sint(tp[2], &len)) {
+		goto badarg;
+	    }
+	    if (len < 0) {
+		Sint lentmp = -len;
+		/* overflow */
+		if (lentmp == len || lentmp < 0 || -lentmp != len) {
+		    goto badarg;
+		}
+		len = lentmp;
+		pos -= len;
+	    }
+	    /* overflow */
+	    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) {
+		goto badarg;
+	    }
+	    *endp = len + pos;
+	    *posp = pos;
+	    if ((orig_size = binary_size(bin)) < pos ||
+		orig_size < (*endp)) {
+		goto badarg;
+	    }
+	    l = CDR(list_val(l));
+	}
+	return 0;
+    } else {
+    badarg:
+	return 1;
+    }
+}
+
+static BIF_RETTYPE binary_match_trap(BIF_ALIST_3)
+{
+    int runres;
+    Eterm result;
+    Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
+    runres = do_binary_match(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result);
+    if (runres == DO_BIN_MATCH_OK) {
+	BIF_RET(result);
+    } else {
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result,
+		  BIF_ARG_3);
+    }
+}
+
+static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3)
+{
+    int runres;
+    Eterm result;
+    Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
+    runres = do_binary_matches(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result);
+    if (runres == DO_BIN_MATCH_OK) {
+	BIF_RET(result);
+    } else {
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result,
+		  BIF_ARG_3);
+    }
+}
+
+BIF_RETTYPE binary_match_3(BIF_ALIST_3)
+{
+    Uint hsstart;
+    Uint hsend;
+    Eterm *tp;
+    Eterm type;
+    Binary *bin;
+    Eterm bin_term = NIL;
+    int runres;
+    Eterm result;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	goto badarg;
+    }
+    if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) {
+	goto badarg;
+    }
+    if (hsend == 0) {
+	BIF_RET(am_nomatch);
+    }
+    if (is_tuple(BIF_ARG_2)) {
+	tp = tuple_val(BIF_ARG_2);
+	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
+	    goto badarg;
+	}
+	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
+	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
+	    goto badarg;
+	}
+	type = tp[1];
+	bin = ((ProcBin *) binary_val(tp[2]))->val;
+	if (type == am_bm &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
+	    goto badarg;
+	}
+	if (type == am_ac &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
+	    goto badarg;
+	}
+	bin_term = tp[2];
+    } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) {
+	goto badarg;
+    }
+    runres = do_binary_match(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin,NIL,&result);
+    if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
+	Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE);
+	bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin);
+    } else if (bin_term == NIL) {
+	erts_bin_free(bin);
+    }
+    switch (runres) {
+    case DO_BIN_MATCH_OK:
+	BIF_RET(result);
+    case DO_BIN_MATCH_RESTART:
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, bin_term);
+    default:
+	goto badarg;
+    }
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+BIF_RETTYPE binary_matches_3(BIF_ALIST_3)
+{
+    Uint hsstart, hsend;
+    Eterm *tp;
+    Eterm type;
+    Binary *bin;
+    Eterm bin_term = NIL;
+    int runres;
+    Eterm result;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	goto badarg;
+    }
+    if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) {
+	goto badarg;
+    }
+    if (hsend == 0) {
+	BIF_RET(NIL);
+    }
+    if (is_tuple(BIF_ARG_2)) {
+	tp = tuple_val(BIF_ARG_2);
+	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
+	    goto badarg;
+	}
+	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
+	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
+	    goto badarg;
+	}
+	type = tp[1];
+	bin = ((ProcBin *) binary_val(tp[2]))->val;
+	if (type == am_bm &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
+	    goto badarg;
+	}
+	if (type == am_ac &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
+	    goto badarg;
+	}
+	bin_term = tp[2];
+    } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) {
+	goto badarg;
+    }
+    runres = do_binary_matches(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin,
+			       NIL,&result);
+    if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
+	Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE);
+	bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin);
+    } else if (bin_term == NIL) {
+	erts_bin_free(bin);
+    }
+    switch (runres) {
+    case DO_BIN_MATCH_OK:
+	BIF_RET(result);
+    case DO_BIN_MATCH_RESTART:
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result,
+		  bin_term);
+    default:
+	goto badarg;
+    }
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+
+BIF_RETTYPE binary_match_2(BIF_ALIST_2)
+{
+    return binary_match_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0));
+}
+
+
+BIF_RETTYPE binary_matches_2(BIF_ALIST_2)
+{
+    return binary_matches_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0));
+}
+
+
+BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen)
+{
+    Uint pos;
+    Sint len;
+    size_t orig_size;
+    Eterm orig;
+    Uint offset;
+    Uint bit_offset;
+    Uint bit_size;
+    Eterm* hp;
+    ErlSubBin* sb;
+
+    if (is_not_binary(binary)) {
+	goto badarg;
+    }
+    if (!term_to_Uint(epos, &pos)) {
+	goto badarg;
+    }
+    if (!term_to_Sint(elen, &len)) {
+	goto badarg;
+    }
+    if (len < 0) {
+	Sint lentmp = -len;
+	/* overflow */
+	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	    goto badarg;
+	}
+	len = lentmp;
+	if (len > pos) {
+	    goto badarg;
+	}
+	pos -= len;
+    }
+    /* overflow */
+    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)){
+	goto badarg;
+    }
+    if ((orig_size = binary_size(binary)) < pos ||
+	orig_size < (pos + len)) {
+	goto badarg;
+    }
+
+
+
+    hp = HAlloc(p, ERL_SUB_BIN_SIZE);
+
+    ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size);
+    sb = (ErlSubBin *) hp;
+    sb->thing_word = HEADER_SUB_BIN;
+    sb->size = len;
+    sb->offs = offset + pos;
+    sb->orig = orig;
+    sb->bitoffs = bit_offset;
+    sb->bitsize = 0;
+    sb->is_writable = 0;
+
+    BIF_RET(make_binary(sb));
+
+ badarg:
+    BIF_ERROR(p, BADARG);
+}
+
+#define ERTS_NEED_GC(p, need) ((HEAP_LIMIT((p)) - HEAP_TOP((p))) <= (need))
+
+BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple)
+{
+    Uint pos;
+    Sint len;
+    size_t orig_size;
+    Eterm orig;
+    Uint offset;
+    Uint bit_offset;
+    Uint bit_size;
+    Eterm* hp;
+    ErlSubBin* sb;
+    Eterm binary;
+    Eterm *tp;
+    Eterm epos, elen;
+    int extra_args;
+
+
+    if (range_is_tuple) {
+	Eterm tpl = reg[live];
+	extra_args = 1;
+	if (is_not_tuple(tpl)) {
+	    goto badarg;
+	}
+	tp = tuple_val(tpl);
+	if (arityval(*tp) != 2) {
+	    goto badarg;
+	}
+
+	epos = tp[1];
+	elen = tp[2];
+    } else {
+	extra_args = 2;
+	epos = reg[live-1];
+	elen = reg[live];
+    }
+    binary = reg[live-extra_args];
+
+    if (is_not_binary(binary)) {
+	goto badarg;
+    }
+    if (!term_to_Uint(epos, &pos)) {
+	goto badarg;
+    }
+    if (!term_to_Sint(elen, &len)) {
+	goto badarg;
+    }
+    if (len < 0) {
+	Sint lentmp = -len;
+	/* overflow */
+	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	    goto badarg;
+	}
+	len = lentmp;
+	if (len > pos) {
+	    goto badarg;
+	}
+	pos -= len;
+    }
+    /* overflow */
+    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) {
+	goto badarg;
+    }
+    if ((orig_size = binary_size(binary)) < pos ||
+	orig_size < (pos + len)) {
+	goto badarg;
+    }
+
+    if (ERTS_NEED_GC(p, ERL_SUB_BIN_SIZE)) {
+	erts_garbage_collect(p, ERL_SUB_BIN_SIZE, reg, live+1-extra_args); /* I don't need the tuple
+									      or indices any more */
+	binary = reg[live-extra_args];
+    }
+
+    hp = p->htop;
+    p->htop += ERL_SUB_BIN_SIZE;
+
+    ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size);
+
+    sb = (ErlSubBin *) hp;
+    sb->thing_word = HEADER_SUB_BIN;
+    sb->size = len;
+    sb->offs = offset + pos;
+    sb->orig = orig;
+    sb->bitoffs = bit_offset;
+    sb->bitsize = 0;
+    sb->is_writable = 0;
+
+    BIF_RET(make_binary(sb));
+
+ badarg:
+    BIF_ERROR(p, BADARG);
+}
+/*************************************************************
+ * The actual guard BIFs are in erl_bif_guard.c
+ * but the implementation of both the non-gc and the gc
+ * variants are here. Note that the functions are named so that they do
+ * not clash with the guard bif's erlang:binary_part/2,3
+ *************************************************************/
+
+BIF_RETTYPE binary_binary_part_3(BIF_ALIST_3)
+{
+    return erts_binary_part(BIF_P,BIF_ARG_1,BIF_ARG_2, BIF_ARG_3);
+}
+
+BIF_RETTYPE binary_binary_part_2(BIF_ALIST_2)
+{
+    Eterm *tp;
+    if (is_not_tuple(BIF_ARG_2)) {
+	goto badarg;
+    }
+    tp = tuple_val(BIF_ARG_2);
+    if (arityval(*tp) != 2) {
+	goto badarg;
+    }
+    return erts_binary_part(BIF_P,BIF_ARG_1,tp[1], tp[2]);
+ badarg:
+   BIF_ERROR(BIF_P,BADARG);
+}
+
+typedef struct {
+    int type;            /* CL_TYPE_XXX */
+    byte *temp_alloc;    /* Used for erts_get/free_aligned, i.e. CL_TYPE_ALIGNED */
+    unsigned char *buff; /* Used for all types, malloced if CL_TYPE_HEAP */
+    Uint bufflen;        /* The length (in bytes) of buffer */
+} CommonData;
+
+#define COMMON_LOOP_FACTOR 10
+
+#define DIRECTION_PREFIX 0
+#define DIRECTION_SUFFIX 1
+
+#define CL_OK 0
+#define CL_RESTART 1
+
+/* The type field in the above structure */
+#define CL_TYPE_EMPTY 0 /* End of array */
+#define CL_TYPE_HEAP 1
+#define CL_TYPE_ALIGNED 2
+#define CL_TYPE_COMMON 3 /* emacsulated */
+#define CL_TYPE_HEAP_NOALLOC 4 /* Will need allocating when trapping */
+
+
+static int do_search_forward(CommonData *cd, Uint *posp, Uint *redsp)
+{
+    Uint pos = *posp;
+    Sint reds = (Sint) *redsp;
+    int i;
+    unsigned char current = 0;
+
+    for(;;) {
+	for(i = 0; cd[i].type != CL_TYPE_EMPTY; ++i) {
+	    if (pos >= cd[i].bufflen) {
+		*posp = pos;
+		if (reds > 0) {
+		    *redsp = (Uint) reds;
+		} else {
+		    *redsp = 0;
+		}
+		return CL_OK;
+	    }
+	    if (i == 0) {
+		current = cd[i].buff[pos];
+	    } else {
+		if (cd[i].buff[pos] != current) {
+		    *posp = pos;
+		    if (reds > 0) {
+			*redsp = (Uint) reds;
+		    } else {
+			*redsp = 0;
+		    }
+		    return CL_OK;
+		}
+	    }
+	    --reds;
+	}
+	++pos;
+	if (reds <= 0) {
+	    *posp = pos;
+	    *redsp = 0;
+	    return CL_RESTART;
+	}
+    }
+}
+static int do_search_backward(CommonData *cd, Uint *posp, Uint *redsp)
+{
+    Uint pos = *posp;
+    Sint reds = (Sint) *redsp;
+    int i;
+    unsigned char current = 0;
+
+    for(;;) {
+	for(i = 0; cd[i].type != CL_TYPE_EMPTY; ++i) {
+	    if (pos >= cd[i].bufflen) {
+		*posp = pos;
+		if (reds > 0) {
+		    *redsp = (Uint) reds;
+		} else {
+		    *redsp = 0;
+		}
+		return CL_OK;
+	    }
+	    if (i == 0) {
+		current = cd[i].buff[cd[i].bufflen - 1 - pos];
+	    } else {
+		if (cd[i].buff[cd[i].bufflen - 1 - pos] != current) {
+		    *posp = pos;
+		    if (reds > 0) {
+			*redsp = (Uint) reds;
+		    } else {
+			*redsp = 0;
+		    }
+		    return CL_OK;
+		}
+	    }
+	    --reds;
+	}
+	++pos;
+	if (reds <= 0) {
+	    *posp = pos;
+	    *redsp = 0;
+	    return CL_RESTART;
+	}
+    }
+}
+
+static void cleanup_common_data(Binary *bp)
+{
+    int i;
+    CommonData *cd;
+    cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bp);
+    for (i=0;cd[i].type != CL_TYPE_EMPTY;++i) {
+	switch (cd[i].type) {
+	case CL_TYPE_HEAP:
+	    erts_free(ERTS_ALC_T_BINARY_BUFFER,cd[i].buff);
+	    break;
+	case CL_TYPE_ALIGNED:
+	    erts_free_aligned_binary_bytes_extra(cd[i].temp_alloc, ERTS_ALC_T_BINARY_BUFFER);
+	    break;
+	default:
+	    break;
+	}
+    }
+    return;
+}
+
+static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction)
+{
+    Eterm l = list;
+    int n = 0;
+    Binary *mb;
+    CommonData *cd;
+    int i = 0;
+    Uint reds = get_reds(p, COMMON_LOOP_FACTOR);
+    Uint save_reds = reds;
+    int res;
+    Export *trapper;
+    Uint pos;
+    Eterm epos;
+    Eterm *hp;
+    Eterm bin_term;
+    Eterm b;
+
+    /* First just count the number of binaries */
+    while (is_list(l)) {
+	b = CAR(list_val(l));
+	if (!is_binary(b)) {
+	    goto badarg;
+	}
+	++n;
+	l = CDR(list_val(l));
+    }
+    if (l != NIL || n == 0) {
+	goto badarg;
+    }
+
+    /* OK, now create a buffer of the right size, we can do a magic binary right away,
+       that's not too costly. */
+    mb = erts_create_magic_binary((n+1)*sizeof(CommonData),cleanup_common_data);
+    cd = (CommonData *) ERTS_MAGIC_BIN_DATA(mb);
+    l = list;
+    while (is_list(l)) {
+	Uint bitoffs;
+	Uint bitsize;
+	Uint offset;
+	Eterm real_bin;
+	ProcBin* pb;
+
+	cd[i].type = CL_TYPE_EMPTY;
+	b = CAR(list_val(l));
+	ERTS_GET_REAL_BIN(b, real_bin, offset, bitoffs, bitsize);
+	if (bitsize != 0) {
+	    erts_bin_free(mb);
+	    goto badarg;
+	}
+	cd[i].bufflen = binary_size(b);
+	cd[i].temp_alloc = NULL;
+	if (*(binary_val(real_bin)) == HEADER_PROC_BIN) {
+	    pb = (ProcBin *) binary_val(real_bin);
+	    if (pb->flags) {
+		erts_emasculate_writable_binary(pb);
+	    }
+	    cd[i].buff = erts_get_aligned_binary_bytes_extra(b, &(cd[i].temp_alloc),
+							     ERTS_ALC_T_BINARY_BUFFER,0);
+	    cd[i].type = (cd[i].temp_alloc != NULL) ? CL_TYPE_ALIGNED : CL_TYPE_COMMON;
+	} else { /* Heap binary */
+	    cd[i].buff = erts_get_aligned_binary_bytes_extra(b, &(cd[i].temp_alloc),
+							     ERTS_ALC_T_BINARY_BUFFER,0);
+	    /* CL_TYPE_HEAP_NOALLOC means you have to copy if trapping */
+	    cd[i].type = (cd[i].temp_alloc != NULL) ? CL_TYPE_ALIGNED : CL_TYPE_HEAP_NOALLOC;
+	}
+	++i;
+	l = CDR(list_val(l));
+    }
+    cd[i].type = CL_TYPE_EMPTY;
+#if defined(DEBUG) || defined(VALGRIND)
+    cd[i].temp_alloc = NULL;
+    cd[i].buff = NULL;
+    cd[i].bufflen = 0;
+#endif
+
+    pos = 0;
+    if (direction == DIRECTION_PREFIX) {
+	trapper = &binary_longest_prefix_trap_export;
+	res = do_search_forward(cd,&pos,&reds);
+    } else {
+	ASSERT(direction == DIRECTION_SUFFIX);
+	trapper = &binary_longest_suffix_trap_export;
+	res = do_search_backward(cd,&pos,&reds);
+    }
+    epos = erts_make_integer(pos,p);
+    if (res == CL_OK) {
+	erts_bin_free(mb);
+	BUMP_REDS(p, (save_reds - reds) / COMMON_LOOP_FACTOR);
+	BIF_RET(epos);
+    } else {
+	ASSERT(res == CL_RESTART);
+	/* Copy all heap binaries that are not already copied (aligned) */
+	for(i = 0; i < n; ++i) {
+	    if (cd[i].type == CL_TYPE_HEAP_NOALLOC) {
+		unsigned char *tmp = cd[i].buff;
+		cd[i].buff = erts_alloc(ERTS_ALC_T_BINARY_BUFFER, cd[i].bufflen);
+		memcpy(cd[i].buff,tmp,cd[i].bufflen);
+		cd[i].type = CL_TYPE_HEAP;
+	    }
+	}
+	hp = HAlloc(p, PROC_BIN_SIZE);
+	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb);
+	BUMP_ALL_REDS(p);
+	BIF_TRAP3(trapper, p, bin_term, epos,list);
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+static BIF_RETTYPE do_longest_common_trap(Process *p, Eterm bin_term, Eterm current_pos,
+					  Eterm orig_list, int direction)
+{
+    Uint reds = get_reds(p, COMMON_LOOP_FACTOR);
+    Uint save_reds = reds;
+    Uint pos;
+    Binary *bin;
+    CommonData *cd;
+    int res;
+    Eterm epos;
+    Export *trapper;
+
+#ifdef DEBUG
+    int r;
+    r = term_to_Uint(current_pos, &pos);
+    ASSERT(r != 0);
+#else
+    term_to_Uint(current_pos, &pos);
+#endif
+    ASSERT(ERTS_TERM_IS_MAGIC_BINARY(bin_term));
+    bin = ((ProcBin *) binary_val(bin_term))->val;
+    cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bin);
+    if (direction == DIRECTION_PREFIX) {
+	trapper = &binary_longest_prefix_trap_export;
+	res = do_search_forward(cd,&pos,&reds);
+    } else {
+	ASSERT(direction == DIRECTION_SUFFIX);
+	trapper = &binary_longest_suffix_trap_export;
+	res = do_search_backward(cd,&pos,&reds);
+    }
+    epos = erts_make_integer(pos,p);
+    if (res == CL_OK) {
+	BUMP_REDS(p, (save_reds - reds) / COMMON_LOOP_FACTOR);
+	BIF_RET(epos);
+    } else {
+	ASSERT(res == CL_RESTART);
+	/* Copy all heap binaries that are not already copied (aligned) */
+	BUMP_ALL_REDS(p);
+	BIF_TRAP3(trapper, p, bin_term, epos, orig_list);
+    }
+}
+
+static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3)
+{
+    return do_longest_common_trap(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3,DIRECTION_PREFIX);
+}
+
+static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3)
+{
+    return do_longest_common_trap(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3,DIRECTION_SUFFIX);
+}
+
+BIF_RETTYPE binary_longest_common_prefix_1(BIF_ALIST_1)
+{
+    return do_longest_common(BIF_P,BIF_ARG_1,DIRECTION_PREFIX);
+}
+
+BIF_RETTYPE binary_longest_common_suffix_1(BIF_ALIST_1)
+{
+    return do_longest_common(BIF_P,BIF_ARG_1,DIRECTION_SUFFIX);
+}
+
+BIF_RETTYPE binary_first_1(BIF_ALIST_1)
+{
+    byte* bytes;
+    Uint byte_size;
+    Uint bit_offs;
+    Uint bit_size;
+    Uint res;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	goto badarg;
+    }
+    byte_size = binary_size(BIF_ARG_1);
+    if (!byte_size) {
+	goto badarg;
+    }
+    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
+    if (bit_size) {
+	goto badarg;
+    }
+    if (bit_offs) {
+	res = ((((Uint) bytes[0]) << bit_offs) | (((Uint) bytes[1]) >> (8-bit_offs))) & 0xFF;
+    } else {
+	res = bytes[0];
+    }
+    BIF_RET(make_small(res));
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+BIF_RETTYPE binary_last_1(BIF_ALIST_1)
+{
+    byte* bytes;
+    Uint byte_size;
+    Uint bit_offs;
+    Uint bit_size;
+    Uint res;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	goto badarg;
+    }
+    byte_size = binary_size(BIF_ARG_1);
+    if (!byte_size) {
+	goto badarg;
+    }
+    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
+    if (bit_size) {
+	goto badarg;
+    }
+    if (bit_offs) {
+	res = ((((Uint) bytes[byte_size-1]) << bit_offs) |
+	       (((Uint) bytes[byte_size]) >> (8-bit_offs))) & 0xFF;
+    } else {
+	res = bytes[byte_size-1];
+    }
+    BIF_RET(make_small(res));
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+BIF_RETTYPE binary_at_2(BIF_ALIST_2)
+{
+    byte* bytes;
+    Uint byte_size;
+    Uint bit_offs;
+    Uint bit_size;
+    Uint res;
+    Uint index;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	goto badarg;
+    }
+    byte_size = binary_size(BIF_ARG_1);
+    if (!byte_size) {
+	goto badarg;
+    }
+    if (!term_to_Uint(BIF_ARG_2, &index)) {
+	goto badarg;
+    }
+    if (index >= byte_size) {
+	goto badarg;
+    }
+    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
+    if (bit_size) {
+	goto badarg;
+    }
+    if (bit_offs) {
+	res = ((((Uint) bytes[index]) << bit_offs) |
+	       (((Uint) bytes[index+1]) >> (8-bit_offs))) & 0xFF;
+    } else {
+	res = bytes[index];
+    }
+    BIF_RET(make_small(res));
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+#define BIN_TO_LIST_OK 0
+#define BIN_TO_LIST_TRAP 1
+/* No badarg, checked before call */
+
+#define BIN_TO_LIST_LOOP_FACTOR 10
+
+static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs,
+			  Uint start, Sint *lenp, Eterm *termp)
+{
+    Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */
+    Uint len = *lenp;
+    Uint loops;
+    Eterm *hp;
+    Eterm term = *termp;
+    Uint n;
+
+    ASSERT(reds > 0);
+
+    loops = MIN(reds,len);
+
+    BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR);
+
+    hp = HAlloc(p,2*loops);
+    while (loops--) {
+	--len;
+	if (bit_offs) {
+	    n = ((((Uint) bytes[start+len]) << bit_offs) |
+		 (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF;
+	} else {
+	    n = bytes[start+len];
+	}
+
+	term = CONS(hp,make_small(n),term);
+	hp +=2;
+    }
+    *termp = term;
+    *lenp = len;
+    if (len) {
+	BUMP_ALL_REDS(p);
+	return BIN_TO_LIST_TRAP;
+    }
+    return BIN_TO_LIST_OK;
+}
+
+
+static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary,
+				       Uint start, Sint len, Eterm sofar)
+{
+    Eterm *hp;
+    Eterm blob;
+
+    hp = HAlloc(p,3);
+    hp[0] = make_pos_bignum_header(2);
+    hp[1] = start;
+    hp[2] = (Uint) len;
+    blob = make_big(hp);
+    BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar);
+}
+
+static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3)
+{
+    Eterm *ptr;
+    Uint start;
+    Sint len;
+    byte *bytes;
+    Uint bit_offs;
+    Uint bit_size;
+    Eterm res = BIF_ARG_3;
+
+    ptr  = big_val(BIF_ARG_2);
+    start = ptr[1];
+    len = (Sint) ptr[2];
+
+    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
+    if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) ==
+	BIN_TO_LIST_OK) {
+	BIF_RET(res);
+    }
+    return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res);
+}
+
+static BIF_RETTYPE binary_bin_to_list_common(Process *p,
+					     Eterm bin,
+					     Eterm epos,
+					     Eterm elen)
+{
+    Uint pos;
+    Sint len;
+    size_t sz;
+    byte *bytes;
+    Uint bit_offs;
+    Uint bit_size;
+    Eterm res = NIL;
+
+    if (is_not_binary(bin)) {
+	goto badarg;
+    }
+    if (!term_to_Uint(epos, &pos)) {
+	goto badarg;
+    }
+    if (!term_to_Sint(elen, &len)) {
+	goto badarg;
+    }
+    if (len < 0) {
+	Sint lentmp = -len;
+	/* overflow */
+	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	    goto badarg;
+	}
+	len = lentmp;
+	if (len > pos) {
+	    goto badarg;
+	}
+	pos -= len;
+    }
+    /* overflow */
+    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) {
+	goto badarg;
+    }
+    sz = binary_size(bin);
+
+    if (pos+len > sz) {
+	goto badarg;
+    }
+    ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size);
+    if (bit_size != 0) {
+	goto badarg;
+    }
+    if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) ==
+       BIN_TO_LIST_OK) {
+	BIF_RET(res);
+    }
+    return do_trap_bin_to_list(p,bin,pos,len,res);
+
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3)
+{
+    return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3);
+}
+
+BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2)
+{
+    Eterm *tp;
+
+    if (is_not_tuple(BIF_ARG_2)) {
+	goto badarg;
+    }
+    tp = tuple_val(BIF_ARG_2);
+    if (arityval(*tp) != 2) {
+	goto badarg;
+    }
+    return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]);
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1)
+{
+    Uint pos = 0;
+    Sint len;
+    byte *bytes;
+    Uint bit_offs;
+    Uint bit_size;
+    Eterm res = NIL;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	goto badarg;
+    }
+    len = binary_size(BIF_ARG_1);
+    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
+    if (bit_size != 0) {
+	goto badarg;
+    }
+    if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) ==
+       BIN_TO_LIST_OK) {
+	BIF_RET(res);
+    }
+    return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res);
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+/*
+ * Ok, erlang:list_to_binary does not interrupt, and we really don't want
+ * an alternative implementation for the exact same thing, why we
+ * have descided to use the old non-restarting implementation for now.
+ * In reality, there are seldom many iterations involved in doing this, so the
+ * problem of long-running bifs is not really that big in this case.
+ * So, for now we use the old implementation also in the module binary.
+ */
+
+BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1)
+{
+    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1);
+}
+
+typedef struct {
+    Uint times_left;
+    Uint source_size;
+    int source_type;
+    byte *source;
+    byte *temp_alloc;
+    Uint result_pos;
+    Binary *result;
+} CopyBinState;
+
+#define BC_TYPE_EMPTY 0
+#define BC_TYPE_HEAP 1
+#define BC_TYPE_ALIGNED 2 /* May or may not point to (emasculated) binary, temp_alloc field is set
+			     so that erts_free_aligned_binary_bytes_extra can handle either */
+
+
+#define BINARY_COPY_LOOP_FACTOR 100
+
+static void cleanup_copy_bin_state(Binary *bp)
+{
+    CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(bp);
+    if (cbs->result != NULL) {
+	erts_bin_free(cbs->result);
+	cbs->result = NULL;
+    }
+    switch (cbs->source_type) {
+    case BC_TYPE_HEAP:
+	erts_free(ERTS_ALC_T_BINARY_BUFFER,cbs->source);
+	break;
+    case BC_TYPE_ALIGNED:
+	erts_free_aligned_binary_bytes_extra(cbs->temp_alloc,
+					     ERTS_ALC_T_BINARY_BUFFER);
+	break;
+    default:
+	/* otherwise do nothing */
+	break;
+    }
+    cbs->source_type =  BC_TYPE_EMPTY;
+}
+
+/*
+ * Binary *erts_bin_nrml_alloc(Uint size);
+ * Binary *erts_bin_realloc(Binary *bp, Uint size);
+ * void erts_bin_free(Binary *bp);
+ */
+static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en)
+{
+    Uint n;
+    byte *bytes;
+    Uint bit_offs;
+    Uint bit_size;
+    size_t size;
+    Uint reds = get_reds(p, BINARY_COPY_LOOP_FACTOR);
+    Uint target_size;
+    byte *t;
+    Uint pos;
+
+
+    if (is_not_binary(bin)) {
+	goto badarg;
+    }
+    if (!term_to_Uint(en, &n)) {
+	goto badarg;
+    }
+    if (!n) {
+	Eterm res_term = erts_new_heap_binary(p,NULL,0,&bytes);
+	BIF_RET(res_term);
+    }
+    ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size);
+    if (bit_size != 0) {
+	goto badarg;
+    }
+
+    size = binary_size(bin);
+    target_size = size * n;
+
+    if ((target_size - size) >= reds) {
+	Eterm orig;
+	Uint offset;
+	Uint bit_offset;
+	Uint bit_size;
+	CopyBinState *cbs;
+	Eterm *hp;
+	Eterm trap_term;
+	int i;
+
+	/* We will trap, set up the structure for trapping right away */
+	Binary *mb = erts_create_magic_binary(sizeof(CopyBinState),
+					      cleanup_copy_bin_state);
+	cbs = ERTS_MAGIC_BIN_DATA(mb);
+
+	cbs->temp_alloc = NULL;
+	cbs->source = NULL;
+
+	ERTS_GET_REAL_BIN(bin, orig, offset, bit_offset, bit_size);
+	if (*(binary_val(orig)) == HEADER_PROC_BIN) {
+	    ProcBin* pb = (ProcBin *) binary_val(orig);
+	    if (pb->flags) {
+		erts_emasculate_writable_binary(pb);
+	    }
+	    cbs->source =
+		erts_get_aligned_binary_bytes_extra(bin,
+						    &(cbs->temp_alloc),
+						    ERTS_ALC_T_BINARY_BUFFER,
+						    0);
+	    cbs->source_type = BC_TYPE_ALIGNED;
+	} else { /* Heap binary */
+	    cbs->source =
+		erts_get_aligned_binary_bytes_extra(bin,
+						    &(cbs->temp_alloc),
+						    ERTS_ALC_T_BINARY_BUFFER,
+						    0);
+	    if (!(cbs->temp_alloc)) { /* alignment not needed, need to copy */
+		byte *tmp = erts_alloc(ERTS_ALC_T_BINARY_BUFFER,size);
+		memcpy(tmp,cbs->source,size);
+		cbs->source = tmp;
+		cbs->source_type = BC_TYPE_HEAP;
+	    } else {
+		cbs->source_type = BC_TYPE_ALIGNED;
+	    }
+	}
+	cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap
+							   if trapping */
+	cbs->result->flags = 0;
+	cbs->result->orig_size = target_size;
+	erts_refc_init(&(cbs->result->refc), 1);
+	t = (byte *) cbs->result->orig_bytes; /* No offset or anything */
+	pos = 0;
+	i = 0;
+	while (pos < reds) {
+	    memcpy(t+pos,cbs->source, size);
+	    pos += size;
+	    ++i;
+	}
+	cbs->source_size = size;
+	cbs->result_pos = pos;
+	cbs->times_left = n-i;
+	hp = HAlloc(p,PROC_BIN_SIZE);
+	trap_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb);
+	BUMP_ALL_REDS(p);
+	BIF_TRAP2(&binary_copy_trap_export, p, bin, trap_term);
+    } else {
+	Eterm res_term;
+	byte *temp_alloc = NULL;
+	byte *source =
+	    erts_get_aligned_binary_bytes(bin,
+					  &temp_alloc);
+	if (target_size <= ERL_ONHEAP_BIN_LIMIT) {
+	    res_term = erts_new_heap_binary(p,NULL,target_size,&t);
+	} else {
+	    res_term = erts_new_mso_binary(p,NULL,target_size);
+	    t = ((ProcBin *) binary_val(res_term))->bytes;
+	}
+	pos = 0;
+	while (pos < target_size) {
+	    memcpy(t+pos,source, size);
+	    pos += size;
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BUMP_REDS(p,pos / BINARY_COPY_LOOP_FACTOR);
+	BIF_RET(res_term);
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+BIF_RETTYPE binary_copy_trap(BIF_ALIST_2)
+{
+    Uint n;
+    size_t size;
+    Uint reds = get_reds(BIF_P, BINARY_COPY_LOOP_FACTOR);
+    byte *t;
+    Uint pos;
+    Binary *mb = ((ProcBin *) binary_val(BIF_ARG_2))->val;
+    CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(mb);
+    Uint opos;
+
+    /* swapout... */
+    n = cbs->times_left;
+    size = cbs->source_size;
+    opos = pos = cbs->result_pos;
+    t = (byte *) cbs->result->orig_bytes; /* "well behaved" binary */
+    if ((n-1) * size >= reds) {
+	Uint i = 0;
+	while ((pos - opos) < reds) {
+	    memcpy(t+pos,cbs->source, size);
+	    pos += size;
+	    ++i;
+	}
+	cbs->result_pos = pos;
+	cbs->times_left -= i;
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP2(&binary_copy_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2);
+    } else {
+	Binary *save;
+	ProcBin* pb;
+	Uint target_size = cbs->result->orig_size;
+	while (pos < target_size) {
+	    memcpy(t+pos,cbs->source, size);
+	    pos += size;
+	}
+	save =  cbs->result;
+	cbs->result = NULL;
+	cleanup_copy_bin_state(mb); /* now cbs is dead */
+	pb = (ProcBin *) HAlloc(BIF_P, PROC_BIN_SIZE);
+	pb->thing_word = HEADER_PROC_BIN;
+	pb->size = target_size;
+	pb->next = MSO(BIF_P).first;
+	MSO(BIF_P).first = (struct erl_off_heap_header*) pb;
+	pb->val = save;
+	pb->bytes = t;
+	pb->flags = 0;
+
+	OH_OVERHEAD(&(MSO(BIF_P)), target_size / sizeof(Eterm));
+	BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR);
+
+	BIF_RET(make_binary(pb));
+    }
+}
+
+
+BIF_RETTYPE binary_copy_1(BIF_ALIST_1)
+{
+    return do_binary_copy(BIF_P,BIF_ARG_1,make_small(1));
+}
+
+BIF_RETTYPE binary_copy_2(BIF_ALIST_2)
+{
+    return do_binary_copy(BIF_P,BIF_ARG_1,BIF_ARG_2);
+}
+
+BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1)
+{
+    ErlSubBin *sb;
+    ProcBin *pb;
+    Eterm res;
+    Eterm bin = BIF_ARG_1;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    sb = (ErlSubBin *) binary_val(bin);
+    if (sb->thing_word == HEADER_SUB_BIN) {
+	bin = sb->orig;
+    }
+    pb = (ProcBin *) binary_val(bin);
+    if (pb->thing_word == HEADER_PROC_BIN) {
+	/* XXX:PaN - Halfword - orig_size is a long, we should handle that */
+	res = erts_make_integer((Uint) pb->val->orig_size, BIF_P);
+    } else { /* heap binary */
+	res = erts_make_integer((Uint) ((ErlHeapBin *) pb)->size, BIF_P);
+    }
+    BIF_RET(res);
+}
+
+#define END_BIG 0
+#define END_SMALL 1
+
+#ifdef WORDS_BIGENDIAN
+#define END_NATIVE END_BIG
+#else
+#define END_NATIVE END_SMALL
+#endif
+
+static int get_need(Uint u) {
+#if defined(ARCH_64) && !HALFWORD_HEAP
+    if (u > 0xFFFFFFFFUL) {
+	if (u > 0xFFFFFFFFFFFFUL) {
+	    if (u > 0xFFFFFFFFFFFFFFUL) {
+		return 8;
+	    }
+	    return 7;
+	}
+	if (u > 0xFFFFFFFFFFUL) {
+	    return 6;
+	}
+	return 5;
+    }
+#endif
+    if (u > 0xFFFFUL) {
+	if (u > 0xFFFFFFUL) {
+	    return 4;
+	}
+	return 3;
+    }
+    if (u > 0xFFUL) {
+	return 2;
+    }
+    return 1;
+}
+
+static BIF_RETTYPE do_encode_unsigned(Process *p, Eterm uns, Eterm endianess)
+{
+    Eterm res;
+    if ((is_not_small(uns) && is_not_big(uns)) || is_not_atom(endianess) ||
+	(endianess != am_big && endianess != am_little)) {
+	goto badarg;
+    }
+    if (is_small(uns)) {
+	Sint x = signed_val(uns);
+	Uint u;
+	int n,i;
+	byte *b;
+
+	if (x < 0) {
+	    goto badarg;
+	}
+
+	u = (Uint) x;
+	n = get_need(u);
+	ASSERT(n <= ERL_ONHEAP_BIN_LIMIT);
+	res = erts_new_heap_binary(p, NULL, n, &b);
+	if (endianess == am_big) {
+	    for(i=n-1;i>=0;--i) {
+		b[i] = u & 0xFF;
+		u >>= 8;
+	    }
+	} else {
+	    for(i=0;i<n;++i) {
+		b[i] = u & 0xFF;
+		u >>= 8;
+	    }
+	}
+	BIF_RET(res);
+    } else {
+	/* Big */
+	Eterm *bigp = big_val(uns);
+	Uint n;
+	dsize_t num_parts = BIG_SIZE(bigp);
+	Eterm res;
+	byte *b;
+	ErtsDigit d;
+
+	if(BIG_SIGN(bigp)) {
+	    goto badarg;
+	}
+	n = (num_parts-1)*sizeof(ErtsDigit)+get_need(BIG_DIGIT(bigp,(num_parts-1)));
+	if (n <= ERL_ONHEAP_BIN_LIMIT) {
+	    res = erts_new_heap_binary(p,NULL,n,&b);
+	} else {
+	    res = erts_new_mso_binary(p,NULL,n);
+	    b = ((ProcBin *) binary_val(res))->bytes;
+	}
+
+	if (endianess == am_big) {
+	    Sint i,j;
+	    j = 0;
+	    d = BIG_DIGIT(bigp,0);
+	    for (i=n-1;i>=0;--i) {
+		b[i] = d & 0xFF;
+		if (!((++j) % sizeof(ErtsDigit))) {
+		    d = BIG_DIGIT(bigp,j / sizeof(ErtsDigit));
+		} else {
+		    d >>= 8;
+		}
+	    }
+	} else {
+	    Sint i,j;
+	    j = 0;
+	    d = BIG_DIGIT(bigp,0);
+	    for (i=0;i<n;++i) {
+		b[i] = d & 0xFF;
+		if (!((++j) % sizeof(ErtsDigit))) {
+		    d = BIG_DIGIT(bigp,j / sizeof(ErtsDigit));
+		} else {
+		    d >>= 8;
+		}
+	    }
+
+	}
+	BIF_RET(res);
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+static BIF_RETTYPE do_decode_unsigned(Process *p, Eterm uns, Eterm endianess)
+{
+    byte *bytes;
+    Uint bitoffs, bitsize;
+    Uint size;
+    Eterm res;
+
+    if (is_not_binary(uns) || is_not_atom(endianess) ||
+	(endianess != am_big && endianess != am_little)) {
+	goto badarg;
+    }
+    ERTS_GET_BINARY_BYTES(uns, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	goto badarg;
+    }
+    /* align while rolling */
+    size = binary_size(uns);
+    if (bitoffs) {
+	if (endianess == am_big) {
+	    while (size && (((((Uint) bytes[0]) << bitoffs) |
+			    (((Uint) bytes[1]) >> (8-bitoffs))) & 0xFF) == 0) {
+		++bytes;
+		--size;
+	    }
+	} else {
+	    while(size &&
+		  (((((Uint) bytes[size-1]) << bitoffs) |
+		    (((Uint) bytes[size]) >> (8-bitoffs))) & 0xFF) == 0) {
+		--size;
+	    }
+	}
+    } else {
+	if (endianess == am_big) {
+	    while (size && *bytes == 0) {
+		++bytes;
+		--size;
+	    }
+	} else {
+	    while(size && bytes[size-1] == 0) {
+		--size;
+	    }
+	}
+    }
+    if (!size) {
+	BIF_RET(make_small(0));
+    }
+
+    if (size <= sizeof(Uint)) {
+	Uint u = 0;
+	Sint i;
+
+	if (endianess == am_big) {
+		if (bitoffs) {
+		    for(i=0;i<size;++i) {
+			u <<=8;
+			u |= (((((Uint) bytes[i]) << bitoffs) |
+			       (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF);
+		    }
+		} else {
+		    for(i=0;i<size;++i) {
+			u <<=8;
+			u |= bytes[i];
+		    }
+		}
+	} else {
+
+		if (bitoffs) {
+		    for(i=size-1;i>=0;--i) {
+			u <<=8;
+			u |= (((((Uint) bytes[i]) << bitoffs) |
+			       (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF);
+		    }
+		} else {
+		    for(i=size-1;i>=0;--i) {
+			u <<=8;
+			u |= bytes[i];
+		    }
+		}
+	}
+	res = erts_make_integer(u,p);
+	BIF_RET(res);
+    } else {
+	/* Assume big, as we stripped away all zeroes from the MSB part of the binary */
+	dsize_t num_parts = size / sizeof(ErtsDigit) + !!(size % sizeof(ErtsDigit));
+	Eterm *bigp;
+
+	bigp = HAlloc(p, BIG_NEED_SIZE(num_parts));
+	*bigp = make_pos_bignum_header(num_parts);
+	res = make_big(bigp);
+
+	if (endianess == am_big) {
+	    Sint i,j;
+	    ErtsDigit *d;
+	    j = size;
+	    d = &(BIG_DIGIT(bigp,num_parts - 1));
+	    *d = 0;
+	    i = 0;
+	    if(bitoffs) {
+		for (;;){
+		    (*d) <<= 8;
+		    (*d) |= (((((Uint) bytes[i]) << bitoffs) |
+			      (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF);
+		    if (++i >= size) {
+			break;
+		    }
+		    if (!(--j % sizeof(ErtsDigit))) {
+			--d;
+			*d = 0;
+		    }
+		}
+	    } else {
+		for (;;){
+		    (*d) <<= 8;
+		    (*d) |= bytes[i];
+		    if (++i >= size) {
+			break;
+		    }
+		    if (!(--j % sizeof(ErtsDigit))) {
+			--d;
+			*d = 0;
+		    }
+		}
+	    }
+	} else {
+	    Sint i,j;
+	    ErtsDigit *d;
+	    j = size;
+	    d = &(BIG_DIGIT(bigp,num_parts - 1));
+	    *d = 0;
+	    i = size-1;
+	    if (bitoffs) {
+		for (;;){
+		    (*d) <<= 8;
+		    (*d) |= (((((Uint) bytes[i]) << bitoffs) |
+			      (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF);
+		    if (--i < 0) {
+			break;
+		    }
+		    if (!(--j % sizeof(ErtsDigit))) {
+			--d;
+			*d = 0;
+		    }
+		}
+	    } else {
+		for (;;){
+		    (*d) <<= 8;
+		    (*d) |= bytes[i];
+		    if (--i < 0) {
+			break;
+		    }
+		    if (!(--j % sizeof(ErtsDigit))) {
+			--d;
+			*d = 0;
+		    }
+		}
+	    }
+	}
+	BIF_RET(res);
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+BIF_RETTYPE binary_encode_unsigned_1(BIF_ALIST_1)
+{
+    return do_encode_unsigned(BIF_P,BIF_ARG_1,am_big);
+}
+
+BIF_RETTYPE binary_encode_unsigned_2(BIF_ALIST_2)
+{
+    return do_encode_unsigned(BIF_P,BIF_ARG_1,BIF_ARG_2);
+}
+
+BIF_RETTYPE binary_decode_unsigned_1(BIF_ALIST_1)
+{
+    return do_decode_unsigned(BIF_P,BIF_ARG_1,am_big);
+}
+
+BIF_RETTYPE binary_decode_unsigned_2(BIF_ALIST_2)
+{
+    return do_decode_unsigned(BIF_P,BIF_ARG_1,BIF_ARG_2);
+}
+
+/*
+ * Hard debug functions (dump) for the search structures
+ */
+
+#ifdef HARDDEBUG
+static void dump_bm_data(BMData *bm)
+{
+    int i,j;
+    erts_printf("Dumping Boyer-Moore structure.\n");
+    erts_printf("=============================\n");
+    erts_printf("Searchstring [%ld]:\n", bm->len);
+    erts_printf("<<");
+    for (i = 0; i < bm->len; ++i) {
+	if (i > 0) {
+	    erts_printf(", ");
+	}
+	erts_printf("%d", (int) bm->x[i]);
+	if (bm->x[i] >= 'A') {
+	    erts_printf(" ($%c)",(char) bm->x[i]);
+	}
+    }
+    erts_printf(">>\n");
+    erts_printf("GoodShift array:\n");
+    for (i = 0; i < bm->len; ++i) {
+	erts_printf("GoodShift[%d]: %ld\n", i, bm->goodshift[i]);
+    }
+    erts_printf("BadShift array:\n");
+    j = 0;
+    for (i = 0; i < ALPHABET_SIZE; i += j) {
+	for (j = 0; i + j < ALPHABET_SIZE && j < 6; ++j) {
+	    erts_printf("BS[%03d]:%02ld, ", i+j, bm->badshift[i+j]);
+	}
+	erts_printf("\n");
+    }
+}
+
+static void dump_ac_node(ACNode *node, int indent, int ch) {
+    int i;
+    char *spaces = erts_alloc(ERTS_ALC_T_TMP, 10 * indent + 1);
+    memset(spaces,' ',10*indent);
+    spaces[10*indent] = '\0';
+    erts_printf("%s-> %c\n",spaces,ch);
+    erts_printf("%sId: %u\n",spaces,(unsigned) node->id);
+    erts_printf("%sD: %u\n",spaces,(unsigned)node->d);
+    erts_printf("%sFinal: %d\n",spaces,(int)node->final);
+    erts_printf("%sFail: %u\n",spaces,(unsigned)node->h->id);
+    erts_free(ERTS_ALC_T_TMP,spaces);
+    for(i=0;i<ALPHABET_SIZE;++i) {
+	if (node->g[i] != NULL && node->g[i] != node) {
+	    dump_ac_node(node->g[i],indent+1,i);
+	}
+    }
+}
+
+
+static void dump_ac_trie(ACTrie *act)
+{
+    erts_printf("Aho Corasick Trie dump.\n");
+    erts_printf("=======================\n");
+    erts_printf("Node counter: %u\n", (unsigned) act->idc);
+    erts_printf("Searchstring counter: %u\n", (unsigned) act->counter);
+    erts_printf("Trie:\n");
+    dump_ac_node(act->root, 0, '0');
+    return;
+}
+#endif
diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c
index 445ba00ca7..06b7ffdf32 100644
--- a/erts/emulator/beam/erl_bif_chksum.c
+++ b/erts/emulator/beam/erl_bif_chksum.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -49,9 +49,9 @@ void erts_init_bif_chksum(void)
     chksum_md5_2_exp.code[1] = am_atom_put("md5_trap",8);
     chksum_md5_2_exp.code[2] = 2;
     chksum_md5_2_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     chksum_md5_2_exp.code[4] = 
-	(Eterm) &md5_2;
+	(BeamInstr) &md5_2;
 }
     
 
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 9d5f0d9c02..9631fb50db 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -146,7 +146,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 				Eterm name_term, Eterm options)
 {
     char *path = NULL;
-    int path_len;
+    Uint path_len;
     char *name = NULL;
     DE_Handle *dh;
     erts_driver_t *drv;
@@ -221,9 +221,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 	goto error;
     }
 
-    path_len = io_list_len(path_term);
-
-    if (path_len <= 0) {
+    if (erts_iolist_size(path_term, &path_len)) {
 	goto error;
     }
     path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1);
@@ -1193,7 +1191,7 @@ int erts_ddll_driver_ok(DE_Handle *dh)
 static void ddll_no_more_references(void *vdh)
 {
     DE_Handle *dh = (DE_Handle *) vdh;
-    int x;
+    erts_aint_t x;
 
     lock_drv_list();
 
@@ -1604,7 +1602,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
 	erts_sys_ddll_close(dh->handle);
 	return ERL_DE_LOAD_ERROR_BAD_NAME;
     }
-    erts_smp_atomic_init(&(dh->refc), (long) 0);
+    erts_smp_atomic_init(&(dh->refc), (erts_aint_t) 0);
     dh->port_count = 0;
     dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1);
     sys_strcpy(dh->full_path, path);
@@ -1646,7 +1644,8 @@ static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name)
 	    if (save_name != NULL) {
 		*save_name = mkatom(q->name);
 	    } 
-	    /* XXX:PaN Future locking problems? Don't dare to let go of the diver_list lock here!*/
+	    /* Future locking problems? Don't dare to let go of the
+	       diver_list lock here!*/
 	    if (q->finish) {
 		int fpe_was_unmasked = erts_block_fpe();
 		(*(q->finish))();
@@ -1671,7 +1670,7 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name)
     dh->handle = NULL;
     dh->procs = NULL;
     dh->port_count = 0;
-    erts_refc_init(&(dh->refc), (long) 0);
+    erts_refc_init(&(dh->refc), (erts_aint_t) 0);
     dh->status = -1;
     dh->reload_full_path = NULL;
     dh->reload_driver_name = NULL;
@@ -1877,7 +1876,7 @@ static Eterm mkatom(char *str)
 static char *pick_list_or_atom(Eterm name_term)
 { 
     char *name = NULL;
-    int name_len;
+    Uint name_len;
     if (is_atom(name_term)) {
 	Atom *ap = atom_tab(atom_val(name_term));
 	if (ap->len == 0) {
@@ -1889,8 +1888,7 @@ static char *pick_list_or_atom(Eterm name_term)
 	memcpy(name,ap->name,ap->len);
 	name[ap->len] = '\0';
     } else {
-	name_len = io_list_len(name_term);
-	if (name_len <= 0) {
+	if (erts_iolist_size(name_term, &name_len)) {
 	    goto error;
 	}
 	name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1);
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index 440b0b4f14..01e6977a2c 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -314,6 +314,30 @@ double_to_integer(Process* p, double x)
     return res;
 }
 
+/********************************************************************************
+ * binary_part guards. The actual implementation is in erl_bif_binary.c
+ ********************************************************************************/
+BIF_RETTYPE binary_part_3(BIF_ALIST_3)
+{
+    return erts_binary_part(BIF_P,BIF_ARG_1,BIF_ARG_2, BIF_ARG_3);
+}
+
+BIF_RETTYPE binary_part_2(BIF_ALIST_2)
+{
+    Eterm *tp;
+    if (is_not_tuple(BIF_ARG_2)) {
+	goto badarg;
+    }
+    tp = tuple_val(BIF_ARG_2);
+    if (arityval(*tp) != 2) {
+	goto badarg;
+    }
+    return erts_binary_part(BIF_P,BIF_ARG_1,tp[1], tp[2]);
+ badarg:
+   BIF_ERROR(BIF_P,BADARG);
+}
+
+
 /*
  * The following code is used when a guard that may build on the
  * heap is called directly. They must not use HAlloc(), but must
@@ -630,3 +654,16 @@ gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live)
     }
     return res;
 }
+
+/********************************************************************************
+ * binary_part guards. The actual implementation is in erl_bif_binary.c
+ ********************************************************************************/
+Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live)
+{
+    return erts_gc_binary_part(p,reg,live,0);
+}
+
+Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live)
+{
+    return erts_gc_binary_part(p,reg,live,1);
+}
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a34d400ed8..f264bf44df 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -38,9 +38,7 @@
 #include "erl_instrument.h"
 #include "dist.h"
 #include "erl_gc.h"
-#ifdef ELIB_ALLOC_IS_CLIB
-#include "elib_stat.h"
-#endif
+#include "erl_cpu_topology.h"
 #ifdef HIPE
 #include "hipe_arch.h"
 #endif
@@ -59,16 +57,23 @@
 /* Keep erts_system_version as a global variable for easy access from a core */
 static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE
 				     " (erts-" ERLANG_VERSION ")"
+#if !HEAP_ON_C_STACK && !HALFWORD_HEAP
+				     " [no-c-stack-objects]"
+#endif
 #ifndef OTP_RELEASE
 				     " [source]"
 #endif	
 #ifdef ARCH_64
+#if HALFWORD_HEAP
+				     " [64-bit halfword]"
+#else
 				     " [64-bit]"
 #endif
+#endif
 #ifdef ERTS_SMP
-				     " [smp:%bpu:%bpu]"
+				     " [smp:%beu:%beu]"
 #endif
-				     " [rq:%bpu]"
+				     " [rq:%beu]"
 #ifdef USE_THREADS
 				     " [async-threads:%d]"
 #endif
@@ -115,22 +120,26 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE
 #endif
 
 static Eterm
-bld_bin_list(Uint **hpp, Uint *szp, ProcBin* pb)
+bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
 {
+    struct erl_off_heap_header* ohh;
     Eterm res = NIL;
     Eterm tuple;
 
-    for (; pb; pb = pb->next) {
-	Eterm val = erts_bld_uint(hpp, szp, (Uint) pb->val);
-	Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size);
-
-	if (szp)
-	    *szp += 4+2;
-	if (hpp) {
-	    Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc);
-	    tuple = TUPLE3(*hpp, val, orig_size, make_small(refc));
-	    res = CONS(*hpp + 4, tuple, res);
-	    *hpp += 4+2;
+    for (ohh = oh->first; ohh; ohh = ohh->next) {
+	if (ohh->thing_word == HEADER_PROC_BIN) {
+	    ProcBin* pb = (ProcBin*) ohh;
+	    Eterm val = erts_bld_uword(hpp, szp, (UWord) pb->val);
+	    Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size);
+    
+	    if (szp)
+		*szp += 4+2;
+	    if (hpp) {
+		Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc);
+		tuple = TUPLE3(*hpp, val, orig_size, make_small(refc));
+		res = CONS(*hpp + 4, tuple, res);
+		*hpp += 4+2;
+	    }
 	}
     }
     return res;
@@ -169,10 +178,10 @@ static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc)
     Eterm tup;
     Eterm r = (IS_CONST(mon->ref)
 	       ? mon->ref
-	       : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->ref));
+	       : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref));
     Eterm p = (IS_CONST(mon->pid)
 	       ? mon->pid
-	       : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->pid));
+	       : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->pid));
     tup = TUPLE5(pmlc->hp, pmlc->tag, make_small(mon->type), r, p, mon->name);
     pmlc->hp += 6;
     pmlc->res = CONS(pmlc->hp, tup, pmlc->res);
@@ -233,7 +242,7 @@ static void do_make_one_lnk_element(ErtsLink *lnk, void * vpllc)
     Eterm old_res, targets = NIL;
     Eterm p = (IS_CONST(lnk->pid)
 	       ? lnk->pid
-	       : STORE_NC(&(pllc->hp), &MSO(pllc->p).externals, lnk->pid));
+	       : STORE_NC(&(pllc->hp), &MSO(pllc->p), lnk->pid));
     if (lnk->type == LINK_NODE) {
 	targets = make_small(ERTS_LINK_REFC(lnk));
     } else if (ERTS_LINK_ROOT(lnk) != NULL) {
@@ -624,12 +633,18 @@ static Eterm pi_1_keys[] = {
 #define ERTS_PI_1_NO_OF_KEYS (sizeof(pi_1_keys)/sizeof(Eterm))
 
 static Eterm pi_1_keys_list;
-static Uint pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS];
+#if HEAP_ON_C_STACK
+static Eterm pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS];
+#endif
 
 static void
 process_info_init(void)
 {
+#if HEAP_ON_C_STACK
     Eterm *hp = &pi_1_keys_list_heap[0];
+#else
+    Eterm *hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM,sizeof(Eterm)*2*ERTS_PI_1_NO_OF_KEYS);
+#endif
     int i;
 
     pi_1_keys_list = NIL;
@@ -998,7 +1013,7 @@ process_info_aux(Process *BIF_P,
 	    hp = HAlloc(BIF_P, 3);
 	    res = am_undefined;
 	} else {
-	    Eterm* current;
+	    BeamInstr* current;
 
 	    if (rp->current[0] == am_erlang &&
 		rp->current[1] == am_process_info &&
@@ -1127,9 +1142,9 @@ process_info_aux(Process *BIF_P,
 			}
 			else {
 			    /* Make our copy of the message */
-			    ASSERT(size_object(msg) == hfp->size);
+			    ASSERT(size_object(msg) == hfp->used_size);
 			    msg = copy_struct(msg,
-					      hfp->size,
+					      hfp->used_size,
 					      &hp,
 					      &MSO(BIF_P));
 			}
@@ -1212,7 +1227,7 @@ process_info_aux(Process *BIF_P,
 	hp = HAlloc(BIF_P, 3 + mic.sz);
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
-	    item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); 
+	    item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); 
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
@@ -1245,9 +1260,7 @@ process_info_aux(Process *BIF_P,
 	    else {
 		/* Monitor by pid. Build {process, Pid} and cons it. */
 		Eterm t;
-		Eterm pid = STORE_NC(&hp,
-				     &MSO(BIF_P).externals,
-				     mic.mi[i].entity);
+		Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity);
 		t = TUPLE2(hp, am_process, pid);
 		hp += 3;
 		res = CONS(hp, t, res);
@@ -1269,7 +1282,7 @@ process_info_aux(Process *BIF_P,
 
 	res = NIL;
 	for (i = 0; i < mic.mi_i; ++i) {
-	    item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); 
+	    item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); 
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
@@ -1478,7 +1491,7 @@ process_info_aux(Process *BIF_P,
     case am_group_leader: {
 	int sz = NC_HEAP_SIZE(rp->group_leader);
 	hp = HAlloc(BIF_P, 3 + sz);
-	res = STORE_NC(&hp, &MSO(BIF_P).externals, rp->group_leader);
+	res = STORE_NC(&hp, &MSO(BIF_P), rp->group_leader);
 	break;
     }
 
@@ -1503,9 +1516,9 @@ process_info_aux(Process *BIF_P,
 
     case am_binary: {
 	Uint sz = 3;
-	(void) bld_bin_list(NULL, &sz, MSO(rp).mso);
+	(void) bld_bin_list(NULL, &sz, &MSO(rp));
 	hp = HAlloc(BIF_P, sz);
-	res = bld_bin_list(&hp, NULL, MSO(rp).mso);
+	res = bld_bin_list(&hp, NULL, &MSO(rp));
 	break;
     }
 
@@ -1532,7 +1545,7 @@ process_info_aux(Process *BIF_P,
     case am_backtrace: {
 	erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
 	erts_stack_dump(ERTS_PRINT_DSBUF, (void *) dsbufp, rp);
-	res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len);
+	res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len);
 	erts_destroy_tmp_dsbuf(dsbufp);
 	hp = HAlloc(BIF_P, 3);
 	break;
@@ -1622,6 +1635,14 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 
     if (sel == am_allocator_sizes && arity == 2) {
 	return erts_allocator_info_term(BIF_P, *tp, 1);
+    } else if (sel == am_wordsize && arity == 2) {
+	if (tp[0] == am_internal) {
+	    return make_small(sizeof(Eterm));
+	}
+	if (tp[0] == am_external) {
+	    return make_small(sizeof(UWord));
+	}
+	goto badarg;
     } else if (sel == am_allocated) {
 	if (arity == 2) {
 	    Eterm res = THE_NON_VALUE;
@@ -1667,6 +1688,8 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 	return erts_get_cpu_topology_term(BIF_P, *tp);
     } else if (ERTS_IS_ATOM_STR("cpu_topology", sel) && arity == 2) {
 	Eterm res = erts_get_cpu_topology_term(BIF_P, *tp);
+	if (res == THE_NON_VALUE)
+	    goto badarg;
 	ERTS_BIF_PREP_TRAP1(ret, erts_format_cpu_topology_trap, BIF_P, res);
 	return ret;
 #if defined(PURIFY) || defined(VALGRIND)
@@ -1700,17 +1723,23 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 	} else if (is_list(*tp)) {
 #if defined(PURIFY)
 #define ERTS_ERROR_CHECKER_PRINTF purify_printf
+#define ERTS_ERROR_CHECKER_PRINTF_XML purify_printf
 #elif defined(VALGRIND)
 #define ERTS_ERROR_CHECKER_PRINTF VALGRIND_PRINTF
+#  ifndef HAVE_VALGRIND_PRINTF_XML
+#    define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF
+#  else
+#    define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML
+#  endif
 #endif
-	    int buf_size = 8*1024; /* Try with 8KB first */
+	    Uint buf_size = 8*1024; /* Try with 8KB first */
 	    char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
 	    int r = io_list_to_buf(*tp, (char*) buf, buf_size - 1);
 	    if (r < 0) {
 		erts_free(ERTS_ALC_T_TMP, (void *) buf);
-		buf_size = io_list_len(*tp);
-		if (buf_size < 0)
+		if (erts_iolist_size(*tp, &buf_size)) {
 		    goto badarg;
+		}
 		buf_size++;
 		buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
 		r = io_list_to_buf(*tp, (char*) buf, buf_size - 1);
@@ -1718,8 +1747,8 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 	    }
 	    buf[buf_size - 1 - r] = '\0';
 	    if (check_if_xml()) {
-		ERTS_ERROR_CHECKER_PRINTF("<erlang_info_log>"
-					  "%s</erlang_info_log>\n", buf);
+		ERTS_ERROR_CHECKER_PRINTF_XML("<erlang_info_log>"
+					      "%s</erlang_info_log>\n", buf);
 	    } else {
 		ERTS_ERROR_CHECKER_PRINTF("%s\n", buf);
 	    }
@@ -1871,6 +1900,37 @@ c_compiler_used(Eterm **hpp, Uint *szp)
 
 }
 
+static int is_snif_term(Eterm module_atom) {
+    int i;
+    Atom *a = atom_tab(atom_val(module_atom));
+    char *aname = (char *) a->name;
+
+    /* if a->name has a '.' then the bif (snif) is bogus i.e a package */
+    for (i = 0; i < a->len; i++) {
+	if (aname[i] == '.')
+	    return 0;
+    }
+
+    return 1;
+}
+
+static Eterm build_snif_term(Eterm **hpp, Uint *szp, int ix, Eterm res) {
+    Eterm tup;
+    tup = erts_bld_tuple(hpp, szp, 3, bif_table[ix].module, bif_table[ix].name, make_small(bif_table[ix].arity));
+    res = erts_bld_cons( hpp, szp, tup, res);
+    return res;
+}
+
+static Eterm build_snifs_term(Eterm **hpp, Uint *szp, Eterm res) {
+    int i;
+    for (i = 0; i < BIF_SIZE; i++) {
+	if (is_snif_term(bif_table[i].module)) {
+	    res = build_snif_term(hpp, szp, i, res);
+	}
+    }
+    return res;
+}
+
 BIF_RETTYPE system_info_1(BIF_ALIST_1)
 {
     Eterm res;
@@ -1904,6 +1964,35 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 		    : am_enabled);
 	}
 #endif
+    } else if (BIF_ARG_1 == am_build_type) {
+#if defined(DEBUG)
+	ERTS_DECL_AM(debug);
+	BIF_RET(AM_debug);
+#elif defined(PURIFY)
+	ERTS_DECL_AM(purify);
+	BIF_RET(AM_purify);
+#elif defined(QUANTIFY)
+	ERTS_DECL_AM(quantify);
+	BIF_RET(AM_quantify);
+#elif defined(PURECOV)
+	ERTS_DECL_AM(purecov);
+	BIF_RET(AM_purecov);
+#elif defined(ERTS_GCOV)
+	ERTS_DECL_AM(gcov);
+	BIF_RET(AM_gcov);
+#elif defined(VALGRIND)
+	ERTS_DECL_AM(valgrind);
+	BIF_RET(AM_valgrind);
+#elif defined(GPROF)
+	ERTS_DECL_AM(gprof);
+	BIF_RET(AM_gprof);
+#elif defined(ERTS_ENABLE_LOCK_COUNT)
+	ERTS_DECL_AM(lcnt);
+	BIF_RET(AM_lcnt);
+#else
+	BIF_RET(am_opt);
+#endif
+	BIF_RET(res);
     } else if (BIF_ARG_1 == am_allocated_areas) {
 	res = erts_allocated_areas(NULL, NULL, BIF_P);
 	BIF_RET(res);
@@ -1919,6 +2008,17 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(db_get_trace_control_word_0(BIF_P));
     } else if (ERTS_IS_ATOM_STR("ets_realloc_moves", BIF_ARG_1)) {
  	BIF_RET((erts_ets_realloc_always_moves) ? am_true : am_false);
+    } else if (ERTS_IS_ATOM_STR("ets_always_compress", BIF_ARG_1)) {
+	BIF_RET((erts_ets_always_compress) ? am_true : am_false);
+    } else if (ERTS_IS_ATOM_STR("snifs", BIF_ARG_1)) {
+	Uint size = 0;
+	Uint *szp;
+
+	szp = &size;
+	build_snifs_term(NULL, szp, NIL);
+	hp = HAlloc(BIF_P, size);
+	res = build_snifs_term(&hp, NULL, NIL);
+	BIF_RET(res);
     } else if (BIF_ARG_1 == am_sequential_tracer) {
 	val = erts_get_system_seq_tracer();
 	ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false)
@@ -1926,7 +2026,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	res = TUPLE2(hp, am_sequential_tracer, val);
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_garbage_collection){
-	Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs);
+	Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs);
 	Eterm tup;
 	hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2);
 
@@ -1941,7 +2041,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_fullsweep_after){
-	Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs);
+	Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs);
 	hp = HAlloc(BIF_P, 3);
 	res = TUPLE2(hp, am_fullsweep_after, make_small(val));
 	BIF_RET(res);
@@ -1980,7 +2080,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	ASSERT(dsbufp && dsbufp->str);
-	res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len);
+	res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len);
 	erts_destroy_info_dsbuf(dsbufp);
 	BIF_RET(res);
     } else if (ERTS_IS_ATOM_STR("dist_ctrl", BIF_ARG_1)) {
@@ -2063,86 +2163,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(erts_alloc_util_allocators((void *) BIF_P));
     }
     else if (BIF_ARG_1 == am_elib_malloc) {
-#ifdef ELIB_ALLOC_IS_CLIB
-	struct elib_stat stat;
-	DECL_AM(heap_size);
-	DECL_AM(max_alloced_size);
-	DECL_AM(alloced_size);
-	DECL_AM(free_size);
-	DECL_AM(no_alloced_blocks);
-	DECL_AM(no_free_blocks);
-	DECL_AM(smallest_alloced_block);
-	DECL_AM(largest_free_block);
-	Eterm atoms[8];
-	Eterm ints[8];
-	Uint **hpp;
-	Uint sz;
-	Uint *szp;
-	int length;
-#ifdef DEBUG
-	Uint *endp;
-#endif
-
-	elib_stat(&stat);
-
-	/* First find out the heap size needed ... */
-	hpp = NULL;
-	szp = &sz;
-	sz = 0;
-
-    build_elib_malloc_term:
-	length = 0;
-	atoms[length] = AM_heap_size;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.mem_total*sizeof(Uint));
-	atoms[length] = AM_max_alloced_size;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.mem_max_alloc*sizeof(Uint));
-	atoms[length] = AM_alloced_size;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.mem_alloc*sizeof(Uint));
-	atoms[length] = AM_free_size;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.mem_free*sizeof(Uint));
-	atoms[length] = AM_no_alloced_blocks;
-	ints[length++] = erts_bld_uint(hpp, szp, (Uint) stat.mem_blocks);
-	atoms[length] = AM_no_free_blocks;
-	ints[length++] = erts_bld_uint(hpp, szp, (Uint) stat.free_blocks);
-	atoms[length] = AM_smallest_alloced_block;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.min_used*sizeof(Uint));
-	atoms[length] = AM_largest_free_block;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.max_free*sizeof(Uint));
-
-
-
-	ASSERT(length <= sizeof(atoms)/sizeof(Eterm));
-	ASSERT(length <= sizeof(ints)/sizeof(Eterm));
-
-	res = erts_bld_2tup_list(hpp, szp, length, atoms, ints);
-
-	if (szp) {
-	    /* ... and then build the term */
-	    hp = HAlloc(BIF_P, sz);
-#ifdef DEBUG
-	    endp = hp + sz;
-#endif
-
-	    szp = NULL;
-	    hpp = &hp;
-	    goto build_elib_malloc_term;
-	}
-
-#ifdef DEBUG
-	ASSERT(endp == hp);
-#endif
-
-#else /* #ifdef ELIB_ALLOC_IS_CLIB */
-	res = am_false;
-#endif /* #ifdef ELIB_ALLOC_IS_CLIB */
-
-	BIF_RET(res);
+	/* To be removed in R15 */
+        BIF_RET(am_false);
     }
     else if (BIF_ARG_1 == am_os_version) {
        int major, minor, build;
@@ -2253,6 +2275,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     } else if (ERTS_IS_ATOM_STR("cpu_topology", BIF_ARG_1)) {
 	res = erts_get_cpu_topology_term(BIF_P, am_used);
 	BIF_TRAP1(erts_format_cpu_topology_trap, BIF_P, res);
+    } else if (ERTS_IS_ATOM_STR("update_cpu_info", BIF_ARG_1)) {
+	if (erts_update_cpu_info()) {
+	    ERTS_DECL_AM(changed);
+	    BIF_RET(AM_changed);
+	}
+	else {
+	    ERTS_DECL_AM(unchanged);
+	    BIF_RET(AM_unchanged);
+	}
 #if defined(__GNUC__) && defined(HAVE_SOLARIS_SPARC_PERFMON)
     } else if (ERTS_IS_ATOM_STR("ultrasparc_read_tick1", BIF_ARG_1)) {
 	register unsigned high asm("%l0");
@@ -2324,7 +2355,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     }
     /* Arguments that are unusual follow ... */
     else if (ERTS_IS_ATOM_STR("logical_processors", BIF_ARG_1)) {
-	int no = erts_get_cpu_configured(erts_cpuinfo);
+	int no;
+	erts_get_logical_processors(&no, NULL, NULL);
 	if (no > 0)
 	    BIF_RET(make_small((Uint) no));
 	else {
@@ -2333,7 +2365,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	}
     }
     else if (ERTS_IS_ATOM_STR("logical_processors_online", BIF_ARG_1)) {
-	int no = erts_get_cpu_online(erts_cpuinfo);
+	int no;
+	erts_get_logical_processors(NULL, &no, NULL);
 	if (no > 0)
 	    BIF_RET(make_small((Uint) no));
 	else {
@@ -2342,7 +2375,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	}
     }
     else if (ERTS_IS_ATOM_STR("logical_processors_available", BIF_ARG_1)) {
-	int no = erts_get_cpu_available(erts_cpuinfo);
+	int no;
+	erts_get_logical_processors(NULL, NULL, &no);
 	if (no > 0)
 	    BIF_RET(make_small((Uint) no));
 	else {
@@ -2502,6 +2536,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(erts_sched_stat_term(BIF_P, 1));
     } else if (ERTS_IS_ATOM_STR("taints", BIF_ARG_1)) {
 	BIF_RET(erts_nif_taints(BIF_P));
+    } else if (ERTS_IS_ATOM_STR("reader_groups_map", BIF_ARG_1)) {
+	BIF_RET(erts_get_reader_groups_map(BIF_P));
+    } else if (ERTS_IS_ATOM_STR("dist_buf_busy_limit", BIF_ARG_1)) {
+	Uint hsz = 0;
+
+ 	(void) erts_bld_uint(NULL, &hsz, erts_dist_buf_busy_limit);
+	hp = hsz ? HAlloc(BIF_P, hsz) : NULL;
+	res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit);
+	BIF_RET(res);
     }
 
     BIF_ERROR(BIF_P, BADARG);
@@ -2617,7 +2660,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
 	hp = HAlloc(BIF_P, 3 + mic.sz);
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
-	    item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); 
+	    item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); 
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
@@ -2637,7 +2680,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
 	    Eterm t;
-	    item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); 
+	    item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); 
 	    t = TUPLE2(hp, am_process, item);
 	    hp += 3;
 	    res = CONS(hp, t, res);
@@ -2694,7 +2737,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
 	erts_doforall_links(prt->nlinks, &one_link_size, &size);
 
 	for (bp = prt->bp; bp; bp = bp->next)
-	    size += sizeof(ErlHeapFragment) + (bp->size - 1)*sizeof(Eterm);
+	    size += sizeof(ErlHeapFragment) + (bp->alloc_size - 1)*sizeof(Eterm);
 
 	if (prt->linebuf)
 	    size += sizeof(LineBuf) + prt->linebuf->ovsiz;
@@ -2817,7 +2860,7 @@ fun_info_2(Process* p, Eterm fun, Eterm what)
 	    goto error;
 	}
     } else if (is_export(fun)) {
-	Export* exp = (Export *) (export_val(fun))[1];
+	Export* exp = (Export *) ((UWord) (export_val(fun))[1]);
 	switch (what) {
 	case am_type:
 	    hp = HAlloc(p, 3);
@@ -3010,11 +3053,11 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	res = erts_run_queues_len(NULL);
 	BIF_RET(make_small(res));
     } else if (BIF_ARG_1 == am_wall_clock) {
-	Uint w1, w2;
+	UWord w1, w2;
 	Eterm b1, b2;
 	wall_clock_elapsed_time_both(&w1, &w2);
-	b1 = erts_make_integer(w1,BIF_P);
-	b2 = erts_make_integer(w2,BIF_P);
+	b1 = erts_make_integer((Uint) w1,BIF_P);
+	b2 = erts_make_integer((Uint) w2,BIF_P);
 	hp = HAlloc(BIF_P,3);
 	res = TUPLE2(hp, b1, b2);
 	BIF_RET(res);
@@ -3365,6 +3408,16 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    else if (ERTS_IS_ATOM_STR("fake_scheduler_bindings", tp[1])) {
 		return erts_fake_scheduler_bindings(BIF_P, tp[2]);
 	    }
+	    else if (ERTS_IS_ATOM_STR("reader_groups_map", tp[1])) {
+		Sint groups;
+		if (is_not_small(tp[2]))
+		    BIF_ERROR(BIF_P, BADARG);
+		groups = signed_val(tp[2]);
+		if (groups < (Sint) 1 || groups > (Sint) INT_MAX)
+		    BIF_ERROR(BIF_P, BADARG);
+
+		BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups));
+	    }
 	    break;
 	}
 	default:
@@ -3383,8 +3436,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
      */
     if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1)
 	&& (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)) {
-	long on = (long) (BIF_ARG_2 == am_true);
-	long prev_on = erts_smp_atomic_xchg(&available_internal_state, on);
+	erts_aint_t on = (erts_aint_t) (BIF_ARG_2 == am_true);
+	erts_aint_t prev_on = erts_smp_atomic_xchg(&available_internal_state, on);
 	if (on) {
 	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	    erts_dsprintf(dsbufp, "Process %T ", BIF_P->id);
@@ -3546,6 +3599,17 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
                 }
 	    }
 	}
+	else if (ERTS_IS_ATOM_STR("binary_loop_limit", BIF_ARG_1)) {
+	    /* Used by binary_module_SUITE (stdlib) */
+	    Uint max_loops;
+	    if (is_atom(BIF_ARG_2) && ERTS_IS_ATOM_STR("default", BIF_ARG_2)) {
+		max_loops = erts_binary_set_loop_limit(-1);
+		BIF_RET(make_small(max_loops));
+	    } else if (term_to_Uint(BIF_ARG_2, &max_loops) != 0) {
+		max_loops = erts_binary_set_loop_limit(max_loops);
+		BIF_RET(make_small(max_loops));
+	    }
+	}
 	else if (ERTS_IS_ATOM_STR("re_loop_limit", BIF_ARG_1)) {
 	    /* Used by re_SUITE (stdlib) */
 	    Uint max_loops;
@@ -3570,7 +3634,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	}
 	else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_suspend", BIF_ARG_1)) {
 	    /* Used by hipe test suites */
-	    long flag = erts_smp_atomic_read(&hipe_test_reschedule_flag);
+	    erts_aint_t flag = erts_smp_atomic_read(&hipe_test_reschedule_flag);
 	    if (!flag && BIF_ARG_2 != am_false) {
 		erts_smp_atomic_set(&hipe_test_reschedule_flag, 1);
 		erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
@@ -3645,7 +3709,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_stats_t *stats, Eterm res) {
-    unsigned long tries = 0, colls = 0;
+    Uint tries = 0, colls = 0;
     unsigned long timer_s = 0, timer_ns = 0, timer_n = 0;
     unsigned int  line = 0;
     
@@ -3658,8 +3722,8 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s
      * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}]
      */
     
-    ethr_atomic_read(&stats->tries, (long *)&tries);
-    ethr_atomic_read(&stats->colls, (long *)&colls);
+    tries = (Uint) ethr_atomic_read(&stats->tries);
+    colls = (Uint) ethr_atomic_read(&stats->colls);
    
     line     = stats->line; 
     timer_s  = stats->timer.s;
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index a9e8dd86f7..47c48e74d6 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -89,13 +89,14 @@ BIF_RETTYPE append_2(BIF_ALIST_2)
     BIF_RET(copy);
 }
 
+#define SMALL_VEC_SIZE 10
 BIF_RETTYPE subtract_2(BIF_ALIST_2)
 {
     Eterm  list;
     Eterm* hp;
     Uint  need;
     Eterm  res;
-    Eterm  small_vec[10];	/* Preallocated memory for small lists */
+    Eterm small_vec[SMALL_VEC_SIZE];	/* Preallocated memory for small lists */
     Eterm* vec_p;
     Eterm* vp;
     int     i;
@@ -115,7 +116,7 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2)
 	BIF_RET(BIF_ARG_1);
     
     /* allocate element vector */
-    if (n <= sizeof(small_vec)/sizeof(small_vec[0]))
+    if (n <= SMALL_VEC_SIZE)
 	vec_p = small_vec;
     else
 	vec_p = (Eterm*) erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm));
@@ -377,7 +378,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List)
 		Eterm *tuple_ptr = tuple_val(term);
 		if (pos <= arityval(*tuple_ptr)) {
 		    Eterm element = tuple_ptr[pos];
-		    if (cmp(Key, element) == 0) {
+		    if (CMP(Key, element) == 0) {
 			return term;
 		    }
 		}
diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index 6da72dcef9..deda7adc1f 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -251,7 +251,7 @@ BIF_RETTYPE is_function_2(BIF_ALIST_2)
 	    BIF_RET(am_true);
 	}
     } else if (is_export(BIF_ARG_1)) {
-	Export* exp = (Export *) (export_val(BIF_ARG_1))[1];
+	Export* exp = (Export *) EXPAND_POINTER((export_val(BIF_ARG_1))[1]);
 
 	if (exp->code[2] == (Uint) arity) {
 	    BIF_RET(am_true);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index f454f2e12d..3fd35dd963 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -21,10 +21,6 @@
 #  include "config.h"
 #endif
 
-#ifdef _OSE_
-#  include "ose.h"
-#endif
-
 #include <ctype.h>
 
 #define ERTS_WANT_EXTERNAL_TAGS
@@ -583,8 +579,8 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
     if (prt->bp == NULL) {	/* MUST be CONST! */
 	res = prt->data;
     } else {
-	Eterm* hp = HAlloc(BIF_P, prt->bp->size);
-	res = copy_struct(prt->data, prt->bp->size, &hp, &MSO(BIF_P));
+	Eterm* hp = HAlloc(BIF_P, prt->bp->used_size);
+	res = copy_struct(prt->data, prt->bp->used_size, &hp, &MSO(BIF_P));
     }
     erts_smp_port_unlock(prt);
     BIF_RET(res);
@@ -614,6 +610,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
     int binary_io;
     int soft_eof;
     Sint linebuf;
+    Eterm edir = NIL;
     byte dir[MAXPATHLEN];
 
     /* These are the defaults */
@@ -690,19 +687,10 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
 
 		} else if (option == am_arg0) {
 		    char *a0;
-		    int n;
-		    if (is_nil(*tp)) {
-			n = 0;
-		    } else if( (n = is_string(*tp)) == 0) {
+
+		    if ((a0 = erts_convert_filename_to_native(*tp, ERTS_ALC_T_TMP, 1)) == NULL) {
 			goto badarg;
 		    }
-		    a0 = (char *) erts_alloc(ERTS_ALC_T_TMP, 
-					    (n + 1) * sizeof(byte));
-		    if (intlist_to_buf(*tp, a0, n) != n) {
-			erl_exit(1, "%s:%d: Internal error\n",
-				 __FILE__, __LINE__);
-		    }
-		    a0[n] = '\0';		    
 		    if (opts.argv == NULL) {
 			opts.argv = erts_alloc(ERTS_ALC_T_TMP, 
 					       2 * sizeof(char **));
@@ -715,20 +703,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
 			opts.argv[0] = a0;
 		    }
 		} else if (option == am_cd) {
-		    Eterm iolist;
-		    Eterm heap[4];
-		    int r;
-
-		    heap[0] = *tp;
-		    heap[1] = make_list(heap+2);
-		    heap[2] = make_small(0);
-		    heap[3] = NIL;
-		    iolist = make_list(heap);
-		    r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN);
-		    if (r < 0) {
-			goto badarg;
-		    }
-		    opts.wd = (char *) dir;
+		    edir = *tp;
 		} else {
 		    goto badarg;
 		}
@@ -840,19 +815,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
 		goto badarg;
 	    }
 	    name = tp[1];
-	    if (is_atom(name)) {
-		name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP,
-					       atom_tab(atom_val(name))->len+1);
-		sys_memcpy((void *) name_buf,
-			   (void *) atom_tab(atom_val(name))->name, 
-			   atom_tab(atom_val(name))->len);
-		name_buf[atom_tab(atom_val(name))->len] = '\0';
-	    } else if ((i = is_string(name))) {
-		name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1);
-		if (intlist_to_buf(name, name_buf, i) != i)
-		    erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
-		name_buf[i] = '\0';
-	    } else {
+	    if ((name_buf = erts_convert_filename_to_native(name,ERTS_ALC_T_TMP,0)) == NULL) {
 		goto badarg;
 	    }
 	    opts.spawn_type = ERTS_SPAWN_EXECUTABLE;
@@ -894,7 +857,33 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
 	/* Argument vector only if explicit spawn_executable */
 	goto badarg;
     }
-	
+
+    if (edir != NIL) {
+	/* A working directory is expressed differently if spawn_executable, i.e. Unicode is handles 
+	   for spawn_executable... */
+	if (opts.spawn_type != ERTS_SPAWN_EXECUTABLE) {
+	    Eterm iolist;
+	    DeclareTmpHeap(heap,4,p);
+	    int r;
+	    
+	    UseTmpHeap(4,p);
+	    heap[0] = edir;
+	    heap[1] = make_list(heap+2);
+	    heap[2] = make_small(0);
+	    heap[3] = NIL;
+	    iolist = make_list(heap);
+	    r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN);
+	    UnUseTmpHeap(4,p);
+	    if (r < 0) {
+		goto badarg;
+	    }
+	    opts.wd = (char *) dir;
+	} else {
+	    if ((opts.wd = erts_convert_filename_to_native(edir,ERTS_ALC_T_TMP,0)) == NULL) {
+		goto badarg;
+	    }
+	}
+    }
 
     if (driver != &spawn_driver && opts.exit_status) {
 	goto badarg;
@@ -943,6 +932,9 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
     if (opts.argv) {
 	free_args(opts.argv);
     }
+    if (opts.wd && opts.wd != ((char *)dir)) {
+	erts_free(ERTS_ALC_T_TMP, (void *) opts.wd);
+    }
     return port_num;
     
  badarg:
@@ -952,6 +944,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
 #undef OPEN_PORT_ERROR
 }
 
+/* Arguments can be given i unicode and as raw binaries, convert filename is used to convert */
 static char **convert_args(Eterm l)
 {
     char **pp;
@@ -968,22 +961,14 @@ static char **convert_args(Eterm l)
     pp[i++] = erts_default_arg0;
     while (is_list(l)) {
 	str = CAR(list_val(l));
-
-	if (is_nil(str)) {
-	    n = 0;
-	} else if( (n = is_string(str)) == 0) {
-	    /* Not a string... */
+	if ((b = erts_convert_filename_to_native(str,ERTS_ALC_T_TMP,1)) == NULL) {
 	    int j;
 	    for (j = 1; j < i; ++j)
 		erts_free(ERTS_ALC_T_TMP, pp[j]);
 	    erts_free(ERTS_ALC_T_TMP, pp);
 	    return NULL;
-	}
-	b = (char *) erts_alloc(ERTS_ALC_T_TMP, (n + 1) * sizeof(byte));
-	pp[i++] = (char *) b;
-	if (intlist_to_buf(str, b, n) != n)
-	    erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
-	b[n] = '\0';
+	}	    
+	pp[i++] = b;
 	l = CDR(list_val(l));
     }
     pp[i] = NULL;
@@ -1011,6 +996,7 @@ static byte* convert_environment(Process* p, Eterm env)
     Eterm* hp;
     Uint heap_size;
     int n;
+    Uint size;
     byte* bytes;
 
     if ((n = list_length(env)) < 0) {
@@ -1054,15 +1040,15 @@ static byte* convert_environment(Process* p, Eterm env)
     if (is_not_nil(env)) {
 	goto done;
     }
-    if ((n = io_list_len(all)) < 0) {
+    if (erts_iolist_size(all, &size)) {
 	goto done;
     }
 
     /*
      * Put the result in a binary (no risk for a memory leak that way).
      */
-    (void) erts_new_heap_binary(p, NULL, n, &bytes);
-    io_list_to_buf(all, (char*)bytes, n);
+    (void) erts_new_heap_binary(p, NULL, size, &bytes);
+    io_list_to_buf(all, (char*)bytes, size);
 
  done:
     erts_free(ERTS_ALC_T_TMP, temp_heap);
@@ -1077,27 +1063,33 @@ struct packet_callback_args
     Eterm res;   /* Out */
     int string_as_bin; /* return strings as binaries (http_bin): */
     byte* aligned_ptr;
+    Uint bin_sz;
     Eterm orig;
     Uint bin_offs;
     byte bin_bitoffs;
 };
 
+#define in_area(ptr,start,nbytes) \
+    ((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes))
+
 static Eterm
 http_bld_string(struct packet_callback_args* pca, Uint **hpp, Uint *szp,
 		const char *str, Sint len)
 {
     Eterm res = THE_NON_VALUE;
     Uint size;
+    int make_subbin;
 
     if (pca->string_as_bin) {
 	size = heap_bin_size(len);
-    
+	make_subbin = (size > ERL_SUB_BIN_SIZE
+		       && in_area(str, pca->aligned_ptr, pca->bin_sz));
 	if (szp) {
-	    *szp += (size > ERL_SUB_BIN_SIZE) ? ERL_SUB_BIN_SIZE : size;	
+	    *szp += make_subbin ? ERL_SUB_BIN_SIZE : size;
 	}
 	if (hpp) {
 	    res = make_binary(*hpp);
-	    if (size > ERL_SUB_BIN_SIZE) {
+	    if (make_subbin) {
 		ErlSubBin* bin = (ErlSubBin*) *hpp;
 		bin->thing_word = HEADER_SUB_BIN;
 		bin->size = len;
@@ -1328,7 +1320,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
     int packet_sz;           /*-------Binaries involved: ------------------*/
     byte* bin_ptr;           /*| orig: original binary                     */
     byte bin_bitsz;          /*| bin: BIF_ARG_2, may be sub-binary of orig */
-    Uint bin_sz;             /*| packet: prefix of bin                     */
+	                     /*| packet: prefix of bin                     */
     char* body_ptr;          /*| body: part of packet to return            */
     int body_sz;             /*| rest: bin without packet                  */
     struct packet_callback_args pca;
@@ -1389,18 +1381,18 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
     }
 
 
-    bin_sz = binary_size(BIF_ARG_2);
+    pca.bin_sz = binary_size(BIF_ARG_2);
     ERTS_GET_BINARY_BYTES(BIF_ARG_2, bin_ptr, pca.bin_bitoffs, bin_bitsz);  
     if (pca.bin_bitoffs != 0) {
-        pca.aligned_ptr = erts_alloc(ERTS_ALC_T_TMP, bin_sz);
-        erts_copy_bits(bin_ptr, pca.bin_bitoffs, 1, pca.aligned_ptr, 0, 1, bin_sz*8);
+        pca.aligned_ptr = erts_alloc(ERTS_ALC_T_TMP, pca.bin_sz);
+        erts_copy_bits(bin_ptr, pca.bin_bitoffs, 1, pca.aligned_ptr, 0, 1, pca.bin_sz*8);
     }
     else {
         pca.aligned_ptr = bin_ptr;
     }
-    packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, bin_sz,
+    packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, pca.bin_sz,
                                   max_plen, trunc_len, &http_state);
-    if (!(packet_sz > 0 && packet_sz <= bin_sz)) {
+    if (!(packet_sz > 0 && packet_sz <= pca.bin_sz)) {
         if (packet_sz < 0) {
 	    goto error;
         }
@@ -1456,7 +1448,7 @@ error:
 
     rest = (ErlSubBin *) hp;
     rest->thing_word = HEADER_SUB_BIN;
-    rest->size = bin_sz - packet_sz;
+    rest->size = pca.bin_sz - packet_sz;
     rest->offs = pca.bin_offs + packet_sz;
     rest->orig = pca.orig;
     rest->bitoffs = pca.bin_bitoffs;
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index c027cd5984..26891c4348 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -76,8 +76,8 @@ void erts_init_bif_re(void)
     re_exec_trap_export.code[0] = am_erlang;
     re_exec_trap_export.code[1] = am_re_run_trap;
     re_exec_trap_export.code[2] = 3;
-    re_exec_trap_export.code[3] = (Eterm) em_apply_bif;
-    re_exec_trap_export.code[4] = (Eterm) &re_exec_trap;
+    re_exec_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    re_exec_trap_export.code[4] = (BeamInstr) &re_exec_trap;
 
     grun_trap_exportp =  erts_export_put(am_re,am_grun,3);
     urun_trap_exportp =  erts_export_put(am_re,am_urun,3);
@@ -103,7 +103,7 @@ Sint erts_re_set_loop_limit(Sint limit)
 
 static int term_to_int(Eterm term, int *sp)
 {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 
     if (is_small(term)) {
 	Uint x = signed_val(term);
@@ -154,7 +154,7 @@ static int term_to_int(Eterm term, int *sp)
 
 static Eterm make_signed_integer(int x, Process *p)
 {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     return make_small(x);
 #else
     Eterm* hp;
@@ -417,7 +417,7 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con
 BIF_RETTYPE
 re_compile_2(BIF_ALIST_2)
 {
-    int slen;
+    Uint slen;
     char *expr;
     pcre *result;
     int errcode = 0;
@@ -444,7 +444,7 @@ re_compile_2(BIF_ALIST_2)
 	BIF_TRAP2(ucompile_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2);
     }
 
-    if ((slen = io_list_len(BIF_ARG_1)) < 0) {
+    if (erts_iolist_size(BIF_ARG_1, &slen)) {
         BIF_ERROR(BIF_P,BADARG);
     }
     expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1);
@@ -795,8 +795,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 			memcpy(tmpb,ap->name,ap->len);
 			tmpb[ap->len] = '\0';
 		    } else {
-			int slen = io_list_len(val);
-			if (slen < 0) {
+			Uint slen;
+			if (erts_iolist_size(val, &slen)) {
 			    goto error;
 			}
 			if ((slen + 1) > tmpbsiz) {
@@ -851,7 +851,7 @@ re_run_3(BIF_ALIST_3)
     const pcre *code_tmp;
     RestartContext restart;
     byte *temp_alloc = NULL;
-    int slength;
+    Uint slength;
     int startoffset = 0;
     int options = 0, comp_options = 0;
     int ovsize;
@@ -875,7 +875,7 @@ re_run_3(BIF_ALIST_3)
     if (is_not_tuple(BIF_ARG_2) || (arityval(*tuple_val(BIF_ARG_2)) != 4)) {
 	if (is_binary(BIF_ARG_2) || is_list(BIF_ARG_2) || is_nil(BIF_ARG_2)) { 
 	    /* Compile from textual RE */
-	    int slen;
+	    Uint slen;
 	    char *expr;
 	    pcre *result;
 	    int errcode = 0;
@@ -889,7 +889,7 @@ re_run_3(BIF_ALIST_3)
 		BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
 	    }
 	    
-	    if ((slen = io_list_len(BIF_ARG_2)) < 0) {
+	    if (erts_iolist_size(BIF_ARG_2, &slen)) {
 		BIF_ERROR(BIF_P,BADARG);
 	    }
 	    
@@ -1027,7 +1027,7 @@ re_run_3(BIF_ALIST_3)
 	restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY;
     } else {
 handle_iolist:
-	if ((slength = io_list_len(BIF_ARG_1)) < 0) {
+	if (erts_iolist_size(BIF_ARG_1, &slength)) {
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector);
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code);
 	    if (restart.ret_info != NULL) {
diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index 172bb37952..db771bd216 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -155,7 +155,7 @@ create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len)
 	erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
     }
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     hp[0] = make_ref_thing_header(len/2 + 1);
     datap = (Uint32 *) &hp[1];
     *(datap++) = len;
@@ -173,13 +173,13 @@ create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len)
 static int
 eq_non_standard_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2)
 {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define MAX_REF_HEAP_SZ (1+(ERTS_MAX_REF_NUMBERS/2+1))
 #else
 #define MAX_REF_HEAP_SZ (1+ERTS_MAX_REF_NUMBERS)
 #endif
-    Uint r1_hp[MAX_REF_HEAP_SZ];
-    Uint r2_hp[MAX_REF_HEAP_SZ];
+    DeclareTmpHeapNoproc(r1_hp,(MAX_REF_HEAP_SZ * 2));
+    Eterm *r2_hp = r1_hp +MAX_REF_HEAP_SZ;
 
     return eq(create_ref(r1_hp, rn1, len1), create_ref(r2_hp, rn2, len2));
 #undef MAX_REF_HEAP_SZ
@@ -357,7 +357,7 @@ bif_timer_timeout(ErtsBifTimer* btm)
 						 rp,
 						 &rp_locks);
 		} else {
-		    Eterm old_size = bp->size;
+		    Eterm old_size = bp->used_size;
 		    bp = erts_resize_message_buffer(bp, old_size + wrap_size,
 						    &message, 1);
 		    hp = &bp->mem[0] + old_size;
@@ -398,7 +398,7 @@ setup_bif_timer(Uint32 xflags,
     
     if (!term_to_Uint(time, &timeout))
 	return THE_NON_VALUE;
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     if ((timeout >> 32) != 0)
 	return THE_NON_VALUE;
 #endif
@@ -478,7 +478,7 @@ setup_bif_timer(Uint32 xflags,
     tab_insert(btm);
     ASSERT(btm == tab_find(ref));
     btm->tm.active = 0; /* MUST be initalized */
-    erl_set_timer(&btm->tm,
+    erts_set_timer(&btm->tm,
 		  (ErlTimeoutProc) bif_timer_timeout,
 		  (ErlCancelProc) bif_timer_cleanup,
 		  (void *) btm,
@@ -550,7 +550,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1)
 	res = am_false;
     }
     else {
-	Uint left = time_left(&btm->tm);
+	Uint left = erts_time_left(&btm->tm);
 	if (!(btm->flags & BTM_FLG_BYNAME)) {
 	    erts_smp_proc_lock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ);
 	    unlink_proc(btm);
@@ -558,7 +558,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1)
 	}
 	tab_remove(btm);
 	ASSERT(!tab_find(BIF_ARG_1));
-	erl_cancel_timer(&btm->tm);
+	erts_cancel_timer(&btm->tm);
 	erts_smp_btm_rwunlock();
 	res = erts_make_integer(left, BIF_P);
     }
@@ -587,7 +587,7 @@ BIF_RETTYPE read_timer_1(BIF_ALIST_1)
 	res = am_false;
     }
     else {
-	Uint left = time_left(&btm->tm);
+	Uint left = erts_time_left(&btm->tm);
 	res = erts_make_integer(left, BIF_P);
     }
 
@@ -613,7 +613,8 @@ erts_print_bif_timer_info(int to, void *to_arg)
 			      : btm->receiver.proc.ess->id);
 	    erts_print(to, to_arg, "=timer:%T\n", receiver);
 	    erts_print(to, to_arg, "Message: %T\n", btm->message);
-	    erts_print(to, to_arg, "Time left: %d ms\n", time_left(&btm->tm));
+	    erts_print(to, to_arg, "Time left: %u ms\n",
+		       erts_time_left(&btm->tm));
 	}
     }
 
@@ -640,7 +641,7 @@ erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks)
 	tab_remove(btm);
 	tmp_btm = btm;
 	btm = btm->receiver.proc.next;
-	erl_cancel_timer(&tmp_btm->tm);
+	erts_cancel_timer(&tmp_btm->tm);
     }
 
     p->bif_timers = NULL;
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 7dff5e0eeb..0509e51a6f 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -40,8 +40,7 @@
 
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
-static erts_smp_mtx_t              trace_pattern_mutex;
-const struct trace_pattern_flags   erts_trace_pattern_flags_off = {0, 0, 0, 0};
+const struct trace_pattern_flags   erts_trace_pattern_flags_off = {0, 0, 0, 0, 0};
 static int                         erts_default_trace_pattern_is_on;
 static Binary                     *erts_default_match_spec;
 static Binary                     *erts_default_meta_match_spec;
@@ -65,7 +64,6 @@ static void clear_trace_bif(int bif_index);
 void
 erts_bif_trace_init(void)
 {
-    erts_smp_mtx_init(&trace_pattern_mutex, "trace_pattern");
     erts_default_trace_pattern_is_on = 0;
     erts_default_match_spec = NULL;
     erts_default_meta_match_spec = NULL;
@@ -86,7 +84,7 @@ trace_pattern_2(Process* p, Eterm MFA, Eterm Pattern)
 Eterm
 trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)       
 {
-    Eterm mfa[3];
+    DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */
     int i;
     int matches = 0;
     int specified = 0;
@@ -101,6 +99,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
     erts_smp_block_system(0);
 
+    UseTmpHeap(3,p);
     /*
      * Check and compile the match specification.
      */
@@ -185,6 +184,14 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 		flags.breakpoint = 1;
 		flags.call_count = 1;
 		break;
+	    case am_call_time:
+		if (is_global) {
+		    goto error;
+		}
+		flags.breakpoint = 1;
+		flags.call_time = 1;
+		break;
+
 	    default:
 		goto error;
 	    }
@@ -194,8 +201,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	goto error;
     }
     
-    if (match_prog_set && !flags.local && !flags.meta && flags.call_count) {
-	/* A match prog is not allowed with just call_count */
+    if (match_prog_set && !flags.local && !flags.meta && (flags.call_count || flags.call_time)) {
+	/* A match prog is not allowed with just call_count or call_time*/
 	goto error;
     }
 
@@ -234,6 +241,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 		    |= flags.meta;
 		erts_default_trace_pattern_flags.call_count
 		    |= (on == 1) ? flags.call_count : 0;
+		erts_default_trace_pattern_flags.call_time
+		    |= (on == 1) ? flags.call_time : 0;
 	    } else {
 		erts_default_trace_pattern_flags.local
 		    &= ~flags.local;
@@ -241,10 +250,13 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 		    &= ~flags.meta;
 		erts_default_trace_pattern_flags.call_count
 		    &= ~flags.call_count;
+		erts_default_trace_pattern_flags.call_time
+		    &= ~flags.call_time;
 		if (! (erts_default_trace_pattern_flags.breakpoint =
 		       erts_default_trace_pattern_flags.local |
 		       erts_default_trace_pattern_flags.meta |
-		       erts_default_trace_pattern_flags.call_count)) {
+		       erts_default_trace_pattern_flags.call_count |
+		       erts_default_trace_pattern_flags.call_time)) {
 		    erts_default_trace_pattern_is_on = !!on; /* i.e off */
 		}
 	    }
@@ -266,8 +278,9 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	    if (on) {
 		if (on != 1) {
 		    flags.call_count = 0;
+		    flags.call_time  = 0;
 		}
-		flags.breakpoint = flags.local | flags.meta | flags.call_count;
+		flags.breakpoint = flags.local | flags.meta | flags.call_count | flags.call_time;
 		erts_default_trace_pattern_flags = flags; /* Struct copy */
 		erts_default_trace_pattern_is_on = !!flags.breakpoint;
 	    }
@@ -312,7 +325,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
     MatchSetUnref(match_prog_set);
 
  done:
-
+    UnUseTmpHeap(3,p);
     erts_smp_release_system();
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 
@@ -322,6 +335,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 
     MatchSetUnref(match_prog_set);
 
+    UnUseTmpHeap(3,p);
     erts_smp_release_system();
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     BIF_ERROR(p, BADARG);
@@ -334,7 +348,6 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
 			       struct trace_pattern_flags *trace_pattern_flags,
 			       Eterm *meta_tracer_pid)
 {
-    erts_smp_mtx_lock(&trace_pattern_mutex);
     if (trace_pattern_is_on)
 	*trace_pattern_is_on = erts_default_trace_pattern_is_on;
     if (match_spec)
@@ -345,12 +358,10 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
 	*trace_pattern_flags = erts_default_trace_pattern_flags;
     if (meta_tracer_pid)
 	*meta_tracer_pid = erts_default_meta_tracer_pid;
-    erts_smp_mtx_unlock(&trace_pattern_mutex);
 }
 
 
 
-
 Uint 
 erts_trace_flag2bit(Eterm flag) 
 {
@@ -378,7 +389,7 @@ erts_trace_flag2bit(Eterm flag)
     default: return 0;
     }
 }
-
+
 /* Scan the argument list and sort out the trace flags.
 **
 ** Returns !0 on success, 0 on failure.
@@ -929,6 +940,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
 #define FUNC_TRACE_LOCAL_TRACE  (1<<2)
 #define FUNC_TRACE_META_TRACE   (1<<3)
 #define FUNC_TRACE_COUNT_TRACE  (1<<4)
+#define FUNC_TRACE_TIME_TRACE   (1<<5)
 /*
  * Returns either FUNC_TRACE_NOEXIST, FUNC_TRACE_UNTRACED,
  * FUNC_TRACE_GLOBAL_TRACE, or,
@@ -943,16 +955,18 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
  *
  * If the return value contains FUNC_TRACE_COUNT_TRACE, *count is set.
  */
-static int function_is_traced(Eterm mfa[3], 
-			      Binary **ms, /* out */
-			      Binary **ms_meta,  /* out */
+static int function_is_traced(Process *p,
+			      Eterm mfa[3],
+			      Binary **ms,              /* out */
+			      Binary **ms_meta,         /* out */
 			      Eterm   *tracer_pid_meta, /* out */
-			      Sint    *count)    /* out */
+			      Sint    *count,           /* out */
+			      Eterm   *call_time)       /* out */
 {
     Export e;
     Export* ep;
     int i;
-    Uint *code;
+    BeamInstr *code;
 
     /* First look for an export entry */
     e.code[0] = mfa[0];
@@ -960,12 +974,12 @@ static int function_is_traced(Eterm mfa[3],
     e.code[2] = mfa[2];
     if ((ep = export_get(&e)) != NULL) {
 	if (ep->address == ep->code+3 &&
-	    ep->code[3] != (Uint) em_call_error_handler) {
-	    if (ep->code[3] == (Uint) em_call_traced_function) {
+	    ep->code[3] != (BeamInstr) em_call_error_handler) {
+	    if (ep->code[3] == (BeamInstr) em_call_traced_function) {
 		*ms = ep->match_prog_set;
 		return FUNC_TRACE_GLOBAL_TRACE;
 	    }
-	    if (ep->code[3] == (Uint) em_apply_bif) {
+	    if (ep->code[3] == (BeamInstr) em_apply_bif) {
 		for (i = 0; i < BIF_SIZE; ++i) {
 		    if (bif_export[i] == ep) {
 			int r = 0;
@@ -978,10 +992,13 @@ static int function_is_traced(Eterm mfa[3],
 				r |= FUNC_TRACE_LOCAL_TRACE;
 				*ms = ep->match_prog_set;
 			    }
-			    if (erts_is_mtrace_bif(ep->code+3, ms_meta, 
+			    if (erts_is_mtrace_break(ep->code+3, ms_meta,
 						   tracer_pid_meta)) {
 				r |= FUNC_TRACE_META_TRACE;
 			    }
+			    if (erts_is_time_break(p, ep->code+3, call_time)) {
+				r |= FUNC_TRACE_TIME_TRACE;
+			    }
 			}
 			return r ? r : FUNC_TRACE_UNTRACED;
 		    }
@@ -999,7 +1016,9 @@ static int function_is_traced(Eterm mfa[3],
 	    | (erts_is_mtrace_break(code, ms_meta, tracer_pid_meta)
 	       ? FUNC_TRACE_META_TRACE : 0)
 	    | (erts_is_count_break(code, count)
-	       ? FUNC_TRACE_COUNT_TRACE : 0);
+	       ? FUNC_TRACE_COUNT_TRACE : 0)
+	    | (erts_is_time_break(p, code, call_time)
+	       ? FUNC_TRACE_TIME_TRACE : 0);
 	
 	return r ? r : FUNC_TRACE_UNTRACED;
     } 
@@ -1011,15 +1030,19 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 {
     Eterm* tp;
     Eterm* hp;
-    Eterm mfa[3];
+    DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */
     Binary *ms = NULL, *ms_meta = NULL;
     Sint count = 0;
     Eterm traced = am_false;
     Eterm match_spec = am_false;
     Eterm retval = am_false;
     Eterm meta = am_false;
+    Eterm call_time = NIL;
     int r;
 
+
+    UseTmpHeap(3,p);
+
     if (!is_tuple(func_spec)) {
 	goto error;
     }
@@ -1034,12 +1057,29 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
     mfa[1] = tp[2];
     mfa[2] = signed_val(tp[3]);
 
-    r = function_is_traced(mfa, &ms, &ms_meta, &meta, &count);
+#ifdef ERTS_SMP
+    if ( (key == am_call_time) || (key == am_all)) {
+	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+	erts_smp_block_system(0);
+    }
+#endif
+
+    r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time);
+
+#ifdef ERTS_SMP
+    if ( (key == am_call_time) || (key == am_all)) {
+	erts_smp_release_system();
+	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+    }
+#endif
+
     switch (r) {
     case FUNC_TRACE_NOEXIST:
+	UnUseTmpHeap(3,p);
 	hp = HAlloc(p, 3);
 	return TUPLE2(hp, key, am_undefined);
     case FUNC_TRACE_UNTRACED:
+	UnUseTmpHeap(3,p);
 	hp = HAlloc(p, 3);
 	return TUPLE2(hp, key, am_false);
     case FUNC_TRACE_GLOBAL_TRACE:
@@ -1085,8 +1125,13 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 		erts_make_integer(count, p);
 	}
 	break;
+    case am_call_time:
+	if (r & FUNC_TRACE_TIME_TRACE) {
+	    retval = call_time;
+	}
+	break;
     case am_all: {
-	Eterm match_spec_meta = am_false, c = am_false, t;
+	Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false;
 	
 	if (ms) {
 	    match_spec = MatchSetGetSource(ms);
@@ -1104,10 +1149,15 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 		erts_make_integer(-count-1, p) : 
 		erts_make_integer(count, p);
 	}
-	hp = HAlloc(p, (3+2)*5);
+	if (r & FUNC_TRACE_TIME_TRACE) {
+	    ct = call_time;
+	}
+	hp = HAlloc(p, (3+2)*6);
 	retval = NIL;
 	t = TUPLE2(hp, am_call_count, c); hp += 3;
 	retval = CONS(hp, t, retval); hp += 2;
+	t = TUPLE2(hp, am_call_time, ct); hp += 3;
+	retval = CONS(hp, t, retval); hp += 2;
 	t = TUPLE2(hp, am_meta_match_spec, match_spec_meta); hp += 3;
 	retval = CONS(hp, t, retval); hp += 2;
 	t = TUPLE2(hp, am_meta, meta); hp += 3;
@@ -1120,10 +1170,12 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
     default:
 	goto error;
     }
+    UnUseTmpHeap(3,p);
     hp = HAlloc(p, 3);
     return TUPLE2(hp, key, retval);
 
  error:
+    UnUseTmpHeap(3,p);
     BIF_ERROR(p, BADARG);
 }
 
@@ -1201,6 +1253,13 @@ trace_info_on_load(Process* p, Eterm key)
 	} else {
 	    return TUPLE2(hp, key, am_false);
 	}
+    case am_call_time:
+	hp = HAlloc(p, 3);
+	if (erts_default_trace_pattern_flags.call_time) {
+	    return TUPLE2(hp, key, am_true);
+	} else {
+	    return TUPLE2(hp, key, am_false);
+	}
     case am_all:
 	{
 	    Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t;
@@ -1275,6 +1334,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 	for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) {
 	    /* Empty loop body */
 	}
+
 	if (j == specified) {
 	    if (on) {
 		if (! flags.breakpoint)
@@ -1312,7 +1372,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 		    if (erts_bif_trace_flags[i] & BIF_TRACE_AS_META) {
 			ASSERT(ExportIsBuiltIn(bif_export[i]));
 			erts_clear_mtrace_bif
-			    ((Uint *)bif_export[i]->code + 3);
+			    ((BeamInstr *)bif_export[i]->code + 3);
 			erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_META;
 		    }
 		    set_trace_bif(i, match_prog_set);
@@ -1341,12 +1401,18 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 		    }
 		    if (flags.meta) {
 			erts_set_mtrace_bif
-			    ((Uint *)bif_export[i]->code + 3,
+			    ((BeamInstr *)bif_export[i]->code + 3,
 			     meta_match_prog_set, meta_tracer_pid);
 			erts_bif_trace_flags[i] |= BIF_TRACE_AS_META;
 			erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_GLOBAL;
 			m = 1;
 		    }
+		    if (flags.call_time) {
+			erts_set_time_trace_bif(bif_export[i]->code + 3, on);
+			/* I don't want to remove any other tracers */
+			erts_bif_trace_flags[i] |= BIF_TRACE_AS_CALL_TIME;
+			m = 1;
+		    }
 		    if (erts_bif_trace_flags[i]) {
 			setup_bif_trace(i);
 		    }
@@ -1361,11 +1427,16 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 		    if (flags.meta) {
 			if (erts_bif_trace_flags[i] & BIF_TRACE_AS_META) {
 			    erts_clear_mtrace_bif
-				((Uint *)bif_export[i]->code + 3);
+				((BeamInstr *)bif_export[i]->code + 3);
 			    erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_META;
 			}
 			m = 1;
 		    }
+		    if (flags.call_time) {
+			erts_clear_time_trace_bif(bif_export[i]->code + 3);
+			erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_CALL_TIME;
+			m = 1;
+		    }
 		    if (! erts_bif_trace_flags[i]) {
 			reset_bif_trace(i);
 		    }
@@ -1383,6 +1454,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 	    erts_clear_trace_break(mfa, specified);
 	    erts_clear_mtrace_break(mfa, specified);
 	    erts_clear_count_break(mfa, specified);
+	    erts_clear_time_break(mfa, specified);
 	} else {
 	    int m = 0;
 	    if (flags.local) {
@@ -1396,6 +1468,9 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 	    if (flags.call_count) {
 		m = erts_set_count_break(mfa, specified, on);
 	    }
+	    if (flags.call_time) {
+		m = erts_set_time_break(mfa, specified, on);
+	    }
 	    /* All assignments to 'm' above should give the same value,
 	     * so just use the last */
 	    matches += m;
@@ -1411,6 +1486,9 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 	if (flags.call_count) {
 	    m = erts_clear_count_break(mfa, specified);
 	}
+	if (flags.call_time) {
+	    m = erts_clear_time_break(mfa, specified);
+	}
 	/* All assignments to 'm' above should give the same value,
 	 * so just use the last */
 	matches += m;
@@ -1430,9 +1508,9 @@ static int
 setup_func_trace(Export* ep, void* match_prog)
 {
     if (ep->address == ep->code+3) {
-	if (ep->code[3] == (Uint) em_call_error_handler) {
+	if (ep->code[3] == (BeamInstr) em_call_error_handler) {
 	    return 0;
-	} else if (ep->code[3] == (Uint) em_call_traced_function) {
+	} else if (ep->code[3] == (BeamInstr) em_call_traced_function) {
 	    MatchSetUnref(ep->match_prog_set);
 	    ep->match_prog_set = match_prog;
 	    MatchSetRef(ep->match_prog_set);
@@ -1452,8 +1530,8 @@ setup_func_trace(Export* ep, void* match_prog)
 	return 0;
     }
     
-    ep->code[3] = (Uint) em_call_traced_function;
-    ep->code[4] = (Uint) ep->address;
+    ep->code[3] = (BeamInstr) em_call_traced_function;
+    ep->code[4] = (BeamInstr) ep->address;
     ep->address = ep->code+3;
     ep->match_prog_set = match_prog;
     MatchSetRef(ep->match_prog_set);
@@ -1465,7 +1543,7 @@ static void setup_bif_trace(int bif_index) {
     
     ASSERT(ExportIsBuiltIn(ep));
     ASSERT(ep->code[4]);
-    ep->code[4] = (Uint) bif_table[bif_index].traced;
+    ep->code[4] = (BeamInstr) bif_table[bif_index].traced;
 }
 
 static void set_trace_bif(int bif_index, void* match_prog) {
@@ -1492,9 +1570,9 @@ static int
 reset_func_trace(Export* ep)
 {
     if (ep->address == ep->code+3) {
-	if (ep->code[3] == (Uint) em_call_error_handler) {
+	if (ep->code[3] == (BeamInstr) em_call_error_handler) {
 	    return 0;
-	} else if (ep->code[3] == (Uint) em_call_traced_function) {
+	} else if (ep->code[3] == (BeamInstr) em_call_traced_function) {
 	    ep->address = (Uint *) ep->code[4];
 	    MatchSetUnref(ep->match_prog_set);
 	    ep->match_prog_set = NULL;
@@ -1527,8 +1605,8 @@ static void reset_bif_trace(int bif_index) {
     ASSERT(ExportIsBuiltIn(ep));
     ASSERT(ep->code[4]);
     ASSERT(! ep->match_prog_set);
-    ASSERT(! erts_is_mtrace_bif((Uint *)ep->code+3, NULL, NULL));
-    ep->code[4] = (Uint) bif_table[bif_index].f;
+    ASSERT(! erts_is_mtrace_break((BeamInstr *)ep->code+3, NULL, NULL));
+    ep->code[4] = (BeamInstr) bif_table[bif_index].f;
 }
 
 static void clear_trace_bif(int bif_index) {
@@ -2083,7 +2161,7 @@ trace_delivered_1(BIF_ALIST_1)
 #ifdef ERTS_SMP
     bp = new_message_buffer(REF_THING_SIZE + 4);
     hp = &bp->mem[0];
-    msg_ref = STORE_NC(&hp, &bp->off_heap.externals, ref);
+    msg_ref = STORE_NC(&hp, &bp->off_heap, ref);
 #else
     hp = HAlloc(BIF_P, 4);
     msg_ref = ref;
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 1f948a9684..506c4813fa 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,6 +21,7 @@
 #define __ERL_BINARY_H
 
 #include "erl_threads.h"
+#include "bif.h"
 
 /*
  * Maximum number of bytes to place in a heap binary.
@@ -70,6 +71,7 @@ typedef struct erl_heap_bin {
  */
 
 #define binary_size(Bin) (binary_val(Bin)[1])
+#define binary_size_rel(Bin,BasePtr) (binary_val_rel(Bin,BasePtr)[1])
 
 #define binary_bitsize(Bin)			\
   ((*binary_val(Bin) == HEADER_SUB_BIN) ?	\
@@ -92,9 +94,12 @@ typedef struct erl_heap_bin {
  * Bitsize: output variable (Uint)
  */
 
-#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize)		\
+#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \
+     ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,NULL)
+
+#define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr)    \
 do {									\
-    Eterm* _real_bin = binary_val(Bin);					\
+    Eterm* _real_bin = binary_val_rel(Bin,BasePtr);			\
     Uint _offs = 0;							\
     Bitoffs = Bitsize = 0;						\
     if (*_real_bin == HEADER_SUB_BIN) {					\
@@ -102,7 +107,7 @@ do {									\
 	_offs = _sb->offs;						\
         Bitoffs = _sb->bitoffs;						\
         Bitsize = _sb->bitsize;						\
-	_real_bin = binary_val(_sb->orig);				\
+	_real_bin = binary_val_rel(_sb->orig,BasePtr);			\
     }									\
     if (*_real_bin == HEADER_PROC_BIN) {				\
 	Bytep = ((ProcBin *) _real_bin)->bytes + _offs;			\
@@ -124,9 +129,12 @@ do {									\
  * BitSize: Extra bit size (Uint)
  */
 
-#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize)	\
+#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \
+     ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, NULL)
+
+#define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \
   do {									\
-    ErlSubBin* _sb = (ErlSubBin *) binary_val(Bin);			\
+    ErlSubBin* _sb = (ErlSubBin *) binary_val_rel(Bin,BasePtr);	        \
     if (_sb->thing_word == HEADER_SUB_BIN) {				\
       RealBin = _sb->orig;						\
       ByteOffset = _sb->offs;						\
@@ -150,7 +158,18 @@ do {									\
 
 void erts_init_binary(void);
 
-byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, unsigned extra);
+byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, ErtsAlcType_t, unsigned extra);
+/* Used by unicode module */
+Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs);
+
+/*
+ * Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1
+ */
+
+BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg);
+BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple);
+BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen);
+
 
 #if defined(__i386__) || !defined(__GNUC__)
 /*
@@ -164,10 +183,11 @@ byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, unsigned extra);
 #endif
 
 #define ERTS_CHK_BIN_ALIGNMENT(B) \
-  do { ASSERT(!(B) || (((Uint) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((Uint) 0)) } while(0)
+  do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)) } while(0)
 
 ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr);
 ERTS_GLB_INLINE void erts_free_aligned_binary_bytes(byte* buf);
+ERTS_GLB_INLINE void erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t);
 ERTS_GLB_INLINE Binary *erts_bin_drv_alloc_fnf(Uint size);
 ERTS_GLB_INLINE Binary *erts_bin_drv_alloc(Uint size);
 ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size);
@@ -184,17 +204,23 @@ ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size,
 ERTS_GLB_INLINE byte*
 erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr)
 {
-    return erts_get_aligned_binary_bytes_extra(bin, base_ptr, 0);
+    return erts_get_aligned_binary_bytes_extra(bin, base_ptr, ERTS_ALC_T_TMP, 0);
 }
 
 ERTS_GLB_INLINE void
-erts_free_aligned_binary_bytes(byte* buf)
+erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t allocator)
 {
     if (buf) {
-	erts_free(ERTS_ALC_T_TMP, (void *) buf);
+	erts_free(allocator, (void *) buf);
     }
 }
 
+ERTS_GLB_INLINE void
+erts_free_aligned_binary_bytes(byte* buf)
+{
+    erts_free_aligned_binary_bytes_extra(buf,ERTS_ALC_T_TMP);
+}
+
 /* Explicit extra bytes allocated to counter buggy drivers.
 ** These extra bytes where earlier (< R13B04) added by an alignment-bug
 ** in this code. Do we dare remove this in some major release (R14?) maybe?
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index e4f5d50ddf..e56084b9cb 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -177,7 +177,6 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
     byte* LSB;
     byte* MSB;
     Uint* hp;
-    Uint* hp_end;
     Uint words_needed;
     Uint actual;
     Uint v32;
@@ -255,7 +254,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
 	 * Simply shift whole bytes into the result.
 	 */
 	switch (BYTE_OFFSET(n)) {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	case 7: w = (w << 8) | *bp++;
 	case 6: w = (w << 8) | *bp++;
 	case 5: w = (w << 8) | *bp++;
@@ -360,7 +359,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
     case 3: 
 	v32 = LSB[0] + (LSB[1]<<8) + (LSB[2]<<16); 
 	goto big_small;
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
     case 4:
 	v32 = (LSB[0] + (LSB[1]<<8) + (LSB[2]<<16) + (LSB[3]<<24));
 	if (!IS_USMALL(sgn, v32)) {
@@ -405,7 +404,6 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
     default:
 	words_needed = 1+WSIZE(bytes);
 	hp = HeapOnlyAlloc(p, words_needed);
-	hp_end = hp + words_needed;
 	res = bytes_to_big(LSB, bytes, sgn, hp); 
 	if (is_small(res)) {
 	    p->htop = hp;
@@ -425,7 +423,6 @@ Eterm
 erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb)
 {
     ErlSubBin* sb;
-    size_t num_bytes;		/* Number of bytes in binary. */
 
     if (mb->size - mb->offset < num_bits) {	/* Asked for too many bits.  */
 	return THE_NON_VALUE;
@@ -435,7 +432,6 @@ erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffe
      * From now on, we can't fail.
      */
 
-    num_bytes = NBYTES(num_bits);
     sb = (ErlSubBin *) HeapOnlyAlloc(p, ERL_SUB_BIN_SIZE);
     
     sb->thing_word = HEADER_SUB_BIN;
@@ -555,10 +551,11 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags)
 {
     unsigned long offs;
 
-    ASSERT(size != 0);
     offs = BIT_OFFSET(size);
     if (is_small(val)) {
 	Sint v = signed_val(val);
+
+	ASSERT(size != 0);	  /* Tested by caller */
 	if (flags & BSF_LITTLE) { /* Little endian */
 	    sz--;
 	    COPY_VAL(buf,1,v,sz);
@@ -578,6 +575,9 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags)
 	ErtsDigit* dp = big_v(val);
 	int n = MIN(sz,ds);
 
+	if (size == 0) {
+	    return 0;
+	}
 	if (flags & BSF_LITTLE) {
 	    sz -= n;                       /* pad with this amount */
 	    if (sign) {
@@ -729,15 +729,13 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag
     Uint b;
     byte *iptr;
 
-    if (num_bits == 0) {
-	return 1;
-    }
-
     bit_offset = BIT_OFFSET(bin_offset);
     if (is_small(arg)) {
 	Uint rbits = 8 - bit_offset;
 
-	if (bit_offset + num_bits <= 8) {
+	if (num_bits == 0) {
+	    return 1;
+	} else if (bit_offset + num_bits <= 8) {
 	    /*
 	     * All bits are in the same byte.
 	     */ 
@@ -1335,12 +1333,12 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
 	hp += PROC_BIN_SIZE;
 	pb->thing_word = HEADER_PROC_BIN;
 	pb->size = used_size_in_bytes;
-	pb->next = MSO(c_p).mso;
-	MSO(c_p).mso = pb;
+	pb->next = MSO(c_p).first;
+	MSO(c_p).first = (struct erl_off_heap_header*)pb;
 	pb->val = bptr;
 	pb->bytes = (byte*) bptr->orig_bytes;
 	pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
-	MSO(c_p).overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
 
 	/*
 	 * Now allocate the sub binary and set its size to include the
@@ -1506,12 +1504,12 @@ erts_bs_init_writable(Process* p, Eterm sz)
     hp += PROC_BIN_SIZE;
     pb->thing_word = HEADER_PROC_BIN;
     pb->size = 0;
-    pb->next = MSO(p).mso;
-    MSO(p).mso = pb;
+    pb->next = MSO(p).first;
+    MSO(p).first = (struct erl_off_heap_header*) pb;
     pb->val = bptr;
     pb->bytes = (byte*) bptr->orig_bytes;
     pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     
     /*
      * Now allocate the sub binary.
@@ -1555,7 +1553,6 @@ Uint32
 erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb)
 {
     Uint bytes;
-    Uint bits;
     Uint offs;
     byte bigbuf[4];
     byte* LSB;
@@ -1565,7 +1562,6 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb)
     ASSERT(mb->size - mb->offset >= 32);
 
     bytes = 4;
-    bits = 8;
     offs = 0;
 
     LSB = bigbuf;
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index e3f8e0b679..0f67733fa4 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -63,7 +63,7 @@ typedef struct erl_bin_match_struct{
 #define HEADER_NUM_SLOTS(hdr) (header_arity(hdr)-sizeof(ErlBinMatchState)/sizeof(Eterm)+1)
 
 #define make_matchstate(_Ms) make_boxed((Eterm*)(_Ms))  
-#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*)(_Ms - TAG_PRIMARY_BOXED))->mb)
+#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb)
 
 
 #if defined(ERTS_SMP)
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
new file mode 100644
index 0000000000..bcf8bcf270
--- /dev/null
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -0,0 +1,2361 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description:	CPU topology and related functionality
+ *
+ * Author: 	Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include <ctype.h>
+
+#include "global.h"
+#include "error.h"
+#include "bif.h"
+#include "erl_cpu_topology.h"
+
+#define ERTS_MAX_READER_GROUPS 8
+
+/*
+ * Cpu topology hierarchy.
+ */
+#define ERTS_TOPOLOGY_NODE		0
+#define ERTS_TOPOLOGY_PROCESSOR		1
+#define ERTS_TOPOLOGY_PROCESSOR_NODE	2
+#define ERTS_TOPOLOGY_CORE		3
+#define ERTS_TOPOLOGY_THREAD		4
+#define ERTS_TOPOLOGY_LOGICAL		5
+
+#define ERTS_TOPOLOGY_MAX_DEPTH		6
+
+typedef struct {
+    int bind_id;
+    int bound_id;
+} ErtsCpuBindData;
+
+static erts_cpu_info_t *cpuinfo;
+
+static int max_main_threads;
+static int reader_groups;
+
+static ErtsCpuBindData *scheduler2cpu_map;
+static erts_smp_rwmtx_t cpuinfo_rwmtx;
+
+typedef enum {
+    ERTS_CPU_BIND_UNDEFINED,
+    ERTS_CPU_BIND_SPREAD,
+    ERTS_CPU_BIND_PROCESSOR_SPREAD,
+    ERTS_CPU_BIND_THREAD_SPREAD,
+    ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD,
+    ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD,
+    ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD,
+    ERTS_CPU_BIND_NO_SPREAD,
+    ERTS_CPU_BIND_NONE
+} ErtsCpuBindOrder;
+
+#define ERTS_CPU_BIND_DEFAULT_BIND \
+  ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD
+
+static int no_cpu_groups_callbacks;
+static ErtsCpuBindOrder cpu_bind_order;
+
+static erts_cpu_topology_t *user_cpudata;
+static int user_cpudata_size;
+static erts_cpu_topology_t *system_cpudata;
+static int system_cpudata_size;
+
+typedef struct {
+    int level[ERTS_TOPOLOGY_MAX_DEPTH+1];
+} erts_avail_cput;
+
+typedef struct {
+    int id;
+    int sub_levels;
+    int cpu_groups;
+} erts_cpu_groups_count_t;
+
+typedef struct {
+    int logical;
+    int cpu_group;
+} erts_cpu_groups_map_array_t;
+
+typedef struct erts_cpu_groups_callback_list_t_ erts_cpu_groups_callback_list_t;
+struct erts_cpu_groups_callback_list_t_ {
+    erts_cpu_groups_callback_list_t *next;
+    erts_cpu_groups_callback_t callback;
+    void *arg;
+};
+
+typedef struct erts_cpu_groups_map_t_ erts_cpu_groups_map_t;
+struct erts_cpu_groups_map_t_ {
+    erts_cpu_groups_map_t *next;
+    int groups;
+    erts_cpu_groups_map_array_t *array;
+    int size;
+    int logical_processors;
+    erts_cpu_groups_callback_list_t *callback_list;
+};
+
+typedef struct {
+    erts_cpu_groups_callback_t callback;
+    int ix;
+    void *arg;
+} erts_cpu_groups_callback_call_t;
+
+static erts_cpu_groups_map_t *cpu_groups_maps;
+
+static erts_cpu_groups_map_t *reader_groups_map;
+
+#define ERTS_TOPOLOGY_CG ERTS_TOPOLOGY_MAX_DEPTH
+
+#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff)
+
+#ifdef ERTS_SMP
+static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
+				int size,
+				ErtsCpuBindOrder bind_order,
+				int mk_seq);
+static void write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size);
+#endif
+
+static void reader_groups_callback(int, ErtsSchedulerData *, int, void *);
+static erts_cpu_groups_map_t *add_cpu_groups(int groups,
+					     erts_cpu_groups_callback_t callback,
+					     void *arg);
+static void update_cpu_groups_maps(void);
+static void make_cpu_groups_map(erts_cpu_groups_map_t *map, int test);
+static int cpu_groups_lookup(erts_cpu_groups_map_t *map,
+			     ErtsSchedulerData *esdp);
+
+static void create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata,
+					 int *cpudata_size);
+static void destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata);
+
+static int
+int_cmp(const void *vx, const void *vy)
+{
+    return *((int *) vx) - *((int *) vy);
+}
+
+static int
+cpu_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->processor_node != y->processor_node)
+	return x->processor_node - y->processor_node;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    if (x->node != y->node)
+	return x->node - y->node;
+    return 0;
+}
+
+static int
+cpu_processor_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->processor_node != y->processor_node)
+	return x->processor_node - y->processor_node;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    return 0;
+}
+
+static int
+cpu_thread_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    if (x->processor_node != y->processor_node)
+	return x->processor_node - y->processor_node;
+    if (x->core != y->core)
+	return x->core - y->core;
+    return 0;
+}
+
+static int
+cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    return 0;
+}
+
+static int
+cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    return 0;
+}
+
+static int
+cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    if (x->core != y->core)
+	return x->core - y->core;
+    return 0;
+}
+
+static int
+cpu_no_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    if (x->processor_node != y->processor_node)
+	return x->processor_node - y->processor_node;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    return 0;
+}
+
+static ERTS_INLINE void
+make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node)
+{
+    int ix;
+    int node = -1;
+    int processor = -1;
+    int processor_node = -1;
+    int processor_node_node = -1;
+    int core = -1;
+    int thread = -1;
+    int old_node = -1;
+    int old_processor = -1;
+    int old_processor_node = -1;
+    int old_core = -1;
+    int old_thread = -1;
+
+    for (ix = 0; ix < size; ix++) {
+	if (!no_node || cpudata[ix].node >= 0) {
+	    if (old_node == cpudata[ix].node)
+		cpudata[ix].node = node;
+	    else {
+		old_node = cpudata[ix].node;
+		old_processor = processor = -1;
+		if (!no_node)
+		    old_processor_node = processor_node = -1;
+		old_core = core = -1;
+		old_thread = thread = -1;
+		if (no_node || cpudata[ix].node >= 0)
+		    cpudata[ix].node = ++node;
+	    }
+	}
+	if (old_processor == cpudata[ix].processor)
+	    cpudata[ix].processor = processor;
+	else {
+	    old_processor = cpudata[ix].processor;
+	    if (!no_node)
+		processor_node_node = old_processor_node = processor_node = -1;
+	    old_core = core = -1;
+	    old_thread = thread = -1;
+	    cpudata[ix].processor = ++processor;
+	}
+	if (no_node && cpudata[ix].processor_node < 0)
+	    old_processor_node = -1;
+	else {
+	    if (old_processor_node == cpudata[ix].processor_node) {
+		if (no_node)
+		    cpudata[ix].node = cpudata[ix].processor_node = node;
+		else {
+		    if (processor_node_node >= 0)
+			cpudata[ix].node = processor_node_node;
+		    cpudata[ix].processor_node = processor_node;
+		}
+	    }
+	    else {
+		old_processor_node = cpudata[ix].processor_node;
+		old_core = core = -1;
+		old_thread = thread = -1;
+		if (no_node)
+		    cpudata[ix].node = cpudata[ix].processor_node = ++node;
+		else {
+		    cpudata[ix].node = processor_node_node = ++node;
+		    cpudata[ix].processor_node = ++processor_node;
+		}
+	    }
+	}
+	if (!no_node && cpudata[ix].processor_node < 0)
+	    cpudata[ix].processor_node = 0;
+	if (old_core == cpudata[ix].core)
+	    cpudata[ix].core = core;
+	else {
+	    old_core = cpudata[ix].core;
+	    old_thread = thread = -1;
+	    cpudata[ix].core = ++core;
+	}
+	if (old_thread == cpudata[ix].thread)
+	    cpudata[ix].thread = thread;
+	else
+	    old_thread = cpudata[ix].thread = ++thread;
+    }
+}
+
+static void
+cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
+		    int size,
+		    ErtsCpuBindOrder bind_order,
+		    int mk_seq)
+{
+    if (size > 1) {
+	int no_node = 0;
+	int (*cmp_func)(const void *, const void *);
+	switch (bind_order) {
+	case ERTS_CPU_BIND_SPREAD:
+	    cmp_func = cpu_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_PROCESSOR_SPREAD:
+	    cmp_func = cpu_processor_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_THREAD_SPREAD:
+	    cmp_func = cpu_thread_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD:
+	    no_node = 1;
+	    cmp_func = cpu_thread_no_node_processor_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD:
+	    no_node = 1;
+	    cmp_func = cpu_no_node_processor_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD:
+	    no_node = 1;
+	    cmp_func = cpu_no_node_thread_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_NO_SPREAD:
+	    cmp_func = cpu_no_spread_order_cmp;
+	    break;
+	default:
+	    cmp_func = NULL;
+	    erl_exit(ERTS_ABORT_EXIT,
+		     "Bad cpu bind type: %d\n",
+		     (int) cpu_bind_order);
+	    break;
+	}
+
+	if (mk_seq)
+	    make_cpudata_id_seq(cpudata, size, no_node);
+
+	qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func);
+    }
+}
+
+static int
+processor_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->processor_node != y->processor_node)
+	return x->processor_node - y->processor_node;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    return 0;
+}
+
+#ifdef ERTS_SMP
+void
+erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp)
+{
+    erts_cpu_groups_map_t *cgm;
+    erts_cpu_groups_callback_list_t *cgcl;
+    erts_cpu_groups_callback_call_t *cgcc;
+    int cgcc_ix;
+
+    /* Unbind from cpu */
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    if (scheduler2cpu_map[esdp->no].bound_id >= 0
+	&& erts_unbind_from_cpu(cpuinfo) == 0) {
+	esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1;
+    }
+
+    cgcc = erts_alloc(ERTS_ALC_T_TMP,
+		      (no_cpu_groups_callbacks
+		       * sizeof(erts_cpu_groups_callback_call_t)));
+    cgcc_ix = 0;
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
+	for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) {
+	    cgcc[cgcc_ix].callback = cgcl->callback;
+	    cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp);
+	    cgcc[cgcc_ix].arg = cgcl->arg;
+	    cgcc_ix++;
+	}
+    }
+    ASSERT(no_cpu_groups_callbacks == cgcc_ix);
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+
+    for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++)
+	cgcc[cgcc_ix].callback(1,
+			       esdp,
+			       cgcc[cgcc_ix].ix,
+			       cgcc[cgcc_ix].arg);
+
+    erts_free(ERTS_ALC_T_TMP, cgcc);
+
+    if (esdp->no <= max_main_threads)
+	erts_thr_set_main_status(0, 0);
+
+}
+
+void
+erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp)
+{
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue));
+
+    if (esdp->no <= max_main_threads)
+	erts_thr_set_main_status(1, (int) esdp->no);
+
+    /* Make sure we check if we should bind to a cpu or not... */
+    if (esdp->run_queue->flags & ERTS_RUNQ_FLG_SHARED_RUNQ)
+	erts_smp_atomic32_set(&esdp->chk_cpu_bind, 1);
+    else
+	esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
+}
+
+#endif
+
+void
+erts_sched_check_cpu_bind(ErtsSchedulerData *esdp)
+{
+    int res, cpu_id, cgcc_ix;
+    erts_cpu_groups_map_t *cgm;
+    erts_cpu_groups_callback_list_t *cgcl;
+    erts_cpu_groups_callback_call_t *cgcc;
+#ifdef ERTS_SMP
+    if (erts_common_run_queue)
+	erts_smp_atomic32_set(&esdp->chk_cpu_bind, 0);
+    else {
+	esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND;
+    }
+#endif
+    erts_smp_runq_unlock(esdp->run_queue);
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    cpu_id = scheduler2cpu_map[esdp->no].bind_id;
+    if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) {
+	res = erts_bind_to_cpu(cpuinfo, cpu_id);
+	if (res == 0)
+	    esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id;
+	else {
+	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+	    erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n",
+			  (int) esdp->no, cpu_id, erl_errno_id(-res));
+	    erts_send_error_to_logger_nogl(dsbufp);
+	    if (scheduler2cpu_map[esdp->no].bound_id >= 0)
+		goto unbind;
+	}
+    }
+    else if (cpu_id < 0) {
+    unbind:
+	/* Get rid of old binding */
+	res = erts_unbind_from_cpu(cpuinfo);
+	if (res == 0)
+	    esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1;
+	else if (res != -ENOTSUP) {
+	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+	    erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n",
+			  (int) esdp->no, cpu_id, erl_errno_id(-res));
+	    erts_send_error_to_logger_nogl(dsbufp);
+	}
+    }
+
+    cgcc = erts_alloc(ERTS_ALC_T_TMP,
+		      (no_cpu_groups_callbacks
+		       * sizeof(erts_cpu_groups_callback_call_t)));
+    cgcc_ix = 0;
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
+	for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) {
+	    cgcc[cgcc_ix].callback = cgcl->callback;
+	    cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp);
+	    cgcc[cgcc_ix].arg = cgcl->arg;
+	    cgcc_ix++;
+	}
+    }
+
+    ASSERT(no_cpu_groups_callbacks == cgcc_ix);
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+
+    for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++)
+	cgcc[cgcc_ix].callback(0,
+			       esdp,
+			       cgcc[cgcc_ix].ix,
+			       cgcc[cgcc_ix].arg);
+
+    erts_free(ERTS_ALC_T_TMP, cgcc);
+
+    erts_smp_runq_lock(esdp->run_queue);
+}
+
+#ifdef ERTS_SMP
+void
+erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp)
+{
+    int cgcc_ix;
+    erts_cpu_groups_map_t *cgm;
+    erts_cpu_groups_callback_list_t *cgcl;
+    erts_cpu_groups_callback_call_t *cgcc;
+
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+
+    cgcc = erts_alloc(ERTS_ALC_T_TMP,
+		      (no_cpu_groups_callbacks
+		       * sizeof(erts_cpu_groups_callback_call_t)));
+    cgcc_ix = 0;
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
+	for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) {
+	    cgcc[cgcc_ix].callback = cgcl->callback;
+	    cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp);
+	    cgcc[cgcc_ix].arg = cgcl->arg;
+	    cgcc_ix++;
+	}
+    }
+
+    ASSERT(no_cpu_groups_callbacks == cgcc_ix);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+
+    for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++)
+	cgcc[cgcc_ix].callback(0,
+			       esdp,
+			       cgcc[cgcc_ix].ix,
+			       cgcc[cgcc_ix].arg);
+
+    erts_free(ERTS_ALC_T_TMP, cgcc);
+
+    if (esdp->no <= max_main_threads)
+	erts_thr_set_main_status(1, (int) esdp->no);
+}
+#endif
+
+static void
+write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size)
+{
+    int s_ix = 1;
+    int cpu_ix;
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+
+    if (cpu_bind_order != ERTS_CPU_BIND_NONE && size) {
+
+	cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1);
+
+	for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++)
+	    if (erts_is_cpu_available(cpuinfo, cpudata[cpu_ix].logical))
+		scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical;
+    }
+
+    if (s_ix <= erts_no_schedulers)
+	for (; s_ix <= erts_no_schedulers; s_ix++)
+	    scheduler2cpu_map[s_ix].bind_id = -1;
+}
+
+int
+erts_init_scheduler_bind_type_string(char *how)
+{
+    if (sys_strcmp(how, "u") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_NONE;
+    else if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP)
+	return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED;
+    else if (!system_cpudata && !user_cpudata)
+	return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY;
+    else if (sys_strcmp(how, "db") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND;
+    else if (sys_strcmp(how, "s") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_SPREAD;
+    else if (sys_strcmp(how, "ps") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
+    else if (sys_strcmp(how, "ts") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
+    else if (sys_strcmp(how, "tnnps") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
+    else if (sys_strcmp(how, "nnps") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
+    else if (sys_strcmp(how, "nnts") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
+    else if (sys_strcmp(how, "ns") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
+    else
+	return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE;
+    return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS;
+}
+
+static Eterm
+bound_schedulers_term(ErtsCpuBindOrder order)
+{
+    switch (order) {
+    case ERTS_CPU_BIND_SPREAD: {
+	ERTS_DECL_AM(spread);
+	return AM_spread;
+    }
+    case ERTS_CPU_BIND_PROCESSOR_SPREAD: {
+	ERTS_DECL_AM(processor_spread);
+	return AM_processor_spread;
+    }
+    case ERTS_CPU_BIND_THREAD_SPREAD: {
+	ERTS_DECL_AM(thread_spread);
+	return AM_thread_spread;
+    }
+    case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: {
+	ERTS_DECL_AM(thread_no_node_processor_spread);
+	return AM_thread_no_node_processor_spread;
+    }
+    case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: {
+	ERTS_DECL_AM(no_node_processor_spread);
+	return AM_no_node_processor_spread;
+    }
+    case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: {
+	ERTS_DECL_AM(no_node_thread_spread);
+	return AM_no_node_thread_spread;
+    }
+    case ERTS_CPU_BIND_NO_SPREAD: {
+	ERTS_DECL_AM(no_spread);
+	return AM_no_spread;
+    }
+    case ERTS_CPU_BIND_NONE: {
+	ERTS_DECL_AM(unbound);
+	return AM_unbound;
+    }
+    default:
+	ASSERT(0);
+	return THE_NON_VALUE;
+    }
+}
+
+Eterm
+erts_bound_schedulers_term(Process *c_p)
+{
+    ErtsCpuBindOrder order;
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    order = cpu_bind_order;
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    return bound_schedulers_term(order);
+}
+
+Eterm
+erts_bind_schedulers(Process *c_p, Eterm how)
+{
+    int notify = 0;
+    Eterm res;
+    erts_cpu_topology_t *cpudata;
+    int cpudata_size;
+    ErtsCpuBindOrder old_cpu_bind_order;
+
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+
+    if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) {
+	if (cpu_bind_order == ERTS_CPU_BIND_NONE
+	    && ERTS_IS_ATOM_STR("unbound", how)) {
+	    res = bound_schedulers_term(ERTS_CPU_BIND_NONE);
+	    goto done;
+	}
+	ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP);
+    }
+    else {
+
+	old_cpu_bind_order = cpu_bind_order;
+
+	if (ERTS_IS_ATOM_STR("default_bind", how))
+	    cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND;
+	else if (ERTS_IS_ATOM_STR("spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_SPREAD;
+	else if (ERTS_IS_ATOM_STR("processor_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
+	else if (ERTS_IS_ATOM_STR("thread_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
+	else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
+	else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
+	else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
+	else if (ERTS_IS_ATOM_STR("no_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
+	else if (ERTS_IS_ATOM_STR("unbound", how))
+	    cpu_bind_order = ERTS_CPU_BIND_NONE;
+	else {
+	    cpu_bind_order = old_cpu_bind_order;
+	    ERTS_BIF_PREP_ERROR(res, c_p, BADARG);
+	    goto done;
+	}
+
+	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
+
+	if (!cpudata) {
+	    cpu_bind_order = old_cpu_bind_order;
+	    ERTS_BIF_PREP_ERROR(res, c_p, BADARG);
+	    goto done;
+	}
+
+	write_schedulers_bind_change(cpudata, cpudata_size);
+	notify = 1;
+
+	destroy_tmp_cpu_topology_copy(cpudata);
+    
+	res = bound_schedulers_term(old_cpu_bind_order);
+    }
+
+ done:
+
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+
+    if (notify)
+	erts_sched_notify_check_cpu_bind();
+
+    return res;
+}
+
+int
+erts_sched_bind_atthrcreate_prepare(void)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    return esdp != NULL && erts_is_scheduler_bound(esdp);
+}
+
+int
+erts_sched_bind_atthrcreate_child(int unbind)
+{
+    int res = 0;
+    if (unbind) {
+	erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+	res = erts_unbind_from_cpu(cpuinfo);
+	erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    }
+    return res;
+}
+
+void
+erts_sched_bind_atthrcreate_parent(int unbind)
+{
+
+}
+
+int
+erts_sched_bind_atfork_prepare(void)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    int unbind = esdp != NULL && erts_is_scheduler_bound(esdp);
+    if (unbind)
+	erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    return unbind;
+}
+
+int
+erts_sched_bind_atfork_child(int unbind)
+{
+    if (unbind) {
+	ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
+			   || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+	return erts_unbind_from_cpu(cpuinfo);
+    }
+    return 0;
+}
+
+char *
+erts_sched_bind_atvfork_child(int unbind)
+{
+    if (unbind) {
+	ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
+			   || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+	return erts_get_unbind_from_cpu_str(cpuinfo);
+    }
+    return "false";
+}
+
+void
+erts_sched_bind_atfork_parent(int unbind)
+{
+    if (unbind)
+	erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+}
+
+Eterm
+erts_fake_scheduler_bindings(Process *p, Eterm how)
+{
+    ErtsCpuBindOrder fake_cpu_bind_order;
+    erts_cpu_topology_t *cpudata;
+    int cpudata_size;
+    Eterm res;
+
+    if (ERTS_IS_ATOM_STR("default_bind", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND;
+    else if (ERTS_IS_ATOM_STR("spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD;
+    else if (ERTS_IS_ATOM_STR("processor_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
+    else if (ERTS_IS_ATOM_STR("thread_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
+    else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
+    else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
+    else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
+    else if (ERTS_IS_ATOM_STR("no_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
+    else if (ERTS_IS_ATOM_STR("unbound", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_NONE;
+    else {
+	ERTS_BIF_PREP_ERROR(res, p, BADARG);
+	return res;
+    }
+
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+
+    if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE)
+	ERTS_BIF_PREP_RET(res, am_false);
+    else {
+	int i;
+	Eterm *hp;
+	
+	cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1);
+
+#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA
+
+	erts_fprintf(stderr, "node:          ");
+	for (i = 0; i < cpudata_size; i++)
+	    erts_fprintf(stderr, " %2d", cpudata[i].node);
+	erts_fprintf(stderr, "\n");
+	erts_fprintf(stderr, "processor:     ");
+	for (i = 0; i < cpudata_size; i++)
+	    erts_fprintf(stderr, " %2d", cpudata[i].processor);
+	erts_fprintf(stderr, "\n");
+	if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD
+	    && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD
+	    && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) {
+	    erts_fprintf(stderr, "processor_node:");
+	    for (i = 0; i < cpudata_size; i++)
+		erts_fprintf(stderr, " %2d", cpudata[i].processor_node);
+	    erts_fprintf(stderr, "\n");
+	}
+	erts_fprintf(stderr, "core:          ");
+	for (i = 0; i < cpudata_size; i++)
+	    erts_fprintf(stderr, " %2d", cpudata[i].core);
+	erts_fprintf(stderr, "\n");
+	erts_fprintf(stderr, "thread:        ");
+	for (i = 0; i < cpudata_size; i++)
+	    erts_fprintf(stderr, " %2d", cpudata[i].thread);
+	erts_fprintf(stderr, "\n");
+	erts_fprintf(stderr, "logical:       ");
+	for (i = 0; i < cpudata_size; i++)
+	    erts_fprintf(stderr, " %2d", cpudata[i].logical);
+	erts_fprintf(stderr, "\n");
+#endif
+
+	hp = HAlloc(p, cpudata_size+1);
+	ERTS_BIF_PREP_RET(res, make_tuple(hp));
+	*hp++ = make_arityval((Uint) cpudata_size);
+	for (i = 0; i < cpudata_size; i++)
+	    *hp++ = make_small((Uint) cpudata[i].logical);
+    }
+
+    destroy_tmp_cpu_topology_copy(cpudata);
+
+    return res;
+}
+
+Eterm
+erts_get_schedulers_binds(Process *c_p)
+{
+    int ix;
+    ERTS_DECL_AM(unbound);
+    Eterm *hp = HAlloc(c_p, erts_no_schedulers+1);
+    Eterm res = make_tuple(hp);
+
+    *(hp++) = make_arityval(erts_no_schedulers);
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    for (ix = 1; ix <= erts_no_schedulers; ix++)
+	*(hp++) = (scheduler2cpu_map[ix].bound_id >= 0
+		   ? make_small(scheduler2cpu_map[ix].bound_id)
+		   : AM_unbound);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    return res;
+}
+
+/*
+ * CPU topology
+ */
+
+typedef struct {
+    int *id;
+    int used;
+    int size;
+} ErtsCpuTopIdSeq;
+
+typedef struct {
+    ErtsCpuTopIdSeq logical;
+    ErtsCpuTopIdSeq thread;
+    ErtsCpuTopIdSeq core;
+    ErtsCpuTopIdSeq processor_node;
+    ErtsCpuTopIdSeq processor;
+    ErtsCpuTopIdSeq node;
+} ErtsCpuTopEntry;
+
+static void
+init_cpu_top_entry(ErtsCpuTopEntry *cte)
+{
+    int size = 10;
+    cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+				 sizeof(int)*size);
+    cte->logical.size = size;
+    cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+				sizeof(int)*size);
+    cte->thread.size = size;
+    cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+			      sizeof(int)*size);
+    cte->core.size = size;
+    cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+					sizeof(int)*size);
+    cte->processor_node.size = size;
+    cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+				   sizeof(int)*size);
+    cte->processor.size = size;
+    cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+			      sizeof(int)*size);
+    cte->node.size = size;
+}
+
+static void
+destroy_cpu_top_entry(ErtsCpuTopEntry *cte)
+{
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id);
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id);
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id);
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id);
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id);
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id);
+}
+
+static int
+get_cput_value_or_range(int *v, int *vr, char **str)
+{
+    long l;
+    char *c = *str;
+    errno = 0;
+    if (!isdigit((unsigned char)*c))
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID;
+    l = strtol(c, &c, 10);
+    if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID;
+    *v = (int) l;
+    if (*c == '-') {
+	c++;
+	if (!isdigit((unsigned char)*c))
+	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+	l = strtol(c, &c, 10);
+	if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l)
+	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+	*vr = (int) l;
+    }
+    *str = c;
+    return ERTS_INIT_CPU_TOPOLOGY_OK;
+}
+
+static int
+get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str)
+{
+    int ix = 0;
+    int need_size = 0;
+    char *c = *str;
+
+    while (1) {
+	int res;
+	int val;
+	int nids;
+	int val_range = -1;
+	res = get_cput_value_or_range(&val, &val_range, &c);
+	if (res != ERTS_INIT_CPU_TOPOLOGY_OK)
+	    return res;
+	if (val_range < 0 || val_range == val)
+	    nids = 1;
+	else {
+	    if (val_range > val)
+		nids = val_range - val + 1;
+	    else
+		nids = val - val_range + 1;
+	}
+	need_size += nids;
+	if (need_size > idseq->size) {
+	    idseq->size = need_size + 10;
+	    idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS,
+				      idseq->id,
+				      sizeof(int)*idseq->size);
+	}
+	if (nids == 1)
+	    idseq->id[ix++] = val;
+	else if (val_range > val) {
+	    for (; val <= val_range; val++)
+		idseq->id[ix++] = val;
+	}
+	else {
+	    for (; val >= val_range; val--)
+		idseq->id[ix++] = val;
+	}
+	if (*c != ',')
+	    break;
+	c++;
+    }
+    *str = c;
+    idseq->used = ix;
+    return ERTS_INIT_CPU_TOPOLOGY_OK;
+}
+
+static int
+get_cput_entry(ErtsCpuTopEntry *cput, char **str)
+{
+    int h;
+    char *c = *str;
+
+    cput->logical.used = 0;
+    cput->thread.id[0] = 0;
+    cput->thread.used = 1;
+    cput->core.id[0] = 0;
+    cput->core.used = 1;
+    cput->processor_node.id[0] = -1;
+    cput->processor_node.used = 1;
+    cput->processor.id[0] = 0;
+    cput->processor.used = 1;
+    cput->node.id[0] = -1;
+    cput->node.used = 1;
+
+    h = ERTS_TOPOLOGY_MAX_DEPTH;
+    while (*c != ':' && *c != '\0') {
+	int res;
+	ErtsCpuTopIdSeq *idseqp;
+	switch (*c++) {
+	case 'L':
+	    if (h <= ERTS_TOPOLOGY_LOGICAL)
+		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+	    idseqp = &cput->logical;
+	    h = ERTS_TOPOLOGY_LOGICAL;
+	    break;
+	case 't':
+	case 'T':
+	    if (h <= ERTS_TOPOLOGY_THREAD)
+		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+	    idseqp = &cput->thread;
+	    h = ERTS_TOPOLOGY_THREAD;
+	    break;
+	case 'c':
+	case 'C':
+	    if (h <= ERTS_TOPOLOGY_CORE)
+		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+	    idseqp = &cput->core;
+	    h = ERTS_TOPOLOGY_CORE;
+	    break;
+	case 'p':
+	case 'P':
+	    if (h <= ERTS_TOPOLOGY_PROCESSOR)
+		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+	    idseqp = &cput->processor;
+	    h = ERTS_TOPOLOGY_PROCESSOR;
+	    break;
+	case 'n':
+	case 'N':
+	    if (h <= ERTS_TOPOLOGY_PROCESSOR) {
+	    do_node:
+		if (h <= ERTS_TOPOLOGY_NODE)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+		idseqp = &cput->node;
+		h = ERTS_TOPOLOGY_NODE;
+	    }
+	    else {
+		int p_node = 0;
+		char *p_chk = c;
+		while (*p_chk != '\0' && *p_chk != ':') {
+		    if (*p_chk == 'p' || *p_chk == 'P') {
+			p_node = 1;
+			break;
+		    }
+		    p_chk++;
+		}
+		if (!p_node)
+		    goto do_node;
+		if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+		idseqp = &cput->processor_node;
+		h = ERTS_TOPOLOGY_PROCESSOR_NODE;
+	    }
+	    break;
+	default:
+	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE;
+	}
+	res = get_cput_id_seq(idseqp, &c);
+	if (res != ERTS_INIT_CPU_TOPOLOGY_OK)
+		return res;
+    }
+
+    if (cput->logical.used < 1)
+	return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID;
+
+    if (*c == ':') {
+	c++;
+    }
+
+    if (cput->thread.used != 1
+	&& cput->thread.used != cput->logical.used)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+    if (cput->core.used != 1
+	&& cput->core.used != cput->logical.used)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+    if (cput->processor_node.used != 1
+	&& cput->processor_node.used != cput->logical.used)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+    if (cput->processor.used != 1
+	&& cput->processor.used != cput->logical.used)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+    if (cput->node.used != 1
+	&& cput->node.used != cput->logical.used)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+
+    *str = c;
+    return ERTS_INIT_CPU_TOPOLOGY_OK;
+}
+
+static int
+verify_topology(erts_cpu_topology_t *cpudata, int size)
+{
+    if (size > 0) {
+	int *logical;
+	int node, processor, no_nodes, i;
+
+	/* Verify logical ids */
+	logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size);
+
+	for (i = 0; i < size; i++)
+	    logical[i] = cpudata[i].logical;
+
+	qsort(logical, size, sizeof(int), int_cmp);
+	for (i = 0; i < size-1; i++) {
+	    if (logical[i] == logical[i+1]) {
+		erts_free(ERTS_ALC_T_TMP, logical);
+		return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS;
+	    }
+	}
+
+	erts_free(ERTS_ALC_T_TMP, logical);
+
+	qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp);
+
+	/* Verify unique entities */
+
+	for (i = 1; i < size; i++) {
+	    if (cpudata[i-1].processor == cpudata[i].processor
+		&& cpudata[i-1].node == cpudata[i].node
+		&& (cpudata[i-1].processor_node
+		    == cpudata[i].processor_node)
+		&& cpudata[i-1].core == cpudata[i].core
+		&& cpudata[i-1].thread == cpudata[i].thread) {
+		return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES;
+	    }
+	}
+
+	/* Verify numa nodes */
+	node = cpudata[0].node;
+	processor = cpudata[0].processor;
+	no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0;
+	for (i = 1; i < size; i++) {
+	    if (no_nodes) {
+		if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
+	    }
+	    else {
+		if (cpudata[i].processor == processor && cpudata[i].node != node)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
+		node = cpudata[i].node;
+		processor = cpudata[i].processor;
+		if (node >= 0 && cpudata[i].processor_node >= 0)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
+		if (node < 0 && cpudata[i].processor_node < 0)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
+	    }
+	}
+    }
+
+    return ERTS_INIT_CPU_TOPOLOGY_OK;
+}
+
+int
+erts_init_cpu_topology_string(char *topology_str)
+{
+    ErtsCpuTopEntry cput;
+    int need_size;
+    char *c;
+    int ix;
+    int error = ERTS_INIT_CPU_TOPOLOGY_OK;
+
+    if (user_cpudata)
+	erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
+    user_cpudata_size = 10;
+
+    user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
+			      (sizeof(erts_cpu_topology_t)
+			       * user_cpudata_size));
+
+    init_cpu_top_entry(&cput);
+
+    ix = 0;
+    need_size = 0;
+
+    c = topology_str;
+    if (*c == '\0') {
+	error = ERTS_INIT_CPU_TOPOLOGY_MISSING;
+	goto fail;
+    }
+    do {
+	int r;
+	error = get_cput_entry(&cput, &c);
+	if (error != ERTS_INIT_CPU_TOPOLOGY_OK)
+	    goto fail;
+	need_size += cput.logical.used;
+	if (user_cpudata_size < need_size) {
+	    user_cpudata_size = need_size + 10;
+	    user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA,
+					user_cpudata,
+					(sizeof(erts_cpu_topology_t)
+					 * user_cpudata_size));
+	}
+
+	ASSERT(cput.thread.used == 1
+	       || cput.thread.used == cput.logical.used);
+	ASSERT(cput.core.used == 1
+	       || cput.core.used == cput.logical.used);
+	ASSERT(cput.processor_node.used == 1
+	       || cput.processor_node.used == cput.logical.used);
+	ASSERT(cput.processor.used == 1
+	       || cput.processor.used == cput.logical.used);
+	ASSERT(cput.node.used == 1
+	       || cput.node.used == cput.logical.used);
+
+	for (r = 0; r < cput.logical.used; r++) {
+	    user_cpudata[ix].logical = cput.logical.id[r];
+	    user_cpudata[ix].thread =
+		cput.thread.id[cput.thread.used == 1 ? 0 : r];
+	    user_cpudata[ix].core =
+		cput.core.id[cput.core.used == 1 ? 0 : r];
+	    user_cpudata[ix].processor_node =
+		cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r];
+	    user_cpudata[ix].processor =
+		cput.processor.id[cput.processor.used == 1 ? 0 : r];
+	    user_cpudata[ix].node =
+		cput.node.id[cput.node.used == 1 ? 0 : r];
+	    ix++;
+	}
+    } while (*c != '\0');
+
+    if (user_cpudata_size != ix) {
+	user_cpudata_size = ix;
+	user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA,
+				    user_cpudata,
+				    (sizeof(erts_cpu_topology_t)
+				     * user_cpudata_size));
+    }
+
+    error = verify_topology(user_cpudata, user_cpudata_size);
+    if (error == ERTS_INIT_CPU_TOPOLOGY_OK) {
+	destroy_cpu_top_entry(&cput);
+	return ERTS_INIT_CPU_TOPOLOGY_OK;
+    }
+
+ fail:
+    if (user_cpudata)
+	erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
+    user_cpudata_size = 0;
+    destroy_cpu_top_entry(&cput);
+    return error;
+}
+
+#define ERTS_GET_CPU_TOPOLOGY_ERROR		-1
+#define ERTS_GET_USED_CPU_TOPOLOGY		0
+#define ERTS_GET_DETECTED_CPU_TOPOLOGY		1
+#define ERTS_GET_DEFINED_CPU_TOPOLOGY		2
+
+static Eterm get_cpu_topology_term(Process *c_p, int type);
+
+Eterm
+erts_set_cpu_topology(Process *c_p, Eterm term)
+{
+    erts_cpu_topology_t *cpudata = NULL;
+    int cpudata_size = 0;
+    Eterm res;
+
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY);
+    if (term == am_undefined) {
+	if (user_cpudata)
+	    erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
+	user_cpudata = NULL;
+	user_cpudata_size = 0;
+
+	if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) {
+	    cpudata_size = system_cpudata_size;
+	    cpudata = erts_alloc(ERTS_ALC_T_TMP,
+				 (sizeof(erts_cpu_topology_t)
+				  * cpudata_size));
+
+	    sys_memcpy((void *) cpudata,
+		       (void *) system_cpudata,
+		       sizeof(erts_cpu_topology_t)*cpudata_size);
+	}
+    }
+    else if (is_not_list(term)) {
+    error:
+	erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+	res = THE_NON_VALUE;
+	goto done;
+    }
+    else {
+	Eterm list = term;
+	int ix = 0;
+
+	cpudata_size = 100;
+	cpudata = erts_alloc(ERTS_ALC_T_TMP,
+			     (sizeof(erts_cpu_topology_t)
+			      * cpudata_size));
+
+	while (is_list(list)) {
+	    Eterm *lp = list_val(list);
+	    Eterm cpu = CAR(lp);
+	    Eterm* tp;
+	    Sint id;
+		
+	    if (is_not_tuple(cpu))
+		goto error;
+
+	    tp = tuple_val(cpu);
+
+	    if (arityval(tp[0]) != 7 || tp[1] != am_cpu)
+		goto error;
+
+	    if (ix >= cpudata_size) {
+		cpudata_size += 100;
+		cpudata = erts_realloc(ERTS_ALC_T_TMP,
+				       cpudata,
+				       (sizeof(erts_cpu_topology_t)
+					* cpudata_size));
+	    }
+
+	    id = signed_val(tp[2]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].node = (int) id;
+
+	    id = signed_val(tp[3]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].processor = (int) id;
+
+	    id = signed_val(tp[4]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].processor_node = (int) id;
+
+	    id = signed_val(tp[5]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].core = (int) id;
+
+	    id = signed_val(tp[6]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].thread = (int) id;
+
+	    id = signed_val(tp[7]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].logical = (int) id;
+
+	    list = CDR(lp);
+	    ix++;
+	}
+
+	if (is_not_nil(list))
+	    goto error;
+	
+	cpudata_size = ix;
+
+	if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size))
+	    goto error;
+
+	if (user_cpudata_size != cpudata_size) {
+	    if (user_cpudata)
+		erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
+	    user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
+				      sizeof(erts_cpu_topology_t)*cpudata_size);
+	    user_cpudata_size = cpudata_size;
+	}
+
+	sys_memcpy((void *) user_cpudata,
+		   (void *) cpudata,
+		   sizeof(erts_cpu_topology_t)*cpudata_size);
+    }
+
+    update_cpu_groups_maps();
+
+    write_schedulers_bind_change(cpudata, cpudata_size);
+
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    erts_sched_notify_check_cpu_bind();
+
+ done:
+
+    if (cpudata)
+	erts_free(ERTS_ALC_T_TMP, cpudata);
+
+    return res;
+}
+
+static void
+create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size)
+{
+    if (user_cpudata) {
+	*cpudata_size = user_cpudata_size;
+	*cpudata = erts_alloc(ERTS_ALC_T_TMP,
+			      (sizeof(erts_cpu_topology_t)
+			       * (*cpudata_size)));
+	sys_memcpy((void *) *cpudata,
+		   (void *) user_cpudata,
+		   sizeof(erts_cpu_topology_t)*(*cpudata_size));
+    }
+    else if (system_cpudata) {
+	*cpudata_size = system_cpudata_size;
+	*cpudata = erts_alloc(ERTS_ALC_T_TMP,
+			      (sizeof(erts_cpu_topology_t)
+			       * (*cpudata_size)));
+	sys_memcpy((void *) *cpudata,
+		   (void *) system_cpudata,
+		   sizeof(erts_cpu_topology_t)*(*cpudata_size));
+    }
+    else {
+	*cpudata = NULL;
+	*cpudata_size = 0;
+    }
+}
+
+static void
+destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata)
+{
+    if (cpudata)
+	erts_free(ERTS_ALC_T_TMP, cpudata);
+}
+
+
+static Eterm
+bld_topology_term(Eterm **hpp,
+		  Uint *hszp,
+		  erts_cpu_topology_t *cpudata,
+		  int size)
+{
+    Eterm res = NIL;
+    int i;
+
+    if (size == 0)
+	return am_undefined;
+
+    for (i = size-1; i >= 0; i--) {
+	res = erts_bld_cons(hpp,
+			    hszp,
+			    erts_bld_tuple(hpp,
+					   hszp,
+					   7,
+					   am_cpu,
+					   make_small(cpudata[i].node),
+					   make_small(cpudata[i].processor),
+					   make_small(cpudata[i].processor_node),
+					   make_small(cpudata[i].core),
+					   make_small(cpudata[i].thread),
+					   make_small(cpudata[i].logical)),
+			    res);
+    }
+    return res;
+}
+
+static Eterm
+get_cpu_topology_term(Process *c_p, int type)
+{
+#ifdef DEBUG
+    Eterm *hp_end;
+#endif
+    Eterm *hp;
+    Uint hsz;
+    Eterm res = THE_NON_VALUE;
+    erts_cpu_topology_t *cpudata = NULL;
+    int size = 0;
+
+    switch (type) {
+    case ERTS_GET_USED_CPU_TOPOLOGY:
+	if (user_cpudata)
+	    goto defined;
+	else
+	    goto detected;
+    case ERTS_GET_DETECTED_CPU_TOPOLOGY:
+    detected:
+	if (!system_cpudata)
+	    res = am_undefined;
+	else {
+	    size = system_cpudata_size;
+	    cpudata = erts_alloc(ERTS_ALC_T_TMP,
+				 (sizeof(erts_cpu_topology_t)
+				  * size));
+	    sys_memcpy((void *) cpudata,
+		       (void *) system_cpudata,
+		       sizeof(erts_cpu_topology_t)*size);
+	}
+	break;
+    case ERTS_GET_DEFINED_CPU_TOPOLOGY:
+    defined:
+	if (!user_cpudata)
+	    res = am_undefined;
+	else {
+	    size = user_cpudata_size;
+	    cpudata = user_cpudata;
+	}
+	break;
+    default:
+	erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type);
+	break;
+    }
+
+    if (res == am_undefined) {
+	ASSERT(!cpudata);
+	return res;
+    }
+
+    hsz = 0;
+
+    bld_topology_term(NULL, &hsz,
+		      cpudata, size);
+
+    hp = HAlloc(c_p, hsz);
+
+#ifdef DEBUG
+    hp_end = hp + hsz;
+#endif
+
+    res = bld_topology_term(&hp, NULL,
+			    cpudata, size);
+
+    ASSERT(hp_end == hp);
+
+    if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata)
+	erts_free(ERTS_ALC_T_TMP, cpudata);
+
+    return res;
+}
+
+Eterm
+erts_get_cpu_topology_term(Process *c_p, Eterm which)
+{
+    Eterm res;
+    int type;
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    if (ERTS_IS_ATOM_STR("used", which))
+	type = ERTS_GET_USED_CPU_TOPOLOGY;
+    else if (ERTS_IS_ATOM_STR("detected", which))
+	type = ERTS_GET_DETECTED_CPU_TOPOLOGY;
+    else if (ERTS_IS_ATOM_STR("defined", which))
+	type = ERTS_GET_DEFINED_CPU_TOPOLOGY;
+    else
+	type = ERTS_GET_CPU_TOPOLOGY_ERROR;
+    if (type == ERTS_GET_CPU_TOPOLOGY_ERROR)
+	res = THE_NON_VALUE;
+    else
+	res = get_cpu_topology_term(c_p, type);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    return res;
+}
+
+static void
+get_logical_processors(int *conf, int *onln, int *avail)
+{
+    if (conf)
+	*conf = erts_get_cpu_configured(cpuinfo);
+    if (onln)
+	*onln = erts_get_cpu_online(cpuinfo);
+    if (avail)
+	*avail = erts_get_cpu_available(cpuinfo);
+}
+
+void
+erts_get_logical_processors(int *conf, int *onln, int *avail)
+{
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    get_logical_processors(conf, onln, avail);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+}
+
+void
+erts_pre_early_init_cpu_topology(int *max_rg_p,
+				 int *conf_p,
+				 int *onln_p,
+				 int *avail_p)
+{
+    cpu_groups_maps = NULL;
+    no_cpu_groups_callbacks = 0;
+    *max_rg_p = ERTS_MAX_READER_GROUPS;
+    cpuinfo = erts_cpu_info_create();
+    get_logical_processors(conf_p, onln_p, avail_p);
+}
+
+void
+erts_early_init_cpu_topology(int no_schedulers,
+			     int *max_main_threads_p,
+			     int max_reader_groups,
+			     int *reader_groups_p)
+{
+    user_cpudata = NULL;
+    user_cpudata_size = 0;
+
+    system_cpudata_size = erts_get_cpu_topology_size(cpuinfo);
+    system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
+				(sizeof(erts_cpu_topology_t)
+				 * system_cpudata_size));
+
+    cpu_bind_order = ERTS_CPU_BIND_UNDEFINED;
+
+    if (!erts_get_cpu_topology(cpuinfo, system_cpudata)
+	|| ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata,
+							system_cpudata_size)) {
+	erts_free(ERTS_ALC_T_CPUDATA, system_cpudata);
+	system_cpudata = NULL;
+	system_cpudata_size = 0;
+    }
+
+    max_main_threads = erts_get_cpu_configured(cpuinfo);
+    if (max_main_threads > no_schedulers)
+	max_main_threads = no_schedulers;
+    *max_main_threads_p = max_main_threads;
+
+    reader_groups = max_main_threads;
+    if (reader_groups <= 1 || max_reader_groups <= 1)
+	reader_groups = 0;
+    if (reader_groups > max_reader_groups)
+	reader_groups = max_reader_groups;
+    *reader_groups_p = reader_groups;
+}
+
+void
+erts_init_cpu_topology(void)
+{
+    int ix;
+
+    erts_smp_rwmtx_init(&cpuinfo_rwmtx, "cpu_info");
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+
+    scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA,
+				   (sizeof(ErtsCpuBindData)
+				    * (erts_no_schedulers+1)));
+    for (ix = 1; ix <= erts_no_schedulers; ix++) {
+	scheduler2cpu_map[ix].bind_id = -1;
+	scheduler2cpu_map[ix].bound_id = -1;
+    }
+
+    if (cpu_bind_order == ERTS_CPU_BIND_UNDEFINED) {
+	int ncpus = erts_get_cpu_configured(cpuinfo);
+	if (ncpus < 1 || erts_no_schedulers < ncpus)
+	    cpu_bind_order = ERTS_CPU_BIND_NONE;
+	else
+	    cpu_bind_order = ((system_cpudata || user_cpudata)
+			      && (erts_bind_to_cpu(cpuinfo, -1) != -ENOTSUP)
+			      ? ERTS_CPU_BIND_DEFAULT_BIND
+			      : ERTS_CPU_BIND_NONE);
+    }
+
+    reader_groups_map = add_cpu_groups(reader_groups,
+				       reader_groups_callback,
+				       NULL);
+
+    if (cpu_bind_order == ERTS_CPU_BIND_NONE)
+	erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    else {
+	erts_cpu_topology_t *cpudata;
+	int cpudata_size;
+	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
+	write_schedulers_bind_change(cpudata, cpudata_size);
+	erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+	erts_sched_notify_check_cpu_bind();
+	destroy_tmp_cpu_topology_copy(cpudata);
+    }
+}
+
+int
+erts_update_cpu_info(void)
+{
+    int changed;
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    changed = erts_cpu_info_update(cpuinfo);
+    if (changed) {
+	erts_cpu_topology_t *cpudata;
+	int cpudata_size;
+
+	if (system_cpudata)
+	    erts_free(ERTS_ALC_T_CPUDATA, system_cpudata);
+
+	system_cpudata_size = erts_get_cpu_topology_size(cpuinfo);
+	if (!system_cpudata_size)
+	    system_cpudata = NULL;
+	else {
+	    system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
+					(sizeof(erts_cpu_topology_t)
+					 * system_cpudata_size));
+
+	    if (!erts_get_cpu_topology(cpuinfo, system_cpudata)
+		|| (ERTS_INIT_CPU_TOPOLOGY_OK
+		    != verify_topology(system_cpudata,
+				       system_cpudata_size))) {
+		erts_free(ERTS_ALC_T_CPUDATA, system_cpudata);
+		system_cpudata = NULL;
+		system_cpudata_size = 0;
+	    }
+	}
+
+	update_cpu_groups_maps();
+
+	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
+	write_schedulers_bind_change(cpudata, cpudata_size);
+	destroy_tmp_cpu_topology_copy(cpudata);
+    }
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    if (changed)
+	erts_sched_notify_check_cpu_bind();
+    return changed;
+}
+
+/*
+ * reader groups map
+ */
+
+void
+reader_groups_callback(int suspending,
+		       ErtsSchedulerData *esdp,
+		       int group,
+		       void *unused)
+{
+    if (reader_groups && esdp->no <= max_main_threads)
+	erts_smp_rwmtx_set_reader_group(suspending ? 0 : group+1);
+}
+
+static Eterm get_cpu_groups_map(Process *c_p,
+				erts_cpu_groups_map_t *map,
+				int offset);
+Eterm
+erts_debug_reader_groups_map(Process *c_p, int groups)
+{
+    Eterm res;
+    erts_cpu_groups_map_t test;
+
+    test.array = NULL;
+    test.groups = groups;
+    make_cpu_groups_map(&test, 1);
+    if (!test.array)
+	res = NIL;
+    else {
+	res = get_cpu_groups_map(c_p, &test, 1);
+	erts_free(ERTS_ALC_T_TMP, test.array);
+    }
+    return res;
+}
+
+
+Eterm
+erts_get_reader_groups_map(Process *c_p)
+{
+    Eterm res;
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    res = get_cpu_groups_map(c_p, reader_groups_map, 1);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    return res;
+}
+
+/*
+ * CPU groups
+ */
+
+static Eterm
+get_cpu_groups_map(Process *c_p,
+		   erts_cpu_groups_map_t *map,
+		   int offset)
+{
+#ifdef DEBUG
+    Eterm *endp;
+#endif
+    Eterm res = NIL, tuple;
+    Eterm *hp;
+    int i;
+
+    hp = HAlloc(c_p, map->logical_processors*(2+3));
+#ifdef DEBUG
+    endp = hp + map->logical_processors*(2+3);
+#endif
+    for (i = map->size - 1; i >= 0; i--) {
+	if (map->array[i].logical >= 0) {
+	    tuple = TUPLE2(hp,
+			   make_small(map->array[i].logical),
+			   make_small(map->array[i].cpu_group + offset));
+	    hp += 3;
+	    res = CONS(hp, tuple, res);
+	    hp += 2;
+	}
+    }
+    ASSERT(hp == endp);
+    return res;
+}
+
+static void
+make_available_cpu_topology(erts_avail_cput *no,
+			    erts_avail_cput *avail,
+			    erts_cpu_topology_t *cpudata,
+			    int *size,
+			    int test)
+{
+    int len = *size;
+    erts_cpu_topology_t last;
+    int a, i, j;
+
+    no->level[ERTS_TOPOLOGY_NODE] = -1;
+    no->level[ERTS_TOPOLOGY_PROCESSOR] = -1;
+    no->level[ERTS_TOPOLOGY_PROCESSOR_NODE] = -1;
+    no->level[ERTS_TOPOLOGY_CORE] = -1;
+    no->level[ERTS_TOPOLOGY_THREAD] = -1;
+    no->level[ERTS_TOPOLOGY_LOGICAL] = -1;
+
+    last.node = INT_MIN;
+    last.processor = INT_MIN;
+    last.processor_node = INT_MIN;
+    last.core = INT_MIN;
+    last.thread = INT_MIN;
+    last.logical = INT_MIN;
+
+    a = 0;
+
+    for (i = 0; i < len; i++) {
+
+	if (!test && !erts_is_cpu_available(cpuinfo, cpudata[i].logical))
+	    continue;
+
+	if (last.node != cpudata[i].node)
+	    goto node;
+	if (last.processor != cpudata[i].processor)
+	    goto processor;
+	if (last.processor_node != cpudata[i].processor_node)
+	    goto processor_node;
+	if (last.core != cpudata[i].core)
+	    goto core;
+	ASSERT(last.thread != cpudata[i].thread);
+	goto thread;
+
+    node:
+	no->level[ERTS_TOPOLOGY_NODE]++;
+    processor:
+	no->level[ERTS_TOPOLOGY_PROCESSOR]++;
+    processor_node:
+	no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++;
+    core:
+	no->level[ERTS_TOPOLOGY_CORE]++;
+    thread:
+	no->level[ERTS_TOPOLOGY_THREAD]++;
+
+	no->level[ERTS_TOPOLOGY_LOGICAL]++;
+
+	for (j = 0; j < ERTS_TOPOLOGY_LOGICAL; j++)
+	    avail[a].level[j] = no->level[j];
+
+	avail[a].level[ERTS_TOPOLOGY_LOGICAL] = cpudata[i].logical;
+	avail[a].level[ERTS_TOPOLOGY_CG] = 0;
+
+	ASSERT(last.logical != cpudata[i].logical);
+
+	last = cpudata[i];
+	a++;
+    }
+
+    no->level[ERTS_TOPOLOGY_NODE]++;
+    no->level[ERTS_TOPOLOGY_PROCESSOR]++;
+    no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++;
+    no->level[ERTS_TOPOLOGY_CORE]++;
+    no->level[ERTS_TOPOLOGY_THREAD]++;
+    no->level[ERTS_TOPOLOGY_LOGICAL]++;
+
+    *size = a;
+}
+
+static void
+cpu_group_insert(erts_cpu_groups_map_t *map,
+		 int logical, int cpu_group)
+{
+    int start = logical % map->size;
+    int ix = start;
+
+    do {
+	if (map->array[ix].logical < 0) {
+	    map->array[ix].logical = logical;
+	    map->array[ix].cpu_group = cpu_group;
+	    return;
+	}
+	ix++;
+	if (ix == map->size)
+	    ix = 0;
+    } while (ix != start);
+
+    erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n");
+}
+
+
+static int
+sub_levels(erts_cpu_groups_count_t *cgc, int level, int aix,
+	   int avail_sz, erts_avail_cput *avail)
+{
+    int sub_level = level+1;
+    int last = -1;
+    cgc->sub_levels = 0;
+
+    do {
+	if (last != avail[aix].level[sub_level]) {
+	    cgc->sub_levels++;
+	    last = avail[aix].level[sub_level];
+	}
+	aix++;
+    }
+    while (aix < avail_sz && cgc->id == avail[aix].level[level]);
+    cgc->cpu_groups = 0;
+    return aix;
+}
+
+static int
+write_cpu_groups(int *cgp, erts_cpu_groups_count_t *cgcp,
+		    int level, int a,
+		    int avail_sz, erts_avail_cput *avail)
+{
+    int cg = *cgp;
+    int sub_level = level+1;
+    int sl_per_gr = cgcp->sub_levels / cgcp->cpu_groups;
+    int xsl = cgcp->sub_levels % cgcp->cpu_groups;
+    int sls = 0;
+    int last = -1;
+    int xsl_cg_lim = (cgcp->cpu_groups - xsl) + cg + 1;
+
+    ASSERT(level < 0 || avail[a].level[level] == cgcp->id);
+
+    do {
+	if (last != avail[a].level[sub_level]) {
+	    if (!sls) {
+		sls = sl_per_gr;
+		cg++;
+		if (cg >= xsl_cg_lim)
+		    sls++;
+	    }
+	    last = avail[a].level[sub_level];
+	    sls--;
+	}
+	avail[a].level[ERTS_TOPOLOGY_CG] = cg;
+	a++;
+    } while (a < avail_sz && (level < 0
+			      || avail[a].level[level] == cgcp->id));
+
+    ASSERT(cgcp->cpu_groups == cg - *cgp);
+
+    *cgp = cg;
+
+    return a;
+}
+
+static int
+cg_count_sub_levels_compare(const void *vx, const void *vy)
+{
+    erts_cpu_groups_count_t *x = (erts_cpu_groups_count_t *) vx;
+    erts_cpu_groups_count_t *y = (erts_cpu_groups_count_t *) vy;
+    if (x->sub_levels != y->sub_levels)
+	return y->sub_levels - x->sub_levels;
+    return x->id - y->id;
+}
+
+static int
+cg_count_id_compare(const void *vx, const void *vy)
+{
+    erts_cpu_groups_count_t *x = (erts_cpu_groups_count_t *) vx;
+    erts_cpu_groups_count_t *y = (erts_cpu_groups_count_t *) vy;
+    return x->id - y->id;
+}
+
+static void
+make_cpu_groups_map(erts_cpu_groups_map_t *map, int test)
+{
+    int i, spread_level, avail_sz;
+    erts_avail_cput no, *avail;
+    erts_cpu_topology_t *cpudata;
+    ErtsAlcType_t alc_type = (test
+			      ? ERTS_ALC_T_TMP
+			      : ERTS_ALC_T_CPU_GRPS_MAP);
+
+    if (map->array)
+	erts_free(alc_type, map->array);
+
+    map->array = NULL;
+    map->logical_processors = 0;
+    map->size = 0;
+
+    if (!map->groups)
+	return;
+
+    create_tmp_cpu_topology_copy(&cpudata, &avail_sz);
+
+    if (!cpudata)
+	return;
+
+    cpu_bind_order_sort(cpudata,
+			avail_sz,
+			ERTS_CPU_BIND_NO_SPREAD,
+			1);
+
+    avail = erts_alloc(ERTS_ALC_T_TMP,
+		       sizeof(erts_avail_cput)*avail_sz);
+
+    make_available_cpu_topology(&no, avail, cpudata,
+				&avail_sz, test);
+
+    destroy_tmp_cpu_topology_copy(cpudata);
+
+    map->size = avail_sz*2+1;
+
+    map->array = erts_alloc(alc_type,
+			    (sizeof(erts_cpu_groups_map_array_t)
+			     * map->size));;
+    map->logical_processors = avail_sz;
+
+    for (i = 0; i < map->size; i++) {
+	map->array[i].logical = -1;
+	map->array[i].cpu_group = -1;
+    }
+
+    spread_level = ERTS_TOPOLOGY_CORE;
+    for (i = ERTS_TOPOLOGY_NODE; i < ERTS_TOPOLOGY_THREAD; i++) {
+	if (no.level[i] > map->groups) {
+	    spread_level = i;
+	    break;
+	}
+    }
+
+    if (no.level[spread_level] <= map->groups) {
+	int a, cg, last = -1;
+	cg = -1;
+	ASSERT(spread_level == ERTS_TOPOLOGY_CORE);
+	for (a = 0; a < avail_sz; a++) {
+	    if (last != avail[a].level[spread_level]) {
+		cg++;
+		last = avail[a].level[spread_level];
+	    }
+	    cpu_group_insert(map,
+			     avail[a].level[ERTS_TOPOLOGY_LOGICAL],
+			     cg);
+	}
+    }
+    else { /* map->groups < no.level[spread_level] */
+	erts_cpu_groups_count_t *cg_count;
+	int a, cg, tl, toplevels;
+
+	tl = spread_level-1;
+
+	if (spread_level == ERTS_TOPOLOGY_NODE)
+	    toplevels = 1;
+	else
+	    toplevels = no.level[tl];
+
+	cg_count = erts_alloc(ERTS_ALC_T_TMP,
+			      toplevels*sizeof(erts_cpu_groups_count_t));
+
+	if (toplevels == 1) {
+	    cg_count[0].id = 0;
+	    cg_count[0].sub_levels = no.level[spread_level];
+	    cg_count[0].cpu_groups = map->groups;
+	}
+	else {
+	    int cgs_per_tl, cgs;
+	    cgs = map->groups;
+	    cgs_per_tl = cgs / toplevels;
+
+	    a = 0;
+	    for (i = 0; i < toplevels; i++) {
+		cg_count[i].id = avail[a].level[tl];
+		a = sub_levels(&cg_count[i], tl, a, avail_sz, avail);
+	    }
+
+	    qsort(cg_count,
+		  toplevels,
+		  sizeof(erts_cpu_groups_count_t),
+		  cg_count_sub_levels_compare);
+
+	    for (i = 0; i < toplevels; i++) {
+		if (cg_count[i].sub_levels < cgs_per_tl) {
+		    cg_count[i].cpu_groups = cg_count[i].sub_levels;
+		    cgs -= cg_count[i].sub_levels;
+		}
+		else {
+		    cg_count[i].cpu_groups = cgs_per_tl;
+		    cgs -= cgs_per_tl;
+		}
+	    }
+
+	    while (cgs > 0) {
+		for (i = 0; i < toplevels; i++) {
+		    if (cg_count[i].sub_levels == cg_count[i].cpu_groups)
+			break;
+		    else {
+			cg_count[i].cpu_groups++;
+			if (--cgs == 0)
+			    break;
+		    }
+		}
+	    }
+
+	    qsort(cg_count,
+		  toplevels,
+		  sizeof(erts_cpu_groups_count_t),
+		  cg_count_id_compare);
+	}
+
+	a = i = 0;
+	cg = -1;
+	while (a < avail_sz) {
+	    a = write_cpu_groups(&cg, &cg_count[i], tl,
+				 a, avail_sz, avail);
+	    i++;
+	}
+
+	ASSERT(map->groups == cg + 1);
+
+	for (a = 0; a < avail_sz; a++)
+	    cpu_group_insert(map,
+			     avail[a].level[ERTS_TOPOLOGY_LOGICAL],
+			     avail[a].level[ERTS_TOPOLOGY_CG]);
+
+	erts_free(ERTS_ALC_T_TMP, cg_count);
+    }
+
+    erts_free(ERTS_ALC_T_TMP, avail);
+}
+
+static erts_cpu_groups_map_t *
+add_cpu_groups(int groups,
+	       erts_cpu_groups_callback_t callback,
+	       void *arg)
+{
+    int use_groups = groups;
+    erts_cpu_groups_callback_list_t *cgcl;
+    erts_cpu_groups_map_t *cgm;
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+
+    if (use_groups > max_main_threads)
+	use_groups = max_main_threads;
+
+    if (!use_groups)
+	return NULL;
+
+    no_cpu_groups_callbacks++;
+    cgcl = erts_alloc(ERTS_ALC_T_CPU_GRPS_MAP,
+		      sizeof(erts_cpu_groups_callback_list_t));
+    cgcl->callback = callback;
+    cgcl->arg = arg;
+
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
+	if (cgm->groups == use_groups) {
+	    cgcl->next = cgm->callback_list;
+	    cgm->callback_list = cgcl;
+	    return cgm;
+	}
+    }
+
+
+    cgm = erts_alloc(ERTS_ALC_T_CPU_GRPS_MAP,
+		     sizeof(erts_cpu_groups_map_t));
+    cgm->next = cpu_groups_maps;
+    cgm->groups = use_groups;
+    cgm->array = NULL;
+    cgm->size = 0;
+    cgm->logical_processors = 0;
+    cgm->callback_list = cgcl;
+
+    cgcl->next = NULL;
+
+    make_cpu_groups_map(cgm, 0);
+
+    cpu_groups_maps = cgm;
+
+    return cgm;
+}
+
+static void
+remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg)
+{
+    erts_cpu_groups_map_t *prev_cgm, *cgm;
+    erts_cpu_groups_callback_list_t *prev_cgcl, *cgcl;
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+
+    no_cpu_groups_callbacks--;
+
+    prev_cgm = NULL;
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
+	prev_cgcl = NULL;
+	for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) {
+	    if (cgcl->callback == callback && cgcl->arg == arg) {
+		if (prev_cgcl)
+		    prev_cgcl->next = cgcl->next;
+		else
+		    cgm->callback_list = cgcl->next;
+		erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgcl);
+		if (!cgm->callback_list) {
+		    if (prev_cgm)
+			prev_cgm->next = cgm->next;
+		    else
+			cpu_groups_maps = cgm->next;
+		    if (cgm->array)
+			erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm->array);
+		    erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm);
+		}
+		return;
+	    }
+	    prev_cgcl = cgcl;
+	}
+	prev_cgm = cgm;
+    }
+
+    erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n");
+}
+
+static int
+cpu_groups_lookup(erts_cpu_groups_map_t *map,
+		  ErtsSchedulerData *esdp)
+{
+    int start, logical, ix;
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
+		       || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+
+    if (esdp->cpu_id < 0)
+	return (((int) esdp->no) - 1) % map->groups;
+
+    logical = esdp->cpu_id;
+    start = logical % map->size;
+    ix = start;
+
+    do {
+	if (map->array[ix].logical == logical) {
+	    int group = map->array[ix].cpu_group;
+	    ASSERT(0 <= group && group < map->groups);
+	    return group;
+	}
+	ix++;
+	if (ix == map->size)
+	    ix = 0;
+    } while (ix != start);
+
+    erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical);
+}
+
+static void
+update_cpu_groups_maps(void)
+{
+    erts_cpu_groups_map_t *cgm;
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next)
+	make_cpu_groups_map(cgm, 0);
+}
+
+void
+erts_add_cpu_groups(int groups,
+		    erts_cpu_groups_callback_t callback,
+		    void *arg)
+{
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    add_cpu_groups(groups, callback, arg);
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+}
+
+void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback,
+			    void *arg)
+{
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    remove_cpu_groups(callback, arg);
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+}
diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h
new file mode 100644
index 0000000000..c5a9520b61
--- /dev/null
+++ b/erts/emulator/beam/erl_cpu_topology.h
@@ -0,0 +1,105 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description:	CPU topology and related functionality
+ *
+ * Author: 	Rickard Green
+ */
+
+#ifndef ERL_CPU_TOPOLOGY_H__
+#define ERL_CPU_TOPOLOGY_H__
+
+void erts_pre_early_init_cpu_topology(int *max_rg_p,
+				      int *conf_p,
+				      int *onln_p,
+				      int *avail_p);
+void erts_early_init_cpu_topology(int no_schedulers,
+				  int *max_main_threads_p,
+				  int max_reader_groups,
+				  int *reader_groups_p);
+void erts_init_cpu_topology(void);
+
+
+#define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS		0
+#define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED		1
+#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY	2
+#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE	3
+
+int erts_init_scheduler_bind_type_string(char *how);
+
+
+#define ERTS_INIT_CPU_TOPOLOGY_OK			0
+#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID		1
+#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE		2
+#define ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY	3
+#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE		4
+#define ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES		5
+#define ERTS_INIT_CPU_TOPOLOGY_MISSING_LID		6
+#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS		7
+#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES	8
+#define ERTS_INIT_CPU_TOPOLOGY_MISSING			9
+
+int erts_init_cpu_topology_string(char *topology_str);
+
+void erts_sched_check_cpu_bind(ErtsSchedulerData *esdp);
+#ifdef ERTS_SMP
+void erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp);
+void erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp);
+void erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp);
+#endif
+
+int erts_update_cpu_info(void);
+
+Eterm erts_bind_schedulers(Process *c_p, Eterm how);
+Eterm erts_get_schedulers_binds(Process *c_p);
+
+Eterm erts_get_reader_groups_map(Process *c_p);
+
+Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
+Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
+
+int erts_update_cpu_info(void);
+void erts_get_logical_processors(int *conf, int *onln, int *avail);
+
+int erts_sched_bind_atthrcreate_prepare(void);
+int erts_sched_bind_atthrcreate_child(int unbind);
+void erts_sched_bind_atthrcreate_parent(int unbind);
+
+int erts_sched_bind_atfork_prepare(void);
+int erts_sched_bind_atfork_child(int unbind);
+char *erts_sched_bind_atvfork_child(int unbind);
+void erts_sched_bind_atfork_parent(int unbind);
+
+Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);
+Eterm erts_debug_cpu_groups_map(Process *c_p, int groups);
+
+
+typedef void (*erts_cpu_groups_callback_t)(int,
+					   ErtsSchedulerData *,
+					   int,
+					   void *);
+
+void erts_add_cpu_groups(int groups,
+			 erts_cpu_groups_callback_t callback,
+			 void *arg);
+void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback,
+			    void *arg);
+
+#endif
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 15b1c6bb56..e0a6aa05c6 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -78,16 +78,24 @@ enum DbIterSafety {
 ** The main meta table, containing all ets tables.
 */
 #ifdef ERTS_SMP
-#  define META_MAIN_TAB_LOCK_CNT 16
-static union {
-    erts_smp_spinlock_t lck;
-    byte _cache_line_alignment[64];
-}meta_main_tab_locks[META_MAIN_TAB_LOCK_CNT];
+
+#define ERTS_META_MAIN_TAB_LOCK_TAB_BITS 8
+#define ERTS_META_MAIN_TAB_LOCK_TAB_SIZE (1 << ERTS_META_MAIN_TAB_LOCK_TAB_BITS)
+#define ERTS_META_MAIN_TAB_LOCK_TAB_MASK (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE - 1)
+
+typedef union {
+    erts_smp_rwmtx_t rwmtx;
+    byte cache_line_align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(
+				sizeof(erts_smp_rwmtx_t))];
+} erts_meta_main_tab_lock_t;
+
+static erts_meta_main_tab_lock_t *meta_main_tab_locks;
+
 #endif
 static struct {
     union {
 	DbTable *tb;     /* Only directly readable if slot is ALIVE */
-	Uint next_free;  /* (index<<2)|1 if slot is FREE */
+	UWord next_free;  /* (index<<2)|1 if slot is FREE */
     }u;
 } *meta_main_tab;
 
@@ -104,17 +112,13 @@ static struct {
 #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2)
 #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */
 
-static ERTS_INLINE void meta_main_tab_lock(unsigned slot)
-{
-#ifdef ERTS_SMP
-    erts_smp_spin_lock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck);
-#endif
-}
-
-static ERTS_INLINE void meta_main_tab_unlock(unsigned slot)
+static ERTS_INLINE erts_smp_rwmtx_t *
+get_meta_main_tab_lock(unsigned slot)
 {
 #ifdef ERTS_SMP
-    erts_smp_spin_unlock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck);
+    return &meta_main_tab_locks[slot & ERTS_META_MAIN_TAB_LOCK_TAB_MASK].rwmtx;
+#else
+    return NULL;
 #endif
 }
 
@@ -166,7 +170,8 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name,
 typedef enum {
     LCK_READ=1,     /* read only access */
     LCK_WRITE=2,    /* exclusive table write access */
-    LCK_WRITE_REC=3 /* record write access */
+    LCK_WRITE_REC=3, /* record write access */
+    LCK_NONE=4
 } db_lock_kind_t;
 
 extern DbTableMethod db_hash;
@@ -174,6 +179,7 @@ extern DbTableMethod db_tree;
 
 int user_requested_db_max_tabs;
 int erts_ets_realloc_always_moves;
+int erts_ets_always_compress;
 static int db_max_tabs;
 static DbTable *meta_pid_to_tab; /* Pid mapped to owned tables */
 static DbTable *meta_pid_to_fixed_tab; /* Pid mapped to fixed tables */
@@ -187,7 +193,7 @@ static Eterm ms_delete_all_buff[8]; /* To compare with for deletion
 
 static void fix_table_locked(Process* p, DbTable* tb);
 static void unfix_table_locked(Process* p,  DbTable* tb, db_lock_kind_t* kind);
-static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data);
+static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data);
 static void free_heir_data(DbTable*);
 static void free_fixations_locked(DbTable *tb);
 
@@ -213,61 +219,87 @@ Export ets_select_continue_exp;
  * Static traps
  */
 static Export ets_delete_continue_exp;
-
-static ERTS_INLINE DbTable* db_ref(DbTable* tb)
-{
-    if (tb != NULL) {
-	erts_refc_inc(&tb->common.ref, 2);
-    }
-    return tb;
-}
-
-static ERTS_INLINE DbTable* db_unref(DbTable* tb)
+	
+static void
+free_dbtable(DbTable* tb)
 {
-    if (!erts_refc_dectest(&tb->common.ref, 0)) {
 #ifdef HARDDEBUG
 	if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) {
-	    erts_fprintf(stderr, "ets: db_unref memory remain=%ld fix=%x\n",
-			 erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), 
+	    erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n",
+			 erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable),
 			 tb->common.fixations);
 	}
-	erts_fprintf(stderr, "ets: db_unref(%T) deleted!!!\r\n", 
+	erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n",
 		     tb->common.id);
 
-	erts_fprintf(stderr, "ets: db_unref: meta_pid_to_tab common.memory_size = %ld\n",
+	erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n",
 		     erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size));
 	print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab);
 
 
-	erts_fprintf(stderr, "ets: db_unref: meta_pid_to_fixed_tab common.memory_size = %ld\n",
+	erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n",
 		     erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size));
 	print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab);
-	
 #endif
 #ifdef ERTS_SMP
 	erts_smp_rwmtx_destroy(&tb->common.rwlock);
 	erts_smp_mtx_destroy(&tb->common.fixlock);
 #endif
 	ASSERT(is_immed(tb->common.heir_data));
-	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));		     
+	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
 	ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
-	return NULL;
-    }
-    return tb;
 }
 
-static ERTS_INLINE void db_init_lock(DbTable* tb, char *rwname, char* fixname)
+#ifdef ERTS_SMP
+static void
+chk_free_dbtable(void *vtb)
+{
+    DbTable * tb = (DbTable *) vtb;
+    ERTS_THR_MEMORY_BARRIER;
+    if (erts_refc_dectest(&tb->common.ref, 0) == 0)
+	free_dbtable(tb);
+}
+#endif
+
+static void schedule_free_dbtable(DbTable* tb)
 {
-    erts_refc_init(&tb->common.ref, 1);
-    erts_refc_init(&tb->common.fixref, 0);
+    /*
+     * NON-SMP case: Caller is *not* allowed to access the *tb
+     *               structure after this function has returned!          
+     * SMP case:     Caller is allowed to access the *tb structure
+     *               until the bif has returned (we typically
+     *               need to unlock the table lock after this
+     *               function has returned).
+     */
 #ifdef ERTS_SMP
-    erts_smp_rwmtx_init_x(&tb->common.rwlock, rwname, tb->common.the_name);
+    int scheds = erts_get_max_no_executing_schedulers();
+    ASSERT(scheds >= 1);
+    ASSERT(erts_refc_read(&tb->common.ref, 0) == 0);
+    erts_refc_init(&tb->common.ref, scheds);
+    ERTS_THR_MEMORY_BARRIER;
+    erts_smp_schedule_misc_aux_work(0, scheds, chk_free_dbtable, tb);
+#else
+    free_dbtable(tb);
+#endif
+}
+
+static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
+				     char *rwname, char* fixname)
+{
+#ifdef ERTS_SMP
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    if (use_frequent_read_lock)
+	rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+#endif
+#ifdef ERTS_SMP
+    erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt,
+			      rwname, tb->common.the_name);
     erts_smp_mtx_init_x(&tb->common.fixlock, fixname, tb->common.the_name);
     tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED);
 #endif
 }
 
-static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind)
+static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
 {
 #ifdef ERTS_SMP
     ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
@@ -295,16 +327,13 @@ static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind)
 #endif
 }
 
-static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
-{
-    (void) db_ref(tb);
-#ifdef ERTS_SMP
-    db_lock_take_over_ref(tb, kind);
-#endif
-}
-
 static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
 {
+    /*
+     * In NON-SMP case tb may refer to an already deallocated
+     * DbTable structure. That is, ONLY the SMP case is allowed
+     * to follow the tb pointer!
+     */
 #ifdef ERTS_SMP
     ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
 
@@ -331,7 +360,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
 	}
     }
 #endif
-    (void) db_unref(tb); /* May delete table... */
 }
 
 
@@ -349,56 +377,85 @@ static ERTS_INLINE void db_meta_unlock(DbTable* tb, db_lock_kind_t kind)
 }
 
 static ERTS_INLINE
-DbTable* db_get_table(Process *p,
-		      Eterm id,
-		      int what,
-		      db_lock_kind_t kind)
+DbTable* db_get_table_aux(Process *p,
+			  Eterm id,
+			  int what,
+			  db_lock_kind_t kind,
+			  int meta_already_locked)
 {
     DbTable *tb = NULL;
+    erts_smp_rwmtx_t *mtl = NULL;
+
+    /*
+     * IMPORTANT: Only scheduler threads are allowed
+     *            to access tables. Memory management
+     *            depend on it.
+     */
+    ASSERT(erts_get_scheduler_data());
 
     if (is_small(id)) {
 	Uint slot = unsigned_val(id) & meta_main_tab_slot_mask;
-	meta_main_tab_lock(slot);
-	if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) {
-	    /* SMP: inc to prevent race, between unlock of meta_main_tab_lock
-	     * and the table locking outside the meta_main_tab_lock
-	     */
-	    tb = db_ref(meta_main_tab[slot].u.tb);
+	if (!meta_already_locked) {
+	    mtl = get_meta_main_tab_lock(slot);
+	    erts_smp_rwmtx_rlock(mtl);
 	}
-	meta_main_tab_unlock(slot);
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+	else {
+	    erts_smp_rwmtx_t *test_mtl = get_meta_main_tab_lock(slot);
+	    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(test_mtl)
+			       || erts_lc_rwmtx_is_rwlocked(test_mtl));
+	}
+#endif
+	if (slot < db_max_tabs && IS_SLOT_ALIVE(slot))
+	    tb = meta_main_tab[slot].u.tb;
     }
     else if (is_atom(id)) {
-	erts_smp_rwmtx_t* rwlock;
-	struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&rwlock);
-	erts_smp_rwmtx_rlock(rwlock);
+	struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl);
+	if (!meta_already_locked)
+	    erts_smp_rwmtx_rlock(mtl);
+	else{
+	    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl)
+			       || erts_lc_rwmtx_is_rwlocked(mtl));
+	    mtl = NULL;
+	}
+
 	if (bucket->pu.tb != NULL) {
 	    if (is_atom(bucket->u.name_atom)) { /* single */
-		if (bucket->u.name_atom == id) {
-		    tb = db_ref(bucket->pu.tb);
-		}
+		if (bucket->u.name_atom == id)
+		    tb = bucket->pu.tb;
 	    }
 	    else { /* multi */
 		Uint cnt = unsigned_val(bucket->u.mcnt);
 		Uint i;
 		for (i=0; i<cnt; i++) {
 		    if (bucket->pu.mvec[i].u.name_atom == id) {
-			tb = db_ref(bucket->pu.mvec[i].pu.tb);
+			tb = bucket->pu.mvec[i].pu.tb;
 			break;
 		    }
 		}
 	    }
 	}
-	erts_smp_rwmtx_runlock(rwlock);
     }
     if (tb) {
-	db_lock_take_over_ref(tb, kind);
-	if (tb->common.id == id && ((tb->common.status & what) != 0 || 
-				    p->id == tb->common.owner)) {
-	    return tb;
+	db_lock(tb, kind);
+	if (tb->common.id != id
+	    || ((tb->common.status & what) == 0 && p->id != tb->common.owner)) {
+	    db_unlock(tb, kind);
+	    tb = NULL;
 	}
-	db_unlock(tb, kind);
     }
-    return NULL;
+    if (mtl)
+	erts_smp_rwmtx_runlock(mtl);
+    return tb;
+}
+
+static ERTS_INLINE
+DbTable* db_get_table(Process *p,
+		      Eterm id,
+		      int what,
+		      db_lock_kind_t kind)
+{
+    return db_get_table_aux(p, id, what, kind, 0);
 }
 
 /* Requires meta_main_tab_locks[slot] locked.
@@ -413,15 +470,15 @@ static ERTS_INLINE void free_slot(int slot)
     erts_smp_spin_unlock(&meta_main_tab_main_lock);
 }
 
-static int insert_named_tab(Eterm name_atom, DbTable* tb)
+static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock)
 {
     int ret = 0;
     erts_smp_rwmtx_t* rwlock;
     struct meta_name_tab_entry* new_entry;
     struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom,
 							      &rwlock);
-
-    erts_smp_rwmtx_rwlock(rwlock);
+    if (!have_lock)
+	erts_smp_rwmtx_rwlock(rwlock);
 
     if (bucket->pu.tb == NULL) { /* empty */
 	new_entry = bucket;
@@ -468,17 +525,32 @@ static int insert_named_tab(Eterm name_atom, DbTable* tb)
     ret = 1; /* Ok */
 
 done:
-    erts_smp_rwmtx_rwunlock(rwlock);
+    if (!have_lock)
+	erts_smp_rwmtx_rwunlock(rwlock);
     return ret;
 }
 
-static int remove_named_tab(Eterm name_atom)
+static int remove_named_tab(DbTable *tb, int have_lock)
 {
     int ret = 0;
     erts_smp_rwmtx_t* rwlock;
+    Eterm name_atom = tb->common.id;
     struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom,
 							      &rwlock);
-    erts_smp_rwmtx_rwlock(rwlock);
+#ifdef ERTS_SMP
+    if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) {
+	/*
+	 * We keep our increased refc over this op in order to
+	 * prevent the table from disapearing.
+	 */
+	erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
+	erts_smp_rwmtx_rwlock(rwlock);
+	erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+    }
+#endif
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock));
+
     if (bucket->pu.tb == NULL) {
 	goto done;
     }
@@ -529,7 +601,8 @@ static int remove_named_tab(Eterm name_atom)
     ret = 1; /* Ok */
 
 done:
-    erts_smp_rwmtx_rwunlock(rwlock);
+    if (!have_lock)
+	erts_smp_rwmtx_rwunlock(rwlock);
     return ret;
 }
 
@@ -538,11 +611,11 @@ done:
 */
 static ERTS_INLINE void local_fix_table(DbTable* tb)
 {
-    erts_refc_inc(&tb->common.fixref, 1);
+    erts_refc_inc(&tb->common.ref, 1);
 }	    
 static ERTS_INLINE void local_unfix_table(DbTable* tb)
 {	
-    if (erts_refc_dectest(&tb->common.fixref, 0) == 0) {
+    if (erts_refc_dectest(&tb->common.ref, 0) == 0) {
 	ASSERT(IS_HASH_TABLE(tb->common.status));
 	db_unfix_table_hash(&(tb->hash));
     }
@@ -704,12 +777,13 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3)
     int cret = DB_ERROR_BADITEM;
     Eterm list;
     Eterm iter;
-    Eterm cell[2];
+    DeclareTmpHeap(cell,2,BIF_P);
     DbUpdateHandle handle;
 
     if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
 	BIF_ERROR(BIF_P, BADARG);
     }
+    UseTmpHeap(2,BIF_P);
     if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
 	goto bail_out;
     }
@@ -762,6 +836,7 @@ finalize:
     tb->common.meth->db_finalize_dbterm(&handle);
 
 bail_out:
+    UnUseTmpHeap(2,BIF_P);
     db_unlock(tb, LCK_WRITE_REC);
 
     switch (cret) {
@@ -794,8 +869,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     Eterm* ret_list_currp = NULL;
     Eterm* ret_list_prevp = NULL;
     Eterm iter;
-    Eterm cell[2];
-    Eterm tuple[3];
+    DeclareTmpHeap(cell,5,BIF_P);
+    Eterm *tuple = cell+2;
     DbUpdateHandle handle;
     Uint halloc_size = 0; /* overestimated heap usage */
     Eterm* htop;          /* actual heap usage */
@@ -805,6 +880,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
 	BIF_ERROR(BIF_P, BADARG);
     }
+
+    UseTmpHeap(5,BIF_P);
+
     if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
 	goto bail_out;
     }
@@ -832,7 +910,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 	Eterm upop;
 	Eterm* tpl;
 	Sint position;
-	Eterm incr, warp, oldcnt;
+	Eterm incr, warp;
+	Wterm oldcnt;
 
 	if (is_not_list(iter)) {
 	    goto finalize;
@@ -871,7 +950,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 		position > arityval(handle.dbterm->tpl[0])) {
 		goto finalize;
 	    }
-	    oldcnt = handle.dbterm->tpl[position];
+	    oldcnt = db_do_read_element(&handle, position);
 	    if (is_big(oldcnt)) {
 		halloc_size += BIG_NEED_SIZE(big_arity(oldcnt));
 	    }
@@ -907,7 +986,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 	Eterm* tpl = tuple_val(CAR(list_val(iter)));
 	Sint position = signed_val(tpl[1]);
 	Eterm incr = tpl[2];
-	Eterm oldcnt = handle.dbterm->tpl[position];
+	Wterm oldcnt = db_do_read_element(&handle,position);
 	Eterm newcnt = db_add_counter(&htop, oldcnt, incr);
 
 	if (newcnt == NIL) {
@@ -920,9 +999,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 
 	if (arityval(*tpl) == 4) { /* Maybe warp it */
 	    Eterm threshold = tpl[3];
-	    if ((cmp(incr,make_small(0)) < 0) ? /* negative increment? */
-		(cmp(newcnt,threshold) < 0) :  /* if negative, check if below */
-		(cmp(newcnt,threshold) > 0)) { /* else check if above threshold */
+	    if ((CMP(incr,make_small(0)) < 0) ? /* negative increment? */
+		(CMP(newcnt,threshold) < 0) :  /* if negative, check if below */
+		(CMP(newcnt,threshold) > 0)) { /* else check if above threshold */
 
 		newcnt = tpl[4];
 	    }
@@ -951,6 +1030,7 @@ finalize:
     tb->common.meth->db_finalize_dbterm(&handle);
 
 bail_out:
+    UnUseTmpHeap(5,BIF_P);
     db_unlock(tb, LCK_WRITE_REC);
 
     switch (cret) {
@@ -1127,6 +1207,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 {
     DbTable* tb;
     Eterm ret;
+    erts_smp_rwmtx_t *lck1, *lck2;
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
@@ -1135,34 +1216,65 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 		BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]);
 #endif
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) {
+
+    if (is_not_atom(BIF_ARG_2)) {
 	BIF_ERROR(BIF_P, BADARG);
     }
 
-    if (is_not_atom(BIF_ARG_2)) {
-	goto badarg;
+    (void) meta_name_tab_bucket(BIF_ARG_2, &lck1);
+
+    if (is_small(BIF_ARG_1)) {
+	Uint slot = unsigned_val(BIF_ARG_1) & meta_main_tab_slot_mask;
+	lck2 = get_meta_main_tab_lock(slot);
+    }
+    else if (is_atom(BIF_ARG_1)) {
+	(void) meta_name_tab_bucket(BIF_ARG_1, &lck2);
+	if (lck1 == lck2)
+	    lck2 = NULL;
+	else if (lck1 > lck2) {
+	    erts_smp_rwmtx_t *tmp = lck1;
+	    lck1 = lck2;
+	    lck2 = tmp;
+	}
     }
+    else {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+
+    erts_smp_rwmtx_rwlock(lck1);
+    if (lck2)
+	erts_smp_rwmtx_rwlock(lck2);
+
+    tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1);
+    if (!tb)
+	goto badarg;
 
     if (is_not_atom(tb->common.id)) { /* Not a named table */
 	tb->common.the_name = BIF_ARG_2;
 	goto done;
     }
 
-    if (!insert_named_tab(BIF_ARG_2,tb)) {
+    if (!insert_named_tab(BIF_ARG_2, tb, 1))
 	goto badarg;
-    }
-    if (!remove_named_tab(tb->common.id)) {
+
+    if (!remove_named_tab(tb, 1))
 	erl_exit(1,"Could not find named tab %s", tb->common.id);
-    }
 
     tb->common.id = tb->common.the_name = BIF_ARG_2;
 
  done:
     ret = tb->common.id;
     db_unlock(tb, LCK_WRITE);
+    erts_smp_rwmtx_rwunlock(lck1);
+    if (lck2)
+	erts_smp_rwmtx_rwunlock(lck2);
     BIF_RET(ret);
  badarg:
-    db_unlock(tb, LCK_WRITE);
+    if (tb)
+	db_unlock(tb, LCK_WRITE);
+    erts_smp_rwmtx_rwunlock(lck1);
+    if (lck2)
+	erts_smp_rwmtx_rwunlock(lck2);
     BIF_ERROR(BIF_P, BADARG);    
 }
 
@@ -1180,13 +1292,14 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     Eterm val;
     Eterm ret;
     Eterm heir;
-    Eterm heir_data;
+    UWord heir_data;
     Uint32 status;
     Sint keypos;
-    int is_named, is_fine_locked;
+    int is_named, is_fine_locked, frequent_read, is_compressed;
     int cret;
-    Eterm meta_tuple[3];
+    DeclareTmpHeap(meta_tuple,3,BIF_P);
     DbTableMethod* meth;
+    erts_smp_rwmtx_t *mmtl;
 
     if (is_not_atom(BIF_ARG_1)) {
 	BIF_ERROR(BIF_P, BADARG);
@@ -1199,8 +1312,10 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     keypos = 1;
     is_named = 0;
     is_fine_locked = 0;
+    frequent_read = 0;
     heir = am_none;
-    heir_data = am_undefined;
+    heir_data = (UWord) am_undefined;
+    is_compressed = erts_ets_always_compress;
 
     list = BIF_ARG_2;
     while(is_list(list)) {
@@ -1232,6 +1347,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 			is_fine_locked = 0;
 		    } else break;
 		}
+		else if (tp[1] == am_read_concurrency) {
+		    if (tp[2] == am_true) {
+			frequent_read = 1;
+		    } else if (tp[2] == am_false) {
+			frequent_read = 0;
+		    } else break;
+		}
 		else if (tp[1] == am_heir && tp[2] == am_none) {
 		    heir = am_none;
 		    heir_data = am_undefined;
@@ -1256,6 +1378,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	else if (val == am_named_table) {
 	    is_named = 1;
 	}
+	else if (val == am_compressed) {
+	    is_compressed = 1;
+	}
 	else if (val == am_set || val == am_protected)
 	    ;
 	else break;
@@ -1280,6 +1405,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	BIF_ERROR(BIF_P, BADARG);
     }
 
+#ifdef ERTS_SMP
+    if (frequent_read && !(status & DB_PRIVATE))
+	status |= DB_FREQ_READ;
+#endif
+
     /* we create table outside any table lock
      * and take the unusal cost of destroy table if it
      * fails to find a slot 
@@ -1302,7 +1432,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.type = status & ERTS_ETS_TABLE_TYPES;
     /* Note, 'type' is *read only* from now on... */
 #endif
-    db_init_lock(tb, "db_tab", "db_tab_fix");
+    erts_refc_init(&tb->common.ref, 0);
+    db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ),
+		 "db_tab", "db_tab_fix");
     tb->common.keypos = keypos;
     tb->common.owner = BIF_P->id;
     set_heir(BIF_P, tb, heir, heir_data);
@@ -1310,6 +1442,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     erts_smp_atomic_init(&tb->common.nitems, 0);
 
     tb->common.fixations = NULL;
+    tb->common.compress = is_compressed;
 
     cret = meth->db_create(BIF_P, tb);
     ASSERT(cret == DB_ERROR_NONE);
@@ -1322,8 +1455,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 				      "** Too many db tables **\n");
 	free_heir_data(tb);
 	tb->common.meth->db_free_table(tb);
-	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
-	ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
+	free_dbtable(tb);
 	BIF_ERROR(BIF_P, SYSTEM_LIMIT);
     }
 
@@ -1345,19 +1477,22 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.id = ret;
     tb->common.slot = slot;           /* store slot for erase */
 
-    meta_main_tab_lock(slot);
+    mmtl = get_meta_main_tab_lock(slot);
+    erts_smp_rwmtx_rwlock(mmtl);
     meta_main_tab[slot].u.tb = tb;
     ASSERT(IS_SLOT_ALIVE(slot));
-    meta_main_tab_unlock(slot);
+    erts_smp_rwmtx_rwunlock(mmtl);
 
-    if (is_named && !insert_named_tab(BIF_ARG_1, tb)) {
-	meta_main_tab_lock(slot);
+    if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) {
+	mmtl = get_meta_main_tab_lock(slot);
+	erts_smp_rwmtx_rwlock(mmtl);
 	free_slot(slot);
-	meta_main_tab_unlock(slot);
+	erts_smp_rwmtx_rwunlock(mmtl);
 
-	db_lock_take_over_ref(tb,LCK_WRITE);
+	db_lock(tb,LCK_WRITE);
 	free_heir_data(tb);
 	tb->common.meth->db_free_table(tb);
+	schedule_free_dbtable(tb);
 	db_unlock(tb,LCK_WRITE);
 	BIF_ERROR(BIF_P, BADARG);
     }
@@ -1375,6 +1510,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 		     erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size));
 #endif
 
+    UseTmpHeap(3,BIF_P);
+
     db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
     if (db_put_hash(meta_pid_to_tab,
 		    TUPLE2(meta_tuple, BIF_P->id, make_small(slot)),
@@ -1383,6 +1520,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     }
     db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
 
+    UnUseTmpHeap(3,BIF_P);
+
     BIF_RET(ret);
 }
 
@@ -1489,6 +1628,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 {
     int trap;
     DbTable* tb;
+    erts_smp_rwmtx_t *mmtl;
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
@@ -1510,16 +1650,26 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE);
     tb->common.status |= DB_DELETE;
 
-    meta_main_tab_lock(tb->common.slot);
+    mmtl = get_meta_main_tab_lock(tb->common.slot);
+#ifdef ERTS_SMP
+    if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
+	/*
+	 * We keep our increased refc over this op in order to
+	 * prevent the table from disapearing.
+	 */
+	erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
+	erts_smp_rwmtx_rwlock(mmtl);
+	erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+    }
+#endif
     /* We must keep the slot, to be found by db_proc_dead() if process dies */
     MARK_SLOT_DEAD(tb->common.slot);
-    meta_main_tab_unlock(tb->common.slot);
-    if (is_atom(tb->common.id)) {
-	remove_named_tab(tb->common.id);
-    }
+    erts_smp_rwmtx_rwunlock(mmtl);
+    if (is_atom(tb->common.id))
+	remove_named_tab(tb, 0);
     
     if (tb->common.owner != BIF_P->id) {
-	Eterm meta_tuple[3];
+	DeclareTmpHeap(meta_tuple,3,BIF_P);
 
 	/*
 	 * The table is being deleted by a process other than its owner.
@@ -1527,6 +1677,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 	 * current process will be killed (e.g. by an EXIT signal), we will
 	 * now transfer the ownership to the current process.
 	 */
+	UseTmpHeap(3,BIF_P);
 	db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
 	db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
 			    make_small(tb->common.slot));
@@ -1538,6 +1689,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 		    TUPLE2(meta_tuple,BIF_P->id,make_small(tb->common.slot)),
 		    0);
 	db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
+	UnUseTmpHeap(3,BIF_P);
     }    
     /* disable inheritance */
     free_heir_data(tb);
@@ -1554,9 +1706,15 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 	 * (it looks like an continuation pointer), but that is will crash the
 	 * emulator if this BIF is call traced.
 	 */
+#if HALFWORD_HEAP
+	Eterm *hp = HAlloc(BIF_P, 3);
+	hp[0] = make_pos_bignum_header(2);
+	*((UWord *) (UWord) (hp+1)) = (UWord) tb;
+#else
 	Eterm *hp = HAlloc(BIF_P, 2);
 	hp[0] = make_pos_bignum_header(1);
 	hp[1] = (Eterm) tb;
+#endif
 	BIF_TRAP1(&ets_delete_continue_exp, BIF_P, make_big(hp));
     }
     else {
@@ -1571,7 +1729,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 {
     Process* to_proc = NULL;
     ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN;
-    Eterm buf[5];
+    DeclareTmpHeap(buf,5,BIF_P);
     Eterm to_pid = BIF_ARG_2;
     Eterm from_pid;
     DbTable* tb = NULL;
@@ -1593,6 +1751,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 	goto badarg;  /* or should we be idempotent? return false maybe */
     }
 
+    UseTmpHeap(5,BIF_P);
     db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
     db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
 			make_small(tb->common.slot));
@@ -1610,6 +1769,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 		      TUPLE4(buf, am_ETS_TRANSFER, tb->common.id, from_pid, BIF_ARG_3), 
 		      0);
     erts_smp_proc_unlock(to_proc, to_locks);
+    UnUseTmpHeap(5,BIF_P);
     BIF_RET(am_true);
 
 badarg:
@@ -1624,11 +1784,12 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2)
     Eterm* tp;
     Eterm opt;
     Eterm heir = THE_NON_VALUE;
-    Eterm heir_data = THE_NON_VALUE;
+    UWord heir_data = (UWord) THE_NON_VALUE;
     Uint32 protection = 0;
-    Eterm fakelist[2];
+    DeclareTmpHeap(fakelist,2,BIF_P);
     Eterm tail;
 
+    UseTmpHeap(2,BIF_P);
     for (tail = is_tuple(BIF_ARG_2) ? CONS(fakelist, BIF_ARG_2, NIL) : BIF_ARG_2;	
 	  is_list(tail);
 	  tail = CDR(list_val(tail))) {
@@ -1681,9 +1842,11 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2)
     }
 
     db_unlock (tb,LCK_WRITE);
+    UnUseTmpHeap(2,BIF_P);
     BIF_RET(am_true);
 
 badarg:
+    UnUseTmpHeap(2,BIF_P);
     if (tb != NULL) {
 	db_unlock(tb,LCK_WRITE);
     }
@@ -1896,14 +2059,15 @@ BIF_RETTYPE ets_all_0(BIF_ALIST_0)
     previous = NIL;
     j = 0;
     for(i = 0; (i < t_max_tabs && j < t_tabs_cnt); i++) {
-	meta_main_tab_lock(i);
+	erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i);
+	erts_smp_rwmtx_rlock(mmtl);
 	if (IS_SLOT_ALIVE(i)) {
 	    j++;
 	    tb = meta_main_tab[i].u.tb;
 	    previous = CONS(hp, tb->common.id, previous);
 	    hp += 2;
 	}
-	meta_main_tab_unlock(i);
+	erts_smp_rwmtx_runlock(mmtl);
     }
     HRelease(BIF_P, hendp, hp);
     BIF_RET(previous);
@@ -1949,29 +2113,37 @@ BIF_RETTYPE ets_match_1(BIF_ALIST_1)
 BIF_RETTYPE ets_match_2(BIF_ALIST_2)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarDollar, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_2(BIF_P, BIF_ARG_1, ms);
+    res = ets_select_2(BIF_P, BIF_ARG_1, ms);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 BIF_RETTYPE ets_match_3(BIF_ALIST_3)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarDollar, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 
@@ -2385,29 +2557,37 @@ BIF_RETTYPE ets_match_object_1(BIF_ALIST_1)
 BIF_RETTYPE ets_match_object_2(BIF_ALIST_2)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarUnderscore, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_2(BIF_P, BIF_ARG_1, ms);
+    res = ets_select_2(BIF_P, BIF_ARG_1, ms);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 BIF_RETTYPE ets_match_object_3(BIF_ALIST_3)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarUnderscore, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 /* 
@@ -2417,7 +2597,7 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3)
 BIF_RETTYPE ets_info_1(BIF_ALIST_1)
 {
     static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table,
-	am_node, am_size, am_name, am_heir, am_owner, am_memory};
+	am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed};
     Eterm results[sizeof(fields)/sizeof(Eterm)];
     DbTable* tb;
     Eterm res;
@@ -2533,7 +2713,6 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
     Binary *mp;
     Eterm res;
     Uint32 dummy;
-    Uint sz;
 
     if (!(is_list(BIF_ARG_1) || BIF_ARG_1 == NIL) || !is_binary(BIF_ARG_2)) {
     error:
@@ -2558,11 +2737,10 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
 	    BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3],
 		      BIF_P,lst,BIF_ARG_2,ret);
 	}
-	res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), 0, &dummy);
+	res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, NULL, 0,
+			    ERTS_PAM_COPY_RESULT, &dummy);
 	if (is_value(res)) {
-	    sz = size_object(res);
-	    hp = HAlloc(BIF_P, sz + 2);
-	    res = copy_struct(res, sz, &hp, &MSO(BIF_P));
+	    hp = HAlloc(BIF_P, 2);
 	    ret = CONS(hp,res,ret);
 	    /*hp += 2;*/
 	} 
@@ -2585,18 +2763,29 @@ void init_db(void)
 {
     DbTable init_tb;
     int i;
-    extern Eterm* em_apply_bif;
+    extern BeamInstr* em_apply_bif;
     Eterm *hp;
     unsigned bits;
     size_t size;
 
 #ifdef ERTS_SMP
-    for (i=0; i<META_MAIN_TAB_LOCK_CNT; i++) {
-	erts_smp_spinlock_init_x(&meta_main_tab_locks[i].lck, "meta_main_tab_slot", make_small(i));
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+    meta_main_tab_locks =
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES,
+					   sizeof(erts_meta_main_tab_lock_t)
+					   * ERTS_META_MAIN_TAB_LOCK_TAB_SIZE);
+
+    for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) {
+	erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt,
+				  "meta_main_tab_slot", make_small(i));
     }
     erts_smp_spinlock_init(&meta_main_tab_main_lock, "meta_main_tab_main");
     for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) {
-	erts_smp_rwmtx_init_x(&meta_name_tab_rwlocks[i].lck, "meta_name_tab", make_small(i));
+	erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt,
+				  "meta_name_tab", make_small(i));
     }
 #endif
 
@@ -2664,9 +2853,9 @@ void init_db(void)
     erts_smp_atomic_init(&meta_pid_to_tab->common.nitems, 0);
     meta_pid_to_tab->common.slot   = -1;
     meta_pid_to_tab->common.meth   = &db_hash;
+    meta_pid_to_tab->common.compress = 0;
 
-    erts_refc_init(&meta_pid_to_tab->common.ref, 1);
-    erts_refc_init(&meta_pid_to_tab->common.fixref, 0);
+    erts_refc_init(&meta_pid_to_tab->common.ref, 0);
     /* Neither rwlock or fixlock used
     db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/
 
@@ -2696,9 +2885,9 @@ void init_db(void)
     erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.nitems, 0);
     meta_pid_to_fixed_tab->common.slot   = -1;
     meta_pid_to_fixed_tab->common.meth   = &db_hash;
+    meta_pid_to_fixed_tab->common.compress = 0;
 
-    erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 1);
-    erts_refc_init(&meta_pid_to_fixed_tab->common.fixref, 0);
+    erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 0);
     /* Neither rwlock or fixlock used
     db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/
 
@@ -2714,9 +2903,9 @@ void init_db(void)
     ets_select_delete_continue_exp.code[1] = am_atom_put("delete_trap",11);
     ets_select_delete_continue_exp.code[2] = 1;
     ets_select_delete_continue_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     ets_select_delete_continue_exp.code[4] = 
-	(Eterm) &ets_select_delete_1;
+	(BeamInstr) &ets_select_delete_1;
 
     /* Non visual BIF to trap to. */
     memset(&ets_select_count_continue_exp, 0, sizeof(Export));
@@ -2726,9 +2915,9 @@ void init_db(void)
     ets_select_count_continue_exp.code[1] = am_atom_put("count_trap",11);
     ets_select_count_continue_exp.code[2] = 1;
     ets_select_count_continue_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     ets_select_count_continue_exp.code[4] = 
-	(Eterm) &ets_select_count_1;
+	(BeamInstr) &ets_select_count_1;
 
     /* Non visual BIF to trap to. */
     memset(&ets_select_continue_exp, 0, sizeof(Export));
@@ -2738,9 +2927,9 @@ void init_db(void)
     ets_select_continue_exp.code[1] = am_atom_put("select_trap",11);
     ets_select_continue_exp.code[2] = 1;
     ets_select_continue_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     ets_select_continue_exp.code[4] = 
-	(Eterm) &ets_select_trap_1;
+	(BeamInstr) &ets_select_trap_1;
 
     /* Non visual BIF to trap to. */
     memset(&ets_delete_continue_exp, 0, sizeof(Export));
@@ -2748,8 +2937,8 @@ void init_db(void)
     ets_delete_continue_exp.code[0] = am_ets;
     ets_delete_continue_exp.code[1] = am_atom_put("delete_trap",11);
     ets_delete_continue_exp.code[2] = 1;
-    ets_delete_continue_exp.code[3] = (Eterm) em_apply_bif;
-    ets_delete_continue_exp.code[4] = (Eterm) &ets_delete_trap;
+    ets_delete_continue_exp.code[3] = (BeamInstr) em_apply_bif;
+    ets_delete_continue_exp.code[4] = (BeamInstr) &ets_delete_trap;
 
     hp = ms_delete_all_buff;
     ms_delete_all = CONS(hp, am_true, NIL);
@@ -2843,9 +3032,9 @@ static int give_away_to_heir(Process* p, DbTable* tb)
 {
     Process* to_proc;
     ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN;
-    Eterm buf[5];
+    DeclareTmpHeap(buf,5,p);
     Eterm to_pid;
-    Eterm heir_data;
+    UWord heir_data;
 
     ASSERT(tb->common.owner == p->id);
     ASSERT(is_internal_pid(tb->common.heir));
@@ -2856,12 +3045,10 @@ retry:
 				to_pid, to_locks,
 				ERTS_P2P_FLG_TRY_LOCK);
     if (to_proc == ERTS_PROC_LOCK_BUSY) {
-	db_ref(tb); /* while unlocked */
 	db_unlock(tb,LCK_WRITE);    
 	to_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
 				to_pid, to_locks);    
 	db_lock(tb,LCK_WRITE);
-	tb = db_unref(tb);
 	ASSERT(tb != NULL);
     
 	if (tb->common.owner != p->id) {
@@ -2888,6 +3075,7 @@ retry:
 	erts_smp_proc_unlock(to_proc, to_locks);
 	return 0; /* heir dead and pid reused, table still mine */
     }
+    UseTmpHeap(5,p);
     db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
     db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
 			make_small(tb->common.slot));
@@ -2899,11 +3087,11 @@ retry:
 		TUPLE2(buf,to_pid,make_small(tb->common.slot)),
 		0);
     db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
-    
+    UnUseTmpHeap(5,p);
     db_unlock(tb,LCK_WRITE);
     heir_data = tb->common.heir_data;
     if (!is_immed(heir_data)) {
-	Eterm* tpv = DBTERM_BUF((DbTerm*)heir_data); /* tuple_val */
+	Eterm* tpv = ((DbTerm*)heir_data)->tpl; /* tuple_val */
 	ASSERT(arityval(*tpv) == 1);
 	heir_data = tpv[1];
     }
@@ -2968,15 +3156,16 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 	    while (state->slots.ix < state->slots.size) {
 		DbTable *tb = NULL;
 		Sint ix = unsigned_val(state->slots.arr[state->slots.ix]);
-		meta_main_tab_lock(ix);
+		erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix);
+		erts_smp_rwmtx_rlock(mmtl);
 		if (!IS_SLOT_FREE(ix)) {
-		    tb = db_ref(GET_ANY_SLOT_TAB(ix));
+		    tb = GET_ANY_SLOT_TAB(ix);
 		    ASSERT(tb);
 		}
-		meta_main_tab_unlock(ix);
+		erts_smp_rwmtx_runlock(mmtl);
 		if (tb) {
 		    int do_yield;
-		    db_lock_take_over_ref(tb, LCK_WRITE);
+		    db_lock(tb, LCK_WRITE);
 		    /* Ownership may have changed since
 		       we looked up the table. */
 		    if (tb->common.owner != pid) {
@@ -3005,7 +3194,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 			    tb->common.status |= DB_DELETE;
 
 			    if (is_atom(tb->common.id))
-				remove_named_tab(tb->common.id);
+				remove_named_tab(tb, 0);
 
 			    free_heir_data(tb);
 			    free_fixations_locked(tb);
@@ -3055,17 +3244,18 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 	    while (state->slots.ix < state->slots.size) {
 		DbTable *tb = NULL;
 		Sint ix = unsigned_val(state->slots.arr[state->slots.ix]);
-		meta_main_tab_lock(ix);
+		erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix);
+		erts_smp_rwmtx_rlock(mmtl);
 		if (IS_SLOT_ALIVE(ix)) {
-		    tb = db_ref(meta_main_tab[ix].u.tb);
+		    tb = meta_main_tab[ix].u.tb;
 		    ASSERT(tb);
 		}
-		meta_main_tab_unlock(ix);
+		erts_smp_rwmtx_runlock(mmtl);
 		if (tb) {
 		    int reds;
 		    DbFixation** pp;
 
-		    db_lock_take_over_ref(tb, LCK_WRITE_REC);
+		    db_lock(tb, LCK_WRITE_REC);
 		    #ifdef ERTS_SMP
 		    erts_smp_mtx_lock(&tb->common.fixlock);
 		    #endif
@@ -3075,7 +3265,8 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 			  pp = &(*pp)->next) {
 			if ((*pp)->pid == pid) {
 			    DbFixation* fix = *pp;
-			    erts_refc_add(&tb->common.fixref,-fix->counter,0);
+			    erts_aint_t diff = -((erts_aint_t) fix->counter);
+			    erts_refc_add(&tb->common.ref,diff,0);
 			    *pp = fix->next;
 			    erts_db_free(ERTS_ALC_T_DB_FIXATION,
 					 tb, fix, sizeof(DbFixation));
@@ -3145,12 +3336,12 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 static void fix_table_locked(Process* p, DbTable* tb)
 {
     DbFixation *fix;
-    Eterm meta_tuple[3];
+    DeclareTmpHeap(meta_tuple,3,p);
 
 #ifdef ERTS_SMP
     erts_smp_mtx_lock(&tb->common.fixlock);
 #endif
-    erts_refc_inc(&tb->common.fixref,1);
+    erts_refc_inc(&tb->common.ref,1);
     fix = tb->common.fixations;
     if (fix == NULL) { 
 	get_now(&(tb->common.megasec),
@@ -3179,12 +3370,15 @@ static void fix_table_locked(Process* p, DbTable* tb)
     erts_smp_mtx_unlock(&tb->common.fixlock);
 #endif
     p->flags |= F_USING_DB;        
+    UseTmpHeap(3,p);
     db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
     if (db_put_hash(meta_pid_to_fixed_tab,
 		    TUPLE2(meta_tuple, p->id, make_small(tb->common.slot)),
 		    0) != DB_ERROR_NONE) {
+	UnUseTmpHeap(3,p);
 	erl_exit(1,"Could not insert ets metadata in safe_fixtable.");
     }	
+    UnUseTmpHeap(3,p);
     db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
 }
 
@@ -3201,7 +3395,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
     for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) {
 	if ((*pp)->pid == p->id) {
 	    DbFixation* fix = *pp;
-	    erts_refc_dec(&tb->common.fixref,0);
+	    erts_refc_dec(&tb->common.ref,0);
 	    --(fix->counter);
 	    ASSERT(fix->counter >= 0);
 	    if (fix->counter > 0) {
@@ -3227,7 +3421,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
 unlocked:
 
     if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)
-	&& erts_smp_atomic_read(&tb->hash.fixdel) != (long)NULL) {
+	&& erts_smp_atomic_read(&tb->hash.fixdel) != (erts_aint_t)NULL) {
 #ifdef ERTS_SMP
 	if (*kind_p == LCK_READ && tb->common.is_thread_safe) {
 	    /* Must have write lock while purging pseudo-deleted (OTP-8166) */
@@ -3249,6 +3443,8 @@ static void free_fixations_locked(DbTable *tb)
 
     fix = tb->common.fixations;
     while (fix != NULL) {
+	erts_aint_t diff = -((erts_aint_t) fix->counter);
+	erts_refc_add(&tb->common.ref,diff,0);
 	next_fix = fix->next;
 	db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
 	db_erase_bag_exact2(meta_pid_to_fixed_tab,
@@ -3264,7 +3460,7 @@ static void free_fixations_locked(DbTable *tb)
     tb->common.fixations = NULL;
 }
 
-static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data)
+static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data)
 {	
     tb->common.heir = heir;
     if (heir == am_none) {
@@ -3285,10 +3481,26 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data)
     }
 
     if (!is_immed(heir_data)) {
-	Eterm tmp[2];
-	/* Make a dummy 1-tuple around data to use db_get_term() */
-	heir_data = (Eterm) db_get_term(&tb->common, NULL, 0,
-					TUPLE1(tmp,heir_data));
+	DeclareTmpHeap(tmp,2,me);
+	Eterm wrap_tpl;
+	int size;
+	DbTerm* dbterm;
+	Eterm* top;
+	ErlOffHeap tmp_offheap;
+
+	UseTmpHeap(2,me);
+	/* Make a dummy 1-tuple around data to use DbTerm */
+	wrap_tpl = TUPLE1(tmp,heir_data);
+	size = size_object(wrap_tpl);
+	dbterm = erts_db_alloc(ERTS_ALC_T_DB_HEIR_DATA, (DbTable *)tb,
+			       (sizeof(DbTerm) + sizeof(Eterm)*(size-1)));
+	dbterm->size = size;
+	top = dbterm->tpl;
+	tmp_offheap.first  = NULL;
+	copy_struct(wrap_tpl, size, &top, &tmp_offheap);
+	dbterm->first_oh = tmp_offheap.first;
+	heir_data = (UWord)dbterm;
+	UnUseTmpHeap(2,me);
 	ASSERT(!is_immed(heir_data));
     }
     tb->common.heir_data = heir_data;
@@ -3298,8 +3510,8 @@ static void free_heir_data(DbTable* tb)
 {
     if (tb->common.heir != am_none && !is_immed(tb->common.heir_data)) {
 	DbTerm* p = (DbTerm*) tb->common.heir_data;
-	db_free_term_data(p);
-	erts_db_free(ERTS_ALC_T_DB_TERM, tb, (void *)p,
+	db_cleanup_offheap_comp(p);
+	erts_db_free(ERTS_ALC_T_DB_HEIR_DATA, tb, (void *)p,
 		     sizeof(DbTerm) + (p->size-1)*sizeof(Eterm));
     }
     #ifdef DEBUG
@@ -3311,10 +3523,13 @@ static BIF_RETTYPE ets_delete_trap(Process *p, Eterm cont)
 {
     int trap;
     Eterm* ptr = big_val(cont);
-    DbTable *tb = (DbTable *) ptr[1];
+    DbTable *tb = *((DbTable **) (UWord) (ptr + 1));
 
+#if HALFWORD_HEAP
+    ASSERT(*ptr == make_pos_bignum_header(2));
+#else
     ASSERT(*ptr == make_pos_bignum_header(1));
-
+#endif
     db_lock(tb, LCK_WRITE);
     trap = free_table_cont(p, tb, 0, 1);
     db_unlock(tb, LCK_WRITE);
@@ -3337,6 +3552,7 @@ static int free_table_cont(Process *p,
 			   int clean_meta_tab)
 {
     Eterm result;
+    erts_smp_rwmtx_t *mmtl;
 
 #ifdef HARDDEBUG
     if (!first) {
@@ -3362,9 +3578,16 @@ static int free_table_cont(Process *p,
 		     tb->common.id);
 #endif
 	/* Completely done - we will not get called again. */
-	meta_main_tab_lock(tb->common.slot);
+	mmtl = get_meta_main_tab_lock(tb->common.slot);
+#ifdef ERTS_SMP
+	if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
+	    erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
+	    erts_smp_rwmtx_rwlock(mmtl);
+	    erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+	}
+#endif
 	free_slot(tb->common.slot);
-	meta_main_tab_unlock(tb->common.slot);
+	erts_smp_rwmtx_rwunlock(mmtl);
 
 	if (clean_meta_tab) {
 	    db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
@@ -3372,7 +3595,7 @@ static int free_table_cont(Process *p,
 				make_small(tb->common.slot));
 	    db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
 	}
-	db_unref(tb);
+	schedule_free_dbtable(tb);
 	BUMP_REDS(p, 100);
 	return 0;
     }
@@ -3420,10 +3643,13 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	ret = erts_this_dist_entry->sysname;
     } else if (What == am_named_table) {
 	ret = is_atom(tb->common.id) ? am_true : am_false;
+    } else if (What == am_compressed) {
+	ret = tb->common.compress ? am_true : am_false;
+    }
     /*
      * For debugging purposes
      */
-    } else if (What == am_data) { 
+    else if (What == am_data) {
 	print_table(ERTS_PRINT_STDOUT, NULL, 1, tb);
 	ret = am_true;
     } else if (What == am_atom_put("fixed",5)) { 
@@ -3511,7 +3737,7 @@ static void print_table(int to, void *to_arg, int show,  DbTable* tb)
 
     erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read(&tb->common.nitems));
     erts_print(to, to_arg, "Words: %bpu\n",
-	       (Uint) ((erts_smp_atomic_read(&tb->common.memory_size)
+	       (UWord) ((erts_smp_atomic_read(&tb->common.memory_size)
 			+ sizeof(Uint)
 			- 1)
 		       / sizeof(Uint)));
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 7da28fad29..e0bdebcb01 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -61,6 +61,7 @@ void erts_db_foreach_offheap(DbTable *,
 
 extern int user_requested_db_max_tabs; /* set in erl_init */
 extern int erts_ets_realloc_always_moves;  /* set in erl_init */
+extern int erts_ets_always_compress;  /* set in erl_init */
 extern Export ets_select_delete_continue_exp;
 extern Export ets_select_count_continue_exp;
 extern Export ets_select_continue_exp;
@@ -82,7 +83,8 @@ Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt);
 
 #define ERTS_DB_ALC_MEM_UPDATE_(TAB, FREE_SZ, ALLOC_SZ)			\
 do {									\
-    long sz__ = ((long) (ALLOC_SZ)) - ((long) (FREE_SZ));		\
+    erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ))			\
+			- ((erts_aint_t) (FREE_SZ)));			\
     ASSERT((TAB));							\
     erts_smp_atomic_add(&(TAB)->common.memory_size, sz__);		\
 } while (0)
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 4141f9766b..fdc82c8b88 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -105,7 +105,7 @@
 #define NSEG_2   256 /* Size of second segment table */
 #define NSEG_INC 128 /* Number of segments to grow after that */
 
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read(&(tb)->segtab))
+#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_acqb(&(tb)->segtab))
 #define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive))
 #define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems))
 
@@ -122,8 +122,8 @@
 */
 static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
 {
-    Uint mask = erts_smp_atomic_read(&tb->szm);
-    Uint ix = hval & mask; 
+    Uint mask = erts_smp_atomic_read_acqb(&tb->szm);
+    Uint ix = hval & mask;
     if (ix >= erts_smp_atomic_read(&tb->nactive)) {
 	ix &= mask>>1;
 	ASSERT(ix < erts_smp_atomic_read(&tb->nactive));
@@ -135,8 +135,8 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
 */
 static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix)
 {
-    long was_next;
-    long exp_next;
+    erts_aint_t was_next;
+    erts_aint_t exp_next;
     FixedDeletion* fixd = (FixedDeletion*) erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL,
 							 (DbTable *) tb,
 							 sizeof(FixedDeletion));
@@ -146,7 +146,9 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix)
     do { /* Lockless atomic insertion in linked list: */
 	exp_next = was_next;
 	fixd->next = (FixedDeletion*) exp_next;
-	was_next = erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixd, exp_next);
+	was_next = erts_smp_atomic_cmpxchg(&tb->fixdel,
+					   (erts_aint_t) fixd,
+					   exp_next);
     }while (was_next != exp_next);
 }
 
@@ -256,22 +258,16 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix,
 }
 
 
-/* 
- * tplp is an untagged pointer to a tuple we know is large enough 
- * and dth is a pointer to a DbTableHash.   
- */
-#define GETKEY(dth, tplp)   (*((tplp) +  (dth)->common.keypos))
-
 /*
  * Some special binary flags
  */
 #define BIN_FLAG_ALL_OBJECTS         BIN_FLAG_USR1
 
-/*
- * Size calculations
- */
-#define SIZ_OVERHEAD ((sizeof(HashDbTerm)/sizeof(Eterm)) - 1)
-#define SIZ_DBTERM(HDT) (SIZ_OVERHEAD + (HDT)->dbterm.size)
+
+static ERTS_INLINE void free_term(DbTableHash *tb, HashDbTerm* p)
+{
+    db_free_term((DbTable*)tb, p, offsetof(HashDbTerm, dbterm));
+}
 
 /*
  * Local types 
@@ -358,10 +354,8 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key,
 			       HashValue hval, HashDbTerm *list);
 static void shrink(DbTableHash* tb, int nactive);
 static void grow(DbTableHash* tb, int nactive);
-static void free_term(DbTableHash *tb, HashDbTerm* p);
-static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2);
-static HashDbTerm* get_term(DbTableHash* tb, HashDbTerm* old, 
-			    Eterm obj, HashValue hval);
+static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
+			   DbTableHash*);
 static int analyze_pattern(DbTableHash *tb, Eterm pattern, 
 			   struct mp_info *mpi);
 
@@ -434,6 +428,9 @@ static ERTS_INLINE void try_shrink(DbTableHash* tb)
     }
 }	
 
+#define EQ_REL(x,y,y_base) \
+    (is_same(x,NULL,y,y_base) || (is_not_both_immed((x),(y)) && eq_rel((x),NULL,(y),y_base)))
+
 /* Is this a live object (not pseodo-deleted) with the specified key? 
 */
 static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b,
@@ -442,7 +439,8 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b,
     if (b->hvalue != hval) return 0;
     else {
 	Eterm itemKey = GETKEY(tb, b->dbterm.tpl);
-	return EQ(key,itemKey);
+	ASSERT(!is_header(itemKey));
+	return EQ_REL(key, itemKey, b->dbterm.tpl);
     }
 }
 
@@ -454,10 +452,38 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b,
     if (b->hvalue != hval && b->hvalue != INVALID_HASH) return 0;
     else {
 	Eterm itemKey = GETKEY(tb, b->dbterm.tpl);
-	return EQ(key,itemKey);
+	ASSERT(!is_header(itemKey));
+	return EQ_REL(key, itemKey, b->dbterm.tpl);
+    }
+}
+
+static ERTS_INLINE HashDbTerm* new_dbterm(DbTableHash* tb, Eterm obj)
+{
+    HashDbTerm* p;
+    if (tb->common.compress) {
+	p = db_store_term_comp(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj);
+    }
+    else {
+	p = db_store_term(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj);
     }
+    return p;
 }
 
+static ERTS_INLINE HashDbTerm* replace_dbterm(DbTableHash* tb, HashDbTerm* old,
+					      Eterm obj)
+{
+    HashDbTerm* ret;
+    ASSERT(old != NULL);
+    if (tb->common.compress) {
+	ret = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj);
+    }
+    else {
+	ret = db_store_term(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj);
+    }
+    return ret;
+}
+
+
 
 /*
 ** External interface 
@@ -514,12 +540,12 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel)
 {
     /*int tries = 0;*/
     DEBUG_WAIT();
-    if (erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixdel,
-				(long)NULL) != (long)NULL) {
+    if (erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel,
+				(erts_aint_t)NULL) != (erts_aint_t)NULL) {
 	/* Oboy, must join lists */    
 	FixedDeletion* last = fixdel;
-	long was_tail;
-	long exp_tail;
+	erts_aint_t was_tail;
+	erts_aint_t exp_tail;
 
 	while (last->next != NULL) last = last->next;	
 	was_tail = erts_smp_atomic_read(&tb->fixdel);
@@ -528,7 +554,7 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel)
 	    last->next = (FixedDeletion*) exp_tail;
 	    /*++tries;*/
 	    DEBUG_WAIT();
-	    was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixdel,
+	    was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel,
 					       exp_tail);
 	}while (was_tail != exp_tail);
     }
@@ -546,7 +572,7 @@ void db_unfix_table_hash(DbTableHash *tb)
 		       || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock)
 			   && !tb->common.is_thread_safe));
 restart:
-    fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (long)NULL);
+    fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (erts_aint_t)NULL);
     while (fixdel != NULL) {
 	FixedDeletion *fx = fixdel;
 	int ix = fx->slot;
@@ -615,20 +641,24 @@ int db_create_hash(Process *p, DbTable *tbl)
 
     erts_smp_atomic_init(&tb->szm, SEGSZ_MASK);
     erts_smp_atomic_init(&tb->nactive, SEGSZ);
-    erts_smp_atomic_init(&tb->fixdel, (long)NULL);
-    erts_smp_atomic_init(&tb->segtab, (long) alloc_ext_seg(tb,0,NULL)->segtab);
+    erts_smp_atomic_init(&tb->fixdel, (erts_aint_t)NULL);
+    erts_smp_atomic_init(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab);
     tb->nsegs = NSEG_1;
     tb->nslots = SEGSZ;
 
     erts_smp_atomic_init(&tb->is_resizing, 0);
 #ifdef ERTS_SMP
     if (tb->common.type & DB_FINE_LOCKED) {
+	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 	int i;
+	if (tb->common.type & DB_FREQ_READ)
+	    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 	tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ 
 							      (DbTable *) tb,
 							      sizeof(DbTableHashFineLocks));	    	    
 	for (i=0; i<DB_HASH_LOCK_CNT; ++i) {
-	    erts_rwmtx_init_x(&tb->locks->lck_vec[i].lck, "db_hash_slot", make_small(i));
+	    erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt,
+				      "db_hash_slot", make_small(i));
 	}
 	/* This important property is needed to guarantee that the buckets
     	 * involved in a grow/shrink operation it protected by the same lock:
@@ -638,6 +668,7 @@ int db_create_hash(Process *p, DbTable *tbl)
     else { /* coarse locking */
 	tb->locks = NULL;
     }
+    ERTS_THR_MEMORY_BARRIER;
 #endif /* ERST_SMP */
     return DB_ERROR_NONE;
 }
@@ -663,9 +694,7 @@ static int db_first_hash(Process *p, DbTable *tbl, Eterm *ret)
 	}
     }
     if (list != NULL) {
-	Eterm key = GETKEY(tb, list->dbterm.tpl);
-	
-	COPY_OBJECT(key, p, ret);
+	*ret = db_copy_key(p, tbl, &list->dbterm);
 	RUNLOCK_HASH(lck);
     }
     else {
@@ -713,7 +742,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 	*ret = am_EOT;
     }
     else {
-	COPY_OBJECT(GETKEY(tb, b->dbterm.tpl), p, ret);
+	*ret = db_copy_key(p, tbl, &b->dbterm);
 	RUNLOCK_HASH(lck);
     }    
     return DB_ERROR_NONE;
@@ -760,7 +789,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
 	    ret = DB_ERROR_BADKEY;
 	    goto Ldone;
 	}
-	q = get_term(tb, b, obj, hval);
+	q = replace_dbterm(tb, b, obj);
 	q->next = bnext;
 	q->hvalue = hval; /* In case of INVALID_HASH */
 	*bp = q;
@@ -780,7 +809,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
 	HashDbTerm** qp = bp;
 	q = b;
 	do {
-	    if (eq(make_tuple(q->dbterm.tpl), obj)) {
+	    if (db_eq(&tb->common,obj,&q->dbterm)) {
 		if (q->hvalue == INVALID_HASH) {
 		    erts_smp_atomic_inc(&tb->common.nitems);
 		    q->hvalue = hval;
@@ -799,7 +828,8 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
     /*else DB_DUPLICATE_BAG */
 
 Lnew:
-    q = get_term(tb, NULL, obj, hval);
+    q = new_dbterm(tb, obj);
+    q->hvalue = hval;
     q->next = b;
     *bp = q;
     nitems = erts_smp_atomic_inctest(&tb->common.nitems);
@@ -840,7 +870,7 @@ int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 		while(b2 != NULL && has_key(tb,b2,key,hval))
 		    b2 = b2->next;
 	    }
-	    copy = put_term_list(p, b1, b2);
+	    copy = build_term_list(p, b1, b2, tb);
 	    CHECK_TABLES();
 	    *ret = copy;
 	    goto done;
@@ -963,13 +993,10 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
 
     while(b1 != 0) {
 	if (has_live_key(tb,b1,key,hval)) {
-	    Eterm copy;
-
 	    if (ndex > arityval(b1->dbterm.tpl[0])) {
 		retval = DB_ERROR_BADITEM;
 		goto done;
 	    }
-
 	    if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) {
 		HashDbTerm* b;
 		HashDbTerm* b2 = b1->next;
@@ -983,15 +1010,12 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
 		    }
 		    b2 = b2->next;
 		}
-
 		b = b1;
 		while(b != b2) {
 		    if (b->hvalue != INVALID_HASH) {
 			Eterm *hp;
-			Uint sz = size_object(b->dbterm.tpl[ndex])+2;
-			
-			hp = HAlloc(p, sz);
-			copy = copy_struct(b->dbterm.tpl[ndex], sz-2, &hp, &MSO(p));
+			Eterm copy = db_copy_element_from_ets(&tb->common, p,
+							      &b->dbterm, ndex, &hp, 2);
 			elem_list = CONS(hp, copy, elem_list);
 			hp += 2;
 		    }
@@ -1000,8 +1024,8 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
 		*ret = elem_list;
 	    }
 	    else {
-		COPY_OBJECT(b1->dbterm.tpl[ndex], p, &copy);
-		*ret = copy;
+		Eterm* hp;
+		*ret = db_copy_element_from_ets(&tb->common, p, &b1->dbterm, ndex, &hp, 0);
 	    }
 	    retval = DB_ERROR_NONE;
 	    goto done;
@@ -1036,6 +1060,7 @@ int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value)
 
     ASSERT(!IS_FIXED(tb));
     ASSERT((tb->common.status & DB_BAG));
+    ASSERT(!tb->common.compress);
 
     while(b != 0) {
 	if (has_live_key(tb,b,key,hval)) {
@@ -1135,7 +1160,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
     while(b != 0) {
 	if (has_live_key(tb,b,key,hval)) {
 	    ++nkeys;
-	    if (eq(object, make_tuple(b->dbterm.tpl))) {
+	    if (db_eq(&tb->common,object, &b->dbterm)) {
 		--nitems_diff;
 		if (nkeys==1 && IS_FIXED(tb)) { /* Pseudo remove */
 		    add_fixed_deletion(tb,ix);
@@ -1184,7 +1209,7 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret)
     lck = RLOCK_HASH(tb, slot);
     nactive = NACTIVE(tb);
     if (slot < nactive) {
-	*ret = put_term_list(p, BUCKET(tb, slot), 0);
+	*ret = build_term_list(p, BUCKET(tb, slot), 0, tb);
 	retval = DB_ERROR_NONE;
     }
     else if (slot == nactive) {
@@ -1228,8 +1253,6 @@ static int db_select_continue_hash(Process *p,
     int num_left = 1000;
     HashDbTerm *current = 0;
     Eterm match_list;
-    Uint32 dummy;
-    unsigned sz;
     Eterm *hp;
     Eterm match_res;
     Sint got;
@@ -1281,26 +1304,14 @@ static int db_select_continue_hash(Process *p,
     }	  
     for(;;) {
 	if (current->hvalue != INVALID_HASH && 
-	    (match_res = 
-	     db_prog_match(p,mp,
-			   make_tuple(current->dbterm.tpl),
-			   0,&dummy),
+	    (match_res = db_match_dbterm(&tb->common, p, mp, all_objects,
+					 &current->dbterm, &hp, 2),
 	     is_value(match_res))) {
-	    if (all_objects) {
-		hp = HAlloc(p, current->dbterm.size + 2);
-		match_res = copy_shallow(DBTERM_BUF(&current->dbterm),
-					 current->dbterm.size,
-					 &hp,
-					 &MSO(p));
-	    } else {
-		sz = size_object(match_res);
-	    
-		hp = HAlloc(p, sz + 2);
-		match_res = copy_struct(match_res, sz, &hp, &MSO(p));
-	    }
-            match_list = CONS(hp, match_res, match_list);
+
+	    match_list = CONS(hp, match_res, match_list);
 	    ++got;
 	}
+
 	--num_left;
 	save_slot_ix = slot_ix;
 	if ((current = next(tb, (Uint*)&slot_ix, &lck, current)) == NULL) {
@@ -1391,9 +1402,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl,
     HashDbTerm *current = 0;
     unsigned current_list_pos = 0;
     Eterm match_list;
-    Uint32 dummy;
     Eterm match_res;
-    unsigned sz;
     Eterm *hp;
     int num_left = 1000;
     Uint got = 0;
@@ -1460,22 +1469,9 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl,
     for(;;) {
 	if (current != NULL) {
 	    if (current->hvalue != INVALID_HASH) {
-		match_res = db_prog_match(p,mpi.mp,
-					  make_tuple(current->dbterm.tpl),
-					  0,&dummy);
+		match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
+					    &current->dbterm, &hp, 2);
 		if (is_value(match_res)) {
-		    if (mpi.all_objects) {
-			hp = HAlloc(p, current->dbterm.size + 2);
-			match_res = copy_shallow(DBTERM_BUF(&current->dbterm),
-						 current->dbterm.size,
-						 &hp,
-						 &MSO(p));
-		    } else {
-			sz = size_object(match_res);
-			
-			hp = HAlloc(p, sz + 2);
-			match_res = copy_struct(match_res, sz, &hp, &MSO(p));
-		    }
 		    match_list = CONS(hp, match_res, match_list);
 		    ++got;
 		}
@@ -1590,7 +1586,6 @@ static int db_select_count_hash(Process *p,
     Uint slot_ix = 0;
     HashDbTerm* current = NULL;
     unsigned current_list_pos = 0;
-    Uint32 dummy;
     Eterm *hp;
     int num_left = 1000;
     Uint got = 0;
@@ -1640,8 +1635,8 @@ static int db_select_count_hash(Process *p,
     for(;;) {
 	if (current != NULL) {
 	    if (current->hvalue != INVALID_HASH) {
-		if (db_prog_match(p, mpi.mp, make_tuple(current->dbterm.tpl),
-				  0, &dummy) == am_true) {
+		if (db_match_dbterm(&tb->common, p, mpi.mp, 0,
+				    &current->dbterm, NULL,0) == am_true) {
 		    ++got;
 		}
 		--num_left;
@@ -1709,7 +1704,6 @@ static int db_select_delete_hash(Process *p,
     Uint slot_ix = 0;
     HashDbTerm **current = NULL;
     unsigned current_list_pos = 0;
-    Uint32 dummy;
     Eterm *hp;
     int num_left = 1000;
     Uint got = 0;
@@ -1719,9 +1713,9 @@ static int db_select_delete_hash(Process *p,
     Eterm mpb;
     Eterm egot;
 #ifdef ERTS_SMP
-    int fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */
+    erts_aint_t fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */
 #else
-    int fixated_by_me = 0;
+    erts_aint_t fixated_by_me = 0;
 #endif
     erts_smp_rwmtx_t* lck;
 
@@ -1790,9 +1784,8 @@ static int db_select_delete_hash(Process *p,
 	} 
 	else {
 	    int did_erase = 0;
-	    if ((db_prog_match(p,mpi.mp,
-			       make_tuple((*current)->dbterm.tpl),
-			       0,&dummy)) == am_true) {
+	    if (db_match_dbterm(&tb->common, p, mpi.mp, 0,
+				&(*current)->dbterm, NULL, 0) == am_true) {
 		if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */
 		    if (slot_ix != last_pseudo_delete) {
 			add_fixed_deletion(tb, slot_ix);
@@ -1855,7 +1848,6 @@ static int db_select_delete_continue_hash(Process *p,
     Uint slot_ix;
     Uint last_pseudo_delete = (Uint)-1;
     HashDbTerm **current = NULL;
-    Uint32 dummy;
     Eterm *hp;
     int num_left = 1000;
     Uint got;
@@ -1903,8 +1895,8 @@ static int db_select_delete_continue_hash(Process *p,
 	} 
 	else {
 	    int did_erase = 0;
-	    if ((db_prog_match(p,mp,make_tuple((*current)->dbterm.tpl),
-			       0,&dummy)) == am_true) {
+	    if (db_match_dbterm(&tb->common, p, mp, 0,
+				&(*current)->dbterm, NULL, 0) == am_true) {
 		if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */
 		    if (slot_ix != last_pseudo_delete) {
 			add_fixed_deletion(tb, slot_ix);
@@ -1966,7 +1958,6 @@ static int db_select_count_continue_hash(Process *p,
     DbTableHash *tb = &tbl->hash;
     Uint slot_ix;
     HashDbTerm* current;
-    Uint32 dummy;
     Eterm *hp;
     int num_left = 1000;
     Uint got;
@@ -2004,8 +1995,8 @@ static int db_select_count_continue_hash(Process *p,
 		current = current->next;
 		continue;
 	    }
-	    if (db_prog_match(p, mp, make_tuple(current->dbterm.tpl),
-			      0,&dummy) == am_true) {
+	    if (db_match_dbterm(&tb->common, p, mp, 0, &current->dbterm,
+				NULL, 0) == am_true) {
 		++got;
 	    }
 	    --num_left;
@@ -2095,7 +2086,14 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl)
 	    while(list != 0) {
 		if (list->hvalue == INVALID_HASH)
 		    erts_print(to, to_arg, "*");
-		erts_print(to, to_arg, "%T", make_tuple(list->dbterm.tpl));
+		if (tb->common.compress) {
+		    Eterm key = GETKEY(tb, list->dbterm.tpl);
+		    erts_print(to, to_arg, "key=%R", key, list->dbterm.tpl);
+		}
+		else {
+		    Eterm obj = make_tuple_rel(list->dbterm.tpl,list->dbterm.tpl);
+		    erts_print(to, to_arg, "%R", obj, list->dbterm.tpl);
+		}
 		if (list->next != 0)
 		    erts_print(to, to_arg, ",");
 		list = list->next;
@@ -2131,11 +2129,11 @@ static int db_free_table_continue_hash(DbTable *tbl)
 		     sizeof(FixedDeletion));
 	ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
 	if (++done >= 2*DELETE_RECORD_LIMIT) {
-	    erts_smp_atomic_set(&tb->fixdel, (long)fixdel);
+	    erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)fixdel);
 	    return 0;		/* Not done */
 	}
     }
-    erts_smp_atomic_set(&tb->fixdel, (long)NULL);
+    erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)NULL);
 
     done /= 2;
     while(tb->nslots != 0) {
@@ -2184,7 +2182,7 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern,
     HashValue hval = NIL;      
     int num_heads = 0;
     int i;
-    
+
     mpi->lists = mpi->dlists;
     mpi->num_lists = 0;
     mpi->key_given = 1;
@@ -2352,7 +2350,7 @@ static int alloc_seg(DbTableHash *tb)
 	    struct ext_segment* eseg;
 	    eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1];
 	    MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment);
-	    erts_smp_atomic_set(&tb->segtab, (long) eseg->segtab);
+	    erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t) eseg->segtab);
 	    tb->nsegs = eseg->nsegs;
 	}
 	ASSERT(seg_ix < tb->nsegs);
@@ -2424,7 +2422,7 @@ static int free_seg(DbTableHash *tb, int free_records)
 	    MY_ASSERT(newtop->s.is_ext_segment);
 	    if (newtop->prev_segtab != NULL) {
 		/* Time to use a smaller segtab */
-		erts_smp_atomic_set(&tb->segtab, (long)newtop->prev_segtab);
+		erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)newtop->prev_segtab);
 		tb->nsegs = seg_ix;
 		ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs);
 	    }
@@ -2441,7 +2439,7 @@ static int free_seg(DbTableHash *tb, int free_records)
     if (seg_ix > 0) {
 	if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL;
     } else {
-	erts_smp_atomic_set(&tb->segtab, (long)NULL);
+	erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)NULL);
     }
 #endif
     tb->nslots -= SEGSZ;
@@ -2450,31 +2448,19 @@ static int free_seg(DbTableHash *tb, int free_records)
 }
 
 
-static HashDbTerm* get_term(DbTableHash* tb, HashDbTerm* old, 
-			    Eterm obj, HashValue hval)
-{
-    HashDbTerm* p = db_get_term((DbTableCommon *) tb,
-				(old != NULL) ? &(old->dbterm) : NULL, 
-				((char *) &(old->dbterm)) - ((char *) old),
-				obj);
-    p->hvalue = hval;
-    /*p->next = NULL;*/ /*No Need */
-    return p;
-}
-
-
 /*
 ** Copy terms from ptr1 until ptr2
 ** works for ptr1 == ptr2 == 0  => []
 ** or ptr2 == 0
 */
-static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2)
+static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
+			   DbTableHash* tb)
 {
     int sz = 0;
     HashDbTerm* ptr;
     Eterm list = NIL;
     Eterm copy;
-    Eterm *hp;
+    Eterm *hp, *hend;
 
     ptr = ptr1;
     while(ptr != ptr2) {
@@ -2486,26 +2472,20 @@ static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2)
     }
 
     hp = HAlloc(p, sz);
+    hend = hp + sz;
 
     ptr = ptr1;
     while(ptr != ptr2) {
 	if (ptr->hvalue != INVALID_HASH) {
-	    copy = copy_shallow(DBTERM_BUF(&ptr->dbterm), ptr->dbterm.size, &hp, &MSO(p));
+	    copy = db_copy_object_from_ets(&tb->common, &ptr->dbterm, &hp, &MSO(p));
 	    list = CONS(hp, copy, list);
 	    hp  += 2;
 	}
 	ptr = ptr->next;
     }
-    return list;
-}
+    HRelease(p,hend,hp);
 
-static void free_term(DbTableHash *tb, HashDbTerm* p)
-{
-    db_free_term_data(&(p->dbterm));
-    erts_db_free(ERTS_ALC_T_DB_TERM,
-		 (DbTable *) tb,
-		 (void *) p,
-		 SIZ_DBTERM(p)*sizeof(Eterm));
+    return list;
 }
 
 /* Grow table with one new bucket.
@@ -2554,9 +2534,9 @@ static void grow(DbTableHash* tb, int nactive)
     }
     erts_smp_atomic_inc(&tb->nactive);
     if (from_ix == 0) {
-	erts_smp_atomic_set(&tb->szm, szm);
+	erts_smp_atomic_set_relb(&tb->szm, szm);
     }
-    erts_smp_atomic_set(&tb->is_resizing, 0);
+    erts_smp_atomic_set_relb(&tb->is_resizing, 0);
 
     /* Finally, let's split the bucket. We try to do it in a smart way
        to keep link order and avoid unnecessary updates of next-pointers */
@@ -2588,7 +2568,7 @@ static void grow(DbTableHash* tb, int nactive)
     return;
    
 abort:
-    erts_smp_atomic_set(&tb->is_resizing, 0);
+    erts_smp_atomic_set_relb(&tb->is_resizing, 0);
 }
 
 
@@ -2632,7 +2612,7 @@ static void shrink(DbTableHash* tb, int nactive)
 	    
 	    erts_smp_atomic_set(&tb->nactive, src_ix);
 	    if (dst_ix == 0) {
-		erts_smp_atomic_set(&tb->szm, low_szm);
+		erts_smp_atomic_set_relb(&tb->szm, low_szm);
 	    }
 	    WUNLOCK_HASH(lck);
 	    
@@ -2646,7 +2626,7 @@ static void shrink(DbTableHash* tb, int nactive)
 
     }
     /*else already done */
-    erts_smp_atomic_set(&tb->is_resizing, 0);
+    erts_smp_atomic_set_relb(&tb->is_resizing, 0);
 }
 
 
@@ -2716,8 +2696,11 @@ static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle
 	    handle->tb = tbl;
 	    handle->bp = (void**) prevp;
 	    handle->dbterm = &b->dbterm;
-	    handle->new_size = b->dbterm.size;
 	    handle->mustResize = 0;
+	    handle->new_size = b->dbterm.size;
+	#if HALFWORD_HEAP
+	    handle->abs_vec = NULL;
+	#endif
 	    handle->lck = lck;
 	    /* KEEP hval WLOCKED, db_finalize_dbterm_hash will WUNLOCK */
 	    return 1;
@@ -2738,39 +2721,14 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle)
     erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck;
 
     ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(&tbl->hash,lck));  /* locked by db_lookup_dbterm_hash */
-    ASSERT(&oldp->dbterm == handle->dbterm);
 
-    if (handle->mustResize) {
-	Eterm* top;
-	Eterm copy;
-	DbTerm* newDbTerm;
-	HashDbTerm* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl,
-					 sizeof(HashDbTerm)+sizeof(Eterm)*(handle->new_size-1));    
-	sys_memcpy(newp, oldp, sizeof(HashDbTerm)-sizeof(DbTerm));  /* copy only hashtab header */
-	*(handle->bp) = newp;
-	newDbTerm = &newp->dbterm;
-    
-	newDbTerm->size = handle->new_size;
-	newDbTerm->off_heap.mso = NULL;
-	newDbTerm->off_heap.externals = NULL;
-    #ifndef HYBRID /* FIND ME! */
-	newDbTerm->off_heap.funs = NULL;
-    #endif
-	newDbTerm->off_heap.overhead = 0;
-	
-	/* make a flat copy */
-	top = DBTERM_BUF(newDbTerm);
-	copy = copy_struct(make_tuple(handle->dbterm->tpl),
-			   handle->new_size,
-			   &top, &newDbTerm->off_heap);
-	DBTERM_SET_TPL(newDbTerm,tuple_val(copy));
+    ASSERT((&oldp->dbterm == handle->dbterm) == !(tbl->common.compress && handle->mustResize));
 
+    if (handle->mustResize) {
+	db_finalize_resize(handle, offsetof(HashDbTerm,dbterm));
 	WUNLOCK_HASH(lck);
-		
-	db_free_term_data(handle->dbterm);
-	erts_db_free(ERTS_ALC_T_DB_TERM, tbl,
-		     (void *) (((char *) handle->dbterm) - (sizeof(HashDbTerm) - sizeof(DbTerm))),
-		     sizeof(HashDbTerm) + sizeof(Eterm)*(handle->dbterm->size-1));
+
+	free_term(&tbl->hash, oldp);
     }
     else {
 	WUNLOCK_HASH(lck);
@@ -2779,7 +2737,7 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle)
     handle->dbterm = 0;
 #endif
     return;
-}   
+}
 
 static int db_delete_all_objects_hash(Process* p, DbTable* tbl)
 {
@@ -2805,7 +2763,11 @@ void db_foreach_offheap_hash(DbTable *tbl,
     for (i = 0; i < nactive; i++) {
 	list = BUCKET(tb,i);
 	while(list != 0) {
-	    (*func)(&(list->dbterm.off_heap), arg);
+	    ErlOffHeap tmp_offheap;
+	    tmp_offheap.first = list->dbterm.first_oh;
+	    tmp_offheap.overhead = 0;
+	    (*func)(&tmp_offheap, arg);
+	    list->dbterm.first_oh = tmp_offheap.first;
 	    list = list->next;
 	}
     }
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index b421da591b..9a0ba3a418 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -48,9 +48,6 @@
 
 #include "erl_db_tree.h"
 
-
-
-#define GETKEY(dtt, tplp)   (*((tplp) +  (dtt)->common.keypos))
 #define GETKEY_WITH_POS(Keypos, Tplp) (*((Tplp) + Keypos))
 #define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems))
 
@@ -114,7 +111,7 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack)
 {
     if (stack == &tb->static_stack) {
 	ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1);
-	erts_smp_atomic_set(&tb->is_stack_busy, 0);
+	erts_smp_atomic_set_relb(&tb->is_stack_busy, 0);
     }
     else {
 	erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb,
@@ -122,12 +119,41 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack)
     }
 }
 
-static void reset_static_stack(DbTableTree* tb)
+static ERTS_INLINE void reset_static_stack(DbTableTree* tb)
 {
     tb->static_stack.pos = 0;
     tb->static_stack.slot = 0;
 }
 
+static ERTS_INLINE void free_term(DbTableTree *tb, TreeDbTerm* p)
+{
+    db_free_term((DbTable*)tb, p, offsetof(TreeDbTerm, dbterm));
+}
+
+static ERTS_INLINE TreeDbTerm* new_dbterm(DbTableTree *tb, Eterm obj)
+{
+    TreeDbTerm* p;
+    if (tb->common.compress) {
+	p = db_store_term_comp(&tb->common, NULL, offsetof(TreeDbTerm,dbterm), obj);
+    }
+    else {
+	p = db_store_term(&tb->common, NULL, offsetof(TreeDbTerm,dbterm), obj);
+    }
+    return p;
+}
+static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableTree *tb, TreeDbTerm* old,
+					      Eterm obj)
+{
+    TreeDbTerm* p;
+    ASSERT(old != NULL);
+    if (tb->common.compress) {
+	p = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj);
+    }
+    else {
+	p = db_store_term(&tb->common, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj);
+    }
+    return p;
+}
 
 /*
 ** Some macros for "direction stacks"
@@ -153,7 +179,7 @@ static void reset_static_stack(DbTableTree* tb)
 static TreeDbTerm *traverse_until(TreeDbTerm *t, int *current, int to);
 static void check_slot_pos(DbTableTree *tb);
 static void check_saved_stack(DbTableTree *tb);
-static int check_table_tree(TreeDbTerm *t);
+static int check_table_tree(DbTableTree* tb, TreeDbTerm *t);
 
 #define TREE_DEBUG
 #endif
@@ -168,8 +194,8 @@ static int check_table_tree(TreeDbTerm *t);
 ** Debugging dump
 */
 
-static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t,
-			  int offset);
+static void do_dump_tree2(DbTableTree*, int to, void *to_arg, int show,
+			  TreeDbTerm *t, int offset);
 
 #else
 
@@ -178,12 +204,6 @@ static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t,
 #endif
 
 /*
- * Size calculations
- */
-#define SIZ_OVERHEAD ((sizeof(TreeDbTerm)/sizeof(Eterm)) - 1)
-#define SIZ_DBTERM(TDT) (SIZ_OVERHEAD + (TDT)->dbterm.size)
-
-/*
 ** Datatypes
 */
 
@@ -259,13 +279,10 @@ struct select_delete_context {
 /*
 ** Forward declarations 
 */
-static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key);
+static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key, Eterm* key_base);
 static TreeDbTerm *linkout_object_tree(DbTableTree *tb, 
 				       Eterm object);
 static int do_free_tree_cont(DbTableTree *tb, int num_left);
-static TreeDbTerm* get_term(DbTableTree *tb,
-			    TreeDbTerm* old, 
-			    Eterm obj);
 static void free_term(DbTableTree *tb, TreeDbTerm* p);
 static int balance_left(TreeDbTerm **this); 
 static int balance_right(TreeDbTerm **this); 
@@ -273,15 +290,15 @@ static int delsub(TreeDbTerm **this);
 static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot);
 static TreeDbTerm *find_node(DbTableTree *tb, Eterm key);
 static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key);
-static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key);
-static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key);
+static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase);
+static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase);
 static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*,
 					 Eterm key);
 static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack*,
 					 Eterm key);
 static void traverse_backwards(DbTableTree *tb,
 			       DbTreeStack*,
-			       Eterm lastkey,
+			       Eterm lastkey, Eterm* lk_base,
 			       int (*doit)(DbTableTree *tb,
 					   TreeDbTerm *,
 					   void *,
@@ -289,7 +306,7 @@ static void traverse_backwards(DbTableTree *tb,
 			       void *context); 
 static void traverse_forward(DbTableTree *tb,
 			     DbTreeStack*,
-			     Eterm lastkey,
+			     Eterm lastkey, Eterm* lk_base,
 			     int (*doit)(DbTableTree *tb,
 					 TreeDbTerm *,
 					 void *,
@@ -297,8 +314,8 @@ static void traverse_forward(DbTableTree *tb,
 			     void *context); 
 static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
 		     Eterm *partly_bound_key);
-static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key);
-static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done);
+static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base);
+static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done);
 
 static int analyze_pattern(DbTableTree *tb, Eterm pattern, 
 			   struct mp_info *mpi);
@@ -318,7 +335,6 @@ static int doit_select_delete(DbTableTree *tb,
 			      TreeDbTerm *this,
 			      void *ptr,
 			      int forward);
-static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t);
 
 static int partly_bound_can_match_lesser(Eterm partly_bound_1, 
 					 Eterm partly_bound_2);
@@ -443,9 +459,9 @@ void db_initialize_tree(void)
     ets_select_reverse_exp.code[1] = am_reverse;
     ets_select_reverse_exp.code[2] = 3;
     ets_select_reverse_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     ets_select_reverse_exp.code[4] = 
-	(Eterm) &ets_select_reverse;
+	(BeamInstr) &ets_select_reverse;
     return;
 };
 
@@ -472,9 +488,6 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret)
     DbTableTree *tb = &tbl->tree;
     DbTreeStack* stack;
     TreeDbTerm *this;
-    Eterm e;
-    Eterm *hp;
-    Uint sz;
 
     if (( this = tb->root ) == NULL) {
 	*ret = am_EOT;
@@ -493,13 +506,7 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret)
 	stack->slot = 1;
 	release_stack(tb,stack);
     }
-    e = GETKEY(tb, this->dbterm.tpl);
-    sz = size_object(e);
-
-    hp = HAlloc(p, sz);
-
-    *ret = copy_struct(e,sz,&hp,&MSO(p));
-    
+    *ret = db_copy_key(p, tbl, &this->dbterm);
     return DB_ERROR_NONE;
 }
 
@@ -508,26 +515,17 @@ static int db_next_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     DbTableTree *tb = &tbl->tree;
     DbTreeStack* stack;
     TreeDbTerm *this;
-    Eterm e;
-    Eterm *hp;
-    Uint sz;
 
     if (is_atom(key) && key == am_EOT)
 	return DB_ERROR_BADKEY;
     stack = get_any_stack(tb);
-    this = find_next(tb, stack, key);
+    this = find_next(tb, stack, key, NULL);
     release_stack(tb,stack);
     if (this == NULL) {
 	*ret = am_EOT;
 	return DB_ERROR_NONE;
     }
-    e = GETKEY(tb, this->dbterm.tpl);
-    sz = size_object(e);
-
-    hp = HAlloc(p, sz);
-
-    *ret = copy_struct(e,sz,&hp,&MSO(p));
-    
+    *ret = db_copy_key(p, tbl, &this->dbterm);
     return DB_ERROR_NONE;
 }
 
@@ -536,9 +534,6 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret)
     DbTableTree *tb = &tbl->tree;
     TreeDbTerm *this;
     DbTreeStack* stack;
-    Eterm e;
-    Eterm *hp;
-    Uint sz;
 
     if (( this = tb->root ) == NULL) {
 	*ret = am_EOT;
@@ -557,13 +552,7 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret)
 	stack->slot = NITEMS(tb);
 	release_stack(tb,stack);
     }
-    e = GETKEY(tb, this->dbterm.tpl);
-    sz = size_object(e);
-
-    hp = HAlloc(p, sz);
-
-    *ret = copy_struct(e,sz,&hp,&MSO(p));
-    
+    *ret = db_copy_key(p, tbl, &this->dbterm);
     return DB_ERROR_NONE;
 }
 
@@ -572,27 +561,33 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     DbTableTree *tb = &tbl->tree;
     TreeDbTerm *this;
     DbTreeStack* stack;
-    Eterm e;
-    Eterm *hp;
-    Uint sz;
 
     if (is_atom(key) && key == am_EOT)
 	return DB_ERROR_BADKEY;
     stack = get_any_stack(tb);
-    this = find_prev(tb, stack, key);
+    this = find_prev(tb, stack, key, NULL);
     release_stack(tb,stack);
     if (this == NULL) {
 	*ret = am_EOT;
 	return DB_ERROR_NONE;
     }
-    e = GETKEY(tb, this->dbterm.tpl);
-    sz = size_object(e);
+    *ret = db_copy_key(p, tbl, &this->dbterm);
+    return DB_ERROR_NONE;
+}
 
-    hp = HAlloc(p, sz);
+static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base,
+			       TreeDbTerm* obj)
+{
+    return cmp_rel(key, key_base,
+		   GETKEY(tb,obj->dbterm.tpl), obj->dbterm.tpl);
+}
 
-    *ret = copy_struct(e,sz,&hp,&MSO(p));
-    
-    return DB_ERROR_NONE;
+static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base,
+				  TreeDbTerm* obj)
+{
+    Eterm obj_key = GETKEY(tb,obj->dbterm.tpl);
+    return is_same(key, key_base, obj_key, obj->dbterm.tpl)
+	|| cmp_rel(key, key_base, obj_key, obj->dbterm.tpl) == 0;
 }
 
 static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
@@ -622,12 +617,12 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
 		erts_smp_atomic_dec(&tb->common.nitems);
 		return DB_ERROR_SYSRES;
 	    }
-	    *this = get_term(tb, NULL, obj);
+	    *this = new_dbterm(tb, obj);
 	    (*this)->balance = 0;
 	    (*this)->left = (*this)->right = NULL;
 	    break;
-	} else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { 
-	    /* go left */
+	} else if ((c = cmp_key(tb, key, NULL, *this)) < 0) {
+	    /* go lefts */
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -636,7 +631,7 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
 	    tstack[tpos++] = this;
 	    this = &((*this)->right);
 	} else if (!key_clash_fail) { /* Equal key and this is a set, replace. */
-	    *this = get_term(tb, *this, obj);
+	    *this = replace_dbterm(tb, *this, obj);
 	    break;
 	} else {
 	    return DB_ERROR_BADKEY; /* key already exists */
@@ -714,7 +709,7 @@ static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 {
     DbTableTree *tb = &tbl->tree;
     Eterm copy;
-    Eterm *hp;
+    Eterm *hp, *hend;
     TreeDbTerm *this;
 
     /*
@@ -728,11 +723,11 @@ static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 	*ret = NIL;
     } else {
 	hp = HAlloc(p, this->dbterm.size + 2);
-	copy = copy_shallow(DBTERM_BUF(&this->dbterm), 
-			    this->dbterm.size, 
-			    &hp, 
-			    &MSO(p));
+	hend = hp + this->dbterm.size + 2;
+	copy = db_copy_object_from_ets(&tb->common, &this->dbterm, &hp, &MSO(p));
 	*ret = CONS(hp, copy, NIL);
+	hp += 2;
+	HRelease(p,hend,hp);
     }
     return DB_ERROR_NONE;
 }
@@ -766,18 +761,10 @@ static int db_get_element_tree(Process *p, DbTable *tbl,
     if (this == NULL) {
 	return DB_ERROR_BADKEY;
     } else {
-	Eterm element;
-	Uint sz;
 	if (ndex > arityval(this->dbterm.tpl[0])) {
 	    return DB_ERROR_BADPARAM;
 	}
-	element = this->dbterm.tpl[ndex];
-	sz = size_object(element);
-	hp = HAlloc(p, sz);
-	*ret = copy_struct(element, 
-			   sz, 
-			   &hp, 
-			   &MSO(p));
+	*ret = db_copy_element_from_ets(&tb->common, p, &this->dbterm, ndex, &hp, 0);
     }
     return DB_ERROR_NONE;
 }
@@ -789,7 +776,7 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret)
 
     *ret = am_true;
 
-    if ((res = linkout_tree(tb, key)) != NULL) {
+    if ((res = linkout_tree(tb, key, NULL)) != NULL) {
 	free_term(tb, res);
     }
     return DB_ERROR_NONE;
@@ -815,7 +802,7 @@ static int db_slot_tree(Process *p, DbTable *tbl,
     DbTableTree *tb = &tbl->tree;
     Sint slot;
     TreeDbTerm *st;
-    Eterm *hp;
+    Eterm *hp, *hend;
     Eterm copy;
 
     /*
@@ -847,11 +834,11 @@ static int db_slot_tree(Process *p, DbTable *tbl,
 	return DB_ERROR_UNSPEC;
     }
     hp = HAlloc(p, st->dbterm.size + 2);
-    copy = copy_shallow(DBTERM_BUF(&st->dbterm), 
-			st->dbterm.size, 
-			&hp, 
-			&MSO(p));
+    hend = hp + st->dbterm.size + 2;
+    copy = db_copy_object_from_ets(&tb->common, &st->dbterm, &hp, &MSO(p));
     *ret = CONS(hp, copy, NIL);
+    hp += 2;
+    HRelease(p,hend,hp);
     return DB_ERROR_NONE;
 }
 
@@ -981,15 +968,15 @@ static int db_select_continue_tree(Process *p,
     stack = get_any_stack(tb);
     if (chunk_size) {
 	if (reverse) {
-	    traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); 
+	    traverse_backwards(tb, stack, lastkey, NULL, &doit_select_chunk, &sc);
 	} else {
-	    traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); 
+	    traverse_forward(tb, stack, lastkey, NULL, &doit_select_chunk, &sc);
 	}
     } else {
 	if (reverse) {
-	    traverse_forward(tb, stack, lastkey, &doit_select, &sc);
+	    traverse_forward(tb, stack, lastkey, NULL, &doit_select, &sc);
 	} else {
-	    traverse_backwards(tb, stack, lastkey, &doit_select, &sc);
+	    traverse_backwards(tb, stack, lastkey, NULL, &doit_select, &sc);
 	}
     }
     release_stack(tb,stack);
@@ -1014,10 +1001,9 @@ static int db_select_continue_tree(Process *p,
 	    }
 
 	    key = GETKEY(tb, sc.lastobj);
-
-	    sz = size_object(key);
+	    sz = size_object_rel(key,sc.lastobj);
 	    hp = HAlloc(p, 9 + sz);
-	    key = copy_struct(key, sz, &hp, &MSO(p));
+	    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
 	    continuation = TUPLE8
 		(hp,
 		 tptr[1],
@@ -1038,8 +1024,8 @@ static int db_select_continue_tree(Process *p,
     key = GETKEY(tb, sc.lastobj);
     if (chunk_size) {
 	if (end_condition != NIL && 
-	    ((!reverse && cmp_partly_bound(end_condition,key) < 0) ||
-	     (reverse && cmp_partly_bound(end_condition,key) > 0))) { 
+	    ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0) ||
+	     (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0))) {
 	    /* done anyway */
 	    if (!sc.got) {
 		RET_TO_BIF(am_EOT, DB_ERROR_NONE);
@@ -1051,16 +1037,16 @@ static int db_select_continue_tree(Process *p,
 	}
     } else {
 	if (end_condition != NIL && 
-	    ((!reverse && cmp_partly_bound(end_condition,key) > 0) ||
-	     (reverse && cmp_partly_bound(end_condition,key) < 0))) { 
+	    ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0) ||
+	     (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0))) {
 	    /* done anyway */
 	    RET_TO_BIF(sc.accum,DB_ERROR_NONE);
 	}
     }
     /* Not done yet, let's trap. */
-    sz = size_object(key);
+    sz = size_object_rel(key,sc.lastobj);
     hp = HAlloc(p, 9 + sz);
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
     continuation = TUPLE8
 	(hp,
 	 tptr[1],
@@ -1081,11 +1067,13 @@ static int db_select_continue_tree(Process *p,
 static int db_select_tree(Process *p, DbTable *tbl, 
 			  Eterm pattern, int reverse, Eterm *ret)
 {
+    /* Strategy: Traverse backwards to build resulting list from tail to head */
     DbTableTree *tb = &tbl->tree;
     DbTreeStack* stack;
     struct select_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
+    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1127,7 +1115,7 @@ static int db_select_tree(Process *p, DbTable *tbl,
     sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	cmp(mpi.least,mpi.most) == 0) {
+	CMP(mpi.least,mpi.most) == 0) {
 	doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */);
 	RET_TO_BIF(sc.accum,DB_ERROR_NONE);
     }
@@ -1137,20 +1125,20 @@ static int db_select_tree(Process *p, DbTable *tbl,
 	if (mpi.some_limitation) {
 	    if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
+		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.most;
 	}
-	
-	traverse_forward(tb, stack, lastkey, &doit_select, &sc);
+	traverse_forward(tb, stack, lastkey, lk_base, &doit_select, &sc);
     } else {
 	if (mpi.some_limitation) {
 	    if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
+		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.least;
 	}
-	
-	traverse_backwards(tb, stack, lastkey, &doit_select, &sc);
+	traverse_backwards(tb, stack, lastkey, lk_base, &doit_select, &sc);
     }
     release_stack(tb,stack);
 #ifdef HARDDEBUG
@@ -1163,9 +1151,9 @@ static int db_select_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, sc.lastobj);
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastobj);
     hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
     mpb=db_make_mp_binary(p,mpi.mp,&hp);
@@ -1246,7 +1234,7 @@ static int db_select_count_continue_tree(Process *p,
     }
 
     stack = get_any_stack(tb);
-    traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc);
+    traverse_backwards(tb, stack, lastkey, NULL, &doit_select_count, &sc);
     release_stack(tb,stack);
 
     BUMP_REDS(p, 1000 - sc.max);
@@ -1256,12 +1244,12 @@ static int db_select_count_continue_tree(Process *p,
     }	
     key = GETKEY(tb, sc.lastobj);
     if (end_condition != NIL && 
-	(cmp_partly_bound(end_condition,key) > 0)) { 
+	(cmp_partly_bound(end_condition,key,sc.lastobj) > 0)) {
 	/* done anyway */
 	RET_TO_BIF(make_small(sc.got),DB_ERROR_NONE);
     }
     /* Not done yet, let's trap. */
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastobj);
     if (IS_USMALL(0, sc.got)) {
 	hp = HAlloc(p, sz + 6);
 	egot = make_small(sc.got);
@@ -1271,7 +1259,7 @@ static int db_select_count_continue_tree(Process *p,
 	egot = uint_to_big(sc.got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
     continuation = TUPLE5
 	(hp,
 	 tptr[1],
@@ -1293,7 +1281,8 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     DbTreeStack* stack;
     struct select_count_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
+    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1334,7 +1323,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	cmp(mpi.least,mpi.most) == 0) {
+	CMP(mpi.least,mpi.most) == 0) {
 	doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */);
 	RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE);
     }
@@ -1343,11 +1332,12 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     if (mpi.some_limitation) {
 	if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
 	    lastkey = GETKEY(tb, this->dbterm.tpl);
+	    lk_base = this->dbterm.tpl;
 	}
 	sc.end_condition = mpi.least;
     }
     
-    traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc);
+    traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_count, &sc);
     release_stack(tb,stack);
     BUMP_REDS(p, 1000 - sc.max);
     if (sc.max > 0) {
@@ -1355,7 +1345,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, sc.lastobj);
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastobj);
     if (IS_USMALL(0, sc.got)) {
 	hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
 	egot = make_small(sc.got);
@@ -1365,7 +1355,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
 	egot = uint_to_big(sc.got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
     mpb = db_make_mp_binary(p,mpi.mp,&hp);
@@ -1395,7 +1385,8 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     DbTreeStack* stack;
     struct select_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
+    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1437,7 +1428,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	cmp(mpi.least,mpi.most) == 0) {
+	CMP(mpi.least,mpi.most) == 0) {
 	doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */);
 	if (sc.accum != NIL) {
 	    hp=HAlloc(p, 3);
@@ -1452,20 +1443,20 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
 	if (mpi.some_limitation) {
 	    if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
+		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.least;
 	}
-
-	traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc);
+	traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc);
     } else {
 	if (mpi.some_limitation) {
 	    if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
+		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.most;
 	}
-
-	traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc);
+	traverse_forward(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc);
     }
     release_stack(tb,stack);
 
@@ -1490,9 +1481,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
 	}
 
 	key = GETKEY(tb, sc.lastobj);
-	sz = size_object(key);
+	sz = size_object_rel(key, sc.lastobj);
 	hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
-	key = copy_struct(key, sz, &hp, &MSO(p));
+	key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
 	if (mpi.all_objects)
 	    (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
 	mpb = db_make_mp_binary(p,mpi.mp,&hp);
@@ -1515,9 +1506,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, sc.lastobj);
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastobj);
     hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
 
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
@@ -1593,7 +1584,7 @@ static int db_select_delete_continue_tree(Process *p,
     sc.keypos = tb->common.keypos;
 
     ASSERT(!erts_smp_atomic_read(&tb->is_stack_busy));
-    traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc);
+    traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc);
 
     BUMP_REDS(p, 1000 - sc.max);
 
@@ -1602,11 +1593,11 @@ static int db_select_delete_continue_tree(Process *p,
     }	
     key = GETKEY(tb, (sc.lastterm)->dbterm.tpl);
     if (end_condition != NIL && 
-	cmp_partly_bound(end_condition,key) > 0) { /* done anyway */
+	cmp_partly_bound(end_condition,key,sc.lastterm->dbterm.tpl) > 0) { /* done anyway */
 	RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE);
     }
     /* Not done yet, let's trap. */
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastterm->dbterm.tpl);
     if (IS_USMALL(0, sc.accum)) {
 	hp = HAlloc(p, sz + 6);
 	eaccsum = make_small(sc.accum);
@@ -1616,7 +1607,7 @@ static int db_select_delete_continue_tree(Process *p,
 	eaccsum = uint_to_big(sc.accum, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL);
     continuation = TUPLE5
 	(hp,
 	 tptr[1],
@@ -1636,7 +1627,8 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     DbTableTree *tb = &tbl->tree;
     struct select_delete_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
+    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1680,7 +1672,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     sc.mp = mpi.mp;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	cmp(mpi.least,mpi.most) == 0) {
+	CMP(mpi.least,mpi.most) == 0) {
 	doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't 
 						      matter */);
 	RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE);
@@ -1689,11 +1681,12 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     if (mpi.some_limitation) {
 	if ((this = find_next_from_pb_key(tb, &tb->static_stack, mpi.most)) != NULL) {
 	    lastkey = GETKEY(tb, this->dbterm.tpl);
+	    lk_base = this->dbterm.tpl;
 	}
 	sc.end_condition = mpi.least;
     }
 
-    traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc);
+    traverse_backwards(tb, &tb->static_stack, lastkey, lk_base, &doit_select_delete, &sc);
     BUMP_REDS(p, 1000 - sc.max);
 
     if (sc.max > 0) {
@@ -1701,7 +1694,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, (sc.lastterm)->dbterm.tpl);
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastterm->dbterm.tpl);
     if (IS_USMALL(0, sc.accum)) {
 	hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
 	eaccsum = make_small(sc.accum);
@@ -1711,7 +1704,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
 	eaccsum = uint_to_big(sc.accum, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL);
     mpb = db_make_mp_binary(p,mpi.mp,&hp);
     
     continuation = TUPLE5
@@ -1737,7 +1730,8 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
 ** Other interface routines (not directly coupled to one bif)
 */
 
-/* Display hash table contents (for dump) */
+
+/* Display tree contents (for dump) */
 static void db_print_tree(int to, void *to_arg, 
 			  int show,
 			  DbTable *tbl)
@@ -1747,13 +1741,12 @@ static void db_print_tree(int to, void *to_arg,
     if (show)
 	erts_print(to, to_arg, "\nTree data dump:\n"
 		   "------------------------------------------------\n");
-    do_dump_tree2(to, to_arg, show, tb->root, 0);
+    do_dump_tree2(&tbl->tree, to, to_arg, show, tb->root, 0);
     if (show)
 	erts_print(to, to_arg, "\n"
 		   "------------------------------------------------\n");
 #else
     erts_print(to, to_arg, "Ordered set (AVL tree), Elements: %d\n", NITEMS(tb));
-    do_dump_tree(to, to_arg, tb->root);
 #endif
 }
 
@@ -1817,15 +1810,19 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt,
 			   void (*func)(ErlOffHeap *, void *),
 			   void * arg)
 {
+    ErlOffHeap tmp_offheap;
     if(!tdbt)
 	return;
     do_db_tree_foreach_offheap(tdbt->left, func, arg);
-    (*func)(&(tdbt->dbterm.off_heap), arg);
+    tmp_offheap.first = tdbt->dbterm.first_oh;
+    tmp_offheap.overhead = 0;
+    (*func)(&tmp_offheap, arg);
+    tdbt->dbterm.first_oh = tmp_offheap.first;
     do_db_tree_foreach_offheap(tdbt->right, func, arg);
 }
 
 static TreeDbTerm *linkout_tree(DbTableTree *tb, 
-				Eterm key)
+				Eterm key, Eterm* key_base)
 {
     TreeDbTerm **tstack[STACK_NEED];
     int tpos = 0;
@@ -1848,7 +1845,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb,
     for (;;) {
 	if (!*this) { /* Failure */
 	    return NULL;
-	} else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { 
+	} else if ((c = cmp_key(tb, key, key_base, *this)) < 0) {
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -1912,7 +1909,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
     for (;;) {
 	if (!*this) { /* Failure */
 	    return NULL;
-	} else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { 
+	} else if ((c = cmp_key(tb,key,NULL,*this)) < 0) {
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -1921,7 +1918,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
 	    tstack[tpos++] = this;
 	    this = &((*this)->right);
 	} else { /* Equal key, found the only possible matching object*/
-	    if (!eq(object,make_tuple((*this)->dbterm.tpl))) {
+	    if (!db_eq(&tb->common,object,&(*this)->dbterm)) {
 		return NULL;
 	    }
 	    q = (*this);
@@ -2065,24 +2062,6 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern,
     return DB_ERROR_NONE;
 }
 
-static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t)
-{
-    if (t != NULL) {
-	do_dump_tree(to, to_arg, t->left);
-	erts_print(to, to_arg, "%T\n", make_tuple(t->dbterm.tpl));
-	do_dump_tree(to, to_arg, t->right); 
-    }
-}
-
-static void free_term(DbTableTree *tb, TreeDbTerm* p)
-{
-    db_free_term_data(&(p->dbterm));
-    erts_db_free(ERTS_ALC_T_DB_TERM,
-		 (DbTable *) tb,
-		 (void *) p,
-		 SIZ_DBTERM(p)*sizeof(Uint));
-}
-
 static int do_free_tree_cont(DbTableTree *tb, int num_left)
 {
     TreeDbTerm *root;
@@ -2113,17 +2092,6 @@ static int do_free_tree_cont(DbTableTree *tb, int num_left)
     return 1;
 }
 
-static TreeDbTerm* get_term(DbTableTree *tb,
-			    TreeDbTerm* old, 
-			    Eterm obj) 
-{
-    TreeDbTerm* p = db_get_term((DbTableCommon *) tb,
-				(old != NULL) ? &(old->dbterm) : NULL, 
-				((char *) &(old->dbterm)) - ((char *) old),
-				obj);
-    return p;
-}
-
 /*
  * Deletion helpers
  */
@@ -2327,14 +2295,15 @@ done:
  * Find next and previous in sort order
  */
 
-static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key)
+static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack,
+			     Eterm key, Eterm* key_base)
 {
     TreeDbTerm *this;
     TreeDbTerm *tmp;
     Sint c;
 
     if(( this = TOP_NODE(stack)) != NULL) {
-	if (!CMP_EQ(GETKEY(tb, this->dbterm.tpl),key)) {
+	if (!cmp_key_eq(tb,key,key_base,this)) {
 	    /* Start from the beginning */
 	    stack->pos = stack->slot = 0;
 	}
@@ -2344,14 +2313,14 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key)
 	    return NULL;
 	for (;;) {
 	    PUSH_NODE(stack, this);
-	    if (( c = cmp(GETKEY(tb, this->dbterm.tpl),key) ) < 0) {
+	    if (( c = cmp_key(tb,key,key_base,this) ) > 0) {
 		if (this->right == NULL) /* We are at the previos 
 					    and the element does
 					    not exist */
 		    break;
 		else
 		    this = this->right;
-	    } else if (c > 0) {
+	    } else if (c < 0) {
 		if (this->left == NULL) /* Done */
 		    return this;
 		else
@@ -2384,14 +2353,15 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key)
     return this;
 }
 
-static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key)
+static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack,
+			     Eterm key, Eterm* key_base)
 {
     TreeDbTerm *this;
     TreeDbTerm *tmp;
     Sint c;
 
     if(( this = TOP_NODE(stack)) != NULL) {
-	if (!CMP_EQ(GETKEY(tb, this->dbterm.tpl),key)) {
+	if (!cmp_key_eq(tb,key,key_base,this)) {
 	    /* Start from the beginning */
 	    stack->pos = stack->slot = 0;
 	}
@@ -2401,14 +2371,14 @@ static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key)
 	    return NULL;
 	for (;;) {
 	    PUSH_NODE(stack, this);
-	    if (( c = cmp(GETKEY(tb, this->dbterm.tpl),key) ) > 0) {
+	    if (( c = cmp_key(tb,key,key_base,this) ) < 0) {
 		if (this->left == NULL) /* We are at the next 
 					   and the element does
 					   not exist */
 		    break;
 		else
 		    this = this->left;
-	    } else if (c < 0) {
+	    } else if (c > 0) {
 		if (this->right == NULL) /* Done */
 		    return this;
 		else
@@ -2454,7 +2424,8 @@ static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack* stack,
 	return NULL;
     for (;;) {
 	PUSH_NODE(stack, this);
-	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl)) ) >= 0) {
+	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl),
+				   this->dbterm.tpl) ) >= 0) {
 	    if (this->right == NULL) {
 		do {
 		    tmp = POP_NODE(stack);
@@ -2487,7 +2458,8 @@ static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack* stack,
 	return NULL;
     for (;;) {
 	PUSH_NODE(stack, this);
-	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl)) ) <= 0) {
+	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl),
+				   this->dbterm.tpl) ) <= 0) {
 	    if (this->left == NULL) {
 		do {
 		    tmp = POP_NODE(stack);
@@ -2517,12 +2489,11 @@ static TreeDbTerm *find_node(DbTableTree *tb, Eterm key)
     Sint res;
     DbTreeStack* stack = get_static_stack(tb);
 
-    if(!stack || EMPTY_NODE(stack) 
-       || !CMP_EQ(GETKEY(tb, ( this = TOP_NODE(stack) )->dbterm.tpl), key)) {
+    if(!stack || EMPTY_NODE(stack)
+       || !cmp_key_eq(tb, key, NULL, (this=TOP_NODE(stack)))) {
 
 	this = tb->root;
-	while (this != NULL && 
-	       ( res = cmp(key, GETKEY(tb, this->dbterm.tpl)) ) != 0) {
+	while (this != NULL && (res = cmp_key(tb,key,NULL,this)) != 0) {
 	    if (res < 0)
 		this = this->left;
 	    else
@@ -2544,8 +2515,7 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key)
     Sint res;
 
     this = &tb->root;
-    while ((*this) != NULL && 
-	   ( res = cmp(key, GETKEY(tb, (*this)->dbterm.tpl)) ) != 0) {
+    while ((*this) != NULL && (res = cmp_key(tb, key, NULL, *this)) != 0) {
 	if (res < 0)
 	    this = &((*this)->left);
 	else
@@ -2565,48 +2535,24 @@ static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle
 
     handle->tb = tbl;
     handle->dbterm = &(*pp)->dbterm;
+    handle->mustResize = 0;
     handle->bp = (void**) pp;
     handle->new_size = (*pp)->dbterm.size;
-    handle->mustResize = 0;
+#if HALFWORD_HEAP
+    handle->abs_vec = NULL;
+#endif
     return 1;
 }
 
 static void db_finalize_dbterm_tree(DbUpdateHandle* handle)
 {
     if (handle->mustResize) {
-	Eterm* top;
-	Eterm copy;
-	DbTerm* newDbTerm;
-	DbTableTree *tb = &handle->tb->tree;
 	TreeDbTerm* oldp = (TreeDbTerm*) *handle->bp;
-	TreeDbTerm* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM,
-					 handle->tb,
-					 sizeof(TreeDbTerm)+sizeof(Eterm)*(handle->new_size-1));
-	memcpy(newp, oldp, sizeof(TreeDbTerm)-sizeof(DbTerm));  /* copy only tree header */
-	*(handle->bp) = newp;
-	reset_static_stack(tb);
-	newDbTerm = &newp->dbterm;
-    
-	newDbTerm->size = handle->new_size;
-	newDbTerm->off_heap.mso = NULL;
-	newDbTerm->off_heap.externals = NULL;
-    #ifndef HYBRID /* FIND ME! */
-	newDbTerm->off_heap.funs = NULL;
-    #endif
-	newDbTerm->off_heap.overhead = 0;
-	
-	/* make a flat copy */
-	top = DBTERM_BUF(newDbTerm);
-	copy = copy_struct(make_tuple(handle->dbterm->tpl),
-			   handle->new_size,
-			   &top, &newDbTerm->off_heap);
-	DBTERM_SET_TPL(newDbTerm,tuple_val(copy));
-    
-	db_free_term_data(handle->dbterm);
-	erts_db_free(ERTS_ALC_T_DB_TERM,
-		     handle->tb,
-		     (void *) (((char *) handle->dbterm) - (sizeof(TreeDbTerm) - sizeof(DbTerm))),
-		     sizeof(TreeDbTerm) + sizeof(Eterm)*(handle->dbterm->size-1));
+
+	db_finalize_resize(handle, offsetof(TreeDbTerm,dbterm));
+	reset_static_stack(&handle->tb->tree);
+
+	free_term(&handle->tb->tree, oldp);
     }
 #ifdef DEBUG
     handle->dbterm = 0;
@@ -2619,7 +2565,7 @@ static void db_finalize_dbterm_tree(DbUpdateHandle* handle)
  */
 static void traverse_backwards(DbTableTree *tb,
 			       DbTreeStack* stack,
-			       Eterm lastkey,
+			       Eterm lastkey, Eterm* lk_base,
 			       int (*doit)(DbTableTree *,
 					   TreeDbTerm *,
 					   void *,
@@ -2628,7 +2574,7 @@ static void traverse_backwards(DbTableTree *tb,
 {
     TreeDbTerm *this, *next;
 
-    if (lastkey == NIL) {
+    if (lastkey == THE_NON_VALUE) {
 	stack->pos = stack->slot = 0;
 	if (( this = tb->root ) == NULL) {
 	    return;
@@ -2638,15 +2584,16 @@ static void traverse_backwards(DbTableTree *tb,
 	    this = this->right;
 	}
 	this = TOP_NODE(stack);
-	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl));
+	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl),
+			 this->dbterm.tpl);
 	if (!((*doit)(tb, this, context, 0)))
 	    return;
     } else {
-	next = find_prev(tb, stack, lastkey);
+	next = find_prev(tb, stack, lastkey, lk_base);
     }
 
     while ((this = next) != NULL) {
-	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl));
+	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
 	if (!((*doit)(tb, this, context, 0)))
 	    return;
     }
@@ -2657,7 +2604,7 @@ static void traverse_backwards(DbTableTree *tb,
  */
 static void traverse_forward(DbTableTree *tb,
 			     DbTreeStack* stack,
-			     Eterm lastkey,
+			     Eterm lastkey, Eterm* lk_base,
 			     int (*doit)(DbTableTree *,
 					 TreeDbTerm *,
 					 void *,
@@ -2666,7 +2613,7 @@ static void traverse_forward(DbTableTree *tb,
 {
     TreeDbTerm *this, *next;
 
-    if (lastkey == NIL) {
+    if (lastkey == THE_NON_VALUE) {
 	stack->pos = stack->slot = 0;
 	if (( this = tb->root ) == NULL) {
 	    return;
@@ -2676,15 +2623,15 @@ static void traverse_forward(DbTableTree *tb,
 	    this = this->left;
 	}
 	this = TOP_NODE(stack);
-	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl));
+	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
 	if (!((*doit)(tb, this, context, 1)))
 	    return;
     } else {
-	next = find_next(tb, stack, lastkey);
+	next = find_next(tb, stack, lastkey, lk_base);
     }
 
     while ((this = next) != NULL) {
-	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl));
+	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
 	if (!((*doit)(tb, this, context, 1)))
 	    return;
     }
@@ -2710,7 +2657,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
 	if (( this = find_node(tb, key) ) == NULL) {
 	    return -1;
 	}
-	*ret = this; 
+	*ret = this;
 	return 1;
     } else if (partly_bound != NULL && key != am_Underscore && 
 	       db_is_variable(key) < 0)
@@ -2721,7 +2668,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
 
 
 
-static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done)
+static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
 {
     Eterm* aa;
     Eterm* bb;
@@ -2735,44 +2682,44 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done)
 	*done = 1;
 	return 0;
     }
-    if (a == b)
+    if (is_same(a,NULL,b,b_base))
 	return 0;
     
     switch (a & _TAG_PRIMARY_MASK) {
     case TAG_PRIMARY_LIST:
 	if (!is_list(b)) {
-	    return cmp(a,b);
+	    return cmp_rel(a,NULL,b,b_base);
 	}
 	aa = list_val(a);
-	bb = list_val(b);
+	bb = list_val_rel(b,b_base);
 	while (1) {
-	    if ((j = do_cmp_partly_bound(*aa++, *bb++, done)) != 0 || *done) 
+	    if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done)
 		return j;
-	    if (*aa==*bb)
+	    if (is_same(*aa, NULL, *bb, b_base))
 		return 0;
 	    if (is_not_list(*aa) || is_not_list(*bb))
-		return do_cmp_partly_bound(*aa, *bb, done);
+		return do_cmp_partly_bound(*aa, *bb, b_base, done);
 	    aa = list_val(*aa);
-	    bb = list_val(*bb);
+	    bb = list_val_rel(*bb,b_base);
 	}
     case TAG_PRIMARY_BOXED:
 	if ((b & _TAG_PRIMARY_MASK) != TAG_PRIMARY_BOXED) {
-	    return cmp(a,b);
+	    return cmp_rel(a,NULL,b,b_base);
 	}
 	a_hdr = ((*boxed_val(a)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
-	b_hdr = ((*boxed_val(b)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
+	b_hdr = ((*boxed_val_rel(b,b_base)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
 	if (a_hdr != b_hdr) {
-	    return cmp(a, b);
+	    return cmp_rel(a, NULL, b, b_base);
 	}
 	if (a_hdr == (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE)) {
 	    aa = tuple_val(a);
-	    bb = tuple_val(b);
+	    bb = tuple_val_rel(b, b_base);
 	    /* compare the arities */
 	    i = arityval(*aa);	/* get the arity*/
 	    if (i < arityval(*bb)) return(-1);
 	    if (i > arityval(*bb)) return(1);
 	    while (i--) {
-		if ((j = do_cmp_partly_bound(*++aa, *++bb, done)) != 0 
+		if ((j = do_cmp_partly_bound(*++aa, *++bb, b_base, done)) != 0
 		    || *done) 
 		    return j;
 	    }
@@ -2780,14 +2727,14 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done)
 	}
 	/* Drop through */
       default:
-	  return cmp(a, b);
+	  return cmp_rel(a, NULL, b, b_base);
     }
 }
 
-static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key) 
+static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base)
 {
     int done = 0;
-    Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, &done);
+    Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, bk_base, &done);
 #ifdef HARDDEBUG
     erts_fprintf(stderr,"\ncmp_partly_bound: %T", partly_bound_key);
     if (ret < 0)
@@ -2796,7 +2743,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key)
 	erts_fprintf(stderr," > ");
     else
 	erts_fprintf(stderr," == ");
-    erts_fprintf(stderr,"%T\n",bound_key);
+    erts_fprintf(stderr,"%R\n", bound_key, bk_base);
 #endif
     return ret;
 }
@@ -2883,7 +2830,7 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b,
 
     if (not_eq_tags(a,b)) {
 	*done = 1;
-	return (cmp(a, b) < 0) ? 1 : 0;
+	return (CMP(a, b) < 0) ? 1 : 0;
     }
 
     /* we now know that tags are the same */
@@ -2919,7 +2866,7 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b,
 	    bb = list_val(*bb);
 	}
     default:
-	if((i = cmp(a, b)) != 0) {
+	if((i = CMP(a, b)) != 0) {
 	    *done = 1;
 	}
 	return (i < 0) ? 1 : 0;
@@ -2954,7 +2901,7 @@ static int do_partly_bound_can_match_greater(Eterm a, Eterm b,
 
     if (not_eq_tags(a,b)) {
 	*done = 1;
-	return (cmp(a, b) > 0) ? 1 : 0;
+	return (CMP(a, b) > 0) ? 1 : 0;
     }
 
     /* we now know that tags are the same */
@@ -2990,7 +2937,7 @@ static int do_partly_bound_can_match_greater(Eterm a, Eterm b,
 	    bb = list_val(*bb);
 	}
     default:
-	if((i = cmp(a, b)) != 0) {
+	if((i = CMP(a, b)) != 0) {
 	    *done = 1;
 	}
 	return (i > 0) ? 1 : 0;
@@ -3006,39 +2953,24 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 {
     struct select_context *sc = (struct select_context *) ptr;
     Eterm ret;
-    Uint32 dummy;
+    Eterm* hp;
 
     sc->lastobj = this->dbterm.tpl;
     
     if (sc->end_condition != NIL && 
 	((forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, 
-					   this->dbterm.tpl)) < 0) ||
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			   this->dbterm.tpl) < 0) ||
 	 (!forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, 
-					   this->dbterm.tpl)) > 0))) {
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			   this->dbterm.tpl) > 0))) {
 	return 0;
     }
-    ret = db_prog_match(sc->p, sc->mp,
-			make_tuple(this->dbterm.tpl), 
-			0, &dummy);
+    ret = db_match_dbterm(&tb->common,sc->p,sc->mp,sc->all_objects,
+			  &this->dbterm, &hp, 2);
     if (is_value(ret)) {
-	Uint sz;
-	Eterm *hp;
-	if (sc->all_objects) {
-	    hp = HAlloc(sc->p, this->dbterm.size + 2);
-	    ret = copy_shallow(DBTERM_BUF(&this->dbterm),
-				     this->dbterm.size,
-				     &hp,
-			             &MSO(sc->p));
-	} else {
-	    sz = size_object(ret);
-	    hp = HAlloc(sc->p, sz + 2);
-	    ret = copy_struct(ret, sz, 
-			      &hp, &MSO(sc->p));
-	}
 	sc->accum = CONS(hp, ret, sc->accum);
     }
     if (MBUF(sc->p)) {
@@ -3059,20 +2991,18 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 {
     struct select_count_context *sc = (struct select_count_context *) ptr;
     Eterm ret;
-    Uint32 dummy;
 
     sc->lastobj = this->dbterm.tpl;
     
     /* Always backwards traversing */
     if (sc->end_condition != NIL && 
 	(cmp_partly_bound(sc->end_condition, 
-			  GETKEY_WITH_POS(sc->keypos, 
-					  this->dbterm.tpl)) > 0)) {
+			  GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			  this->dbterm.tpl) > 0)) {
 	return 0;
     }
-    ret = db_prog_match(sc->p, sc->mp,
-			make_tuple(this->dbterm.tpl), 
-			0, &dummy);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
+			  &this->dbterm, NULL, 0);
     if (ret == am_true) {
 	++(sc->got);
     }
@@ -3087,41 +3017,26 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 {
     struct select_context *sc = (struct select_context *) ptr;
     Eterm ret;
-    Uint32 dummy;
+    Eterm* hp;
 
     sc->lastobj = this->dbterm.tpl;
     
     if (sc->end_condition != NIL && 
 	((forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, 
-					   this->dbterm.tpl)) < 0) ||
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			   this->dbterm.tpl) < 0) ||
 	 (!forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, 
-					   this->dbterm.tpl)) > 0))) {
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			   this->dbterm.tpl) > 0))) {
 	return 0;
     }
 
-    ret = db_prog_match(sc->p, sc->mp,
-			make_tuple(this->dbterm.tpl), 
-			0, &dummy);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, sc->all_objects,
+			  &this->dbterm, &hp, 2);
     if (is_value(ret)) {
-	Uint sz;
-	Eterm *hp;
-
 	++(sc->got);
-	if (sc->all_objects) {
-	    hp = HAlloc(sc->p, this->dbterm.size + 2);
-	    ret = copy_shallow(DBTERM_BUF(&this->dbterm),
-				     this->dbterm.size,
-				     &hp,
-			             &MSO(sc->p));
-	} else {
-	    sz = size_object(ret);
-	    hp = HAlloc(sc->p, sz + 2);
-	    ret = copy_struct(ret, sz, &hp, &MSO(sc->p));
-	}
 	sc->accum = CONS(hp, ret, sc->accum);
     }
     if (MBUF(sc->p)) {
@@ -3143,7 +3058,6 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 {
     struct select_delete_context *sc = (struct select_delete_context *) ptr;
     Eterm ret;
-    Uint32 dummy;
     Eterm key;
 
     if (sc->erase_lastterm)
@@ -3153,15 +3067,14 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr,
     
     if (sc->end_condition != NIL && 
 	cmp_partly_bound(sc->end_condition, 
-			 GETKEY_WITH_POS(sc->keypos, 
-					 this->dbterm.tpl)) > 0)
+			 GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			 this->dbterm.tpl) > 0)
 	return 0;
-    ret = db_prog_match(sc->p, sc->mp,
-			make_tuple(this->dbterm.tpl), 
-			0, &dummy);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
+			  &this->dbterm, NULL, 0);
     if (ret == am_true) {
 	key = GETKEY(sc->tb, this->dbterm.tpl);
-	linkout_tree(sc->tb, key);
+	linkout_tree(sc->tb, key, this->dbterm.tpl);
 	sc->erase_lastterm = 1;
 	++sc->accum;
     }
@@ -3172,19 +3085,28 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 }
 
 #ifdef TREE_DEBUG
-static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t,
-			  int offset)
+static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show,
+			  TreeDbTerm *t, int offset)
 {
     if (t == NULL)
-	return 0;
-    do_dump_tree2(to, to_arg, show, t->right, offset + 4);
+	return;
+    do_dump_tree2(tb, to, to_arg, show, t->right, offset + 4);
     if (show) {
-	erts_print(to, to_arg, "%*s%T (addr = %p, bal = %d)\n"
-		   offset, "", make_tuple(t->dbterm.tpl),
+	const char* prefix;
+	Eterm term;
+	if (tb->common.compress) {
+	    prefix = "key=";
+	    term = GETKEY(tb, t->dbterm.tpl);
+	}
+	else {
+	    prefix = "";
+	    term = make_tuple_rel(t->dbterm.tpl,t->dbterm.tpl);
+	}
+	erts_print(to, to_arg, "%*s%s%R (addr = %p, bal = %d)\n",
+		   offset, "", prefix, term, t->dbterm.tpl,
 		   t, t->balance);
     }
-    do_dump_tree2(to, to_arg, show, t->left, offset + 4); 
-    return sum;
+    do_dump_tree2(tb, to, to_arg, show, t->left, offset + 4); 
 }
 
 #endif
@@ -3194,7 +3116,7 @@ static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t,
 void db_check_table_tree(DbTable *tbl)
 {
     DbTableTree *tb = &tbl->tree;
-    check_table_tree(tb->root);
+    check_table_tree(tb, tb->root);
     check_saved_stack(tb);
     check_slot_pos(tb);
 }
@@ -3225,7 +3147,7 @@ static void check_slot_pos(DbTableTree *tb)
 		   "element position %d is really 0x%08X, when stack says "
 		   "it's 0x%08X\n", tb->stack.slot, t, 
 		   tb->stack.array[tb->stack.pos - 1]);
-	do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
+	do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
     }
 }
 	
@@ -3240,14 +3162,14 @@ static void check_saved_stack(DbTableTree *tb)
      if (t != stack->array[0]) {
 	 erts_fprintf(stderr,"tb->stack[0] is 0x%08X, should be 0x%08X\n",
 		      stack->array[0], t);
-	 do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
+	 do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
 	 return;
      }
      while (n < stack->pos) {
 	 if (t == NULL) {
 	     erts_fprintf(stderr, "NULL pointer in tree when stack not empty,"
 			" stack depth is %d\n", n);
-	     do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
+	     do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
 	     return;
 	 }
 	 n++;
@@ -3261,28 +3183,26 @@ static void check_saved_stack(DbTableTree *tb)
 			    "represent child pointer in tree!"
 			    "(left == 0x%08X, right == 0x%08X\n", 
 			    n, tb->stack[n], t->left, t->right);
-		 do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
+		 do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
 		 return;
 	     }
 	 }
      }
 }
 
-static int check_table_tree(TreeDbTerm *t)
+static int check_table_tree(DbTableTree* tb, TreeDbTerm *t)
 {
     int lh, rh;
     if (t == NULL)
 	return 0;
-    lh = check_table_tree(t->left);
-    rh = check_table_tree(t->right);
+    lh = check_table_tree(tb, t->left);
+    rh = check_table_tree(tb, t->right);
     if ((rh - lh) != t->balance) {
 	erts_fprintf(stderr, "Invalid tree balance for this node:\n");
-	erts_fprintf(stderr,"balance = %d, left = 0x%08X, right = 0x%08X\n"
-		     "data = %T",
-		     t->balance, t->left, t->right,
-		     make_tuple(t->dbterm.tpl));
+	erts_fprintf(stderr,"balance = %d, left = 0x%08X, right = 0x%08X\n",
+		     t->balance, t->left, t->right);
 	erts_fprintf(stderr,"\nDump:\n---------------------------------\n");
-	do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, t, 0);
+	do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, t, 0);
 	erts_fprintf(stderr,"\n---------------------------------\n");
     }
     return ((rh > lh) ? rh : lh) + 1;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 8c373451fd..c3b074f782 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -25,7 +25,6 @@
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
 #endif
-
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
@@ -58,6 +57,7 @@
 DBIF_TABLE_GUARD | DBIF_TABLE_BODY | DBIF_TRACE_GUARD | DBIF_TRACE_BODY
 
 
+#define HEAP_XTRA 100
 
 /*
 ** Some convenience macros for stacks (DMC == db_match_compile)
@@ -117,6 +117,10 @@ do {									\
 	erts_free(ERTS_ALC_T_DB_MC_STK, (Name).data);			\
 } while (0)
 
+
+#define TermWords(t) (((t) / (sizeof(UWord)/sizeof(Eterm))) + !!((t) % (sizeof(UWord)/sizeof(Eterm))))
+
+
 static ERTS_INLINE Process *
 get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks)
 {
@@ -226,6 +230,11 @@ typedef enum {
     matchCall2,
     matchCall3,
     matchPushV,
+#if HALFWORD_HEAP
+    matchPushVGuard,    /* First guard-only variable reference */
+#endif
+    matchPushVResult, /* First variable reference in result, or (if HALFWORD)
+                         in guard if also referenced in result */
     matchPushExpr, /* Push the whole expression we're matching ('$_') */
     matchPushArrayAsList, /* Only when parameter is an Array and 
 			     not an erlang term  (DCOMP_TRACE) */
@@ -281,7 +290,7 @@ typedef struct dmc_guard_bif {
 */
 DMC_DECLARE_STACK_TYPE(Eterm);
 
-DMC_DECLARE_STACK_TYPE(Uint);
+DMC_DECLARE_STACK_TYPE(UWord);
 
 DMC_DECLARE_STACK_TYPE(unsigned);
 
@@ -289,11 +298,19 @@ DMC_DECLARE_STACK_TYPE(unsigned);
 ** Data about the heap during compilation
 */
 
+typedef struct DMCVariable {
+    int is_bound;
+    int is_in_body;
+#if HALFWORD_HEAP
+    int first_guard_label;  /* to maybe change from PushVGuard to PushVResult */
+#endif
+} DMCVariable;
+
 typedef struct DMCHeap {
     int size;
-    unsigned def[DMC_DEFAULT_SIZE];
-    unsigned *data;
-    int used;
+    DMCVariable vars_def[DMC_DEFAULT_SIZE];
+    DMCVariable* vars;
+    int vars_used;
 } DMCHeap;
 
 /*
@@ -320,7 +337,6 @@ typedef struct dmc_context {
     Eterm *bodyexpr;
     int num_match;
     int current_match;
-    int eheap_need;
     Uint cflags;
     int is_guard; /* 1 if in guard, 0 if in body */
     int special; /* 1 if the head in the match was a single expression */ 
@@ -343,9 +359,22 @@ typedef struct dmc_context {
 
 #define ERTS_DEFAULT_MS_HEAP_SIZE 128
 
+/* Runtime info about a $-variable
+*/
+typedef struct MatchVariable {
+    Eterm term;
+#ifdef DEBUG
+    Process* proc;
+    Eterm* base;
+#endif
+} MatchVariable;
+
 typedef struct {
     Process process;
-    Eterm *heap;
+    union {
+	Eterm* heap;
+	MatchVariable* variables;   /* first on "heap" */
+    }u;
     Eterm default_heap[ERTS_DEFAULT_MS_HEAP_SIZE];
 } ErtsMatchPseudoProcess;
 
@@ -359,12 +388,7 @@ static ErtsMatchPseudoProcess *match_pseudo_process;
 static ERTS_INLINE void
 cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap)
 {
-    if (mpsp->process.mbuf
-	|| mpsp->process.off_heap.mso
-#ifndef HYBRID /* FIND ME! */
-	|| mpsp->process.off_heap.funs
-#endif
-	|| mpsp->process.off_heap.externals) {
+    if (mpsp->process.mbuf || mpsp->process.off_heap.first) {
 	erts_cleanup_empty_process(&mpsp->process);
     }
 #ifdef DEBUG
@@ -373,16 +397,16 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap)
     }
 #endif
     if (!keep_heap) {
-	if (mpsp->heap != &mpsp->default_heap[0]) {
+	if (mpsp->u.heap != mpsp->default_heap) {
 	    /* Have to be done *after* call to erts_cleanup_empty_process() */
-	    erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->heap);
-	    mpsp->heap = &mpsp->default_heap[0];
+	    erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->u.heap);
+	    mpsp->u.heap = mpsp->default_heap;
 	}
 #ifdef DEBUG
 	else {
 	    int i;
 	    for (i = 0; i < ERTS_DEFAULT_MS_HEAP_SIZE; i++) {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		mpsp->default_heap[i] = (Eterm) 0xdeadbeefdeadbeef;
 #else
 		mpsp->default_heap[i] = (Eterm) 0xdeadbeef;
@@ -400,7 +424,7 @@ create_match_pseudo_process(void)
     mpsp = (ErtsMatchPseudoProcess *)erts_alloc(ERTS_ALC_T_DB_MS_PSDO_PROC,
 						sizeof(ErtsMatchPseudoProcess));
     erts_init_empty_process(&mpsp->process);
-    mpsp->heap = &mpsp->default_heap[0];
+    mpsp->u.heap = mpsp->default_heap;
     return mpsp;
 }
 
@@ -424,11 +448,11 @@ get_match_pseudo_process(Process *c_p, Uint heap_size)
     mpsp = match_pseudo_process;
     cleanup_match_pseudo_process(mpsp, 0);
 #endif
-    if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE)
-	mpsp->heap = (Eterm *) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP,
-					  heap_size*sizeof(Uint));
+    if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE*sizeof(Eterm)) {
+	mpsp->u.heap = (Eterm*) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, heap_size);
+    }
     else {
-	ASSERT(mpsp->heap == &mpsp->default_heap[0]);
+	ASSERT(mpsp->u.heap == mpsp->default_heap);
     }
     return mpsp;
 }
@@ -469,23 +493,6 @@ erts_match_set_release_result(Process* c_p)
 
 static erts_smp_atomic_t trace_control_word;
 
-
-Eterm
-erts_ets_copy_object(Eterm obj, Process* to)
-{
-    Uint size = size_object(obj);
-    Eterm* hp = HAlloc(to, size);
-    Eterm res;
-
-    res = copy_struct(obj, size, &hp, &MSO(to));
-#ifdef DEBUG
-    if (eq(obj, res) == 0) {
-	erl_exit(1, "copy not equal to source\n");
-    }
-#endif
-    return res;
-}
-
 /* This needs to be here, before the bif table... */
 
 static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm val);
@@ -830,50 +837,50 @@ static Uint my_size_object(Eterm t);
 static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap);
 
 /* Guard compilation */
-static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(Uint) *text,
+static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text,
 			     Eterm t);
 static DMCRet dmc_list(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant);
 static DMCRet dmc_tuple(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant);
 static DMCRet dmc_variable(DMCContext *context,
 			   DMCHeap *heap,
-			   DMC_STACK_TYPE(Uint) *text,
+			   DMC_STACK_TYPE(UWord) *text,
 			   Eterm t,
 			   int *constant);
 static DMCRet dmc_fun(DMCContext *context,
 		      DMCHeap *heap,
-		      DMC_STACK_TYPE(Uint) *text,
+		      DMC_STACK_TYPE(UWord) *text,
 		      Eterm t,
 		      int *constant);
 static DMCRet dmc_expr(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant);
 static DMCRet compile_guard_expr(DMCContext *context,
 				    DMCHeap *heap,
-				    DMC_STACK_TYPE(Uint) *text,
+				    DMC_STACK_TYPE(UWord) *text,
 				    Eterm t);
 /* match expression subroutine */
 static DMCRet dmc_one_term(DMCContext *context, 
 			   DMCHeap *heap,
 			   DMC_STACK_TYPE(Eterm) *stack,
-			   DMC_STACK_TYPE(Uint) *text,
+			   DMC_STACK_TYPE(UWord) *text,
 			   Eterm c);
 
 
 #ifdef DMC_DEBUG
 static int test_disassemble_next = 0;
-static void db_match_dis(Binary *prog);
+void db_match_dis(Binary *prog);
 #define TRACE erts_fprintf(stderr,"Trace: %s:%d\n",__FILE__,__LINE__)
-#define FENCE_PATTERN_SIZE 1
+#define FENCE_PATTERN_SIZE (1*sizeof(Uint))
 #define FENCE_PATTERN 0xDEADBEEFUL
 #else
 #define TRACE /* Nothing */
@@ -891,6 +898,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace);
 
 static Eterm seq_trace_fake(Process *p, Eterm arg1);
 
+static void db_free_tmp_uncompressed(DbTerm* obj);
+
 
 /*
 ** Interface routines.
@@ -915,7 +924,7 @@ BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new)
     if (val != ((Uint32)val))
 	BIF_ERROR(p, BADARG);
     
-    old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (long) val);
+    old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (erts_aint_t) val);
     BIF_RET(erts_make_integer((Uint) old_tcw, p));
 }
 
@@ -1179,14 +1188,14 @@ done:
 }
     
 Eterm erts_match_set_run(Process *p, Binary *mpsp, 
-			 Eterm *args, int num_args, 
+			 Eterm *args, int num_args,
+			 enum erts_pam_run_flags in_flags,
 			 Uint32 *return_flags) 
 {
     Eterm ret;
 
-    ret = db_prog_match(p, mpsp,
-			(Eterm) args, 
-			num_args, return_flags);
+    ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args,
+			in_flags, return_flags);
 #if defined(HARDDEBUG)
     if (is_non_value(ret)) {
 	erts_fprintf(stderr, "Failed\n");
@@ -1204,6 +1213,32 @@ Eterm erts_match_set_run(Process *p, Binary *mpsp,
      */
 }
 
+static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp,
+				    Eterm args, int num_args,
+				    Uint32 *return_flags)
+{
+    Eterm ret;
+
+    ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args,
+			ERTS_PAM_CONTIGUOUS_TUPLE | ERTS_PAM_COPY_RESULT,
+			return_flags);
+#if defined(HARDDEBUG)
+    if (is_non_value(ret)) {
+	erts_fprintf(stderr, "Failed\n");
+    } else {
+	erts_fprintf(stderr, "Returning : %T\n", ret);
+    }
+#endif
+    return ret;
+    /* Returns
+     *   THE_NON_VALUE if no match
+     *   am_false      if {message,false} has been called,
+     *   am_true       if {message,_} has not been called or
+     *                 if {message,true} has been called,
+     *   Msg           if {message,Msg} has been called.
+     */
+}
+
 /*
 ** API Used by other erl_db modules.
 */
@@ -1245,7 +1280,7 @@ Binary *db_match_compile(Eterm *matchexpr,
 {
     DMCHeap heap;
     DMC_STACK_TYPE(Eterm) stack;
-    DMC_STACK_TYPE(Uint) text;
+    DMC_STACK_TYPE(UWord) text;
     DMCContext context;
     MatchProg *ret = NULL;
     Eterm t;
@@ -1254,7 +1289,6 @@ Binary *db_match_compile(Eterm *matchexpr,
     int structure_checked;
     DMCRet res;
     int current_try_label;
-    Uint max_eheap_need;
     Binary *bp = NULL;
     unsigned clause_start;
 
@@ -1267,27 +1301,24 @@ Binary *db_match_compile(Eterm *matchexpr,
     context.matchexpr = matchexpr;
     context.guardexpr = guards;
     context.bodyexpr = body;
-    context.eheap_need = 0;
     context.err_info = err_info;
     context.cflags = flags;
 
     heap.size = DMC_DEFAULT_SIZE;
-    heap.data = heap.def;
+    heap.vars = heap.vars_def;
 
     /*
     ** Compile the match expression
     */
 restart:
-    heap.used = 0;
-    max_eheap_need = 0;
+    heap.vars_used = 0;
     for (context.current_match = 0; 
 	 context.current_match < num_progs; 
 	 ++context.current_match) { /* This loop is long, 
 				       too long */
-	memset(heap.data, 0, heap.size * sizeof(*heap.data));
+	memset(heap.vars, 0, heap.size * sizeof(*heap.vars));
 	t = context.matchexpr[context.current_match];
 	context.stack_used = 0;
-	context.eheap_need = 0;
 	structure_checked = 0;
 	if (context.current_match < num_progs - 1) {
 	    DMC_PUSH(text,matchTryMeElse);
@@ -1380,7 +1411,7 @@ restart:
     
 	/* 
 	** There is one single top variable in the match expression
-	** iff the text is tho Uint's and the single instruction 
+	** iff the text is two Uint's and the single instruction
 	** is 'matchBind' or it is only a skip.
 	*/
 	context.special = 
@@ -1459,10 +1490,6 @@ restart:
 	if (current_try_label >= 0) {
 	    DMC_POKE(text, current_try_label, DMC_STACK_NUM(text));
 	}
-	/* So, how much eheap did this part of the match program need? */
-	if (context.eheap_need > max_eheap_need) {
-	    max_eheap_need = context.eheap_need;
-	}
     } /* for (context.current_match = 0 ...) */
 
 
@@ -1491,23 +1518,20 @@ restart:
     ** A special case is when the match expression is a single binding 
     ** (i.e '$1'), then the field single_variable is set to 1.
     */
-    bp = erts_create_magic_binary(((sizeof(MatchProg) - sizeof(Uint)) +
-				   (DMC_STACK_NUM(text) * sizeof(Uint))),
+    bp = erts_create_magic_binary(((sizeof(MatchProg) - sizeof(UWord)) +
+				   (DMC_STACK_NUM(text) * sizeof(UWord))),
 				  erts_db_match_prog_destructor);
     ret = Binary2MatchProg(bp);
     ret->saved_program_buf = NULL;
     ret->saved_program = NIL;
     ret->term_save = context.save;
-    ret->num_bindings = heap.used;
+    ret->num_bindings = heap.vars_used;
     ret->single_variable = context.special;
     sys_memcpy(ret->text, DMC_STACK_DATA(text), 
-	       DMC_STACK_NUM(text) * sizeof(Uint));
-    ret->heap_size = ((heap.used * sizeof(Eterm)) +
-		      (max_eheap_need * sizeof(Eterm)) +
-		      (context.stack_need * sizeof(Eterm *)) +
-		      (3 * (FENCE_PATTERN_SIZE * sizeof(Eterm *))));
-    ret->eheap_offset = heap.used + FENCE_PATTERN_SIZE;
-    ret->stack_offset = ret->eheap_offset + max_eheap_need + FENCE_PATTERN_SIZE;
+	       DMC_STACK_NUM(text) * sizeof(UWord));
+    ret->stack_offset = heap.vars_used*sizeof(MatchVariable) + FENCE_PATTERN_SIZE;
+    ret->heap_size = ret->stack_offset + context.stack_need * sizeof(Eterm*) + FENCE_PATTERN_SIZE;
+
 #ifdef DMC_DEBUG
     ret->prog_end = ret->text + DMC_STACK_NUM(text);
 #endif
@@ -1517,17 +1541,16 @@ restart:
      */  
     context.save = NULL;
 error: /* Here is were we land when compilation failed. */
-    while (context.save != NULL) {
-	ErlHeapFragment *ll = context.save->next;
+    if (context.save != NULL) {
 	free_message_buffer(context.save);
-	context.save = ll;
+	context.save = NULL;
     }
     DMC_FREE(stack);
     DMC_FREE(text);
     if (context.copy != NULL) 
 	free_message_buffer(context.copy);
-    if (heap.data != heap.def)
-	erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.data);
+    if (heap.vars != heap.vars_def)
+	erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.vars);
     return bp;
 }
 
@@ -1537,15 +1560,11 @@ error: /* Here is were we land when compilation failed. */
 void erts_db_match_prog_destructor(Binary *bprog)
 {
     MatchProg *prog;
-    ErlHeapFragment *tmp, *ll;
     if (bprog == NULL)
 	return;
     prog = Binary2MatchProg(bprog);
-    tmp = prog->term_save; 
-    while (tmp != NULL) {
-	ll = tmp->next;
-	free_message_buffer(tmp);
-	tmp = ll;
+    if (prog->term_save != NULL) {
+	free_message_buffer(prog->term_save); 
     }
     if (prog->saved_program_buf != NULL)
 	free_message_buffer(prog->saved_program_buf);
@@ -1576,7 +1595,7 @@ erts_match_prog_foreach_offheap(Binary *bprog,
 */
 static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
 {
-    Eterm *hp = HAlloc(psp, arity * 2);
+    Eterm *hp = HAllocX(psp, arity * 2, HEAP_XTRA);
     Eterm ret = NIL;
     while (--arity >= 0) {
 	ret = CONS(hp, arr[arity], ret);
@@ -1584,14 +1603,83 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
     }
     return ret;
 }
+
+
+#if HALFWORD_HEAP
+struct heap_checkpoint_t
+{
+    Process *p;
+    Eterm* htop;
+    ErlHeapFragment* mbuf;
+    unsigned used_size;
+    ErlOffHeap off_heap;
+};
+
+static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp)
+{
+    hcp->p = p;
+    hcp->htop = HEAP_TOP(p);
+    hcp->mbuf = MBUF(p);
+    hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0;
+    hcp->off_heap = MSO(p);
+}
+
+static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp)
+{
+    struct erl_off_heap_header* oh = MSO(hcp->p).first;
+
+    if (oh != hcp->off_heap.first) {
+	ASSERT(oh != NULL);
+	if (hcp->off_heap.first) {
+	    while (oh->next != hcp->off_heap.first) {
+		oh = oh->next;
+	    }
+	    oh->next = NULL;
+	}
+	erts_cleanup_offheap(&MSO(hcp->p));
+	MSO(hcp->p) = hcp->off_heap;
+    }
+    if (MBUF(hcp->p) != hcp->mbuf) {
+	ErlHeapFragment* hf = MBUF(hcp->p);
+	ASSERT(hf != NULL);
+	if (hcp->mbuf) {
+	    while (hf->next != hcp->mbuf) {
+		hf = hf->next;
+	    }
+	    hf->next = NULL;
+	}
+	free_message_buffer(MBUF(hcp->p));
+	MBUF(hcp->p) = hcp->mbuf;
+    }
+    if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) {
+	hcp->mbuf->used_size = hcp->used_size;
+    }
+    HEAP_TOP(hcp->p) = hcp->htop;
+}
+#endif /* HALFWORD_HEAP */
+
+static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base)
+{
+    if (!is_immed(term)) {
+	Uint sz = size_object_rel(term, base);
+	Eterm* top = HAllocX(p, sz, HEAP_XTRA);
+	return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL);
+    }
+    return term;
+}
+
+
 /*
 ** Execution of the match program, this is Pam.
 ** May return THE_NON_VALUE, which is a bailout.
-** the para meter 'arity' is only used if 'term' is actually an array,
+** the parameter 'arity' is only used if 'term' is actually an array,
 ** i.e. 'DCOMP_TRACE' was specified 
 */
-Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, 
+Eterm db_prog_match(Process *c_p, Binary *bprog,
+		    Eterm term, Eterm* base,
+		    Eterm *termp,
 		    int arity,
+		    enum erts_pam_run_flags in_flags,
 		    Uint32 *return_flags)
 {
     MatchProg *prog = Binary2MatchProg(bprog);
@@ -1600,8 +1688,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
     Eterm t;
     Eterm **sp;
     Eterm *esp;
-    Eterm *hp;
-    Uint *pc = prog->text;
+    MatchVariable* variables;
+    BeamInstr *cp;
+    UWord *pc = prog->text;
     Eterm *ehp;
     Eterm ret;
     Uint n = 0; /* To avoid warning. */
@@ -1609,19 +1698,24 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
     unsigned do_catch;
     ErtsMatchPseudoProcess *mpsp;
     Process *psp;
+    Process* build_proc;
     Process *tmpp;
     Process *current_scheduled;
     ErtsSchedulerData *esdp;
     Eterm (*bif)(Process*, ...);
     int fail_label;
     int atomic_trace;
+#if HALFWORD_HEAP
+    struct heap_checkpoint_t c_p_checkpoint = {};
+#endif
 #ifdef DMC_DEBUG
-    unsigned long *heap_fence;
-    unsigned long *eheap_fence;
-    unsigned long *stack_fence;
+    Uint *heap_fence;
+    Uint *stack_fence;
     Uint save_op;
 #endif /* DMC_DEBUG */
 
+    ASSERT(base==NULL || HALFWORD_HEAP);
+
     mpsp = get_match_pseudo_process(c_p, prog->heap_size);
     psp = &mpsp->process;
 
@@ -1631,7 +1725,6 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
     esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(c_p);
     ASSERT(esdp != NULL);
     current_scheduled = esdp->current_process;
-    esdp->current_process = psp;
     /* SMP: psp->scheduler_data is set by get_match_pseudo_process */
 
     atomic_trace = 0;
@@ -1654,11 +1747,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
 
 #ifdef DMC_DEBUG
     save_op = 0;
-    heap_fence =  (unsigned long *) mpsp->heap + prog->eheap_offset - 1;
-    eheap_fence = (unsigned long *) mpsp->heap + prog->stack_offset - 1;
-    stack_fence = (unsigned long *) mpsp->heap + prog->heap_size - 1;
+    heap_fence = (Eterm*)((char*) mpsp->u.heap + prog->stack_offset) - 1;
+    stack_fence = (Eterm*)((char*) mpsp->u.heap + prog->heap_size) - 1;
     *heap_fence = FENCE_PATTERN;
-    *eheap_fence = FENCE_PATTERN;
     *stack_fence = FENCE_PATTERN;
 #endif /* DMC_DEBUG */
 
@@ -1672,36 +1763,48 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
 
     *return_flags = 0U;
 
+    variables = mpsp->u.variables;
+#if HALFWORD_HEAP
+    c_p_checkpoint.p = NULL;
+#endif
+
 restart:
     ep = &term;
-    esp = mpsp->heap + prog->stack_offset;
+    esp = (Eterm*)((char*)mpsp->u.heap + prog->stack_offset);
     sp = (Eterm **) esp;
-    hp = mpsp->heap;
-    ehp = mpsp->heap + prog->eheap_offset;
     ret = am_true;
     do_catch = 0;
     fail_label = -1;
+    build_proc = psp;
+    esdp->current_process = psp;
+    ASSERT_HALFWORD(!c_p_checkpoint.p);
+
+#ifdef DEBUG
+    ASSERT(variables == mpsp->u.variables);
+    for (i=0; i<prog->num_bindings; i++) {
+	variables[i].term = THE_NON_VALUE;
+	variables[i].proc = NULL;
+	variables[i].base = base;
+    }
+#endif
 
     for (;;) {
-#ifdef DMC_DEBUG
+
+    #ifdef DMC_DEBUG
 	if (*heap_fence != FENCE_PATTERN) {
 	    erl_exit(1, "Heap fence overwritten in db_prog_match after op "
 		     "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence);
 	}
-	if (*eheap_fence != FENCE_PATTERN) {
-	    erl_exit(1, "Eheap fence overwritten in db_prog_match after op "
-		     "0x%08x, overwritten with 0x%08x.", save_op, 
-		     *eheap_fence);
-	}
 	if (*stack_fence != FENCE_PATTERN) {
 	    erl_exit(1, "Stack fence overwritten in db_prog_match after op "
 		     "0x%08x, overwritten with 0x%08x.", save_op, 
 		     *stack_fence);
 	}
 	save_op = *pc;
-#endif
+    #endif
 	switch (*pc++) {
 	case matchTryMeElse:
+	    ASSERT(fail_label == -1);
 	    fail_label = *pc++;
 	    break;
 	case matchArray: /* only when DCOMP_TRACE, is always first
@@ -1709,16 +1812,17 @@ restart:
 	    n = *pc++;
 	    if ((int) n != arity)
 		FAIL();
-	    ep = (Eterm *) *ep;
+	    ep = termp;
 	    break;
 	case matchArrayBind: /* When the array size is unknown. */
+	    ASSERT(termp);
 	    n = *pc++;
-	    hp[n] = dpm_array_to_list(psp, (Eterm *) term, arity);
+	    variables[n].term = dpm_array_to_list(psp, termp, arity);
 	    break;
 	case matchTuple: /* *ep is a tuple of arity n */
-	    if (!is_tuple(*ep))
+	    if (!is_tuple_rel(*ep,base))
 		FAIL();
-	    ep = tuple_val(*ep);
+	    ep = tuple_val_rel(*ep,base);
 	    n = *pc++;
 	    if (arityval(*ep) != n)
 		FAIL();
@@ -1726,9 +1830,9 @@ restart:
 	    break;
 	case matchPushT: /* *ep is a tuple of arity n, 
 			    push ptr to first element */
-	    if (!is_tuple(*ep))
+	    if (!is_tuple_rel(*ep,base))
 		FAIL();
-	    tp = tuple_val(*ep);
+	    tp = tuple_val_rel(*ep,base);
 	    n = *pc++;
 	    if (arityval(*tp) != n)
 		FAIL();
@@ -1738,12 +1842,12 @@ restart:
 	case matchList:
 	    if (!is_list(*ep))
 		FAIL();
-	    ep = list_val(*ep);
+	    ep = list_val_rel(*ep,base);
 	    break;
 	case matchPushL:
 	    if (!is_list(*ep))
 		FAIL();
-	    *sp++ = list_val(*ep);
+	    *sp++ = list_val_rel(*ep,base);
 	    ++ep;
 	    break;
 	case matchPop:
@@ -1751,52 +1855,61 @@ restart:
 	    break;
 	case matchBind:
 	    n = *pc++;
-	    hp[n] = *ep++;
+	    variables[n].term = *ep++;
 	    break;
 	case matchCmp:
 	    n = *pc++;
-	    if (!eq(hp[n],*ep))
+	    if (!eq_rel(variables[n].term, base, *ep, base))
 		FAIL();
 	    ++ep;
 	    break;
 	case matchEqBin:
 	    t = (Eterm) *pc++;
-	    if (!eq(*ep,t))
+	    if (!eq_rel(t,NULL,*ep,base))
 		FAIL();
 	    ++ep;
 	    break;
 	case matchEqFloat:
-	    if (!is_float(*ep))
+	    if (!is_float_rel(*ep,base))
 		FAIL();
-	    if (memcmp(float_val(*ep) + 1, pc, sizeof(double)))
+	    if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double)))
 		FAIL();
-	    pc += 2;
+	    pc += TermWords(2);
 	    ++ep;
 	    break;
-	case matchEqRef:
-	    if (!is_ref(*ep))
+	case matchEqRef: {
+	    Eterm* epc = (Eterm*)pc;
+	    if (!is_ref_rel(*ep,base))
 		FAIL();
-	    if (!eq(*ep, make_internal_ref(pc)))
+	    if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) {
 		FAIL();
-	    i = thing_arityval(*pc);
-	    pc += i+1;
+	    }
+	    i = thing_arityval(*epc);
+	    pc += TermWords(i+1);
 	    ++ep;
 	    break;
+	}
 	case matchEqBig:
-	    if (!is_big(*ep))
+	    if (!is_big_rel(*ep,base))
 		FAIL();
-	    tp = big_val(*ep);
-	    if (*tp != *pc)
-		FAIL();
-	    i = BIG_ARITY(pc);
-	    while(i--)
-		if (*++tp != *++pc)
+	    tp = big_val_rel(*ep,base);
+	    {
+		Eterm *epc = (Eterm *) pc;
+		if (*tp != *epc)
 		    FAIL();
-	    ++pc;
+		i = BIG_ARITY(epc);
+		pc += TermWords(i+1);
+		while(i--) {
+		    if (*++tp != *++epc) {
+			FAIL();
+		    }
+		}
+	    }
 	    ++ep;
 	    break;
 	case matchEq:
-	    t = (Eterm) *pc++; 
+	    t = (Eterm) *pc++;
+	    ASSERT(is_immed(t));
 	    if (t != *ep++)
 		FAIL();
 	    break;
@@ -1804,25 +1917,32 @@ restart:
 	    ++ep;
 	    break;
 	/* 
-	 * Here comes guard instructions 
+	 * Here comes guard & body instructions
 	 */
 	case matchPushC: /* Push constant */
-	    *esp++ = *pc++;
+	    if ((in_flags & ERTS_PAM_COPY_RESULT)
+		&& do_catch && !is_immed(*pc)) {
+		*esp++ = copy_object(*pc++, c_p);
+	    }
+	    else {
+		*esp++ = *pc++;
+	    }
 	    break;
 	case matchConsA:
-	    ehp[1] = *--esp;
-	    ehp[0] = esp[-1];
+	    ehp = HAllocX(build_proc, 2, HEAP_XTRA);
+	    CDR(ehp) = *--esp;
+	    CAR(ehp) = esp[-1];
 	    esp[-1] = make_list(ehp);
-	    ehp += 2;
 	    break;
 	case matchConsB:
-	    ehp[0] = *--esp;
-	    ehp[1] = esp[-1];
+	    ehp = HAllocX(build_proc, 2, HEAP_XTRA);
+	    CAR(ehp) = *--esp;
+	    CDR(ehp) = esp[-1];
 	    esp[-1] = make_list(ehp);
-	    ehp += 2;
 	    break;
 	case matchMkTuple:
 	    n = *pc++;
+	    ehp = HAllocX(build_proc, n+1, HEAP_XTRA);
 	    t = make_tuple(ehp);
 	    *ehp++ = make_arityval(n);
 	    while (n--) {
@@ -1832,7 +1952,7 @@ restart:
 	    break;
 	case matchCall0:
 	    bif = (Eterm (*)(Process*, ...)) *pc++;
-	    t = (*bif)(psp);
+	    t = (*bif)(build_proc);
 	    if (is_non_value(t)) {
 		if (do_catch)
 		    t = FAIL_TERM;
@@ -1843,7 +1963,7 @@ restart:
 	    break;
 	case matchCall1:
 	    bif = (Eterm (*)(Process*, ...)) *pc++;
-	    t = (*bif)(psp, esp[-1]);
+	    t = (*bif)(build_proc, esp[-1]);
 	    if (is_non_value(t)) {
 		if (do_catch)
 		    t = FAIL_TERM;
@@ -1854,7 +1974,7 @@ restart:
 	    break;
 	case matchCall2:
 	    bif = (Eterm (*)(Process*, ...)) *pc++;
-	    t = (*bif)(psp, esp[-1], esp[-2]);
+	    t = (*bif)(build_proc, esp[-1], esp[-2]);
 	    if (is_non_value(t)) {
 		if (do_catch)
 		    t = FAIL_TERM;
@@ -1866,7 +1986,7 @@ restart:
 	    break;
 	case matchCall3:
 	    bif = (Eterm (*)(Process*, ...)) *pc++;
-	    t = (*bif)(psp, esp[-1], esp[-2], esp[-3]);
+	    t = (*bif)(build_proc, esp[-1], esp[-2], esp[-3]);
 	    if (is_non_value(t)) {
 		if (do_catch)
 		    t = FAIL_TERM;
@@ -1876,15 +1996,73 @@ restart:
 	    esp -= 2;
 	    esp[-1] = t;
 	    break;
+
+	#if HALFWORD_HEAP
+	case matchPushVGuard:
+	    if (!base) goto case_matchPushV;
+	    /* Build NULL-based copy on pseudo heap for easy disposal */
+	    n = *pc++;
+	    ASSERT(is_value(variables[n].term));
+	    ASSERT(!variables[n].proc);
+	    variables[n].term = copy_object_rel(psp, variables[n].term, base);
+	    *esp++ = variables[n].term;
+	    #ifdef DEBUG
+	    variables[n].proc = psp;
+	    variables[n].base = NULL;
+	    #endif
+	    break;
+	#endif
+	case matchPushVResult:
+	    if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV;
+
+	    /* Build (NULL-based) copy on callers heap */
+	#if HALFWORD_HEAP
+	    if (!do_catch && !c_p_checkpoint.p) {
+		heap_checkpoint_init(c_p, &c_p_checkpoint);
+	    }
+        #endif
+	    n = *pc++;
+	    ASSERT(is_value(variables[n].term));
+	    ASSERT(!variables[n].proc);
+	    variables[n].term = copy_object_rel(c_p, variables[n].term, base);
+	    *esp++ = variables[n].term;
+	    #ifdef DEBUG
+	    variables[n].proc = c_p;
+	    variables[n].base = NULL;
+	    #endif
+	    break;
 	case matchPushV:
-	    *esp++ = hp[*pc++];
+	case_matchPushV:
+	    n = *pc++;
+	    ASSERT(is_value(variables[n].term));
+	    ASSERT(!variables[n].base);
+	    *esp++ = variables[n].term;
 	    break;
 	case matchPushExpr:
-	    *esp++ = term;
+	    if (in_flags & ERTS_PAM_COPY_RESULT) {
+		Uint sz;
+		Eterm* top;
+		sz = size_object_rel(term, base);
+		top = HAllocX(build_proc, sz, HEAP_XTRA);
+		if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) {
+		    ASSERT(is_tuple_rel(term,base));
+		    *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz,
+					      &top, &MSO(build_proc), base);
+		}
+		else {
+		    *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc),
+					     base, NULL);
+		}
+	    }
+	    else {
+		*esp = term;
+	    }
 	    break;
 	case matchPushArrayAsList:
+	    ASSERT_HALFWORD(base == NULL);
 	    n = arity; /* Only happens when 'term' is an array */
-	    tp = (Eterm *) term;
+	    tp = termp;
+	    ehp = HAllocX(build_proc, n*2, HEAP_XTRA);
 	    *esp++  = make_list(ehp);
 	    while (n--) {
 		*ehp++ = *tp++;
@@ -1897,7 +2075,8 @@ restart:
 	    break;
 	case matchPushArrayAsListU:
 	    /* This instruction is NOT efficient. */
-	    *esp++  = dpm_array_to_list(psp, (Eterm *) term, arity); 
+	    ASSERT_HALFWORD(base == NULL);
+	    *esp++  = dpm_array_to_list(build_proc, termp, arity);
 	    break;
 	case matchTrue:
 	    if (*--esp != am_true)
@@ -1983,7 +2162,8 @@ restart:
 	case matchProcessDump: {
 	    erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
 	    print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p);
-	    *esp++ = new_binary(psp, (byte *)dsbufp->str, (int)dsbufp->str_len);
+	    *esp++ = new_binary(build_proc, (byte *)dsbufp->str,
+				dsbufp->str_len);
 	    erts_destroy_tmp_dsbuf(dsbufp);
 	    break;
 	}
@@ -2027,29 +2207,24 @@ restart:
 	    if (SEQ_TRACE_TOKEN(c_p) == NIL) 
 		*esp++ = NIL;
 	    else {
+		Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p);
+		Uint sender_sz = is_immed(sender) ? 0 : size_object(sender);
+		ehp = HAllocX(build_proc, 6 + sender_sz, HEAP_XTRA);
+		if (sender_sz) {
+		    sender = copy_struct(sender, sender_sz, &ehp, &MSO(build_proc));
+		}
  		*esp++ = make_tuple(ehp);
  		ehp[0] = make_arityval(5);
  		ehp[1] = SEQ_TRACE_TOKEN_FLAGS(c_p);
  		ehp[2] = SEQ_TRACE_TOKEN_LABEL(c_p);
  		ehp[3] = SEQ_TRACE_TOKEN_SERIAL(c_p);
- 		ehp[4] = SEQ_TRACE_TOKEN_SENDER(c_p);
+		ehp[4] = sender;
  		ehp[5] = SEQ_TRACE_TOKEN_LASTCNT(c_p);
 		ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
 		ASSERT(is_immed(ehp[1]));
 		ASSERT(is_immed(ehp[2]));
 		ASSERT(is_immed(ehp[3]));
 		ASSERT(is_immed(ehp[5]));
-		if(!is_immed(ehp[4])) {
-		    Eterm *sender = &ehp[4];
-		    ehp += 6;
-		    *sender = copy_struct(*sender,
-					  size_object(*sender),
-					  &ehp,
-					  &MSO(psp));
-		}
-		else
-		    ehp += 6;
-
 	    } 
 	    break;
 	case matchEnableTrace:
@@ -2095,17 +2270,17 @@ restart:
 	    }
 	    break;
  	case matchCaller:
- 	    if (!(c_p->cp) || !(hp = find_function_from_pc(c_p->cp))) {
+	    if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) {
  		*esp++ = am_undefined;
  	    } else {
+		ehp = HAllocX(build_proc, 4, HEAP_XTRA);
  		*esp++ = make_tuple(ehp);
- 		ehp[0] = make_arityval(3);
- 		ehp[1] = hp[0];
- 		ehp[2] = hp[1];
- 		ehp[3] = make_small(hp[2]);
- 		ehp += 4;
- 	    }
- 	    break;
+		ehp[0] = make_arityval(3);
+		ehp[1] = cp[0];
+		ehp[2] = cp[1];
+		ehp[3] = make_small((Uint) cp[2]);
+	    }
+	    break;
 	case matchSilent:
 	    --esp;
 	    if (*esp == am_true) {
@@ -2180,8 +2355,12 @@ restart:
 		}
 	    }
 	    break;
-	case matchCatch:
+	case matchCatch:  /* Match success, now build result */
 	    do_catch = 1;
+	    if (in_flags & ERTS_PAM_COPY_RESULT) {
+		build_proc = c_p;
+		esdp->current_process = c_p;
+	    }
 	    break;
 	case matchHalt:
 	    goto success;
@@ -2190,9 +2369,16 @@ restart:
 	}
     }
 fail:
+#if HALFWORD_HEAP
+    if (c_p_checkpoint.p) {
+	/* Dispose garbage built by guards on caller heap */
+	heap_checkpoint_revert(&c_p_checkpoint);
+	c_p_checkpoint.p = NULL;
+    }
+#endif
     *return_flags = 0U;
-    if (fail_label >= 0) { /* We failed during a "TryMeElse", 
-			      lets restart, with the next match 
+    if (fail_label >= 0) { /* We failed during a "TryMeElse",
+			      lets restart, with the next match
 			      program */
 	pc = (prog->text) + fail_label;
 	cleanup_match_pseudo_process(mpsp, 1);
@@ -2206,11 +2392,6 @@ success:
 	erl_exit(1, "Heap fence overwritten in db_prog_match after op "
 		 "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence);
     }
-    if (*eheap_fence != FENCE_PATTERN) {
-	erl_exit(1, "Eheap fence overwritten in db_prog_match after op "
-		 "0x%08x, overwritten with 0x%08x.", save_op, 
-		 *eheap_fence);
-    }
     if (*stack_fence != FENCE_PATTERN) {
 	erl_exit(1, "Stack fence overwritten in db_prog_match after op "
 		 "0x%08x, overwritten with 0x%08x.", save_op, 
@@ -2221,6 +2402,7 @@ success:
     esdp->current_process = current_scheduled;
 
     END_ATOMIC_TRACE(c_p);
+
     return ret;
 #undef FAIL
 #undef FAIL_TERM
@@ -2232,7 +2414,8 @@ success:
 /*
  * Convert a match program to a "magic" binary to return up to erlang
  */
-Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp) {
+Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp)
+{
     return erts_mk_magic_binary_term(hpp, &MSO(p), mp);
 }
 
@@ -2298,13 +2481,13 @@ void db_free_dmc_err_info(DMCErrInfo *ei){
 ** Store bignum in *hpp and increase *hpp accordingly.
 ** *hpp is assumed to be large enough to hold the result.
 */
-Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
+Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr)
 {
-    Eterm big_tmp[2];
+    DeclareTmpHeapNoproc(big_tmp,2);
     Eterm res;
     Sint ires;
-    Eterm arg1;
-    Eterm arg2;
+    Wterm arg1;
+    Wterm arg2;
 
     if (is_both_small(counter,incr)) {
 	ires = signed_val(counter) + signed_val(incr);
@@ -2318,6 +2501,7 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
 	}
     }
     else {
+	UseTmpHeapNoproc(2);
 	switch(NUMBER_CODE(counter, incr)) {
 	case SMALL_BIG:
 	    arg1 = small_to_big(signed_val(counter), big_tmp);
@@ -2332,16 +2516,46 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
 	    arg2 = counter;
 	    break;
 	default:
+	    UnUseTmpHeapNoproc(2);
 	    return THE_NON_VALUE;
 	}
 	res = big_plus(arg1, arg2, *hpp);
 	if (is_big(res)) {
 	    *hpp += BIG_NEED_SIZE(big_size(res));
 	}
+	UnUseTmpHeapNoproc(2);
 	return res;
     }
 }
 
+/* Must be called to read elements after db_lookup_dbterm.
+** Will decompress if needed.
+** HEALFWORD_HEAP:
+** 	Will convert from relative to Wterm format if needed.
+**      (but only on top level, tuples and lists will still contain rterms)
+*/
+Wterm db_do_read_element(DbUpdateHandle* handle, Sint position)
+{
+    Eterm elem = handle->dbterm->tpl[position];
+    if (!is_header(elem)) {
+#if HALFWORD_HEAP
+	if (!is_immed(elem)
+	    && !handle->tb->common.compress
+	    && !(handle->abs_vec && handle->abs_vec[position])) {
+	    return rterm2wterm(elem, handle->dbterm->tpl);
+	}
+#endif
+	return elem;
+    }
+
+    ASSERT(((DbTableCommon*)handle->tb)->compress);
+    ASSERT(!handle->mustResize);
+    handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common,
+					       handle->dbterm);
+    handle->mustResize = 1;
+    return handle->dbterm->tpl[position];
+}
+
 /*
 ** Update one element:
 ** handle:   Initialized by db_lookup_dbterm()
@@ -2358,127 +2572,488 @@ void db_do_update_element(DbUpdateHandle* handle,
     Eterm* oldp;
     Uint newval_sz;
     Uint oldval_sz;
+#if HALFWORD_HEAP
+    Eterm* old_base;
+#endif
 
     if (is_both_immed(newval,oldval)) {
 	handle->dbterm->tpl[position] = newval;
+    #ifdef DEBUG_CLONE
+	if (handle->dbterm->debug_clone) {
+	    handle->dbterm->debug_clone[position] = newval;
+	}
+    #endif
 	return;
     }
-    else if (!handle->mustResize && is_boxed(newval)) {
-	newp = boxed_val(newval);
-	switch (*newp & _TAG_HEADER_MASK) {
-	case _TAG_HEADER_POS_BIG:
-	case _TAG_HEADER_NEG_BIG:
-	case _TAG_HEADER_FLOAT:
-	case _TAG_HEADER_HEAP_BIN:	    
-	    newval_sz = header_arity(*newp) + 1;	    
-	    if (is_boxed(oldval)) {
-		oldp = boxed_val(oldval);
-		switch (*oldp & _TAG_HEADER_MASK) {
+    if (!handle->mustResize) {
+	if (handle->tb->common.compress) {
+	    handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common,
+						       handle->dbterm);
+	    handle->mustResize = 1;
+	    oldval = handle->dbterm->tpl[position];
+        #if HALFWORD_HEAP
+	    old_base = NULL;
+	#endif
+	}
+	else {
+	#if HALFWORD_HEAP
+	    ASSERT(!handle->abs_vec);
+	    old_base = handle->dbterm->tpl;
+	#endif
+	    if (is_boxed(newval)) {
+		newp = boxed_val(newval);
+		switch (*newp & _TAG_HEADER_MASK) {
 		case _TAG_HEADER_POS_BIG:
 		case _TAG_HEADER_NEG_BIG:
 		case _TAG_HEADER_FLOAT:
 		case _TAG_HEADER_HEAP_BIN:
-		    oldval_sz = header_arity(*oldp) + 1;
-		    if (oldval_sz == newval_sz) {
-			/* "self contained" terms of same size, do memcpy */
-			sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm));			
-			return;
+		    newval_sz = header_arity(*newp) + 1;
+		    if (is_boxed(oldval)) {
+			oldp = boxed_val_rel(oldval,old_base);
+			switch (*oldp & _TAG_HEADER_MASK) {
+			case _TAG_HEADER_POS_BIG:
+			case _TAG_HEADER_NEG_BIG:
+			case _TAG_HEADER_FLOAT:
+			case _TAG_HEADER_HEAP_BIN:
+			    oldval_sz = header_arity(*oldp) + 1;
+			    if (oldval_sz == newval_sz) {
+				/* "self contained" terms of same size, do memcpy */
+				    sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm));
+				return;
+			    }
+			    goto both_size_set;
+			}
 		    }
-		    goto both_size_set;
+		    goto new_size_set;
 		}
 	    }
-	    goto new_size_set;
 	}
     }
+#if HALFWORD_HEAP
+    else {
+	old_base = (handle->tb->common.compress
+		    || (handle->abs_vec && handle->abs_vec[position])) ?
+	    NULL : handle->dbterm->tpl;
+    }
+#endif
     /* Not possible for simple memcpy or dbterm is already non-contiguous, */
     /* need to realloc... */
 
     newval_sz = is_immed(newval) ? 0 : size_object(newval);
 new_size_set:
-	
-    oldval_sz = is_immed(oldval) ? 0 : size_object(oldval);
+
+    oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base);
 both_size_set:
 
     handle->new_size = handle->new_size - oldval_sz + newval_sz;
 
-    /* write new value in old dbterm, finalize will make a flat copy */	
+    /* write new value in old dbterm, finalize will make a flat copy */
     handle->dbterm->tpl[position] = newval;
     handle->mustResize = 1;
+
+#if HALFWORD_HEAP
+    if (old_base && newval_sz > 0) {
+	ASSERT(!handle->tb->common.compress);
+	if (!handle->abs_vec) {
+	    int i = header_arity(handle->dbterm->tpl[0]);
+	    handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char));
+	    sys_memset(handle->abs_vec, 0, i+1);
+	    /* abs_vec[0] not used */
+	}
+	handle->abs_vec[position] = 1;
+    }
+#endif
+}
+
+static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old,
+					 Uint old_sz, Uint new_sz, Uint offset)
+{
+    byte* ret;
+    if (erts_ets_realloc_always_moves) {
+	ret = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, new_sz);
+	sys_memcpy(ret, old, offset);
+	erts_db_free(ERTS_ALC_T_DB_TERM, (DbTable*)tb, old, old_sz);
+    } else {
+	ret = erts_db_realloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb,
+			      old, old_sz, new_sz);
+    }
+    return ret;
 }
 
+/* Allocated size of a compressed dbterm
+*/
+static ERTS_INLINE Uint db_alloced_size_comp(DbTerm* obj)
+{
+    return obj->tpl[arityval(*obj->tpl) + 1];
+}
+
+void db_free_term(DbTable *tb, void* basep, Uint offset)
+{
+    DbTerm* db = (DbTerm*) ((byte*)basep + offset);
+    Uint size;
+    if (tb->common.compress) {
+	db_cleanup_offheap_comp(db);
+	size = db_alloced_size_comp(db);
+    }
+    else {
+	ErlOffHeap tmp_oh;
+	tmp_oh.first = db->first_oh;
+	erts_cleanup_offheap(&tmp_oh);
+	size = offset + offsetof(DbTerm,tpl) + db->size*sizeof(Eterm);
+    }
+    erts_db_free(ERTS_ALC_T_DB_TERM, tb, basep, size);
+}
+
+static ERTS_INLINE Uint align_up(Uint value, Uint pow2)
+{
+    ASSERT((pow2 & (pow2-1)) == 0);
+    return (value + (pow2-1)) & ~(pow2-1);
+}
+
+/* Compressed size of an uncompressed term
+*/
+static Uint db_size_dbterm_comp(DbTableCommon* tb, Eterm obj)
+{
+    Eterm* tpl = tuple_val(obj);
+    int i;
+    Uint size = sizeof(DbTerm)
+	+ arityval(*tpl) * sizeof(Eterm)
+        + sizeof(Uint); /* "alloc_size" */
+
+    for (i = arityval(*tpl); i>0; i--) {
+	if (i != tb->keypos && is_not_immed(tpl[i])) {
+	    size += erts_encode_ext_size_ets(tpl[i]);
+	}
+    }
+    size += size_object(tpl[tb->keypos]) * sizeof(Eterm);
+    return align_up(size, sizeof(Uint));
+}
+
+/* Conversion between top tuple element and pointer to compressed data
+*/
+static ERTS_INLINE Eterm ext2elem(Eterm* tpl, byte* ext)
+{
+    return (((Uint)(ext - (byte*)tpl)) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER;
+}
+static ERTS_INLINE byte* elem2ext(Eterm* tpl, Uint ix)
+{
+    ASSERT(is_header(tpl[ix]));
+    return (byte*)tpl + (tpl[ix] >> _TAG_PRIMARY_SIZE);
+}
+
+static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest,
+			  Uint alloc_size)
+{
+    ErlOffHeap tmp_offheap;
+    Eterm* src = tuple_val(obj);
+    Eterm* tpl = dest->tpl;
+    Eterm key = src[tb->keypos];
+    int arity = arityval(src[0]);
+    union {
+	Eterm* ep;
+	byte* cp;
+	UWord ui;
+    }top;
+    int i;
+
+    top.ep = tpl+ 1 + arity + 1;
+    tpl[0] = src[0];
+    tpl[arity + 1] = alloc_size;
+
+    tmp_offheap.first = NULL;
+    tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl);
+    dest->first_oh = tmp_offheap.first;
+    for (i=1; i<=arity; i++) {
+	if (i != tb->keypos) {
+	    if (is_immed(src[i])) {
+		tpl[i] = src[i];
+	    }
+	    else {
+		tpl[i] = ext2elem(tpl, top.cp);
+		top.cp = erts_encode_ext_ets(src[i], top.cp, &dest->first_oh);
+	    }
+	}
+    }
+
+#ifdef DEBUG_CLONE
+    {
+	Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm));
+	dest->debug_clone = dbg_top;
+	tmp_offheap.first = dest->first_oh;
+	copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top);
+	dest->first_oh = tmp_offheap.first;
+	ASSERT(dbg_top == dest->debug_clone + dest->size);
+    }
+#endif
+    return top.cp;
+}
 
 /*
 ** Copy the object into a possibly new DbTerm, 
 ** offset is the offset of the DbTerm from the start
-** of the sysAllocaed structure, The possibly realloced and copied
+** of the allocated structure, The possibly realloced and copied
 ** structure is returned. Make sure (((char *) old) - offset) is a 
 ** pointer to a ERTS_ALC_T_DB_TERM allocated data area.
 */
-void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj)
+void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj)
 {
+    byte* basep;
+    DbTerm* newp;
+    Eterm* top;
     int size = size_object(obj);
-    void *structp = ((char*) old) - offset;
-    DbTerm* p;
-    Eterm copy;
-    Eterm *top;
+    ErlOffHeap tmp_offheap;
 
     if (old != 0) {
-	erts_cleanup_offheap(&old->off_heap);
+	basep = ((byte*) old) - offset;
+	tmp_offheap.first  = old->first_oh;
+	erts_cleanup_offheap(&tmp_offheap);
+	old->first_oh = tmp_offheap.first;
 	if (size == old->size) {
-	    p = old;
-	} else {
+	    newp = old;
+	}
+	else {
 	    Uint new_sz = offset + sizeof(DbTerm) + sizeof(Eterm)*(size-1);
 	    Uint old_sz = offset + sizeof(DbTerm) + sizeof(Eterm)*(old->size-1);
 
-	    if (erts_ets_realloc_always_moves) {
-		void *nstructp = erts_db_alloc(ERTS_ALC_T_DB_TERM,
-					       (DbTable *) tb,
-					       new_sz);
-		memcpy(nstructp,structp,offset);
-		erts_db_free(ERTS_ALC_T_DB_TERM,
-			     (DbTable *) tb,
-			     structp,
-			     old_sz);
-		structp = nstructp;
-	    } else {
-		structp = erts_db_realloc(ERTS_ALC_T_DB_TERM,
-					  (DbTable *) tb,
-					  structp,
-					  old_sz,
-					  new_sz);
-	    }
-	    p = (DbTerm*) ((void *)(((char *) structp) + offset));
+	    basep = db_realloc_term(tb, basep, old_sz, new_sz, offset);
+	    newp = (DbTerm*) (basep + offset);
 	}
     }
     else {
-	structp = erts_db_alloc(ERTS_ALC_T_DB_TERM,
-				(DbTable *) tb,
-				(offset
-				 + sizeof(DbTerm)
-				 + sizeof(Eterm)*(size-1)));
-	p = (DbTerm*) ((void *)(((char *) structp) + offset));
-    }
-    p->size = size;
-    p->off_heap.mso = NULL;
-    p->off_heap.externals = NULL;
-#ifndef HYBRID /* FIND ME! */
-    p->off_heap.funs = NULL;
+	basep = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable *)tb,
+			      (offset + sizeof(DbTerm) + sizeof(Eterm)*(size-1)));
+	newp = (DbTerm*) (basep + offset);
+    }
+    newp->size = size;
+    top = newp->tpl;
+    tmp_offheap.first  = NULL;
+    copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top);
+    newp->first_oh = tmp_offheap.first;
+#ifdef DEBUG_CLONE
+    newp->debug_clone = NULL;
 #endif
-    p->off_heap.overhead = 0;
+    return basep;
+}
+
 
-    top = DBTERM_BUF(p);
-    copy = copy_struct(obj, size, &top, &p->off_heap);
-    DBTERM_SET_TPL(p,tuple_val(copy));
+void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj)
+{
+    Uint new_sz = offset + db_size_dbterm_comp(tb, obj);
+    byte* basep;
+    DbTerm* newp;
+    byte* top;
+
+    ASSERT(tb->compress);
+    if (old != 0) {
+	Uint old_sz = db_alloced_size_comp(old);
+	db_cleanup_offheap_comp(old);
 
-    return structp;
+	basep = ((byte*) old) - offset;
+	if (new_sz == old_sz) {
+	    newp = old;
+	}
+	else {
+	    basep = db_realloc_term(tb, basep, old_sz, new_sz, offset);
+	    newp = (DbTerm*) (basep + offset);
+	}
+    }
+    else {
+	basep = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, new_sz);
+	newp = (DbTerm*) (basep + offset);
+    }
+
+    newp->size = size_object(obj);
+    top = copy_to_comp(tb, obj, newp, new_sz);
+    ASSERT(top <= basep + new_sz);
+
+    /* ToDo: Maybe realloc if ((basep+new_sz) - top) > WASTED_SPACE_LIMIT */
+
+    return basep;
 }
 
 
-void db_free_term_data(DbTerm* p)
+void db_finalize_resize(DbUpdateHandle* handle, Uint offset)
 {
-    erts_cleanup_offheap(&p->off_heap);
+    DbTable* tbl = handle->tb;
+    DbTerm* newDbTerm;
+    Uint alloc_sz = offset +
+	(tbl->common.compress ?
+	 db_size_dbterm_comp(&tbl->common, make_tuple(handle->dbterm->tpl)) :
+	 sizeof(DbTerm)+sizeof(Eterm)*(handle->new_size-1));
+    byte* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl, alloc_sz);
+    byte* oldp = *(handle->bp);
+
+    sys_memcpy(newp, oldp, offset);  /* copy only hash/tree header */
+    *(handle->bp) = newp;
+    newDbTerm = (DbTerm*) (newp + offset);
+    newDbTerm->size = handle->new_size;
+#ifdef DEBUG_CLONE
+    newDbTerm->debug_clone = NULL;
+#endif
+
+    /* make a flat copy */
+
+    if (tbl->common.compress) {
+	copy_to_comp(&tbl->common, make_tuple(handle->dbterm->tpl),
+		     newDbTerm, alloc_sz);
+	db_free_tmp_uncompressed(handle->dbterm);
+    }
+    else {
+	ErlOffHeap tmp_offheap;
+	Eterm* tpl = handle->dbterm->tpl;
+	Eterm* top = newDbTerm->tpl;
+
+	tmp_offheap.first = NULL;
+
+    #if HALFWORD_HEAP
+	if (handle->abs_vec) {
+	    int i, arity = header_arity(handle->dbterm->tpl[0]);
+
+	    top[0] = tpl[0];
+	    top += arity + 1;
+	    for (i=1; i<=arity; i++) {
+		Eterm* src_base = handle->abs_vec[i] ? NULL : tpl;
+
+		newDbTerm->tpl[i] = copy_struct_rel(tpl[i],
+						    size_object_rel(tpl[i],src_base),
+						    &top, &tmp_offheap, src_base,
+						    newDbTerm->tpl);
+	    }
+	    newDbTerm->first_oh = tmp_offheap.first;
+	    ASSERT((byte*)top <= (newp + alloc_sz));
+	    erts_free(ERTS_ALC_T_TMP, handle->abs_vec);
+	}
+	else
+    #endif /* HALFWORD_HEAP */
+	{
+	    copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top,
+			    &tmp_offheap, tpl, top);
+	    newDbTerm->first_oh = tmp_offheap.first;
+	    ASSERT((byte*)top == (newp + alloc_sz));
+	}
+    }
+}
+
+Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
+			     ErlOffHeap* off_heap)
+{
+    Eterm* hp = *hpp;
+    int i, arity = arityval(bp->tpl[0]);
+
+    hp[0] = bp->tpl[0];
+    *hpp += arity + 1;
+
+    hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos],
+				     size_object_rel(bp->tpl[tb->keypos], bp->tpl),
+				     hpp, off_heap, bp->tpl, NULL);
+    for (i=arity; i>0; i--) {
+	if (i != tb->keypos) {
+	    if (is_immed(bp->tpl[i])) {
+		hp[i] = bp->tpl[i];
+	    }
+	    else {
+		hp[i] = erts_decode_ext_ets(hpp, off_heap,
+					    elem2ext(bp->tpl, i));
+	    }
+	}
+    }
+    ASSERT((*hpp - hp) <= bp->size);
+#ifdef DEBUG_CLONE
+    ASSERT(eq_rel(make_tuple(hp),make_tuple(bp->debug_clone),bp->debug_clone));
+#endif
+    return make_tuple(hp);
+}
+
+Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p,
+			       DbTerm* obj, Uint pos,
+			       Eterm** hpp, Uint extra)
+{
+    if (is_immed(obj->tpl[pos])) {
+	*hpp = HAlloc(p, extra);
+	return obj->tpl[pos];
+    }
+    if (tb->compress && pos != tb->keypos) {
+	byte* ext = elem2ext(obj->tpl, pos);
+	Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra;
+	Eterm* hp = HAlloc(p, sz);
+	Eterm* endp = hp + sz;
+	Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext);
+	*hpp = hp;
+	hp += extra;
+	HRelease(p, endp, hp);
+#ifdef DEBUG_CLONE
+	ASSERT(eq_rel(copy, obj->debug_clone[pos], obj->debug_clone));
+#endif
+	return copy;
+    }
+    else {
+	Uint sz = size_object_rel(obj->tpl[pos], obj->tpl);
+	*hpp = HAlloc(p, sz + extra);
+	return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL);
+    }
+}
+
+
+/* Our own "cleanup_offheap"
+ * as refc-binaries may be unaligned in compressed terms
+*/
+void db_cleanup_offheap_comp(DbTerm* obj)
+{
+    union erl_off_heap_ptr u;
+    ProcBin tmp;
+
+    for (u.hdr = obj->first_oh; u.hdr; u.hdr = u.hdr->next) {
+	if ((UWord)u.voidp % sizeof(Uint) != 0) { /* unaligned ptr */
+	    sys_memcpy(&tmp, u.voidp, sizeof(tmp));
+	    /* Warning, must pass (void*)-variable to memcpy. Otherwise it will
+	       cause Bus error on Sparc due to false compile time assumptions
+	       about word aligned memory (type cast is not enough) */
+	    u.pb = &tmp;
+	}
+	switch (thing_subtag(u.hdr->thing_word)) {
+	case REFC_BINARY_SUBTAG:
+	    if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) {
+		erts_bin_free(u.pb->val);
+	    }
+	    break;
+	case FUN_SUBTAG:
+	    ASSERT(u.pb != &tmp);
+	    if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) {
+		erts_erase_fun_entry(u.fun->fe);
+	    }
+	    break;
+	default:
+	    ASSERT(is_external_header(u.hdr->thing_word));
+	    ASSERT(u.pb != &tmp);
+	    erts_deref_node_entry(u.ext->node);
+	    break;
+	}
+    }
+#ifdef DEBUG_CLONE
+    if (obj->debug_clone != NULL) {
+	erts_free(ERTS_ALC_T_DB_TERM, obj->debug_clone);
+	obj->debug_clone = NULL;
+    }
+#endif
 }
 
+int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b)
+{
+    ErlOffHeap tmp_offheap;
+    Eterm* allocp;
+    Eterm* hp;
+    Eterm tmp_b;
+    int is_eq;
+
+    ASSERT(tb->compress);
+    hp = allocp = erts_alloc(ERTS_ALC_T_TMP, b->size*sizeof(Eterm));
+    tmp_offheap.first = NULL;
+    tmp_b = db_copy_from_comp(tb, b, &hp, &tmp_offheap);
+    is_eq = eq(a,tmp_b);
+    erts_cleanup_offheap(&tmp_offheap);
+    erts_free(ERTS_ALC_T_TMP, allocp);
+    return is_eq;
+}
 
 /*
 ** Check if object represents a "match" variable 
@@ -2606,7 +3181,7 @@ static void add_dmc_err(DMCErrInfo *err_info,
 static DMCRet dmc_one_term(DMCContext *context, 
 			   DMCHeap *heap,
 			   DMC_STACK_TYPE(Eterm) *stack,
-			   DMC_STACK_TYPE(Uint) *text,
+			   DMC_STACK_TYPE(UWord) *text,
 			   Eterm c)
 {
     Sint n;
@@ -2624,7 +3199,7 @@ static DMCRet dmc_one_term(DMCContext *context,
 		** Ouch, big integer in match variable.
 		*/
 		Eterm *save_hp;
-		ASSERT(heap->data == heap->def);
+		ASSERT(heap->vars == heap->vars_def);
 		sz = sz2 = sz3 = 0;
 		for (j = 0; j < context->num_match; ++j) {
 		    sz += size_object(context->matchexpr[j]);
@@ -2662,24 +3237,23 @@ static DMCRet dmc_one_term(DMCContext *context,
 		       may be atoms that changed */
 		    context->matchexpr[j] = context->copy->mem[j];
 		}
-		heap->data = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP,
-					heap->size*sizeof(unsigned));
-		sys_memset(heap->data, 0, 
-			   heap->size * sizeof(unsigned));
+		heap->vars = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP,
+					heap->size*sizeof(DMCVariable));
+		sys_memset(heap->vars, 0, heap->size * sizeof(DMCVariable));
 		DMC_CLEAR(*stack);
 		/*DMC_PUSH(*stack,NIL);*/
 		DMC_CLEAR(*text);
 		return retRestart;
 	    }
-	    if (heap->data[n]) { /* already bound ? */
+	    if (heap->vars[n].is_bound) {
 		DMC_PUSH(*text,matchCmp);
 		DMC_PUSH(*text,n);
 	    } else { /* Not bound, bind! */
-		if (n >= heap->used)
-		    heap->used = n + 1;
+		if (n >= heap->vars_used)
+		    heap->vars_used = n + 1;
 		DMC_PUSH(*text,matchBind);
 		DMC_PUSH(*text,n);
-		heap->data[n] = 1;
+		heap->vars[n].is_bound = 1;
 	    }
 	} else if (c == am_Underscore) {
 	    DMC_PUSH(*text, matchSkip);
@@ -2704,27 +3278,84 @@ static DMCRet dmc_one_term(DMCContext *context,
 	    DMC_PUSH(*stack, c);
 	    break;
 	case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
-	    n = thing_arityval(*internal_ref_val(c));
+	{
+	    Eterm* ref_val = internal_ref_val(c);
 	    DMC_PUSH(*text, matchEqRef);
-	    DMC_PUSH(*text, *internal_ref_val(c));
-	    for (i = 1; i <= n; ++i) {
-		DMC_PUSH(*text, (Uint) internal_ref_val(c)[i]);
+#if HALFWORD_HEAP
+	    {
+		union {
+		    UWord u;
+		    Uint t[2];
+		} fiddle;
+		ASSERT(thing_arityval(ref_val[0]) == 3);
+		fiddle.t[0] = ref_val[0];
+		fiddle.t[1] = ref_val[1];
+		DMC_PUSH(*text, fiddle.u);
+		fiddle.t[0] = ref_val[2];
+		fiddle.t[1] = ref_val[3];
+		DMC_PUSH(*text, fiddle.u);
 	    }
+#else
+	    n = thing_arityval(ref_val[0]);
+	    for (i = 0; i <= n; ++i) {
+		DMC_PUSH(*text, ref_val[i]);
+	    }
+#endif
 	    break;
+	}
 	case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):
 	case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):
-	    n = thing_arityval(*big_val(c));
+	{
+	    Eterm* bval = big_val(c);
+	    n = thing_arityval(bval[0]);
 	    DMC_PUSH(*text, matchEqBig);
-	    DMC_PUSH(*text, *big_val(c));
-	    for (i = 1; i <= n; ++i) {
-		DMC_PUSH(*text, (Uint) big_val(c)[i]);
+#if HALFWORD_HEAP
+	    {
+		union {
+		    UWord u;
+		    Uint t[2];
+		} fiddle;
+		ASSERT(n >= 1);
+		fiddle.t[0] = bval[0];
+		fiddle.t[1] = bval[1];
+		DMC_PUSH(*text, fiddle.u);
+		for (i = 2; i <= n; ++i) {
+		    fiddle.t[0] = bval[i];
+		    if (++i <= n) {
+			fiddle.t[1] = bval[i];
+		    } else {
+			fiddle.t[1] = (Uint) 0;
+		    }
+		    DMC_PUSH(*text, fiddle.u);
+		}
 	    }
+#else
+	    for (i = 0; i <= n; ++i) {
+		DMC_PUSH(*text, (Uint) bval[i]);
+	    }
+#endif
 	    break;
+	}
 	case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
 	    DMC_PUSH(*text,matchEqFloat);
+#if HALFWORD_HEAP
+	    {
+		union {
+		    UWord u;
+		    Uint t[2];
+		} fiddle;
+		fiddle.t[0] = float_val(c)[1];
+		fiddle.t[1] = float_val(c)[2];
+		DMC_PUSH(*text, fiddle.u);
+	    }
+#else
 	    DMC_PUSH(*text, (Uint) float_val(c)[1]);
-	    /* XXX: this reads and pushes random junk on ARCH_64 */
+#ifdef ARCH_64
+	    DMC_PUSH(*text, (Uint) 0);
+#else
 	    DMC_PUSH(*text, (Uint) float_val(c)[2]);
+#endif
+#endif
 	    break;
 	default: /* BINARY, FUN, VECTOR, or EXTERNAL */
 	    /*
@@ -2753,7 +3384,7 @@ static DMCRet dmc_one_term(DMCContext *context,
 ** Match guard compilation
 */
 
-static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(Uint) *text,
+static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text,
 			     Eterm t) 
 {
 	int sz;
@@ -2807,7 +3438,7 @@ add_dmc_err((ContextP)->err_info, String, -1, T, dmcWarning)
 
 static DMCRet dmc_list(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant)
 {
@@ -2837,17 +3468,16 @@ static DMCRet dmc_list(DMCContext *context,
 	DMC_PUSH(*text, matchConsB);
     }
     --context->stack_used; /* Two objects on stack becomes one */
-    context->eheap_need += 2;
     return retOk;
 }
 
 static DMCRet dmc_tuple(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant)
 {
-    DMC_STACK_TYPE(Uint) instr_save;
+    DMC_STACK_TYPE(UWord) instr_save;
     int all_constant = 1;
     int textpos = DMC_STACK_NUM(*text);
     Eterm *p = tuple_val(t);
@@ -2896,14 +3526,13 @@ static DMCRet dmc_tuple(DMCContext *context,
     DMC_PUSH(*text, matchMkTuple);
     DMC_PUSH(*text, nelems);
     context->stack_used -= (nelems - 1);
-    context->eheap_need += (nelems + 1);
     *constant = 0;
     return retOk;
 }
 
 static DMCRet dmc_whole_expression(DMCContext *context,
 				   DMCHeap *heap,
-				   DMC_STACK_TYPE(Uint) *text,
+				   DMC_STACK_TYPE(UWord) *text,
 				   Eterm t,
 				   int *constant)
 {
@@ -2914,9 +3543,6 @@ static DMCRet dmc_whole_expression(DMCContext *context,
 	} else { 
 	    ASSERT(is_tuple(context->matchexpr
 			    [context->current_match]));
-	    context->eheap_need += 
-		arityval(*(tuple_val(context->matchexpr
-				     [context->current_match]))) * 2;
 	    DMC_PUSH(*text, matchPushArrayAsList);
 	}
     } else {
@@ -2929,20 +3555,55 @@ static DMCRet dmc_whole_expression(DMCContext *context,
     return retOk;
 }
 
+/* Figure out which PushV instruction to use.
+*/
+static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap,
+				  DMC_STACK_TYPE(UWord) *text, Uint n)
+{
+    DMCVariable* v = &heap->vars[n];
+    MatchOps instr = matchPushV;
+
+    ASSERT(n < heap->vars_used && v->is_bound);
+    if (context->is_guard) {
+	#if HALFWORD_HEAP
+	if (!v->first_guard_label) {
+	    v->first_guard_label = DMC_STACK_NUM(*text);
+	    ASSERT(v->first_guard_label);
+	    instr = matchPushVGuard; /* may be changed to PushVResult below */
+	}
+	#endif
+    }
+    else { /* body */
+	#if HALFWORD_HEAP
+	if (v->first_guard_label) {
+	    /* Avoid double-copy, copy to result heap at first encounter in guard */
+	    DMC_POKE(*text, v->first_guard_label, matchPushVResult);
+	    v->is_in_body = 1;
+	}
+	#endif
+	if (!v->is_in_body) {
+	    instr = matchPushVResult;
+	    v->is_in_body = 1;
+	}
+    }
+    DMC_PUSH(*text, instr);
+    DMC_PUSH(*text, n);
+}
+
 static DMCRet dmc_variable(DMCContext *context,
 			   DMCHeap *heap,
-			   DMC_STACK_TYPE(Uint) *text,
+			   DMC_STACK_TYPE(UWord) *text,
 			   Eterm t,
 			   int *constant)
 {
     Uint n = db_is_variable(t);
-    ASSERT(n >= 0);
-    if (n >= heap->used) 
-	RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant);
-    if (heap->data[n] == 0U)
+
+    if (n >= heap->vars_used || !heap->vars[n].is_bound) {
 	RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant);
-    DMC_PUSH(*text, matchPushV);
-    DMC_PUSH(*text, n);
+    }
+
+    dmc_add_pushv_variant(context, heap, text, n);
+
     ++context->stack_used;
     if (context->stack_used > context->stack_need)
 	context->stack_need = context->stack_used;
@@ -2952,7 +3613,7 @@ static DMCRet dmc_variable(DMCContext *context,
 
 static DMCRet dmc_all_bindings(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -2961,10 +3622,9 @@ static DMCRet dmc_all_bindings(DMCContext *context,
 
     DMC_PUSH(*text, matchPushC);
     DMC_PUSH(*text, NIL);
-    for (i = heap->used - 1; i >= 0; --i) { 
-	if (heap->data[i]) {
-	    DMC_PUSH(*text, matchPushV);
-	    DMC_PUSH(*text, i);
+    for (i = heap->vars_used - 1; i >= 0; --i) {
+	if (heap->vars[i].is_bound) {
+	    dmc_add_pushv_variant(context, heap, text, i);
 	    DMC_PUSH(*text, matchConsB);
 	    heap_used += 2;
 	}
@@ -2972,14 +3632,13 @@ static DMCRet dmc_all_bindings(DMCContext *context,
     ++context->stack_used;
     if ((context->stack_used + 1) > context->stack_need)
 	context->stack_need = (context->stack_used + 1);
-    context->eheap_need += heap_used;
     *constant = 0;
     return retOk;
 }
 
 static DMCRet dmc_const(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant)
 {
@@ -2996,7 +3655,7 @@ static DMCRet dmc_const(DMCContext *context,
 
 static DMCRet dmc_and(DMCContext *context,
 		      DMCHeap *heap,
-		      DMC_STACK_TYPE(Uint) *text,
+		      DMC_STACK_TYPE(UWord) *text,
 		      Eterm t,
 		      int *constant)
 {
@@ -3025,7 +3684,7 @@ static DMCRet dmc_and(DMCContext *context,
 
 static DMCRet dmc_or(DMCContext *context,
 		     DMCHeap *heap,
-		     DMC_STACK_TYPE(Uint) *text,
+		     DMC_STACK_TYPE(UWord) *text,
 		     Eterm t,
 		     int *constant)
 {
@@ -3055,7 +3714,7 @@ static DMCRet dmc_or(DMCContext *context,
 
 static DMCRet dmc_andalso(DMCContext *context,
 			  DMCHeap *heap,
-			  DMC_STACK_TYPE(Uint) *text,
+			  DMC_STACK_TYPE(UWord) *text,
 			  Eterm t,
 			  int *constant)
 {
@@ -3104,7 +3763,7 @@ static DMCRet dmc_andalso(DMCContext *context,
 
 static DMCRet dmc_orelse(DMCContext *context,
 			 DMCHeap *heap,
-			 DMC_STACK_TYPE(Uint) *text,
+			 DMC_STACK_TYPE(UWord) *text,
 			 Eterm t,
 			 int *constant)
 {
@@ -3152,7 +3811,7 @@ static DMCRet dmc_orelse(DMCContext *context,
 
 static DMCRet dmc_message(DMCContext *context,
 			  DMCHeap *heap,
-			  DMC_STACK_TYPE(Uint) *text,
+			  DMC_STACK_TYPE(UWord) *text,
 			  Eterm t,
 			  int *constant)
 {
@@ -3194,7 +3853,7 @@ static DMCRet dmc_message(DMCContext *context,
 
 static DMCRet dmc_self(DMCContext *context,
 		     DMCHeap *heap,
-		     DMC_STACK_TYPE(Uint) *text,
+		     DMC_STACK_TYPE(UWord) *text,
 		     Eterm t,
 		     int *constant)
 {
@@ -3214,7 +3873,7 @@ static DMCRet dmc_self(DMCContext *context,
 
 static DMCRet dmc_return_trace(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -3244,7 +3903,7 @@ static DMCRet dmc_return_trace(DMCContext *context,
 
 static DMCRet dmc_exception_trace(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -3276,7 +3935,7 @@ static DMCRet dmc_exception_trace(DMCContext *context,
 
 static DMCRet dmc_is_seq_trace(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -3302,7 +3961,7 @@ static DMCRet dmc_is_seq_trace(DMCContext *context,
 
 static DMCRet dmc_set_seq_token(DMCContext *context,
 				DMCHeap *heap,
-				DMC_STACK_TYPE(Uint) *text,
+				DMC_STACK_TYPE(UWord) *text,
 				Eterm t,
 				int *constant)
 {
@@ -3351,7 +4010,7 @@ static DMCRet dmc_set_seq_token(DMCContext *context,
 
 static DMCRet dmc_get_seq_token(DMCContext *context,
 				DMCHeap *heap,
-				DMC_STACK_TYPE(Uint) *text,
+				DMC_STACK_TYPE(UWord) *text,
 				Eterm t,
 				int *constant)
 {
@@ -3375,10 +4034,6 @@ static DMCRet dmc_get_seq_token(DMCContext *context,
 
     *constant = 0;
     DMC_PUSH(*text, matchGetSeqToken);
-    context->eheap_need += (6 /* A 5-tuple is built */
-			    + EXTERNAL_THING_HEAD_SIZE + 2 /* Sender can
-							      be an external
-							      pid */);
     if (++context->stack_used > context->stack_need)
  	context->stack_need = context->stack_used;
     return retOk;
@@ -3388,7 +4043,7 @@ static DMCRet dmc_get_seq_token(DMCContext *context,
 
 static DMCRet dmc_display(DMCContext *context,
 			  DMCHeap *heap,
-			  DMC_STACK_TYPE(Uint) *text,
+			  DMC_STACK_TYPE(UWord) *text,
 			  Eterm t,
 			  int *constant)
 {
@@ -3428,7 +4083,7 @@ static DMCRet dmc_display(DMCContext *context,
 
 static DMCRet dmc_process_dump(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -3458,7 +4113,7 @@ static DMCRet dmc_process_dump(DMCContext *context,
 
 static DMCRet dmc_enable_trace(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -3518,7 +4173,7 @@ static DMCRet dmc_enable_trace(DMCContext *context,
 
 static DMCRet dmc_disable_trace(DMCContext *context,
 				DMCHeap *heap,
-				DMC_STACK_TYPE(Uint) *text,
+				DMC_STACK_TYPE(UWord) *text,
 				Eterm t,
 				int *constant)
 {
@@ -3578,7 +4233,7 @@ static DMCRet dmc_disable_trace(DMCContext *context,
 
 static DMCRet dmc_trace(DMCContext *context,
 			DMCHeap *heap,
-			DMC_STACK_TYPE(Uint) *text,
+			DMC_STACK_TYPE(UWord) *text,
 			Eterm t,
 			int *constant)
 {
@@ -3652,7 +4307,7 @@ static DMCRet dmc_trace(DMCContext *context,
 
 static DMCRet dmc_caller(DMCContext *context,
  			 DMCHeap *heap,
- 			 DMC_STACK_TYPE(Uint) *text,
+			 DMC_STACK_TYPE(UWord) *text,
  			 Eterm t,
  			 int *constant)
 {
@@ -3675,7 +4330,6 @@ static DMCRet dmc_caller(DMCContext *context,
     }
     *constant = 0;
     DMC_PUSH(*text, matchCaller); /* Creates binary */
-    context->eheap_need += 4;     /* A 3-tuple is built */
     if (++context->stack_used > context->stack_need)
  	context->stack_need = context->stack_used;
     return retOk;
@@ -3685,7 +4339,7 @@ static DMCRet dmc_caller(DMCContext *context,
   
 static DMCRet dmc_silent(DMCContext *context,
  			 DMCHeap *heap,
- 			 DMC_STACK_TYPE(Uint) *text,
+			 DMC_STACK_TYPE(UWord) *text,
  			 Eterm t,
  			 int *constant)
 {
@@ -3727,7 +4381,7 @@ static DMCRet dmc_silent(DMCContext *context,
 
 static DMCRet dmc_fun(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant)
 {
@@ -3844,7 +4498,7 @@ static DMCRet dmc_fun(DMCContext *context,
 	erl_exit(1,"ets:match() internal error, "
 		 "guard with more than 3 arguments.");
     }
-    DMC_PUSH(*text, (Uint) b->biff);
+    DMC_PUSH(*text, (UWord) b->biff);
     context->stack_used -= (((int) a) - 2);
     if (context->stack_used > context->stack_need)
  	context->stack_need = context->stack_used;
@@ -3853,7 +4507,7 @@ static DMCRet dmc_fun(DMCContext *context,
 
 static DMCRet dmc_expr(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant)
 {
@@ -3916,7 +4570,7 @@ static DMCRet dmc_expr(DMCContext *context,
     
 static DMCRet compile_guard_expr(DMCContext *context,
 				 DMCHeap *heap,
-				 DMC_STACK_TYPE(Uint) *text,
+				 DMC_STACK_TYPE(UWord) *text,
 				 Eterm l)
 {
     DMCRet ret;
@@ -4031,7 +4685,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info)
     DMC_INIT_STACK(heap);
 
     p = expr->mem;
-    i = expr->size;
+    i = expr->used_size;
     while (i--) {
 	if (is_thing(*p)) {
 	    a = thing_arityval(*p);
@@ -4060,7 +4714,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info)
     }
 
     p = expr->mem;
-    i = expr->size;
+    i = expr->used_size;
     while (i--) {
 	if (is_thing(*p)) {
 	    a = thing_arityval(*p);
@@ -4230,7 +4884,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
     Eterm l;
     Uint32 ret_flags;
     Uint sz;
-    Eterm *save_cp;
+    BeamInstr *save_cp;
 
     if (trace && !(is_list(against) || against == NIL)) {
 	return THE_NON_VALUE;
@@ -4271,25 +4925,26 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
 		++n;
 		l = CDR(list_val(l));
 	    }
+	    save_cp = p->cp;
+	    p->cp = NULL;
+	    res = erts_match_set_run(p, mps, arr, n,
+				     ERTS_PAM_COPY_RESULT, &ret_flags);
+	    p->cp = save_cp;
 	} else {
 	    n = 0;
-	    arr = (Eterm *) against;
+	    arr = NULL;
+	    res = erts_match_set_run_ets(p, mps, against, n, &ret_flags);
 	}
 	
 	/* We are in the context of a BIF, 
 	   {caller} should return 'undefined' */
-	save_cp = p->cp;
-	p->cp = NULL;
-	res = erts_match_set_run(p, mps, arr, n, &ret_flags);
-	p->cp = save_cp;
 	if (is_non_value(res)) {
 	    res = am_false;
 	}
-	sz = size_object(res);
+	sz = 0;
 	if (ret_flags & MATCH_SET_EXCEPTION_TRACE) sz += 2;
 	if (ret_flags & MATCH_SET_RETURN_TRACE) sz += 2;
 	hp = HAlloc(p, 5 + sz);
-	res = copy_struct(res, sz, &hp, &MSO(p));
 	flg = NIL;
 	if (ret_flags & MATCH_SET_EXCEPTION_TRACE) {
 	    flg = CONS(hp, am_exception_trace, flg);
@@ -4316,15 +4971,70 @@ static Eterm seq_trace_fake(Process *p, Eterm arg1)
     }
     return result;
 }
-    
+
+DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org)
+{
+    ErlOffHeap tmp_offheap;
+    DbTerm* res = erts_alloc(ERTS_ALC_T_TMP,
+			     sizeof(DbTerm) + org->size*sizeof(Eterm));
+    Eterm* hp = res->tpl;
+    tmp_offheap.first = NULL;
+    db_copy_from_comp(tb, org, &hp, &tmp_offheap);
+    res->first_oh = tmp_offheap.first;
+    res->size = org->size;
+#ifdef DEBUG_CLONE
+    res->debug_clone = NULL;
+#endif
+    return res;
+}
+
+void db_free_tmp_uncompressed(DbTerm* obj)
+{
+    ErlOffHeap off_heap;
+    off_heap.first = obj->first_oh;
+    erts_cleanup_offheap(&off_heap);
+#ifdef DEBUG_CLONE
+    ASSERT(obj->debug_clone == NULL);
+#endif
+    erts_free(ERTS_ALC_T_TMP, obj);
+}
+
+Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
+			     int all, DbTerm* obj, Eterm** hpp, Uint extra)
+{
+    Uint32 dummy;
+    Eterm* base;
+    Eterm res;
+
+    if (tb->compress) {
+	obj = db_alloc_tmp_uncompressed(tb, obj);
+	base = NULL;
+    }
+    else base = HALFWORD_HEAP ? obj->tpl : NULL;
+
+    res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0,
+			ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy);
+
+    if (is_value(res) && hpp!=NULL) {
+	*hpp = HAlloc(c_p, extra);
+    }
+
+    if (tb->compress) {
+	db_free_tmp_uncompressed(obj);
+    }
+    return res;
+}
+
+
 #ifdef DMC_DEBUG
+
 /*
 ** Disassemble match program
 */
-static void db_match_dis(Binary *bp)
+void db_match_dis(Binary *bp)
 {
     MatchProg *prog = Binary2MatchProg(bp);
-    Uint *t = prog->text;
+    UWord *t = prog->text;
     Uint n;
     Eterm p;
     int first;
@@ -4336,31 +5046,31 @@ static void db_match_dis(Binary *bp)
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("TryMeElse\t%bpu\n", n);
+	    erts_printf("TryMeElse\t%beu\n", n);
 	    break;
 	case matchArray:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("Array\t%bpu\n", n);
+	    erts_printf("Array\t%beu\n", n);
 	    break;
 	case matchArrayBind:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("ArrayBind\t%bpu\n", n);
+	    erts_printf("ArrayBind\t%beu\n", n);
 	    break;
 	case matchTuple:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("Tuple\t%bpu\n", n);
+	    erts_printf("Tuple\t%beu\n", n);
 	    break;
 	case matchPushT:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("PushT\t%bpu\n", n);
+	    erts_printf("PushT\t%beu\n", n);
 	    break;
 	case matchPushL:
 	    ++t;
@@ -4374,13 +5084,13 @@ static void db_match_dis(Binary *bp)
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("Bind\t%bpu\n", n);
+	    erts_printf("Bind\t%beu\n", n);
 	    break;
 	case matchCmp:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("Cmp\t%bpu\n", n);
+	    erts_printf("Cmp\t%beu\n", n);
 	    break;
 	case matchEqBin:
 	    ++t;
@@ -4390,41 +5100,48 @@ static void db_match_dis(Binary *bp)
 	    break;
 	case matchEqRef:
 	    ++t;
-	    n = thing_arityval(*t);
-	    ++t;
-	    erts_printf("EqRef\t(%d) {", (int) n);
-	    first = 1;
-	    while (n--) {
-		if (first)
-		    first = 0;
-		else
-		    erts_printf(", ");
-#ifdef ARCH_64
-		erts_printf("0x%016bpx", *t);
+	    {
+		RefThing *rt = (RefThing *) t;
+		int ri;
+		n = thing_arityval(rt->header);
+		erts_printf("EqRef\t(%d) {", (int) n);
+		first = 1;
+		for (ri = 0; ri < n; ++ri) {
+		    if (first)
+			first = 0;
+		    else
+			erts_printf(", ");
+#if defined(ARCH_64) && !HALFWORD_HEAP
+		    erts_printf("0x%016bex", rt->data.ui[ri]);
 #else
-		erts_printf("0x%08bpx", *t);
+		    erts_printf("0x%08bex", rt->data.ui[ri]);
 #endif
-		++t;
+		}
 	    }
+	    t += TermWords(REF_THING_SIZE);
 	    erts_printf("}\n");
 	    break;
 	case matchEqBig:
 	    ++t;
 	    n = thing_arityval(*t);
-	    ++t;
-	    erts_printf("EqBig\t(%d) {", (int) n);
-	    first = 1;
-	    while (n--) {
-		if (first)
-		    first = 0;
-		else
-		    erts_printf(", ");
-#ifdef ARCH_64
-		erts_printf("0x%016bpx", *t);
+	    {
+		Eterm *et = (Eterm *) t;
+		t += TermWords(n+1);
+		erts_printf("EqBig\t(%d) {", (int) n);
+		first = 1;
+		++n;
+		while (n--) {
+		    if (first)
+			first = 0;
+		    else
+			erts_printf(", ");
+#if defined(ARCH_64) && !HALFWORD_HEAP
+		    erts_printf("0x%016bex", *et);
 #else
-		erts_printf("0x%08bpx", *t);
+		    erts_printf("0x%08bex", *et);
 #endif
-		++t;
+		++et;
+		}
 	    }
 	    erts_printf("}\n");
 	    break;
@@ -4432,8 +5149,8 @@ static void db_match_dis(Binary *bp)
 	    ++t;
 	    {
 		double num;
-		memcpy(&num,t, 2 * sizeof(*t));
-		t += 2;
+		memcpy(&num,t,sizeof(double));
+		t += TermWords(2);
 		erts_printf("EqFloat\t%f\n", num);
 	    }
 	    break;
@@ -4473,31 +5190,31 @@ static void db_match_dis(Binary *bp)
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("MkTuple\t%bpu\n", n);
+	    erts_printf("MkTuple\t%beu\n", n);
 	    break;
 	case matchOr:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("Or\t%bpu\n", n);
+	    erts_printf("Or\t%beu\n", n);
 	    break;
 	case matchAnd:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("And\t%bpu\n", n);
+	    erts_printf("And\t%beu\n", n);
 	    break;
 	case matchOrElse:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("OrElse\t%bpu\n", n);
+	    erts_printf("OrElse\t%beu\n", n);
 	    break;
 	case matchAndAlso:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("AndAlso\t%bpu\n", n);
+	    erts_printf("AndAlso\t%beu\n", n);
 	    break;
 	case matchCall0:
 	    ++t;
@@ -4527,7 +5244,19 @@ static void db_match_dis(Binary *bp)
 	    ++t;
 	    n = (Uint) *t;
 	    ++t;
-	    erts_printf("PushV\t%bpu\n", n);
+	    erts_printf("PushV\t%beu\n", n);
+	    break;
+    #if HALFWORD_HEAP
+	case matchPushVGuard:
+	    n = (Uint) *++t;
+	    ++t;
+	    erts_printf("PushVGuard\t%beu\n", n);
+	    break;
+    #endif
+	case matchPushVResult:
+	    n = (Uint) *++t;
+	    ++t;
+	    erts_printf("PushVResult\t%beu\n", n);
 	    break;
 	case matchTrue:
 	    ++t;
@@ -4638,9 +5367,8 @@ static void db_match_dis(Binary *bp)
     }
     erts_printf("}\n");
     erts_printf("num_bindings: %d\n", prog->num_bindings);
-    erts_printf("heap_size: %bpu\n", prog->heap_size);
-    erts_printf("eheap_offset: %bpu\n", prog->eheap_offset);
-    erts_printf("stack_offset: %bpu\n", prog->stack_offset);
+    erts_printf("heap_size: %beu\n", prog->heap_size);
+    erts_printf("stack_offset: %beu\n", prog->stack_offset);
     erts_printf("text: 0x%08x\n", (unsigned long) prog->text);
     erts_printf("stack_size: %d (words)\n", prog->heap_size-prog->stack_offset);
     
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 4fc7b4f52e..bb1751d309 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -52,22 +52,27 @@
 				     is broken.*/
 #define DB_ERROR_UNSPEC   -10    /* Unspecified error */
 
+/*#define DEBUG_CLONE*/
 
 /*
  * A datatype for a database entry stored out of a process heap
  */
 typedef struct db_term {
-    ErlOffHeap off_heap;   /* Off heap data for term. */
-    Uint size;		   /* Size of term in "words" */
-    Eterm tpl[1];          /* Untagged "constant pointer" to top tuple */
-                           /* (assumed to be first in buffer) */
+    struct erl_off_heap_header* first_oh; /* Off heap data for term. */
+    Uint size;		   /* Heap size of term in "words" */
+#ifdef DEBUG_CLONE
+    Eterm* debug_clone;    /* An uncompressed copy */
+#endif
+    Eterm tpl[1];          /* Term data. Top tuple always first */
+
+    /* Compression: is_immed and key element are uncompressed.
+       Compressed elements are stored in external format after each other
+       last in dbterm. The top tuple elements contains byte offsets, to
+       the start of the data, tagged as headers.
+       The allocated size of the dbterm in bytes is stored at tpl[arity+1].
+     */
 } DbTerm;
 
-/* "Assign" a value to DbTerm.tpl */
-#define DBTERM_SET_TPL(dbtermPtr,tplPtr) ASSERT((tplPtr)==(dbtermPtr->tpl))
-/* Get start of term buffer */
-#define DBTERM_BUF(dbtermPtr) ((dbtermPtr)->tpl)
-
 union db_table;
 typedef union db_table DbTable;
 
@@ -81,6 +86,9 @@ typedef struct {
     Uint new_size;
     int mustResize;
     void* lck;
+#if HALFWORD_HEAP
+    unsigned char* abs_vec;  /* [i] true if dbterm->tpl[i] is absolute Eterm */
+#endif
 } DbUpdateHandle;
 
 
@@ -186,6 +194,12 @@ typedef struct db_table_method
 
 } DbTableMethod;
 
+typedef struct db_fixation {
+    Eterm pid;
+    Uint counter;
+    struct db_fixation *next;
+} DbFixation;
+
 /*
  * This structure contains data for all different types of database
  * tables. Note that these fields must match the same fields
@@ -194,16 +208,8 @@ typedef struct db_table_method
  * operations may be the same on different types of tables.
  */
 
-typedef struct db_fixation {
-    Eterm pid;
-    Uint counter;
-    struct db_fixation *next;
-} DbFixation;
-
-
 typedef struct db_table_common {
-    erts_refc_t ref;
-    erts_refc_t fixref;       /* fixation counter */
+    erts_refc_t ref;          /* fixation counter and delete counter */
 #ifdef ERTS_SMP
     erts_smp_rwmtx_t rwlock;  /* rw lock on table */
     erts_smp_mtx_t fixlock;   /* Protects fixations,megasec,sec,microsec */
@@ -212,7 +218,7 @@ typedef struct db_table_common {
 #endif
     Eterm owner;              /* Pid of the creator */
     Eterm heir;               /* Pid of the heir */
-    Eterm heir_data;          /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */
+    UWord heir_data;          /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */
     SysTimeval heir_started;  /* To further identify the heir */
     Eterm the_name;           /* an atom */
     Eterm id;                 /* atom | integer */
@@ -226,6 +232,7 @@ typedef struct db_table_common {
     Uint32 status;            /* bit masks defined  below */
     int slot;                 /* slot index in meta_main_tab */
     int keypos;               /* defaults to 1 */
+    int compress;
 } DbTableCommon;
 
 /* These are status bit patterns */
@@ -240,30 +247,78 @@ typedef struct db_table_common {
 #define DB_DUPLICATE_BAG (1 << 8)
 #define DB_ORDERED_SET   (1 << 9)
 #define DB_DELETE        (1 << 10) /* table is being deleted */
+#define DB_FREQ_READ     (1 << 11)
 
-#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED)
+#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED|DB_FREQ_READ)
 
 #define IS_HASH_TABLE(Status) (!!((Status) & \
 				  (DB_BAG | DB_SET | DB_DUPLICATE_BAG)))
 #define IS_TREE_TABLE(Status) (!!((Status) & \
 				  DB_ORDERED_SET))
-#define NFIXED(T) (erts_refc_read(&(T)->common.fixref,0))
+#define NFIXED(T) (erts_refc_read(&(T)->common.ref,0))
 #define IS_FIXED(T) (NFIXED(T) != 0) 
 
-Eterm erts_ets_copy_object(Eterm, Process*);
+/*
+ * tplp is an untagged pointer to a tuple we know is large enough
+ * and dth is a pointer to a DbTableHash.
+ */
+#define GETKEY(dth, tplp)   (*((tplp) + ((DbTableCommon*)(dth))->keypos))
+
+
+ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj);
+Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
+			ErlOffHeap* off_heap);
+int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b);
+DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org);
+
+ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp,
+					      Eterm** hpp, ErlOffHeap* off_heap);
+ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b);
+Wterm db_do_read_element(DbUpdateHandle* handle, Sint position);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj)
+{
+    Eterm key = GETKEY(tb, obj->tpl);
+    if IS_CONST(key) return key;
+    else {
+	Uint size = size_object_rel(key, obj->tpl);
+	Eterm* hp = HAlloc(p, size);
+	Eterm res = copy_struct_rel(key, size, &hp, &MSO(p), obj->tpl, NULL);
+	ASSERT(eq_rel(res,NULL,key,obj->tpl));
+	return res;
+    }
+}
+
+ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp,
+					      Eterm** hpp, ErlOffHeap* off_heap)
+{
+    if (tb->compress) {
+	return db_copy_from_comp(tb, bp, hpp, off_heap);
+    }
+    else {
+	return copy_shallow_rel(bp->tpl, bp->size, hpp, off_heap, bp->tpl);
+    }
+}
+
+ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b)
+{
+    if (!tb->compress) {
+	return eq_rel(a, NULL, make_tuple_rel(b->tpl,b->tpl), b->tpl);
+    }
+    else {
+	return db_eq_comp(tb, a, b);
+    }
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-/* optimised version of copy_object (normal case? atomic object) */
-#define COPY_OBJECT(obj, p, objp) \
-   if (IS_CONST(obj)) { *(objp) = (obj); } \
-   else { *objp = erts_ets_copy_object(obj, p); }
 
 #define DB_READ  (DB_PROTECTED|DB_PUBLIC)
 #define DB_WRITE DB_PUBLIC
 #define DB_INFO  (DB_PROTECTED|DB_PUBLIC|DB_PRIVATE)
 
-/* tb is an DbTableCommon and obj is an Eterm (tagged) */
-#define TERM_GETKEY(tb, obj) db_getkey((tb)->common.keypos, (obj)) 
-
 #define ONLY_WRITER(P,T) (((T)->common.status & (DB_PRIVATE|DB_PROTECTED)) \
 			  && (T)->common.owner == (P)->id)
 
@@ -276,15 +331,19 @@ Eterm db_set_trace_control_word_1(Process *p, Eterm val);
 
 void db_initialize_util(void);
 Eterm db_getkey(int keypos, Eterm obj);
-void db_free_term_data(DbTerm* p);
-void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj);
+void db_cleanup_offheap_comp(DbTerm* p);
+void db_free_term(DbTable *tb, void* basep, Uint offset);
+void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj);
+void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj);
+Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj,
+			       Uint pos, Eterm** hpp, Uint extra);
 int db_has_variable(Eterm obj);
 int db_is_variable(Eterm obj);
 void db_do_update_element(DbUpdateHandle* handle,
 			  Sint position,
 			  Eterm newval);
-void db_finalize_update_element(DbUpdateHandle* handle);
-Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr);
+void db_finalize_resize(DbUpdateHandle* handle, Uint offset);
+Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr);
 Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags);
 Binary *db_match_set_compile(Process *p, Eterm matchexpr, 
 			     Uint flags);
@@ -301,12 +360,11 @@ typedef struct match_prog {
     struct erl_heap_fragment *saved_program_buf;
     Eterm saved_program;
     Uint heap_size;          /* size of: heap + eheap + stack */
-    Uint eheap_offset;
     Uint stack_offset;
 #ifdef DMC_DEBUG
-    Uint* prog_end;		/* End of program */
+    UWord* prog_end;		/* End of program */
 #endif
-    Uint text[1];		/* Beginning of program */
+    UWord text[1];		/* Beginning of program */
 } MatchProg;
 
 /*
@@ -366,8 +424,15 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards,
 			 Uint flags, 
 			 DMCErrInfo *err_info);
 /* Returns newly allocated MatchProg binary with refc == 0*/
-Eterm db_prog_match(Process *p, Binary *prog, Eterm term, int arity, 
+
+Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
+		      int all, DbTerm* obj, Eterm** hpp, Uint extra);
+
+Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm* base,
+		    Eterm *termp, int arity,
+		    enum erts_pam_run_flags in_flags,
 		    Uint32 *return_flags /* Zeroed on enter */);
+
 /* returns DB_ERROR_NONE if matches, 1 if not matches and some db error on 
    error. */
 DMCErrInfo *db_new_dmc_err_info(void);
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index e5c3c76fdd..d7d6fcf0a2 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -207,11 +207,7 @@ pdisplay1(int to, void *to_arg, Process* p, Eterm obj)
     case FLOAT_DEF: {
 	    FloatDef ff;
 	    GET_DOUBLE(obj, ff);
-#ifdef _OSE_
-	    erts_print(to, to_arg, "%e", ff.fd);
-#else
 	    erts_print(to, to_arg, "%.20e", ff.fd);
-#endif
 	}
 	break;
     case BINARY_DEF:
@@ -235,9 +231,9 @@ pps(Process* p, Eterm* stop)
     }
 
     while(sp >= stop) {
-	erts_print(to, to_arg, "%0*lx: ", PTR_SIZE, (Eterm) sp);
+	erts_print(to, to_arg, "%0*lx: ", PTR_SIZE, (UWord) sp);
 	if (is_catch(*sp)) {
-	    erts_print(to, to_arg, "catch %d", (Uint)catch_pc(*sp));
+	    erts_print(to, to_arg, "catch %ld", (UWord)catch_pc(*sp));
 	} else {
 	    paranoid_display(to, to_arg, p, *sp);
 	}
@@ -265,7 +261,7 @@ static int verify_eterm(Process *p,Eterm element)
             return 1;
 
         for (mbuf = p->mbuf; mbuf; mbuf = mbuf->next) {
-            if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->size)) {
+            if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->used_size)) {
                 return 1;
             }
         }
@@ -312,7 +308,7 @@ void erts_check_stack(Process *p)
 	if (IN_HEAP(p, ptr))
 	    continue;
 	for (mbuf = p->mbuf; mbuf; mbuf = mbuf->next)
-	    if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->size)) {
+	    if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->used_size)) {
 		in_mbuf = 1;
 		break;
 	    }
@@ -750,7 +746,7 @@ static void print_process_memory(Process *p)
                     PTR_SIZE, "heap fragments",
                     dashes, dashes, dashes, dashes);
     while (bp) {
-	print_untagged_memory(bp->mem,bp->mem + bp->size);
+	print_untagged_memory(bp->mem,bp->mem + bp->used_size);
         bp = bp->next;
     }
 }
@@ -895,5 +891,29 @@ void print_memory_info(Process *p)
 #endif
     erts_printf("+-----------------%s-%s-%s-%s-+\n",dashes,dashes,dashes,dashes);
 }
+#if !HEAP_ON_C_STACK && defined(DEBUG)
+Eterm *erts_debug_allocate_tmp_heap(int size, Process *p)
+{
+    ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
+    int offset = sd->num_tmp_heap_used;
+
+    ASSERT(offset+size <= TMP_HEAP_SIZE);
+    return (sd->tmp_heap)+offset;
+}
+void erts_debug_use_tmp_heap(int size, Process *p)
+{
+    ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
+
+    sd->num_tmp_heap_used += size;
+    ASSERT(sd->num_tmp_heap_used <= TMP_HEAP_SIZE);
+}
+void erts_debug_unuse_tmp_heap(int size, Process *p)
+{
+    ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
+
+    sd->num_tmp_heap_used -= size;
+    ASSERT(sd->num_tmp_heap_used >= 0);
+}
+#endif
 #endif
 
diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h
index 74f4a00b63..bdfbaddbbf 100644
--- a/erts/emulator/beam/erl_debug.h
+++ b/erts/emulator/beam/erl_debug.h
@@ -1,26 +1,27 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 #ifndef _ERL_DEBUG_H_
 #define _ERL_DEBUG_H_
-
 #ifdef DEBUG
 
+#include "erl_term.h"
+
 #ifdef HIPE
 #include "hipe_debug.h"
 #endif
@@ -92,6 +93,11 @@ extern void print_tagged_memory(Eterm *start, Eterm *end);
 extern void print_untagged_memory(Eterm *start, Eterm *end);
 extern void print_memory(Process *p);
 extern void print_memory_info(Process *p);
+#if defined(DEBUG) && !HEAP_ON_C_STACK
+extern Eterm *erts_debug_allocate_tmp_heap(int, Process *);
+extern void erts_debug_use_tmp_heap(int, Process *);
+extern void erts_debug_unuse_tmp_heap(int, Process *);
+#endif
 
 #ifdef HYBRID
 extern void print_ma_info(void);
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 489e74d960..401967a8de 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -48,6 +48,10 @@
 #  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
 #  undef SIZEOF_LONG_LONG
 #endif
+#ifdef HALFWORD_HEAP_EMULATOR
+#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
+#  undef HALFWORD_HEAP_EMULATOR
+#endif
 #include "erl_int_sizes_config.h"
 #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR
 #  error SIZEOF_CHAR mismatch
@@ -65,6 +69,11 @@
 #  error SIZEOF_LONG_LONG mismatch
 #endif
 
+/* This is OK to override by the NIF/driver implementor */
+#if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR)
+#define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__
+#endif
+
 #include "erl_drv_nif.h"
 
 #include <stdlib.h>
@@ -141,6 +150,27 @@ typedef struct {
 #define ERL_DRV_FLAG_SOFT_BUSY		(1 << 1)
 
 /*
+ * Integer types
+ */
+
+typedef unsigned long ErlDrvTermData;
+typedef unsigned long ErlDrvUInt;
+typedef signed long ErlDrvSInt;
+
+#if defined(__WIN32__)
+typedef unsigned __int64 ErlDrvUInt64;
+typedef __int64 ErlDrvSInt64;
+#elif SIZEOF_LONG == 8
+typedef unsigned long ErlDrvUInt64;
+typedef long ErlDrvSInt64;
+#elif SIZEOF_LONG_LONG == 8
+typedef unsigned long long ErlDrvUInt64;
+typedef long long ErlDrvSInt64;
+#else
+#error No 64-bit integer type
+#endif
+
+/*
  * A binary as seen in a driver. Note that a binary should never be
  * altered by the driver when it has been sent to Erlang.
  */
@@ -170,26 +200,6 @@ struct erl_drv_event_data {
 #endif
 typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */
 
-/* 
- * Used in monitors...
- */
-typedef unsigned long ErlDrvTermData;
-typedef unsigned long ErlDrvUInt;
-typedef signed long ErlDrvSInt;
-
-#if defined(__WIN32__)
-typedef unsigned __int64 ErlDrvUInt64;
-typedef __int64 ErlDrvSInt64;
-#elif SIZEOF_LONG == 8
-typedef unsigned long ErlDrvUInt64;
-typedef long ErlDrvSInt64;
-#elif SIZEOF_LONG_LONG == 8
-typedef unsigned long long ErlDrvUInt64;
-typedef long long ErlDrvSInt64;
-#else
-#error No 64-bit integer type
-#endif
-
 /*
  * A driver monitor
  */
@@ -272,7 +282,7 @@ typedef struct erl_drv_entry {
 				   the port */
     void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); 
 				/* called when we have input from one of 
-				   the driver's handles) */
+				   the driver's handles */
     void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event);  
 				/* called when output is possible to one of 
 				   the driver's handles */
@@ -284,7 +294,7 @@ typedef struct erl_drv_entry {
     int (*control)(ErlDrvData drv_data, unsigned int command, char *buf, 
 		   int len, char **rbuf, int rlen); 
 				/* "ioctl" for drivers - invoked by 
-				   port_control/3) */
+				   port_control/3 */
     void (*timeout)(ErlDrvData drv_data);	/* Handling of timeout in driver */
     void (*outputv)(ErlDrvData drv_data, ErlIOVec *ev);
 				/* called when we have output from erlang
@@ -297,7 +307,7 @@ typedef struct erl_drv_entry {
 				   before 'stop' can be called */
     int (*call)(ErlDrvData drv_data, unsigned int command, char *buf, 
 		   int len, char **rbuf, int rlen, unsigned int *flags); 
-                                /* Works mostly like 'control', a syncronous
+                                /* Works mostly like 'control', a synchronous
 				   call into the driver. */
     void (*event)(ErlDrvData drv_data, ErlDrvEvent event,
 		  ErlDrvEventData event_data);
@@ -385,9 +395,9 @@ EXTERN int driver_exit (ErlDrvPort port, int err);
 EXTERN ErlDrvPDL driver_pdl_create(ErlDrvPort);
 EXTERN void driver_pdl_lock(ErlDrvPDL);
 EXTERN void driver_pdl_unlock(ErlDrvPDL);
-EXTERN long driver_pdl_get_refc(ErlDrvPDL);
-EXTERN long driver_pdl_inc_refc(ErlDrvPDL);
-EXTERN long driver_pdl_dec_refc(ErlDrvPDL);
+EXTERN ErlDrvSInt driver_pdl_get_refc(ErlDrvPDL);
+EXTERN ErlDrvSInt driver_pdl_inc_refc(ErlDrvPDL);
+EXTERN ErlDrvSInt driver_pdl_dec_refc(ErlDrvPDL);
 
 /*
  * Process monitors
@@ -423,9 +433,9 @@ EXTERN ErlDrvBinary* driver_realloc_binary(ErlDrvBinary *bin, int size);
 EXTERN void driver_free_binary(ErlDrvBinary *bin);
 
 /* Referenc count on driver binaries */
-EXTERN long driver_binary_get_refc(ErlDrvBinary *dbp);
-EXTERN long driver_binary_inc_refc(ErlDrvBinary *dbp);
-EXTERN long driver_binary_dec_refc(ErlDrvBinary *dbp);
+EXTERN ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp);
+EXTERN ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp);
+EXTERN ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp);
 
 /* Allocation interface */
 EXTERN void *driver_alloc(size_t size);
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 50d8c25c46..dc578f6d2a 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -24,6 +24,10 @@
 #include "global.h"
 #include <string.h>
 
+#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
+#define __DARWIN__ 1
+#endif
+
 #define ERL_DRV_THR_OPTS_SIZE(LAST_FIELD) \
   (((size_t) &((ErlDrvThreadOpts *) 0)->LAST_FIELD) \
    + sizeof(((ErlDrvThreadOpts *) 0)->LAST_FIELD))
@@ -186,10 +190,9 @@ int
 erl_drv_mutex_trylock(ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
-    int res = dmtx ? ethr_mutex_trylock(&dmtx->mtx) : EINVAL;
-    if (res != 0 && res != EBUSY)
-	fatal_error(res, "erl_drv_mutex_trylock()");
-    return res;
+    if (!dmtx)
+	fatal_error(EINVAL, "erl_drv_mutex_trylock()");
+    return ethr_mutex_trylock(&dmtx->mtx);
 #else
     return 0;
 #endif
@@ -199,9 +202,9 @@ void
 erl_drv_mutex_lock(ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
-    int res = dmtx ? ethr_mutex_lock(&dmtx->mtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_mutex_lock()");
+    if (!dmtx)
+	fatal_error(EINVAL, "erl_drv_mutex_lock()");
+    ethr_mutex_lock(&dmtx->mtx);
 #endif
 }
 
@@ -209,9 +212,9 @@ void
 erl_drv_mutex_unlock(ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
-    int res = dmtx ? ethr_mutex_unlock(&dmtx->mtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_mutex_unlock()");
+    if (!dmtx)
+	fatal_error(EINVAL, "erl_drv_mutex_unlock()");
+    ethr_mutex_unlock(&dmtx->mtx);
 #endif
 }
 
@@ -256,9 +259,9 @@ void
 erl_drv_cond_signal(ErlDrvCond *dcnd)
 {
 #ifdef USE_THREADS
-    int res = dcnd ? ethr_cond_signal(&dcnd->cnd) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_cond_signal()");
+    if (!dcnd)
+	fatal_error(EINVAL, "erl_drv_cond_signal()");
+    ethr_cond_signal(&dcnd->cnd);
 #endif
 }
 
@@ -266,9 +269,9 @@ void
 erl_drv_cond_broadcast(ErlDrvCond *dcnd)
 {
 #ifdef USE_THREADS
-    int res = dcnd ? ethr_cond_broadcast(&dcnd->cnd) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_cond_broadcast()");
+    if (!dcnd)
+	fatal_error(EINVAL, "erl_drv_cond_broadcast()");
+    ethr_cond_broadcast(&dcnd->cnd);
 #endif
 }
 
@@ -277,18 +280,13 @@ void
 erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
-    int res;
     if (!dcnd || !dmtx) {
-	res = EINVAL;
-    error:
-	fatal_error(res, "erl_drv_cond_wait()");
+	fatal_error(EINVAL, "erl_drv_cond_wait()");
     }
     while (1) {
-	res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx);
+	int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx);
 	if (res == 0)
 	    break;
-	if (res != EINTR)
-	    goto error;
     }
 #endif
 }
@@ -333,10 +331,9 @@ int
 erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_tryrlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0 && res != EBUSY)
-	fatal_error(res, "erl_drv_rwlock_tryrlock()");
-    return res;
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()");
+    return ethr_rwmutex_tryrlock(&drwlck->rwmtx);
 #else
     return 0;
 #endif
@@ -346,9 +343,9 @@ void
 erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_rlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_rwlock_rlock()");
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_rlock()");
+    ethr_rwmutex_rlock(&drwlck->rwmtx);
 #endif
 }
 
@@ -356,9 +353,9 @@ void
 erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_runlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_rwlock_runlock()");
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_runlock()");
+    ethr_rwmutex_runlock(&drwlck->rwmtx);
 #endif
 }
 
@@ -366,10 +363,9 @@ int
 erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_tryrwlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0 && res != EBUSY)
-	fatal_error(res, "erl_drv_rwlock_tryrwlock()");
-    return res;
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()");
+    return ethr_rwmutex_tryrwlock(&drwlck->rwmtx);
 #else
     return 0;
 #endif
@@ -379,9 +375,9 @@ void
 erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_rwlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_rwlock_rwlock()");
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_rwlock()");
+    ethr_rwmutex_rwlock(&drwlck->rwmtx);
 #endif
 }
 
@@ -389,9 +385,9 @@ void
 erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_rwunlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_rwlock_rwunlock()");
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()");
+    ethr_rwmutex_rwunlock(&drwlck->rwmtx);
 #endif
 }
 
@@ -536,7 +532,7 @@ erl_drv_tsd_get(ErlDrvTSDKey key)
     if (!dtid)
 	return NULL;
 #endif
-    if (ERL_DRV_TSD_LEN__ < key)
+    if (ERL_DRV_TSD_LEN__ <= key)
 	return NULL;
     return ERL_DRV_TSD__[key];
 }
@@ -603,11 +599,7 @@ erl_drv_thread_create(char *name,
 	dtid->name = ((char *) dtid) + sizeof(struct ErlDrvTid_);
 	sys_strcpy(dtid->name, name);
     }
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    res = erts_lcnt_thr_create(&dtid->tid, erl_drv_thread_wrapper, dtid, use_opts);
-#else
     res = ethr_thr_create(&dtid->tid, erl_drv_thread_wrapper, dtid, use_opts);
-#endif
 
     if (res != 0) {
 	erts_free(ERTS_ALC_T_DRV_TID, dtid);
@@ -704,3 +696,64 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp)
 #endif
 }
 
+#if defined(__DARWIN__) && defined(USE_THREADS) && defined(ERTS_SMP)
+extern int erts_darwin_main_thread_pipe[2];
+extern int erts_darwin_main_thread_result_pipe[2];
+
+int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp);
+int erl_drv_steal_main_thread(char *name,
+			      ErlDrvTid *dtid,
+			      void* (*func)(void*),
+			      void* arg,
+			      ErlDrvThreadOpts *opts);
+
+
+int
+erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp)
+{
+    void *dummy;
+    void **x;
+    if (respp == NULL)
+	x = &dummy;
+    else
+	x = respp;
+    read(erts_darwin_main_thread_result_pipe[0],x,sizeof(void *));
+    return 0;
+}
+
+int
+erl_drv_steal_main_thread(char *name,
+			  ErlDrvTid *tid,
+			  void* (*func)(void*),
+			  void* arg,
+			  ErlDrvThreadOpts *opts)
+{
+    char buff[sizeof(void* (*)(void*)) + sizeof(void *)];
+    int buff_sz = sizeof(void* (*)(void*)) + sizeof(void *);
+    /*struct ErlDrvTid_ *dtid;
+
+    dtid = erts_alloc_fnf(ERTS_ALC_T_DRV_TID,
+			  (sizeof(struct ErlDrvTid_)
+			   + (name ? sys_strlen(name) + 1 : 0)));
+    if (!dtid)
+	return ENOMEM;
+    memset(dtid,0,sizeof(ErlDrvTid_));
+    dtid->tid = (void * ) -1;
+    dtid->drv_thr = 1;
+    dtid->func = func;
+    dtid->arg = arg;
+    dtid->tsd = NULL;
+    dtid->tsd_len = 0;
+    dtid->name = no_name;
+    *tid = (ErlDrvTid) dtid;
+    */
+    *tid = NULL;
+    /* Ignore options and name... */
+    
+    memcpy(buff,&func,sizeof(void* (*)(void*)));
+    memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *));
+    write(erts_darwin_main_thread_pipe[1],buff,buff_sz);
+    return 0;
+}
+
+#endif
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 79e844b315..88947b5536 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -37,8 +37,6 @@ static erts_smp_rwmtx_t erts_fun_table_lock;
 #define erts_fun_read_unlock()	erts_smp_rwmtx_runlock(&erts_fun_table_lock)
 #define erts_fun_write_lock()	erts_smp_rwmtx_rwlock(&erts_fun_table_lock)
 #define erts_fun_write_unlock()	erts_smp_rwmtx_rwunlock(&erts_fun_table_lock)
-#define erts_fun_init_lock()	erts_smp_rwmtx_init(&erts_fun_table_lock, \
-						    "fun_tab")
 
 static HashValue fun_hash(ErlFunEntry* obj);
 static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2);
@@ -50,15 +48,19 @@ static void fun_free(ErlFunEntry* obj);
  * to unloaded_fun[]. The -1 in unloaded_fun[0] will be interpreted
  * as an illegal arity when attempting to call a fun.
  */
-static Eterm unloaded_fun_code[3] = {NIL, -1, 0};
-static Eterm* unloaded_fun = unloaded_fun_code + 2;
+static BeamInstr unloaded_fun_code[3] = {NIL, -1, 0};
+static BeamInstr* unloaded_fun = unloaded_fun_code + 2;
 
 void
 erts_init_fun_table(void)
 {
     HashFunctions f;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+    erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab");
 
-    erts_fun_init_lock();
     f.hash = (H_FUN) fun_hash;
     f.cmp  = (HCMP_FUN) fun_cmp;
     f.alloc = (HALLOC_FUN) fun_alloc;
@@ -95,7 +97,7 @@ erts_put_fun_entry(Eterm mod, int uniq, int index)
 {
     ErlFunEntry template;
     ErlFunEntry* fe;
-    long refc;
+    erts_aint_t refc;
     ASSERT(is_atom(mod));
     template.old_uniq = uniq;
     template.old_index = index;
@@ -117,7 +119,7 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
 {
     ErlFunEntry template;
     ErlFunEntry* fe;
-    long refc;
+    erts_aint_t refc;
 
     ASSERT(is_atom(mod));
     template.old_uniq = old_uniq;
@@ -155,7 +157,7 @@ erts_get_fun_entry(Eterm mod, int uniq, int index)
     erts_fun_read_lock();
     ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template);
     if (ret) {
-	long refc = erts_refc_inctest(&ret->refc, 1);
+	erts_aint_t refc = erts_refc_inctest(&ret->refc, 1);
 	if (refc < 2) /* Pending delete */
 	    erts_refc_inc(&ret->refc, 1);
     }
@@ -192,22 +194,8 @@ erts_erase_fun_entry(ErlFunEntry* fe)
     erts_fun_write_unlock();
 }
 
-#ifndef HYBRID /* FIND ME! */
-void
-erts_cleanup_funs(ErlFunThing* funp)
-{
-    while (funp) {
-	ErlFunEntry* fe = funp->fe;
-	if (erts_refc_dectest(&fe->refc, 0) == 0) {
-	    erts_erase_fun_entry(fe);
-	}
-	funp = funp->next;
-    }
-}
-#endif
-
 void
-erts_cleanup_funs_on_purge(Eterm* start, Eterm* end)
+erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end)
 {
     int limit;
     HashBucket** bucket;
@@ -222,7 +210,7 @@ erts_cleanup_funs_on_purge(Eterm* start, Eterm* end)
 
 	while (b) {
 	    ErlFunEntry* fe = (ErlFunEntry *) b;
-	    Eterm* addr = fe->address;
+	    BeamInstr* addr = fe->address;
 
 	    if (start <= addr && addr < end) {
 		fe->address = unloaded_fun;
@@ -269,7 +257,7 @@ erts_dump_fun_entries(int to, void *to_arg)
 #ifdef HIPE
 	    erts_print(to, to_arg, "Native_address: %p\n", fe->native_address);
 #endif
-	    erts_print(to, to_arg, "Refc: %d\n", erts_refc_read(&fe->refc, 1));
+	    erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1));
 	    b = b->next;
 	}
     }
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index fb5e75649b..2f165afa06 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -33,10 +33,10 @@ typedef struct erl_fun_entry {
     int index;			/* New style index. */
     int old_uniq;		/* Unique number (old_style) */
     int old_index;		/* Old style index */
-    Eterm* address;		/* Pointer to code for fun */
+    BeamInstr* address;		/* Pointer to code for fun */
 
 #ifdef HIPE
-    Eterm* native_address;	/* Native entry code for fun. */
+    UWord* native_address;	/* Native entry code for fun. */
 #endif
 
     Uint arity;			/* The arity of the fun. */
@@ -53,12 +53,12 @@ typedef struct erl_fun_entry {
 
 typedef struct erl_fun_thing {
     Eterm thing_word;		/* Subtag FUN_SUBTAG. */
+    ErlFunEntry* fe;		/* Pointer to fun entry. */
 #ifndef HYBRID /* FIND ME! */
-    struct erl_fun_thing* next;	/* Next fun in mso list. */
+    struct erl_off_heap_header* next;
 #endif
-    ErlFunEntry* fe;		/* Pointer to fun entry. */
 #ifdef HIPE
-    Eterm* native_address;	/* Native code for the fun. */
+    UWord* native_address;	/* Native code for the fun. */
 #endif
     Uint arity;			/* The arity of the fun. */
     Uint num_free;		/* Number of free variables (in env). */
@@ -86,7 +86,7 @@ void erts_erase_fun_entry(ErlFunEntry* fe);
 #ifndef HYBRID /* FIND ME! */
 void erts_cleanup_funs(ErlFunThing* funp);
 #endif
-void erts_cleanup_funs_on_purge(Eterm* start, Eterm* end);
+void erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end);
 void erts_dump_fun_entries(int, void *);
 
 #endif
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index e9bf37a173..5edcd667e7 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -33,6 +33,7 @@
 #include "erl_gc.h"
 #if HIPE
 #include "hipe_stack.h"
+#include "hipe_mode_switch.h"
 #endif
 
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
@@ -110,9 +111,7 @@ static Uint adjust_after_fullsweep(Process *p, int size_before,
 				   int need, Eterm *objv, int nobj);
 static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
 static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
-static void sweep_proc_bins(Process *p, int fullsweep);
-static void sweep_proc_funs(Process *p, int fullsweep);
-static void sweep_proc_externals(Process *p, int fullsweep);
+static void sweep_off_heap(Process *p, int fullsweep);
 static void offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
 static void offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
 static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
@@ -126,7 +125,7 @@ static void disallow_heap_frag_ref_in_old_heap(Process* p);
 static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj);
 #endif
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 # define MAX_HEAP_SIZES 154
 #else
 # define MAX_HEAP_SIZES 55
@@ -145,6 +144,16 @@ erts_init_gc(void)
 {
     int i = 0;
 
+    ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word));
+    ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word));
+    ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header));
+    ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size));
+    ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size));
+    ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size));
+    ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next));
+    ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next));
+    ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next));
+
     erts_smp_spinlock_init(&info_lck, "gc_info");
     garbage_cols = 0;
     reclaimed = 0;
@@ -206,7 +215,7 @@ erts_next_heap_size(Uint size, Uint offset)
 		low = mid + 1;
 	    }
 	}
-	erl_exit(1, "no next heap size found: %d, offset %d\n", size, offset);
+	erl_exit(1, "no next heap size found: %lu, offset %lu\n", (unsigned long)size, (unsigned long)offset);
     }
     return 0;
 }
@@ -286,25 +295,14 @@ erts_offset_heap_ptr(Eterm* hp, Uint sz, Sint offs,
     offset_heap_ptr(hp, sz, offs, (char *) low, ((char *)high)-((char *)low));
 }
 
+
 #define ptr_within(ptr, low, high) ((ptr) < (high) && (ptr) >= (low))
 
 void
 erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high)
 {
-    if (ohp->mso && ptr_within((Eterm *)ohp->mso, low, high)) {
-        Eterm** uptr = (Eterm**) (void *) &ohp->mso;
-        *uptr += offs;
-    }
-
-#ifndef HYBRID /* FIND ME! */
-    if (ohp->funs && ptr_within((Eterm *)ohp->funs, low, high)) {
-        Eterm** uptr = (Eterm**) (void *) &ohp->funs;
-        *uptr += offs;
-    }
-#endif
-
-    if (ohp->externals && ptr_within((Eterm *)ohp->externals, low, high)) {
-        Eterm** uptr = (Eterm**) (void *) &ohp->externals;
+    if (ohp->first && ptr_within((Eterm *)ohp->first, low, high)) {
+        Eterm** uptr = (Eterm**) (void *) &ohp->first;
         *uptr += offs;
     }
 }
@@ -457,7 +455,6 @@ erts_garbage_collect_hibernate(Process* p)
     Eterm* heap;
     Eterm* htop;
     Rootset rootset;
-    int n;
     char* src;
     Uint src_size;
     Uint actual_size;
@@ -488,7 +485,10 @@ erts_garbage_collect_hibernate(Process* p)
 				    sizeof(Eterm)*heap_size);
     htop = heap;
 
-    n = setup_rootset(p, p->arg_reg, p->arity, &rootset);
+    (void) setup_rootset(p, p->arg_reg, p->arity, &rootset);
+#if HIPE
+    hipe_empty_nstack(p);
+#endif
 
     src = (char *) p->heap;
     src_size = (char *) p->htop - src;
@@ -504,14 +504,8 @@ erts_garbage_collect_hibernate(Process* p)
 
     cleanup_rootset(&rootset);
 
-    if (MSO(p).mso) {
-        sweep_proc_bins(p, 1);
-    }
-    if (MSO(p).funs) {
-        sweep_proc_funs(p, 1);
-    }
-    if (MSO(p).externals) {
-        sweep_proc_externals(p, 1);
+    if (MSO(p).first) {
+        sweep_off_heap(p, 1);
     }
 
     /*
@@ -667,7 +661,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
 	    case TAG_PRIMARY_BOXED:
 		ptr = boxed_val(gval);
 		val = *ptr;
-                if (IS_MOVED(val)) {
+                if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
 		} else if (in_area(ptr, area, area_size)) {
@@ -679,7 +673,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
 	    case TAG_PRIMARY_LIST:
                 ptr = list_val(gval);
                 val = *ptr;
-                if (is_non_value(val)) { /* Moved */
+                if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
 		} else if (in_area(ptr, area, area_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
@@ -752,7 +746,10 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
      * is large enough.
      */
 
-    if (OLD_HEAP(p) && mature <= OLD_HEND(p) - OLD_HTOP(p)) {
+    if (OLD_HEAP(p) &&
+	    ((mature <= OLD_HEND(p) - OLD_HTOP(p)) &&
+	    ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
+	    ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
 	ErlMessage *msgp;
 	Uint size_after;
 	Uint need_after;
@@ -913,7 +910,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
 	    case TAG_PRIMARY_BOXED: {
 		ptr = boxed_val(gval);
                 val = *ptr;
-                if (IS_MOVED(val)) {
+                if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
                 } else if (in_area(ptr, heap, mature_size)) {
@@ -929,7 +926,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
 	    case TAG_PRIMARY_LIST: {
                 ptr = list_val(gval);
                 val = *ptr;
-                if (is_non_value(val)) { /* Moved */
+                if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
                 } else if (in_area(ptr, heap, mature_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
@@ -972,7 +969,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
 	    case TAG_PRIMARY_BOXED: {
 		ptr = boxed_val(gval);
 		val = *ptr;
-		if (IS_MOVED(val)) {
+		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
 		} else if (in_area(ptr, heap, mature_size)) {
@@ -987,7 +984,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
 	    case TAG_PRIMARY_LIST: {
 		ptr = list_val(gval);
 		val = *ptr;
-		if (is_non_value(val)) {
+		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
 		} else if (in_area(ptr, heap, mature_size)) {
 		    MOVE_CONS(ptr,val,old_htop,n_hp++);
@@ -1008,7 +1005,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
 			Eterm* origptr = &(mb->orig);
 			ptr = boxed_val(*origptr);
 			val = *ptr;
-			if (IS_MOVED(val)) {
+			if (IS_MOVED_BOXED(val)) {
 			    *origptr = val;
 			    mb->base = binary_bytes(val);
 			} else if (in_area(ptr, heap, mature_size)) {
@@ -1041,15 +1038,8 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
     OLD_HTOP(p) = old_htop;
     HIGH_WATER(p) = (HEAP_START(p) != HIGH_WATER(p)) ? n_heap : n_htop;
 
-    if (MSO(p).mso) {
-        sweep_proc_bins(p, 0);
-    }
-
-    if (MSO(p).funs) {
-        sweep_proc_funs(p, 0);
-    }
-    if (MSO(p).externals) {
-        sweep_proc_externals(p, 0);
+    if (MSO(p).first) {
+	sweep_off_heap(p, 0);
     }
 
 #ifdef HARDDEBUG
@@ -1161,7 +1151,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	    case TAG_PRIMARY_BOXED: {
 		ptr = boxed_val(gval);
 		val = *ptr;
-		if (IS_MOVED(val)) {
+		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *g_ptr++ = val;
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
@@ -1175,7 +1165,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	    case TAG_PRIMARY_LIST: {
 		ptr = list_val(gval);
 		val = *ptr;
-		if (is_non_value(val)) {
+		if (IS_MOVED_CONS(val)) {
 		    *g_ptr++ = ptr[1];
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
 		    MOVE_CONS(ptr,val,n_htop,g_ptr++);
@@ -1216,7 +1206,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	    case TAG_PRIMARY_BOXED: {
 		ptr = boxed_val(gval);
 		val = *ptr;
-		if (IS_MOVED(val)) {
+		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
@@ -1229,7 +1219,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	    case TAG_PRIMARY_LIST: {
 		ptr = list_val(gval);
 		val = *ptr;
-		if (is_non_value(val)) {
+		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
 		    MOVE_CONS(ptr,val,n_htop,n_hp++);
@@ -1249,7 +1239,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 			origptr = &(mb->orig);
 			ptr = boxed_val(*origptr);
 			val = *ptr;
-			if (IS_MOVED(val)) {
+			if (IS_MOVED_BOXED(val)) {
 			    *origptr = val;
 			    mb->base = binary_bytes(*origptr);
 			} else if (in_area(ptr, src, src_size) ||
@@ -1271,17 +1261,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	}
     }
 
-    if (MSO(p).mso) {
-	sweep_proc_bins(p, 1);
-    }
-    if (MSO(p).funs) {
-	sweep_proc_funs(p, 1);
-    }
-    if (MSO(p).externals) {
-	sweep_proc_externals(p, 1);
+    if (MSO(p).first) {
+	sweep_off_heap(p, 1);
     }
 
-    if (OLD_HEAP(p) != NULL) {
+    if (OLD_HEAP(p) != NULL) {       
 	ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP,
 		       OLD_HEAP(p),
 		       (OLD_HEND(p) - OLD_HEAP(p)) * sizeof(Eterm));
@@ -1305,6 +1289,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     HIGH_WATER(p) = HEAP_TOP(p);
 
     ErtsGcQuickSanityCheck(p);
+
     /*
      * Copy newly received message onto the end of the new heap.
      */
@@ -1392,17 +1377,12 @@ combined_message_size(Process* p)
 static void
 remove_message_buffers(Process* p)
 {
-    ErlHeapFragment* bp = MBUF(p);
-
-    MBUF(p) = NULL;
-    MBUF_SIZE(p) = 0;
-    while (bp != NULL) {
-	ErlHeapFragment* next_bp = bp->next;
-	free_message_buffer(bp);
-	bp = next_bp;
-    }
+    if (MBUF(p) != NULL) {
+	free_message_buffer(MBUF(p));
+	MBUF(p) = NULL;
+    }    
+    MBUF_SIZE(p) = 0;    
 }
-
 #ifdef HARDDEBUG
 
 /*
@@ -1433,12 +1413,12 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
 	case TAG_PRIMARY_BOXED: {
 	    ptr = _unchecked_boxed_val(gval);
 	    val = *ptr;
-	    if (IS_MOVED(val)) {
+	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
 		objv++;
 	    } else {
  		for (qb = mbuf; qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1450,11 +1430,11 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
 	case TAG_PRIMARY_LIST: {
 	    ptr = _unchecked_list_val(gval);
 	    val = *ptr;
-	    if (is_non_value(val)) {
+	    if (IS_MOVED_CONS(val)) {
 		objv++;
 	    } else {
 		for (qb = mbuf; qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1499,7 +1479,7 @@ disallow_heap_frag_ref_in_heap(Process* p)
 	    ptr = _unchecked_boxed_val(val);
 	    if (!in_area(ptr, heap, heap_size)) {
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1509,7 +1489,7 @@ disallow_heap_frag_ref_in_heap(Process* p)
 	    ptr = _unchecked_list_val(val);
 	    if (!in_area(ptr, heap, heap_size)) {
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1551,26 +1531,26 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 	val = *hp++;
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
-	    ptr = (Eterm *) val;
+	    ptr = (Eterm *) EXPAND_POINTER(val);
 	    if (!in_area(ptr, old_heap, old_heap_size)) {
 		if (in_area(ptr, new_heap, new_heap_size)) {
 		    abort();
 		}
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
 	    }
 	    break;
 	case TAG_PRIMARY_LIST:
-	    ptr = (Eterm *) val;
+	    ptr = (Eterm *) EXPAND_POINTER(val);
 	    if (!in_area(ptr, old_heap, old_heap_size)) {
 		if (in_area(ptr, new_heap, new_heap_size)) {
 		    abort();
 		}
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1610,7 +1590,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size)
 	    case TAG_PRIMARY_BOXED: {
 		ptr = boxed_val(gval);
                 val = *ptr;
-                if (IS_MOVED(val)) {
+                if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
                 } else if (in_area(ptr, src, src_size)) {
@@ -1623,7 +1603,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size)
 	    case TAG_PRIMARY_LIST: {
                 ptr = list_val(gval);
                 val = *ptr;
-                if (is_non_value(val)) { /* Moved */
+                if (IS_MOVED_CONS(val)) {
                     *g_ptr++ = ptr[1];
                 } else if (in_area(ptr, src, src_size)) {
                     MOVE_CONS(ptr,val,htop,g_ptr++);
@@ -1657,7 +1637,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 	case TAG_PRIMARY_BOXED: {
 	    ptr = boxed_val(gval);
 	    val = *ptr;
-	    if (IS_MOVED(val)) {
+	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
 		*n_hp++ = val;
 	    } else if (in_area(ptr, src, src_size)) {
@@ -1670,7 +1650,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 	case TAG_PRIMARY_LIST: {
 	    ptr = list_val(gval);
 	    val = *ptr;
-	    if (is_non_value(val)) {
+	    if (IS_MOVED_CONS(val)) {
 		*n_hp++ = ptr[1];
 	    } else if (in_area(ptr, src, src_size)) {
 		MOVE_CONS(ptr,val,n_htop,n_hp++);
@@ -1690,7 +1670,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 		    origptr = &(mb->orig);
 		    ptr = boxed_val(*origptr);
 		    val = *ptr;
-		    if (IS_MOVED(val)) {
+		    if (IS_MOVED_BOXED(val)) {
 			*origptr = val;
 			mb->base = binary_bytes(*origptr);
 		    } else if (in_area(ptr, src, src_size)) {
@@ -1722,7 +1702,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 	case TAG_PRIMARY_BOXED: {
 	    ptr = boxed_val(gval);
 	    val = *ptr;
-	    if (IS_MOVED(val)) {
+	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
 		*heap_ptr++ = val;
 	    } else if (in_area(ptr, src, src_size)) {
@@ -1735,7 +1715,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 	case TAG_PRIMARY_LIST: {
 	    ptr = list_val(gval);
 	    val = *ptr;
-	    if (is_non_value(val)) {
+	    if (IS_MOVED_CONS(val)) {
 		*heap_ptr++ = ptr[1];
 	    } else if (in_area(ptr, src, src_size)) {
 		MOVE_CONS(ptr,val,htop,heap_ptr++);
@@ -1830,28 +1810,6 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop,
     return n_htop;
 }
 
-#ifdef DEBUG
-static Eterm follow_moved(Eterm term)
-{
-    Eterm* ptr;
-    switch (primary_tag(term)) {
-    case TAG_PRIMARY_IMMED1:
-	break;
-    case TAG_PRIMARY_BOXED:
-	ptr = boxed_val(term);
-	if (IS_MOVED(*ptr)) term = *ptr;
-	break;
-    case TAG_PRIMARY_LIST:
-	ptr = list_val(term);
-	if (is_non_value(ptr[0])) term = ptr[1];
-	break;
-    default:
-	abort();
-    }
-    return term;
-}
-#endif
-
 static Uint
 setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 {
@@ -2030,8 +1988,8 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj)
     HEAP_SIZE(p) = new_sz;
 }
 
-static Uint
-do_next_vheap_size(Uint vheap, Uint vheap_sz) {
+static Uint64
+do_next_vheap_size(Uint64 vheap, Uint64 vheap_sz) {
 
     /*                grow
      *
@@ -2048,131 +2006,53 @@ do_next_vheap_size(Uint vheap, Uint vheap_sz) {
      *          ----------------------
      */
 
-    if (vheap > (Uint) (vheap_sz*3/4)) {
+    if ((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+	Uint64 new_vheap_sz = vheap_sz;
 
-	while(vheap > (Uint) (vheap_sz*3/4)) {
-	    vheap_sz = vheap_sz*2;
+	while((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+	    /* the golden ratio = 1.618 */
+	    new_vheap_sz = (Uint64) vheap_sz * 1.618;
+	    if (new_vheap_sz < vheap_sz ) {
+	        return vheap_sz;
+	    }
+	    vheap_sz = new_vheap_sz;
 	}
 
-	return erts_next_heap_size(vheap_sz, 0);
+	return vheap_sz;
     }
 
-    if (vheap < (Uint) (vheap_sz/4)) {
-	return erts_next_heap_size((Uint) (vheap_sz / 2), 0);
+    if (vheap < (Uint64) (vheap_sz/4)) {
+	return (vheap_sz >> 1);
     }
 
     return vheap_sz;
 
 }
 
-static Uint
-next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) {
-    vheap_sz = do_next_vheap_size(vheap, vheap_sz);
-    return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz;
-}
-
-static void
-sweep_proc_externals(Process *p, int fullsweep)
-{
-    ExternalThing** prev;
-    ExternalThing* ptr;
-    char* oh = 0;
-    Uint oh_size = 0;
-
-    if (fullsweep == 0) {
-	oh = (char *) OLD_HEAP(p);
-	oh_size = (char *) OLD_HEND(p) - oh;
-    }
-
-    prev = &MSO(p).externals;
-    ptr = MSO(p).externals;
-
-    while (ptr) {
-        Eterm* ppt = (Eterm *) ptr;
-
-        if (IS_MOVED(*ppt)) {        /* Object is alive */
-            ExternalThing* ro = external_thing_ptr(*ppt);
-
-            *prev = ro;         /* Patch to moved pos */
-            prev = &ro->next;
-            ptr = ro->next;
-        } else if (in_area(ppt, oh, oh_size)) {
-            /*
-             * Object resides on old heap, and we just did a
-             * generational collection - keep object in list.
-             */
-            prev = &ptr->next;
-            ptr = ptr->next;
-        } else {                /* Object has not been moved - deref it */
-	    erts_deref_node_entry(ptr->node);
-            *prev = ptr = ptr->next;
-        }
-    }
-    ASSERT(*prev == NULL);
-}
-
-static void
-sweep_proc_funs(Process *p, int fullsweep)
-{
-    ErlFunThing** prev;
-    ErlFunThing* ptr;
-    char* oh = 0;
-    Uint oh_size = 0;
-
-    if (fullsweep == 0) {
-	oh = (char *) OLD_HEAP(p);
-	oh_size = (char *) OLD_HEND(p) - oh;
-    }
-		      
-    prev = &MSO(p).funs;
-    ptr = MSO(p).funs;
-
-    while (ptr) {
-        Eterm* ppt = (Eterm *) ptr;
-
-        if (IS_MOVED(*ppt)) {        /* Object is alive */
-            ErlFunThing* ro = (ErlFunThing *) fun_val(*ppt);
-
-            *prev = ro;         /* Patch to moved pos */
-            prev = &ro->next;
-            ptr = ro->next;
-        } else if (in_area(ppt, oh, oh_size)) {
-            /*
-             * Object resides on old heap, and we just did a
-             * generational collection - keep object in list.
-             */
-            prev = &ptr->next;
-            ptr = ptr->next;
-        } else {                /* Object has not been moved - deref it */
-            ErlFunEntry* fe = ptr->fe;
-
-            *prev = ptr = ptr->next;
-	    if (erts_refc_dectest(&fe->refc, 0) == 0) {
-                erts_erase_fun_entry(fe);
-            }
-        }
-    }
-    ASSERT(*prev == NULL);
+static Uint64
+next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz) {
+    Uint64 new_vheap_sz = do_next_vheap_size(vheap, vheap_sz);
+    return new_vheap_sz < p->min_vheap_size ? p->min_vheap_size : new_vheap_sz;
 }
 
 struct shrink_cand_data {
-    ProcBin* new_candidates;
-    ProcBin* new_candidates_end;
-    ProcBin* old_candidates;
+    struct erl_off_heap_header* new_candidates;
+    struct erl_off_heap_header* new_candidates_end;
+    struct erl_off_heap_header* old_candidates;
     Uint no_of_candidates;
     Uint no_of_active;
 };
 
 static ERTS_INLINE void
 link_live_proc_bin(struct shrink_cand_data *shrink,
-		   ProcBin ***prevppp,
-		   ProcBin **pbpp,
+		   struct erl_off_heap_header*** prevppp,
+		   struct erl_off_heap_header** currpp,
 		   int new_heap)
 {
-    ProcBin *pbp = *pbpp;
-
-    *pbpp = pbp->next;
+    ProcBin *pbp = (ProcBin*) *currpp;
+    ASSERT(**prevppp == *currpp);
 
+    *currpp = pbp->next;
     if (pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) {
 	ASSERT(((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
 		== (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
@@ -2189,15 +2069,16 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
 	    /* Our allocators are 8 byte aligned, i.e., shrinking with
 	       less than 8 bytes will have no real effect */
 	    if (unused >= 8) { /* A shrink candidate; save in candidate list */
+		**prevppp = pbp->next;
 		if (new_heap) {
 		    if (!shrink->new_candidates)
-			shrink->new_candidates_end = pbp;
+			shrink->new_candidates_end = (struct erl_off_heap_header*)pbp;
 		    pbp->next = shrink->new_candidates;
-		    shrink->new_candidates = pbp;
+		    shrink->new_candidates = (struct erl_off_heap_header*)pbp;
 		}
 		else {
 		    pbp->next = shrink->old_candidates;
-		    shrink->old_candidates = pbp;
+		    shrink->old_candidates = (struct erl_off_heap_header*)pbp;
 		}
 		shrink->no_of_candidates++;
 		return;
@@ -2205,83 +2086,117 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
 	}
     }
 
-    /* Not a shrink candidate; keep in original mso list */
-    **prevppp = pbp;
+    /* Not a shrink candidate; keep in original mso list */ 
     *prevppp = &pbp->next;
-
 }
 
 
-static void 
-sweep_proc_bins(Process *p, int fullsweep)
+static void
+sweep_off_heap(Process *p, int fullsweep)
 {
     struct shrink_cand_data shrink = {0};
-    ProcBin** prev;
-    ProcBin* ptr;
-    Binary* bptr;
-    char* oh = NULL;
-    Uint oh_size = 0;
-    Uint bin_vheap = 0;
+    struct erl_off_heap_header* ptr;
+    struct erl_off_heap_header** prev;
+    char* oheap = NULL;
+    Uint oheap_sz = 0;
+    Uint64 bin_vheap = 0;
+#ifdef DEBUG
+    int seen_mature = 0;
+#endif
 
     if (fullsweep == 0) {
-	oh = (char *) OLD_HEAP(p);
-	oh_size = (char *) OLD_HEND(p) - oh;
+	oheap = (char *) OLD_HEAP(p);
+	oheap_sz = (char *) OLD_HEND(p) - oheap;
     }
 
     BIN_OLD_VHEAP(p) = 0;
 
-    prev = &MSO(p).mso;
-    ptr = MSO(p).mso;
+    prev = &MSO(p).first;
+    ptr = MSO(p).first;
 
-    /*
-     * Note: In R7 we no longer force a fullsweep when we find binaries
-     * on the old heap. The reason is that with the introduction of the
-     * bit syntax we can expect binaries to be used a lot more. Note that
-     * in earlier releases a brand new binary (or any other term) could
-     * be put on the old heap during a gen-gc fullsweep, but this is
-     * no longer the case in R7.
+    /* Firts part of the list will reside on the (old) new-heap.
+     * Keep if moved, otherwise deref.
      */
     while (ptr) {
-        Eterm* ppt = (Eterm *) ptr;
-
-        if (IS_MOVED(*ppt)) {        /* Object is alive */
-	    bin_vheap += ptr->size / sizeof(Eterm);
-            ptr = (ProcBin*) binary_val(*ppt);		   
-	    link_live_proc_bin(&shrink,
-			       &prev,
-			       &ptr,
-			       !in_area(ptr, oh, oh_size)); 
-        } else if (in_area(ppt, oh, oh_size)) {
-            /*
-             * Object resides on old heap, and we just did a
-             * generational collection - keep object in list.
-             */
-            BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
-	    link_live_proc_bin(&shrink, &prev, &ptr, 0); 
-        } else {                /* Object has not been moved - deref it */
-
-            *prev = ptr->next;
-            bptr = ptr->val;
-            if (erts_refc_dectest(&bptr->refc, 0) == 0)
-		erts_bin_free(bptr);
-            ptr = *prev;
-        }
+	if (IS_MOVED_BOXED(ptr->thing_word)) {
+	    ASSERT(!in_area(ptr, oheap, oheap_sz));
+	    *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word);
+	    ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
+	    if (ptr->thing_word == HEADER_PROC_BIN) {
+		int to_new_heap = !in_area(ptr, oheap, oheap_sz);
+		ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1)));
+		if (to_new_heap) {
+		    bin_vheap += ptr->size / sizeof(Eterm);
+		} else {
+		    BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
+		}		
+		link_live_proc_bin(&shrink, &prev, &ptr, to_new_heap);
+	    }
+	    else {
+		prev = &ptr->next;
+		ptr = ptr->next;
+	    }
+	}
+	else if (!in_area(ptr, oheap, oheap_sz)) {
+	    /* garbage */
+	    switch (thing_subtag(ptr->thing_word)) {
+	    case REFC_BINARY_SUBTAG:
+		{
+		    Binary* bptr = ((ProcBin*)ptr)->val;	
+		    if (erts_refc_dectest(&bptr->refc, 0) == 0) {
+			erts_bin_free(bptr);
+		    }
+		    break;
+		}
+	    case FUN_SUBTAG:
+		{
+		    ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe;
+		    if (erts_refc_dectest(&fe->refc, 0) == 0) {
+			erts_erase_fun_entry(fe);
+		    }
+		    break;
+		}
+	    default:
+		ASSERT(is_external_header(ptr->thing_word));
+		erts_deref_node_entry(((ExternalThing*)ptr)->node);
+	    }
+	    *prev = ptr = ptr->next;
+	}
+	else break; /* and let old-heap loop continue */
     }
 
-    if (BIN_OLD_VHEAP(p) >= BIN_OLD_VHEAP_SZ(p)) {
-        FLAGS(p) |= F_NEED_FULLSWEEP;
+    /* The rest of the list resides on old-heap, and we just did a
+     * generational collection - keep objects in list.
+     */
+    while (ptr) {
+	ASSERT(in_area(ptr, oheap, oheap_sz));
+	ASSERT(!IS_MOVED_BOXED(ptr->thing_word));       
+	if (ptr->thing_word == HEADER_PROC_BIN) {
+	    BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
+	    link_live_proc_bin(&shrink, &prev, &ptr, 0);
+	}
+	else {
+	    ASSERT(is_fun_header(ptr->thing_word) ||
+		   is_external_header(ptr->thing_word));
+	    prev = &ptr->next;
+	    ptr = ptr->next;
+	}
     }
 
-    BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
-    BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p));
-    MSO(p).overhead = bin_vheap;
+    if (fullsweep) {
+	BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p) + MSO(p).overhead, BIN_OLD_VHEAP_SZ(p));
+    }
+    BIN_VHEAP_SZ(p)     = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
+    MSO(p).overhead     = bin_vheap;
+    BIN_VHEAP_MATURE(p) = bin_vheap;
 
     /*
      * If we got any shrink candidates, check them out.
      */
 
     if (shrink.no_of_candidates) {
-	ProcBin *candlist[] = {shrink.new_candidates, shrink.old_candidates};
+	ProcBin *candlist[] = { (ProcBin*)shrink.new_candidates,
+	                        (ProcBin*)shrink.old_candidates };
 	Uint leave_unused = 0;
 	int i;
 
@@ -2293,21 +2208,21 @@ sweep_proc_bins(Process *p, int fullsweep)
 	}
 
 	for (i = 0; i < sizeof(candlist)/sizeof(candlist[0]); i++) {
-
-	    for (ptr = candlist[i]; ptr; ptr = ptr->next) {
-		Uint new_size = ptr->size;
+	    ProcBin* pb;
+	    for (pb = candlist[i]; pb; pb = (ProcBin*)pb->next) {
+		Uint new_size = pb->size;
 
 		if (leave_unused) {
 		    new_size += (new_size * 100) / leave_unused;
 		    /* Our allocators are 8 byte aligned, i.e., shrinking with
 		       less than 8 bytes will have no real effect */
-		    if (new_size + 8 >= ptr->val->orig_size)
+		    if (new_size + 8 >= pb->val->orig_size)
 			continue;
 		}
 
-		ptr->val = erts_bin_realloc(ptr->val, new_size);
-		ptr->val->orig_size = new_size;
-		ptr->bytes = (byte *) ptr->val->orig_bytes;
+		pb->val = erts_bin_realloc(pb->val, new_size);
+		pb->val->orig_size = new_size;
+		pb->bytes = (byte *) pb->val->orig_bytes;
 	    }
 	}
 
@@ -2316,21 +2231,20 @@ sweep_proc_bins(Process *p, int fullsweep)
 	 * We now potentially have the mso list divided into three lists:
 	 * - shrink candidates on new heap (inactive writable with unused data)
 	 * - shrink candidates on old heap (inactive writable with unused data)
-	 * - other binaries (read only + active writable ...)
+	 * - other binaries (read only + active writable ...) + funs and externals
 	 *
 	 * Put them back together: new candidates -> other -> old candidates
 	 * This order will ensure that the list only refers from new
 	 * generation to old and never from old to new *which is important*.
 	 */
 	if (shrink.new_candidates) {
-	    if (prev == &MSO(p).mso) /* empty other binaries list */
+	    if (prev == &MSO(p).first) /* empty other binaries list */
 		prev = &shrink.new_candidates_end->next;
 	    else
-		shrink.new_candidates_end->next = MSO(p).mso;
-	    MSO(p).mso = shrink.new_candidates;
+		shrink.new_candidates_end->next = MSO(p).first;
+	    MSO(p).first = shrink.new_candidates;
 	}
     }
-
     *prev = shrink.old_candidates;
 }
 
@@ -2361,15 +2275,17 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 	      tari = thing_arityval(val);
 	      switch (thing_subtag(val)) {
 	      case REFC_BINARY_SUBTAG:
+	      case FUN_SUBTAG:
+	      case EXTERNAL_PID_SUBTAG:
+	      case EXTERNAL_PORT_SUBTAG:
+	      case EXTERNAL_REF_SUBTAG:
 		  {
-		      ProcBin* pb = (ProcBin*) hp;
-		      Eterm** uptr = (Eterm **) (void *) &pb->next;
+		      struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp;
 
-		      if (*uptr && in_area((Eterm *)pb->next, area, area_size)) {
+		      if (in_area(oh->next, area, area_size)) {
+			  Eterm** uptr = (Eterm **) (void *) &oh->next;
 			  *uptr += offs; /* Patch the mso chain */
 		      }
-		      sz -= tari;
-		      hp += tari + 1;
 		  }
 		  break;
 	      case BIN_MATCHSTATE_SUBTAG:
@@ -2380,40 +2296,11 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 		      mb->orig = offset_ptr(mb->orig, offs);
 		      mb->base = binary_bytes(mb->orig);
 		  }
-		  sz -= tari;
-		  hp += tari + 1;
 		}
 		break;
-	      case FUN_SUBTAG:
-		  {
-		      ErlFunThing* funp = (ErlFunThing *) hp;
-		      Eterm** uptr = (Eterm **) (void *) &funp->next;
-
-		      if (*uptr && in_area((Eterm *)funp->next, area, area_size)) {
-			  *uptr += offs;
-		      }
-		      sz -= tari;
-		      hp += tari + 1;
-		  }
-		  break;
-	      case EXTERNAL_PID_SUBTAG:
-	      case EXTERNAL_PORT_SUBTAG:
-	      case EXTERNAL_REF_SUBTAG:
-		  {
-		      ExternalThing* etp = (ExternalThing *) hp;
-		      Eterm** uptr = (Eterm **) (void *) &etp->next;
-
-		      if (*uptr && in_area((Eterm *)etp->next, area, area_size)) {
-			  *uptr += offs;
-		      }
-		      sz -= tari;
-		      hp += tari + 1;
-		  }
-		  break;
-	      default:
-		  sz -= tari;
-		  hp += tari + 1;
 	      }
+	      sz -= tari;
+	      hp += tari + 1;
 	      break;
 	  }
 	  default:
@@ -2450,18 +2337,8 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 static void
 offset_off_heap(Process* p, Sint offs, char* area, Uint area_size)
 {
-    if (MSO(p).mso && in_area((Eterm *)MSO(p).mso, area, area_size)) {
-        Eterm** uptr = (Eterm**) (void *) &MSO(p).mso;
-        *uptr += offs;
-    }
-
-    if (MSO(p).funs && in_area((Eterm *)MSO(p).funs, area, area_size)) {
-        Eterm** uptr = (Eterm**) (void *) &MSO(p).funs;
-        *uptr += offs;
-    }
-
-    if (MSO(p).externals && in_area((Eterm *)MSO(p).externals, area, area_size)) {
-        Eterm** uptr = (Eterm**) (void *) &MSO(p).externals;
+    if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) {
+        Eterm** uptr = (Eterm**) (void *) &MSO(p).first;
         *uptr += offs;
     }
 }
@@ -2542,7 +2419,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop)
         return 1;
     }
     while (bp != NULL) {
-        if (bp->mem <= ptr && ptr < bp->mem + bp->size) {
+        if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) {
             return 1;
         }
         bp = bp->next;
@@ -2556,7 +2433,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop)
 		hfp = erts_dist_ext_trailer(mp->data.dist_ext);
 	    else
 		hfp = NULL;
-	    if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->size)
+	    if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->used_size)
 		return 1;
 	}
         mp = mp->next;
@@ -2582,8 +2459,8 @@ do {							\
 		 __FILE__, __LINE__, #EXP);		\
 } while (0)
 
-#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
-#  define ERTS_EXTERNAL_VISITED_BIT ((Eterm) 1 << 31)
+#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
+#  define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31)
 #endif
 
 
@@ -2593,62 +2470,45 @@ erts_check_off_heap2(Process *p, Eterm *htop)
     Eterm *oheap = (Eterm *) OLD_HEAP(p);
     Eterm *ohtop = (Eterm *) OLD_HTOP(p);
     int old;
-    ProcBin *pb;
-    ErlFunThing *eft;
-    ExternalThing *et;
+    union erl_off_heap_ptr u;
 
     old = 0;
-    for (pb = MSO(p).mso; pb; pb = pb->next) {
-	Eterm *ptr = (Eterm *) pb;
-	long refc = erts_refc_read(&pb->val->refc, 1);
+    for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) {
+	erts_aint_t refc;
+	switch (thing_subtag(u.hdr->thing_word)) {
+	case REFC_BINARY_SUBTAG:
+	    refc = erts_refc_read(&u.pb->val->refc, 1);		
+	    break;
+	case FUN_SUBTAG:
+	    refc = erts_refc_read(&u.fun->fe->refc, 1);
+	    break;
+	case EXTERNAL_PID_SUBTAG:
+	case EXTERNAL_PORT_SUBTAG:
+	case EXTERNAL_REF_SUBTAG:
+	    refc = erts_refc_read(&u.ext->node->refc, 1);
+	    break;
+	default:
+	    ASSERT(!!"erts_check_off_heap2: Invalid thing_word");
+	}
 	ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
+#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
+	ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_EXTERNAL_VISITED_BIT));
+	u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT;
+#endif
 	if (old) {
-	    ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop);
+	    ERTS_CHK_OFFHEAP_ASSERT(oheap <= u.ep && u.ep < ohtop);
 	}
-	else if (oheap <= ptr && ptr < ohtop)
+	else if (oheap <= u.ep && u.ep < ohtop)
 	    old = 1;
 	else {
-	    ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop));
+	    ERTS_CHK_OFFHEAP_ASSERT(within2(u.ep, p, htop));
 	}
     }
 
-    old = 0;
-    for (eft = MSO(p).funs; eft; eft = eft->next) {
-	Eterm *ptr = (Eterm *) eft;
-	long refc = erts_refc_read(&eft->fe->refc, 1);
-	ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
-	if (old)
-	    ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop);
-	else if (oheap <= ptr && ptr < ohtop)
-	    old = 1;
-	else
-	    ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop));
-    }
-
-    old = 0;
-    for (et = MSO(p).externals; et; et = et->next) {
-	Eterm *ptr = (Eterm *) et;
-	long refc = erts_refc_read(&et->node->refc, 1);
-	ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
-#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
-	ERTS_CHK_OFFHEAP_ASSERT(!(et->header & ERTS_EXTERNAL_VISITED_BIT));
-#endif
-	if (old)
-	    ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop);
-	else if (oheap <= ptr && ptr < ohtop)
-	    old = 1;
-	else
-	    ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop));
-#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
-	et->header |= ERTS_EXTERNAL_VISITED_BIT;
-#endif
-    }
-
 #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
-    for (et = MSO(p).externals; et; et = et->next)
-	et->header &= ~ERTS_EXTERNAL_VISITED_BIT;
+    for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next)
+	u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT;
 #endif
-	
 }
 
 void
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index af55b6363f..807ef8ae8d 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2007-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -22,11 +22,12 @@
 
 /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */
 
-#ifdef DEBUG
+#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF
 #  define HARDDEBUG 1
 #endif
 
-#define IS_MOVED(x)	(!is_header((x)))
+#define IS_MOVED_BOXED(x)	(!is_header((x)))
+#define IS_MOVED_CONS(x)	(is_non_value((x)))
 
 #define MOVE_CONS(PTR,CAR,HTOP,ORIG)					\
 do {									\
@@ -69,4 +70,28 @@ extern Uint erts_test_long_gc_sleep;
 int within(Eterm *ptr, Process *p);
 #endif
 
+ERTS_GLB_INLINE Eterm follow_moved(Eterm term);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE Eterm follow_moved(Eterm term)
+{
+    Eterm* ptr;
+    switch (primary_tag(term)) {
+    case TAG_PRIMARY_IMMED1:
+	break;
+    case TAG_PRIMARY_BOXED:
+	ptr = boxed_val(term);
+	if (IS_MOVED_BOXED(*ptr)) term = *ptr;
+	break;
+    case TAG_PRIMARY_LIST:
+	ptr = list_val(term);
+	if (IS_MOVED_CONS(ptr[0])) term = ptr[1];
+	break;
+    default:
+	ASSERT(!"strange tag in follow_moved");
+    }
+    return term;
+}
+#endif
+
 #endif /* __ERL_GC_H__ */
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index ea2ba4d55c..76b206d76f 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -49,30 +49,30 @@
 #define MIN_MBC_FIRST_FREE_SZ		(4*1024)
 
 #define MAX_SUB_MASK_IX \
-  ((((Uint)1) << (NO_OF_BKT_IX_BITS - SUB_MASK_IX_SHIFT)) - 1)
-#define MAX_SUB_BKT_IX ((((Uint)1) << SUB_MASK_IX_SHIFT) - 1)
+  ((((UWord)1) << (NO_OF_BKT_IX_BITS - SUB_MASK_IX_SHIFT)) - 1)
+#define MAX_SUB_BKT_IX ((((UWord)1) << SUB_MASK_IX_SHIFT) - 1)
 #define MAX_BKT_IX (NO_OF_BKTS - 1)
 
-#define MIN_BLK_SZ  UNIT_CEILING(sizeof(GFFreeBlock_t) + sizeof(Uint))
+#define MIN_BLK_SZ  UNIT_CEILING(sizeof(GFFreeBlock_t) + sizeof(UWord))
 
-#define IX2SBIX(IX) ((IX) & (~(~((Uint)0) << SUB_MASK_IX_SHIFT)))
+#define IX2SBIX(IX) ((IX) & (~(~((UWord)0) << SUB_MASK_IX_SHIFT)))
 #define IX2SMIX(IX) ((IX) >> SUB_MASK_IX_SHIFT)
 #define MAKE_BKT_IX(SMIX, SBIX)	\
-  ((((Uint)(SMIX)) << SUB_MASK_IX_SHIFT) | ((Uint)(SBIX)))
+  ((((UWord)(SMIX)) << SUB_MASK_IX_SHIFT) | ((UWord)(SBIX)))
 
 #define SET_BKT_MASK_IX(BM, IX)						\
 do {									\
     int sub_mask_ix__ = IX2SMIX((IX));					\
-    (BM).main |= (((Uint) 1) << sub_mask_ix__);				\
-    (BM).sub[sub_mask_ix__] |= (((Uint)1) << IX2SBIX((IX)));		\
+    (BM).main |= (((UWord) 1) << sub_mask_ix__);				\
+    (BM).sub[sub_mask_ix__] |= (((UWord)1) << IX2SBIX((IX)));		\
 } while (0)
 
 #define UNSET_BKT_MASK_IX(BM, IX)					\
 do {									\
     int sub_mask_ix__ = IX2SMIX((IX));					\
-    (BM).sub[sub_mask_ix__] &= ~(((Uint)1) << IX2SBIX((IX)));		\
+    (BM).sub[sub_mask_ix__] &= ~(((UWord)1) << IX2SBIX((IX)));		\
     if (!(BM).sub[sub_mask_ix__])					\
-	(BM).main &= ~(((Uint)1) << sub_mask_ix__);			\
+	(BM).main &= ~(((UWord)1) << sub_mask_ix__);			\
 } while (0)
 
 /* Buckets ... */
@@ -263,8 +263,8 @@ find_bucket(BucketMask_t *bmask, int min_index)
     while(max != min) {					\
 	mid = ((max - min) >> 1) + min;			\
 	if((BitMask)					\
-	   & (~(~((Uint) 0) << (mid + 1)))		\
-	   & (~((Uint) 0) << min))			\
+	   & (~(~((UWord) 0) << (mid + 1)))		\
+	   & (~((UWord) 0) << min))			\
 	    max = mid;					\
 	else						\
 	    min = mid + 1;				\
@@ -272,21 +272,21 @@ find_bucket(BucketMask_t *bmask, int min_index)
     (MinBit) = min
 
 
-    ASSERT(bmask->main < (((Uint) 1) << (MAX_SUB_MASK_IX+1)));
+    ASSERT(bmask->main < (((UWord) 1) << (MAX_SUB_MASK_IX+1)));
 
     sub_mask_ix = IX2SMIX(min_index);
 
-    if ((bmask->main & (~((Uint) 0) << sub_mask_ix)) == 0)
+    if ((bmask->main & (~((UWord) 0) << sub_mask_ix)) == 0)
 	return -1;
 
     /* There exists a non empty bucket; find it... */
 
-    if (bmask->main & (((Uint) 1) << sub_mask_ix)) {
+    if (bmask->main & (((UWord) 1) << sub_mask_ix)) {
 	sub_bkt_ix = IX2SBIX(min_index);
-	if ((bmask->sub[sub_mask_ix] & (~((Uint) 0) << sub_bkt_ix)) == 0) {
+	if ((bmask->sub[sub_mask_ix] & (~((UWord) 0) << sub_bkt_ix)) == 0) {
 	    sub_mask_ix++;
 	    sub_bkt_ix = 0;
-	    if ((bmask->main & (~((Uint) 0)<< sub_mask_ix)) == 0)
+	    if ((bmask->main & (~((UWord) 0)<< sub_mask_ix)) == 0)
 		return -1;
 	}
 	else
@@ -299,17 +299,17 @@ find_bucket(BucketMask_t *bmask, int min_index)
 
     ASSERT(sub_mask_ix <= MAX_SUB_MASK_IX);
     /* Has to be a bit > sub_mask_ix */
-    ASSERT(bmask->main & (~((Uint) 0) << (sub_mask_ix)));
+    ASSERT(bmask->main & (~((UWord) 0) << (sub_mask_ix)));
     GET_MIN_BIT(sub_mask_ix, bmask->main, sub_mask_ix, MAX_SUB_MASK_IX);
 
  find_sub_bkt_ix:
     ASSERT(sub_mask_ix <= MAX_SUB_MASK_IX);
     ASSERT(sub_bkt_ix <= MAX_SUB_BKT_IX);
 
-    if ((bmask->sub[sub_mask_ix] & (((Uint) 1) << sub_bkt_ix)) == 0) {
+    if ((bmask->sub[sub_mask_ix] & (((UWord) 1) << sub_bkt_ix)) == 0) {
 	ASSERT(sub_mask_ix + 1 <= MAX_SUB_BKT_IX);
 	/* Has to be a bit > sub_bkt_ix */
-	ASSERT(bmask->sub[sub_mask_ix] & (~((Uint) 0) << sub_bkt_ix));
+	ASSERT(bmask->sub[sub_mask_ix] & (~((UWord) 0) << sub_bkt_ix));
 
 	GET_MIN_BIT(sub_bkt_ix,
 		    bmask->sub[sub_mask_ix],
@@ -336,7 +336,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size)
     Uint min_sz;
     Uint blk_sz;
     Uint cand_sz = 0;
-    Uint max_blk_search;
+    UWord max_blk_search;
     GFFreeBlock_t *blk;
     GFFreeBlock_t *cand = NULL;
     int blk_on_lambc;
@@ -615,9 +615,9 @@ check_block(Allctr_t *allctr, Block_t * blk, int free_block)
 	Uint blk_sz = BLK_SZ(blk);
 	bi = BKT_IX(gfallctr, blk_sz);
 
-	ASSERT(gfallctr->bucket_mask.main & (((Uint) 1) << IX2SMIX(bi)));
+	ASSERT(gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi)));
 	ASSERT(gfallctr->bucket_mask.sub[IX2SMIX(bi)]
-	       & (((Uint) 1) << IX2SBIX(bi)));
+	       & (((UWord) 1) << IX2SBIX(bi)));
 		
 	found = 0;
 	for (fblk = gfallctr->buckets[bi]; fblk; fblk = fblk->next)
@@ -648,9 +648,9 @@ check_mbc(Allctr_t *allctr, Carrier_t *mbc)
     int bi;
 
     for(bi = 0; bi < NO_OF_BKTS; bi++) {
-	if ((gfallctr->bucket_mask.main & (((Uint) 1) << IX2SMIX(bi)))
+	if ((gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi)))
 	    && (gfallctr->bucket_mask.sub[IX2SMIX(bi)]
-		& (((Uint) 1) << IX2SBIX(bi)))) {
+		& (((UWord) 1) << IX2SBIX(bi)))) {
 	    ASSERT(gfallctr->buckets[bi] != NULL);
 	}
 	else {
diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h
index 3d1b8c01f6..a554a6f466 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.h
+++ b/erts/emulator/beam/erl_goodfit_alloc.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -28,7 +28,7 @@
 typedef struct GFAllctr_t_ GFAllctr_t;
 
 typedef struct {
-    Uint mbsd;
+    UWord mbsd;
 } GFAllctrInit_t;
 
 #define ERTS_DEFAULT_GF_ALLCTR_INIT {                                      \
@@ -49,18 +49,18 @@ Allctr_t *erts_gfalc_start(GFAllctr_t *, GFAllctrInit_t *, AllctrInit_t *);
 #include "erl_alloc_util.h"
 
 #define NO_OF_BKT_IX_BITS (8)
-#ifdef ARCH_64
+#if defined(ARCH_64)
 #  define SUB_MASK_IX_SHIFT (6)
 #else
 #  define SUB_MASK_IX_SHIFT (5)
 #endif
-#define NO_OF_BKTS (((Uint) 1) << NO_OF_BKT_IX_BITS)
-#define NO_OF_SUB_MASKS (NO_OF_BKTS/(((Uint) 1) << SUB_MASK_IX_SHIFT))
+#define NO_OF_BKTS (((UWord) 1) << NO_OF_BKT_IX_BITS)
+#define NO_OF_SUB_MASKS (NO_OF_BKTS/(((UWord) 1) << SUB_MASK_IX_SHIFT))
 
 typedef struct {
-    Uint main;
-    Uint sub[NO_OF_SUB_MASKS];
-} BucketMask_t; 
+    UWord main;
+    UWord sub[NO_OF_SUB_MASKS];
+} BucketMask_t;
 
 typedef struct GFFreeBlock_t_ GFFreeBlock_t;
 struct GFFreeBlock_t_ {
@@ -74,11 +74,11 @@ struct GFAllctr_t_ {
 
     char *		last_aux_mbc_start;
     char *		last_aux_mbc_end;
-    Uint		bkt_max_size_d;
-    Uint		bkt_intrvl_d;
+    UWord		bkt_max_size_d;
+    UWord		bkt_intrvl_d;
     BucketMask_t	bucket_mask;
     GFFreeBlock_t *	buckets[NO_OF_BKTS];
-    Uint 		max_blk_search;
+    UWord 		max_blk_search;
 
 };
 
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index e97ab328cd..0a57eb6d88 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -41,6 +41,7 @@
 #include "erl_printf_term.h"
 #include "erl_misc_utils.h"
 #include "packet_parser.h"
+#include "erl_cpu_topology.h"
 
 #ifdef HIPE
 #include "hipe_mode_switch.h"	/* for hipe_mode_switch_init() */
@@ -63,6 +64,8 @@ extern void ConNormalExit(void);
 extern void ConWaitForExit(void);
 #endif
 
+static void erl_init(int ncpu);
+
 #define ERTS_MIN_COMPAT_REL 7
 
 #ifdef ERTS_SMP
@@ -76,8 +79,6 @@ int erts_initialized = 0;
 static erts_tid_t main_thread;
 #endif
 
-erts_cpu_info_t *erts_cpuinfo;
-
 int erts_use_sender_punish;
 
 /*
@@ -99,7 +100,7 @@ int erts_backtrace_depth;	/* How many functions to show in a backtrace
 
 int erts_async_max_threads;  /* number of threads for async support */
 int erts_async_thread_suggested_stack_size;
-erts_smp_atomic_t erts_max_gen_gcs;
+erts_smp_atomic32_t erts_max_gen_gcs;
 
 Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, 
 				     am_info or am_warning, am_error is 
@@ -228,18 +229,18 @@ void erl_error(char *fmt, va_list args)
     erts_vfprintf(stderr, fmt, args);
 }
 
-static void early_init(int *argc, char **argv);
+static int early_init(int *argc, char **argv);
 
 void
 erts_short_init(void)
 {
-    early_init(NULL, NULL);
-    erl_init();
+    int ncpu = early_init(NULL, NULL);
+    erl_init(ncpu);
     erts_initialized = 1;
 }
 
-void
-erl_init(void)
+static void
+erl_init(int ncpu)
 {
     init_benchmarking();
 
@@ -249,12 +250,13 @@ erl_init(void)
 
     erts_init_monitors();
     erts_init_gc();
-    init_time();
-    erts_init_process();
+    erts_init_time();
+    erts_init_sys_common_misc();
+    erts_init_process(ncpu);
     erts_init_scheduling(use_multi_run_queue,
 			 no_schedulers,
 			 no_schedulers_online);
-
+    erts_init_cpu_topology(); /* Must be after init_scheduling */
     H_MIN_SIZE      = erts_next_heap_size(H_MIN_SIZE, 0);
     BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0);
 
@@ -281,20 +283,18 @@ erl_init(void)
     init_load();
     erts_init_bif();
     erts_init_bif_chksum();
+    erts_init_bif_binary();
     erts_init_bif_re();
     erts_init_unicode(); /* after RE to get access to PCRE unicode */
     erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2);
     erts_late_init_process();
 #if HAVE_ERTS_MSEG
-    erts_mseg_late_init(); /* Must be after timer (init_time()) and thread
+    erts_mseg_late_init(); /* Must be after timer (erts_init_time()) and thread
 			      initializations */
 #endif
 #ifdef HIPE
     hipe_mode_switch_init(); /* Must be after init_load/beam_catches/init */
 #endif
-#ifdef _OSE_
-    erl_sys_init_final();
-#endif
     packet_parser_init();
     erl_nif_init();
 }
@@ -323,7 +323,7 @@ init_shared_memory(int argc, char **argv)
 #endif
 
     global_gen_gcs = 0;
-    global_max_gen_gcs = erts_smp_atomic_read(&erts_max_gen_gcs);
+    global_max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs);
     global_gc_flags = erts_default_process_flags;
 
     erts_global_offheap.mso = NULL;
@@ -338,59 +338,6 @@ init_shared_memory(int argc, char **argv)
 #endif
 }
 
-
-/*
- * Create the very first process.
- */
-
-void
-erts_first_process(Eterm modname, void* code, unsigned size, int argc, char** argv)
-{
-    int i;
-    Eterm args;
-    Eterm pid;
-    Eterm* hp;
-    Process parent;
-    Process* p;
-    ErlSpawnOpts so;
-    
-    if (erts_find_function(modname, am_start, 1) == NULL) {
-	char sbuf[256];
-	Atom* ap;
-
-	ap = atom_tab(atom_val(modname));
-	memcpy(sbuf, ap->name, ap->len);
-	sbuf[ap->len] = '\0';
-	erl_exit(5, "No function %s:start/1\n", sbuf);
-    }
-
-    /*
-     * We need a dummy parent process to be able to call erl_create_process().
-     */
-    erts_init_empty_process(&parent);
-    hp = HAlloc(&parent, argc*2 + 4);
-    args = NIL;
-    for (i = argc-1; i >= 0; i--) {
-	int len = sys_strlen(argv[i]);
-	args = CONS(hp, new_binary(&parent, (byte*)argv[i], len), args);
-	hp += 2;
-    }
-    args = CONS(hp, new_binary(&parent, code, size), args);
-    hp += 2;
-    args = CONS(hp, args, NIL);
-
-    so.flags = 0;
-    pid = erl_create_process(&parent, modname, am_start, args, &so);
-    p = process_tab[internal_pid_index(pid)];
-    p->group_leader = pid;
-
-    erts_cleanup_empty_process(&parent);
-}
-
-/*
- * XXX Old way of starting. Hopefully soon obsolete.
- */
-
 static void
 erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv)
 {
@@ -560,10 +507,13 @@ void erts_usage(void)
 	       ERTS_MIN_COMPAT_REL, this_rel_num());
 
     erts_fprintf(stderr, "-r          force ets memory block to be moved on realloc\n");
+    erts_fprintf(stderr, "-rg amount  set reader groups limit\n");
     erts_fprintf(stderr, "-sbt type   set scheduler bind type, valid types are:\n");
     erts_fprintf(stderr, "            u|ns|ts|ps|s|nnts|nnps|tnnps|db\n");
     erts_fprintf(stderr, "-sct cput   set cpu topology,\n");
     erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
+    erts_fprintf(stderr, "-swt val    set scheduler wakeup threshold, valid values are:\n");
+    erts_fprintf(stderr, "            very_low|low|medium|high|very_high.\n");
     erts_fprintf(stderr, "-sss size   suggested stack size in kilo words for scheduler threads,\n");
     erts_fprintf(stderr, "            valid range is [%d-%d]\n",
 		 ERTS_SCHED_THREAD_MIN_STACK_SIZE,
@@ -585,7 +535,8 @@ void erts_usage(void)
 
     erts_fprintf(stderr, "-W<i|w>     set error logger warnings mapping,\n");
     erts_fprintf(stderr, "            see error_logger documentation for details\n");
-
+    erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n");
+    erts_fprintf(stderr, "            valid range is [1-%d]\n", INT_MAX/1024);
     erts_fprintf(stderr, "\n");
     erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n");
     erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n");
@@ -593,7 +544,51 @@ void erts_usage(void)
     erl_exit(-1, "");
 }
 
-static void
+#ifdef USE_THREADS
+/*
+ * allocators for thread lib
+ */
+
+static void *ethr_std_alloc(size_t size)
+{
+    return erts_alloc_fnf(ERTS_ALC_T_ETHR_STD, (Uint) size);
+}
+static void *ethr_std_realloc(void *ptr, size_t size)
+{
+    return erts_realloc_fnf(ERTS_ALC_T_ETHR_STD, ptr, (Uint) size);
+}
+static void ethr_std_free(void *ptr)
+{
+    erts_free(ERTS_ALC_T_ETHR_STD, ptr);
+}
+static void *ethr_sl_alloc(size_t size)
+{
+    return erts_alloc_fnf(ERTS_ALC_T_ETHR_SL, (Uint) size);
+}
+static void *ethr_sl_realloc(void *ptr, size_t size)
+{
+    return erts_realloc_fnf(ERTS_ALC_T_ETHR_SL, ptr, (Uint) size);
+}
+static void ethr_sl_free(void *ptr)
+{
+    erts_free(ERTS_ALC_T_ETHR_SL, ptr);
+}
+static void *ethr_ll_alloc(size_t size)
+{
+    return erts_alloc_fnf(ERTS_ALC_T_ETHR_LL, (Uint) size);
+}
+static void *ethr_ll_realloc(void *ptr, size_t size)
+{
+    return erts_realloc_fnf(ERTS_ALC_T_ETHR_LL, ptr, (Uint) size);
+}
+static void ethr_ll_free(void *ptr)
+{
+    erts_free(ERTS_ALC_T_ETHR_LL, ptr);
+}
+
+#endif
+
+static int
 early_init(int *argc, char **argv) /*
 				   * Only put things here which are
 				   * really important initialize
@@ -606,6 +601,10 @@ early_init(int *argc, char **argv) /*
     int ncpuavail;
     int schdlrs;
     int schdlrs_onln;
+    int max_main_threads;
+    int max_reader_groups;
+    int reader_groups;
+
     use_multi_run_queue = 1;
     erts_printf_eterm_func = erts_printf_term;
     erts_disable_tolerant_timeofday = 0;
@@ -621,13 +620,11 @@ early_init(int *argc, char **argv) /*
 
     erts_use_sender_punish = 1;
 
-    erts_cpuinfo = erts_cpu_info_create();
-
-#ifdef ERTS_SMP
-    ncpu = erts_get_cpu_configured(erts_cpuinfo);
-    ncpuonln = erts_get_cpu_online(erts_cpuinfo);
-    ncpuavail = erts_get_cpu_available(erts_cpuinfo);
-#else
+    erts_pre_early_init_cpu_topology(&max_reader_groups,
+				     &ncpu,
+				     &ncpuonln,
+				     &ncpuavail);
+#ifndef ERTS_SMP
     ncpu = 1;
     ncpuonln = 1;
     ncpuavail = 1;
@@ -654,7 +651,7 @@ early_init(int *argc, char **argv) /*
     erts_writing_erl_crash_dump = 0;
 #endif
 
-    erts_smp_atomic_init(&erts_max_gen_gcs, (long)((Uint16) -1));
+    erts_smp_atomic32_init(&erts_max_gen_gcs, (erts_aint32_t) ((Uint16) -1));
 
     erts_pre_init_process();
 #if defined(USE_THREADS) && !defined(ERTS_SMP)
@@ -682,6 +679,24 @@ early_init(int *argc, char **argv) /*
 	    }
 	    if (argv[i][0] == '-') {
 		switch (argv[i][1]) {
+		case 'r': {
+		    char *sub_param = argv[i]+2;
+		    if (has_prefix("g", sub_param)) {
+			char *arg = get_arg(sub_param+1, argv[i+1], &i);
+			if (sscanf(arg, "%d", &max_reader_groups) != 1) {
+			    erts_fprintf(stderr,
+					 "bad reader groups limit: %s\n", arg);
+			    erts_usage();
+			}
+			if (max_reader_groups < 0) {
+			    erts_fprintf(stderr,
+					 "bad reader groups limit: %d\n",
+					 max_reader_groups);
+			    erts_usage();
+			}
+		    }
+		    break;
+		}
 		case 'S' : {
 		    int tot, onln;
 		    char *arg = get_arg(argv[i]+2, argv[i+1], &i);
@@ -750,13 +765,40 @@ early_init(int *argc, char **argv) /*
 
     erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes)
 						 -M flags. */
+    /* Require allocators */
+    erts_early_init_scheduling();
+    erts_init_utils();
+    erts_early_init_cpu_topology(no_schedulers,
+				 &max_main_threads,
+				 max_reader_groups,
+				 &reader_groups);
 
-    erts_early_init_scheduling(); /* Require allocators */
-    erts_init_utils(); /* Require allocators */
+#ifdef USE_THREADS
+    {
+	erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER;
+	elid.mem.std.alloc = ethr_std_alloc;
+	elid.mem.std.realloc = ethr_std_realloc;
+	elid.mem.std.free = ethr_std_free;
+	elid.mem.sl.alloc = ethr_sl_alloc;
+	elid.mem.sl.realloc = ethr_sl_realloc;
+	elid.mem.sl.free = ethr_sl_free;
+	elid.mem.ll.alloc = ethr_ll_alloc;
+	elid.mem.ll.realloc = ethr_ll_realloc;
+	elid.mem.ll.free = ethr_ll_free;
+	elid.main_threads = max_main_threads;
+	elid.reader_groups = reader_groups;
+
+	erts_thr_late_init(&elid);
+    }
+#endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_late_init();
 #endif
+    
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_late_init();
+#endif
 
 #if defined(HIPE)
     hipe_signal_init();	/* must be done very early */
@@ -766,7 +808,10 @@ early_init(int *argc, char **argv) /*
     erl_sys_args(argc, argv);
 
     erts_ets_realloc_always_moves = 0;
+    erts_ets_always_compress = 0;
+    erts_dist_buf_busy_limit = ERTS_DE_BUSY_LIMIT;
 
+    return ncpu;
 }
 
 #ifndef ERTS_SMP
@@ -800,8 +845,7 @@ erl_start(int argc, char **argv)
     char envbuf[21]; /* enough for any 64-bit integer */
     size_t envbufsz;
     int async_max_threads = erts_async_max_threads;
-
-    early_init(&argc, argv);
+    int ncpu = early_init(&argc, argv);
 
     envbufsz = sizeof(envbuf);
     if (erts_sys_getenv(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0)
@@ -812,14 +856,20 @@ erl_start(int argc, char **argv)
     envbufsz = sizeof(envbuf);
     if (erts_sys_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) {
 	Uint16 max_gen_gcs = atoi(envbuf);
-	erts_smp_atomic_set(&erts_max_gen_gcs, (long) max_gen_gcs);
+	erts_smp_atomic32_set(&erts_max_gen_gcs, (erts_aint32_t) max_gen_gcs);
     }
 
     envbufsz = sizeof(envbuf);
     if (erts_sys_getenv("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) {
 	async_max_threads = atoi(envbuf);
     }
-    
+
+#if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__)
+    /*
+     * The default stack size on MacOS X is too small for pcre.
+     */
+    erts_sched_thread_suggested_stack_size = 256;
+#endif
 
 #ifdef DEBUG
     verbose = DEBUG_DEFAULT;
@@ -858,7 +908,27 @@ erl_start(int argc, char **argv)
 	    VERBOSE(DEBUG_SYSTEM,
                     ("using display items %d\n",display_items));
 	    break;
-
+	case 'f':
+	    if (!strncmp(argv[i],"-fn",3)) {
+		arg = get_arg(argv[i]+3, argv[i+1], &i);
+		switch (*arg) {
+		case 'u':
+		    erts_set_user_requested_filename_encoding(ERL_FILENAME_UTF8);
+		    break;
+		case 'l':
+		    erts_set_user_requested_filename_encoding(ERL_FILENAME_LATIN1);
+		    break;
+		case 'a':
+		    erts_set_user_requested_filename_encoding(ERL_FILENAME_UNKNOWN);
+		default:
+		    erts_fprintf(stderr, "bad filename encoding %s, can be (l,u or a)\n", arg);
+		    erts_usage();
+		}
+		break;
+	    } else {
+		erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]);
+		erts_usage();
+	    }
 	case 'l':
 	    display_loads++;
 	    break;
@@ -980,15 +1050,20 @@ erl_start(int argc, char **argv)
 	    break;
 
 	case 'e':
-	    /* set maximum number of ets tables */
-	    arg = get_arg(argv[i]+2, argv[i+1], &i);
-	    if (( user_requested_db_max_tabs = atoi(arg) ) < 0) {
-		erts_fprintf(stderr, "bad maximum number of ets tables %s\n", arg);
-		erts_usage();
+	    if (sys_strcmp("c", argv[i]+2) == 0) {
+		erts_ets_always_compress = 1;
+	    }
+	    else {
+		/* set maximum number of ets tables */
+		arg = get_arg(argv[i]+2, argv[i+1], &i);
+		if (( user_requested_db_max_tabs = atoi(arg) ) < 0) {
+		    erts_fprintf(stderr, "bad maximum number of ets tables %s\n", arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("using maximum number of ets tables %d\n",
+			 user_requested_db_max_tabs));
 	    }
-	    VERBOSE(DEBUG_SYSTEM,
-                    ("using maximum number of ets tables %d\n",
-                     user_requested_db_max_tabs));
 	    break;
 
 	case 'i':
@@ -1052,7 +1127,7 @@ erl_start(int argc, char **argv)
 	    char *sub_param = argv[i]+2;
 	    if (has_prefix("bt", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
-		res = erts_init_scheduler_bind_type(arg);
+		res = erts_init_scheduler_bind_type_string(arg);
 		if (res != ERTS_INIT_SCHED_BIND_TYPE_SUCCESS) {
 		    switch (res) {
 		    case ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED:
@@ -1077,7 +1152,7 @@ erl_start(int argc, char **argv)
 	    }
 	    else if (has_prefix("ct", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
-		res = erts_init_cpu_topology(arg);
+		res = erts_init_cpu_topology_string(arg);
 		if (res != ERTS_INIT_CPU_TOPOLOGY_OK) {
 		    switch (res) {
 		    case ERTS_INIT_CPU_TOPOLOGY_INVALID_ID:
@@ -1120,10 +1195,20 @@ erl_start(int argc, char **argv)
 	    }
 	    else if (sys_strcmp("mrq", sub_param) == 0)
 		use_multi_run_queue = 1;
-	    else if (sys_strcmp("srq", sub_param) == 0)
-		use_multi_run_queue = 0;
 	    else if (sys_strcmp("nsp", sub_param) == 0)
 		erts_use_sender_punish = 0;
+	    else if (sys_strcmp("srq", sub_param) == 0)
+		use_multi_run_queue = 0;
+	    else if (sys_strcmp("wt", sub_param) == 0) {
+		arg = get_arg(sub_param+2, argv[i+1], &i);
+		if (erts_sched_set_wakeup_limit(arg) != 0) {
+		    erts_fprintf(stderr, "scheduler wakeup threshold: %s\n",
+				 arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("scheduler wakup threshold: %s\n", arg));
+	    }
 	    else if (has_prefix("ss", sub_param)) {
 		/* suggested stack size (Kilo Words) for scheduler threads */
 		arg = get_arg(sub_param+2, argv[i+1], &i);
@@ -1238,9 +1323,17 @@ erl_start(int argc, char **argv)
 		     erts_async_thread_suggested_stack_size));
 	    break;
 
- 	case 'r':
-	    erts_ets_realloc_always_moves = 1;
+	case 'r': {
+	    char *sub_param = argv[i]+2;
+	    if (has_prefix("g", sub_param)) {
+		get_arg(sub_param+1, argv[i+1], &i);
+		/* already handled */
+	    }
+	    else {
+		erts_ets_realloc_always_moves = 1;
+	    }
 	    break;
+	}
 	case 'n':   /* XXX obsolete */
 	    break;
 	case 'c':
@@ -1270,6 +1363,26 @@ erl_start(int argc, char **argv)
 	    }
 	    break;
 
+	case 'z': {
+	    char *sub_param = argv[i]+2;
+	    int new_limit;
+
+	    if (has_prefix("dbbl", sub_param)) {
+		arg = get_arg(sub_param+4, argv[i+1], &i);
+		new_limit = atoi(arg);
+		if (new_limit < 1 || INT_MAX/1024 < new_limit) {
+		    erts_fprintf(stderr, "Invalid dbbl limit: %d\n", new_limit);
+		    erts_usage();
+		} else {
+		    erts_dist_buf_busy_limit = new_limit*1024;
+		}
+	    } else {
+		erts_fprintf(stderr, "bad -z option %s\n", argv[i]);
+		erts_usage();
+	    }
+	    break;
+        }
+
 	default:
 	    erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]);
 	    erts_usage();
@@ -1310,7 +1423,7 @@ erl_start(int argc, char **argv)
     boot_argc = argc - i;  /* Number of arguments to init */
     boot_argv = &argv[i];
 
-    erl_init();
+    erl_init(ncpu);
 
     init_shared_memory(boot_argc, boot_argv);
     load_preloaded();
@@ -1325,6 +1438,7 @@ erl_start(int argc, char **argv)
 
     erts_sys_main_thread(); /* May or may not return! */
 #else
+    erts_thr_set_main_status(1, 1);
     set_main_stack_size();
     process_main();
 #endif
@@ -1398,7 +1512,7 @@ system_cleanup(int exit_code)
     erts_cleanup_incgc();
 #endif
 
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
+#if defined(USE_THREADS)
     exit_async();
 #endif
 #if HAVE_ERTS_MSEG
@@ -1447,13 +1561,7 @@ __decl_noreturn void erl_exit0(char *file, int line, int n, char *fmt,...)
     if (fmt != NULL && *fmt != '\0')
 	  erl_error(fmt, args);	/* Print error message. */
     va_end(args);
-#ifdef __WIN32__
-    if(n > 0) ConWaitForExit();
-    else ConNormalExit();
-#endif
-#if !defined(__WIN32__) && !defined(VXWORKS) && !defined(_OSE_)
-    sys_tty_reset();
-#endif
+    sys_tty_reset(n);
 
     if (n == ERTS_INTR_EXIT)
 	exit(0);
@@ -1493,13 +1601,7 @@ __decl_noreturn void erl_exit(int n, char *fmt,...)
     if (fmt != NULL && *fmt != '\0')
 	  erl_error(fmt, args);	/* Print error message. */
     va_end(args);
-#ifdef __WIN32__
-    if(n > 0) ConWaitForExit();
-    else ConNormalExit();
-#endif
-#if !defined(__WIN32__) && !defined(VXWORKS) && !defined(_OSE_)
-    sys_tty_reset();
-#endif
+    sys_tty_reset(n);
 
     if (n == ERTS_INTR_EXIT)
 	exit(0);
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
index 3f022f92b8..f3f3c22933 100644
--- a/erts/emulator/beam/erl_instrument.c
+++ b/erts/emulator/beam/erl_instrument.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -540,18 +540,18 @@ static void dump_memory_map_to_stream(FILE *fp)
 	if (is_internal_pid(bp->pid))
 	    fprintf(fp,
 		    "{%lu, %lu, %lu, {%lu,%lu,%lu}}.\n",
-		    (Uint) bp->type_no,
-		    (Uint) bp->mem,
-		    (Uint) bp->size,
-		    (Uint) pid_channel_no(bp->pid),
-		    (Uint) pid_number(bp->pid),
-		    (Uint) pid_serial(bp->pid));
+		    (UWord) bp->type_no,
+		    (UWord) bp->mem,
+		    (UWord) bp->size,
+		    (UWord) pid_channel_no(bp->pid),
+		    (UWord) pid_number(bp->pid),
+		    (UWord) pid_serial(bp->pid));
 	else
 	    fprintf(fp,
 		    "{%lu, %lu, %lu, undefined}.\n",
-		    (Uint) bp->type_no,
-		    (Uint) bp->mem,
-		    (Uint) bp->size);
+		    (UWord) bp->type_no,
+		    (UWord) bp->mem,
+		    (UWord) bp->size);
     }
 
     if (lock)
@@ -638,7 +638,7 @@ Eterm erts_instr_get_memory_map(Process *proc)
 	    hsz += 4;
 	}
 
-	if ((Uint) bp->mem > MAX_SMALL)
+	if ((UWord) bp->mem > MAX_SMALL)
 	    hsz += BIG_UINT_HEAP_SIZE;
 	if (bp->size > MAX_SMALL)
 	    hsz += BIG_UINT_HEAP_SIZE;
@@ -749,12 +749,12 @@ Eterm erts_instr_get_memory_map(Process *proc)
 #endif
 	type = make_small(bp->type_no);
 
-	if ((Uint) bp->mem > MAX_SMALL) {
-	    ptr = uint_to_big((Uint) bp->mem, hp);
+	if ((UWord) bp->mem > MAX_SMALL) {
+	    ptr = uint_to_big((UWord) bp->mem, hp);
 	    hp += BIG_UINT_HEAP_SIZE;
 	}
 	else
-	    ptr = make_small((Uint) bp->mem);
+	    ptr = make_small((UWord) bp->mem);
 
 	if (bp->size > MAX_SMALL) {
 	    size = uint_to_big(bp->size, hp);
@@ -962,12 +962,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
 
     fprintf(fp,
 	    "{total,[{total,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}]}.\n",
-	    stats->tot.size,
-	    stats->tot.max_size,
-	    stats->tot.max_size_ever,
-	    stats->tot.blocks,
-	    stats->tot.max_blocks,
-	    stats->tot.max_blocks_ever);
+	    (UWord) stats->tot.size,
+	    (UWord) stats->tot.max_size,
+	    (UWord) stats->tot.max_size_ever,
+	    (UWord) stats->tot.blocks,
+	    (UWord) stats->tot.max_blocks,
+	    (UWord) stats->tot.max_blocks_ever);
 
     a_max = 0;
     a_min = ~0;
@@ -992,12 +992,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
 		    "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
 		    i == a_min ? "{allocators,\n [" : "  ",
 		    ERTS_ALC_A2AD(i),
-		    stats->a[i].size,
-		    stats->a[i].max_size,
-		    stats->a[i].max_size_ever,
-		    stats->a[i].blocks,
-		    stats->a[i].max_blocks,
-		    stats->a[i].max_blocks_ever,
+		    (UWord) stats->a[i].size,
+		    (UWord) stats->a[i].max_size,
+		    (UWord) stats->a[i].max_size_ever,
+		    (UWord) stats->a[i].blocks,
+		    (UWord) stats->a[i].max_blocks,
+		    (UWord) stats->a[i].max_blocks_ever,
 		    i == a_max ? "]}.\n" : ",\n");
 	}
     }
@@ -1009,12 +1009,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
 		"%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
 		i == ERTS_ALC_C_MIN ? "{classes,\n [" : "  ",
 		ERTS_ALC_C2CD(i),
-		stats->c[i].size,
-		stats->c[i].max_size,
-		stats->c[i].max_size_ever,
-		stats->c[i].blocks,
-		stats->c[i].max_blocks,
-		stats->c[i].max_blocks_ever,
+		(UWord) stats->c[i].size,
+		(UWord) stats->c[i].max_size,
+		(UWord) stats->c[i].max_size_ever,
+		(UWord) stats->c[i].blocks,
+		(UWord) stats->c[i].max_blocks,
+		(UWord) stats->c[i].max_blocks_ever,
 		i == ERTS_ALC_C_MAX ? "]}.\n" :  ",\n" );
     }
 
@@ -1025,12 +1025,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
 		"%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
 		i == ERTS_ALC_N_MIN ? "{types,\n [" : "  ",
 		ERTS_ALC_N2TD(i),
-		stats->n[i].size,
-		stats->n[i].max_size,
-		stats->n[i].max_size_ever,
-		stats->n[i].blocks,
-		stats->n[i].max_blocks,
-		stats->n[i].max_blocks_ever,
+		(UWord) stats->n[i].size,
+		(UWord) stats->n[i].max_size,
+		(UWord) stats->n[i].max_size_ever,
+		(UWord) stats->n[i].blocks,
+		(UWord) stats->n[i].max_blocks,
+		(UWord) stats->n[i].max_blocks_ever,
 		i == ERTS_ALC_N_MAX ? "]}.\n" :  ",\n" );
     }
 
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 074b08ea57..9180508a49 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -96,16 +96,15 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"proc_status",				"pid"			},
     {	"proc_tab",				NULL			},
     {   "ports_snapshot",                       NULL                    },
-    {	"db_tab",				"address"		},    
-    {	"db_tab_fix",				"address"		},
     {	"meta_name_tab",	         	"address"		},
     {	"meta_main_tab_slot",			"address"		},
+    {	"db_tab",				"address"		},
+    {	"db_tab_fix",				"address"		},
     {	"meta_main_tab_main",			NULL 			},
     {	"db_hash_slot",				"address"		},
     {	"node_table",				NULL			},
     {	"dist_table",				NULL			},
     {	"sys_tracers",				NULL			},
-    {	"trace_pattern",			NULL			},
     {	"module_tab",				NULL			},
     {	"export_tab",				NULL			},
     {	"fun_tab",				NULL			},
@@ -120,17 +119,17 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"child_status",				NULL			},
 #endif
 #ifdef __WIN32__
-    {	"sys_driver_data_lock",			NULL 			}, 
+    {	"sys_driver_data_lock",			NULL 			},
 #endif
-    {	"drv_ev_state_grow",			NULL,   		}, 
+    {	"drv_ev_state_grow",			NULL,   		},
     {	"drv_ev_state",				"address"		},
     {	"safe_hash",				"address"		},
     {   "pollset_rm_list",                      NULL                    },
     {   "removed_fd_pre_alloc_lock",            NULL                    },
     {   "state_prealloc",                       NULL                    },
     {	"schdlr_sspnd",				NULL			},
-    {	"cpu_bind",				NULL			},
     {	"run_queue",				"address"		},
+    {	"cpu_info",				NULL			},
     {	"pollset",				"address"		},
 #ifdef __WIN32__
     {	"pollwaiter",				"address"		},
@@ -154,7 +153,11 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"instr",				NULL			},
     {	"fix_alloc",				"index"			},
     {	"alcu_allocator",			"index"			},
+    {	"alcu_delayed_free",			"index"			},
     {	"mseg",					NULL			},
+#if HALFWORD_HEAP
+    {	"pmmap",				NULL			},
+#endif
 #ifdef ERTS_SMP
     {	"port_task_pre_alloc_lock",		"address"		},
     {	"port_taskq_pre_alloc_lock",		"address"		},
@@ -174,18 +177,22 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"async_id",				NULL			},
     {	"pix_lock",				"address"		},
     {	"run_queues_lists",			NULL			},
+    {	"misc_aux_work_queue",			"index"			},
+    {	"misc_aux_work_pre_alloc_lock",		"address"		},
     {	"sched_stat",				NULL			},
+    {	"run_queue_sleep_list",			"address"		},
 #endif
     {	"alloc_thr_ix_lock",			NULL			},
 #ifdef ERTS_SMP
-    {	"proc_lck_wtr_alloc",			NULL 			},
+    {	"proc_lck_qs_alloc",			NULL 			},
 #endif
 #ifdef __WIN32__
 #ifdef DEBUG
     {   "save_ops_lock",                        NULL                    },
 #endif
 #endif
-    {	"mtrace_buf",				NULL			}
+    {	"mtrace_buf",				NULL			},
+    {	"erts_alloc_hard_debug",		NULL			}
 };
 
 #define ERTS_LOCK_ORDER_SIZE \
@@ -197,6 +204,8 @@ static erts_lc_lock_order_t erts_lock_order[] = {
        & ERTS_LC_FLG_LT_ALL						\
        & ~(ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK)))
 
+static __decl_noreturn void  __noreturn lc_abort(void);
+
 static char *
 lock_type(Uint16 flags)
 {
@@ -220,7 +229,7 @@ rw_op_str(Uint16 flags)
 	return " (r)";
     case ERTS_LC_FLG_LO_WRITE:
 	erts_fprintf(stderr, "\nInternal error\n");
-	abort();
+	lc_abort();
     default:
 	break;
     }
@@ -231,7 +240,7 @@ typedef struct erts_lc_locked_lock_t_ erts_lc_locked_lock_t;
 struct erts_lc_locked_lock_t_ {
     erts_lc_locked_lock_t *next;
     erts_lc_locked_lock_t *prev;
-    Eterm extra;
+    UWord extra;
     Sint16 id;
     Uint16 flags;
 };
@@ -269,28 +278,18 @@ static erts_lc_free_block_t *free_blocks;
 #define ERTS_LC_FB_CHUNK_SIZE 10
 #endif
 
-#ifdef ETHR_HAVE_NATIVE_LOCKS
 static ethr_spinlock_t free_blocks_lock;
-#define ERTS_LC_LOCK	ethr_spin_lock
-#define ERTS_LC_UNLOCK	ethr_spin_unlock
-#else
-static ethr_mutex free_blocks_lock;
-#define ERTS_LC_LOCK	ethr_mutex_lock
-#define ERTS_LC_UNLOCK	ethr_mutex_unlock
-#endif
 
 static ERTS_INLINE void
 lc_lock(void)
 {
-    if (ERTS_LC_LOCK(&free_blocks_lock) != 0)
-	abort();
+    ethr_spin_lock(&free_blocks_lock);
 }
 
 static ERTS_INLINE void
 lc_unlock(void)
 {
-    if (ERTS_LC_UNLOCK(&free_blocks_lock) != 0)
-	abort();
+    ethr_spin_unlock(&free_blocks_lock);
 }
 
 static ERTS_INLINE void lc_free(void *p)
@@ -311,7 +310,7 @@ static void *lc_core_alloc(void)
 {
     lc_unlock();
     erts_fprintf(stderr, "Lock checker out of memory!\n");
-    abort();
+    lc_abort();
 }
 
 #else
@@ -325,7 +324,7 @@ static void *lc_core_alloc(void)
 					  * ERTS_LC_FB_CHUNK_SIZE);
     if (!fbs) {
 	erts_fprintf(stderr, "Lock checker failed to allocate memory!\n");
-	abort();
+	lc_abort();
     }
     for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
 #ifdef DEBUG
@@ -365,11 +364,11 @@ create_locked_locks(char *thread_name)
 {
     erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t));
     if (!l_lcks)
-	abort();
+	lc_abort();
 
     l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown");
     if (!l_lcks->thread_name)
-	abort();
+	lc_abort();
 
     l_lcks->tid = erts_thr_self();
     l_lcks->required.first = NULL;
@@ -442,12 +441,12 @@ new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags)
 }
 
 static void
-print_lock2(char *prefix, Sint16 id, Eterm extra, Uint16 flags, char *suffix)
+print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix)
 {
     char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE
 		   ? erts_lock_order[id].name
 		   : "unknown");
-    if (is_boxed(extra))
+    if (is_not_immed(extra))
 	erts_fprintf(stderr,
 		     "%s'%s:%p%s'%s%s",
 		     prefix,
@@ -511,7 +510,7 @@ uninitialized_lock(void)
 {
     erts_fprintf(stderr, "Performing operations on uninitialized lock!\n");
     print_curr_locks(get_my_locked_locks());
-    abort();
+    lc_abort();
 }
 
 static void
@@ -521,7 +520,7 @@ lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
     erts_fprintf(stderr, "%s%s", prefix, rw_op_str(op_flags));
     print_lock(" ", lck, " lock which is already locked by thread!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -531,7 +530,7 @@ unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
     erts_fprintf(stderr, "Unlocking%s ", rw_op_str(op_flags));
     print_lock("", lck, " lock which mismatch previous lock operation!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -539,7 +538,7 @@ unlock_of_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
 {
     print_lock("Unlocking ", lck, " lock which is not locked by thread!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -548,7 +547,7 @@ lock_order_violation(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
     print_lock("Lock order violation occured when locking ", lck, "!\n");
     print_curr_locks(l_lcks);
     print_lock_order();
-    abort();
+    lc_abort();
 }
 
 static void
@@ -559,7 +558,7 @@ type_order_violation(char *op, erts_lc_locked_locks_t *l_lcks,
     print_lock(op, lck, "!\n");
     ASSERT(l_lcks);
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -611,7 +610,7 @@ lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact,
 	}
     }
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -619,7 +618,7 @@ unlock_of_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
 {
     print_lock("Unlocking required ", lck, " lock!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -627,7 +626,7 @@ unrequire_of_not_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *l
 {
     print_lock("Unrequire on ", lck, " lock not required!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -635,7 +634,7 @@ require_twice(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
 {
     print_lock("Require on ", lck, " lock already required!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -643,7 +642,7 @@ required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
 {
     print_lock("Required ", lck, " lock not locked!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 
@@ -656,13 +655,23 @@ thread_exit_handler(void)
 	    erts_fprintf(stderr,
 			 "Thread exiting while having locked locks!\n");
 	    print_curr_locks(l_lcks);
-	    abort();
+	    lc_abort();
 	}
 	destroy_locked_locks(l_lcks);
 	/* erts_tsd_set(locks_key, NULL); */
     }
 }
 
+static __decl_noreturn void
+lc_abort(void)
+{
+#ifdef __WIN32__
+    DebugBreak();
+#else
+    abort();
+#endif
+}
+
 void
 erts_lc_set_thread_name(char *thread_name)
 {
@@ -674,7 +683,7 @@ erts_lc_set_thread_name(char *thread_name)
 	free((void *) l_lcks->thread_name);
 	l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown");
 	if (!l_lcks->thread_name)
-	    abort();
+	    lc_abort();
     }
 }
 
@@ -684,7 +693,7 @@ erts_lc_assert_failed(char *file, int line, char *assertion)
     erts_fprintf(stderr, "%s:%d: Lock check assertion \"%s\" failed!\n",
 		 file, line, assertion);
     print_curr_locks(get_my_locked_locks());
-    abort();
+    lc_abort();
     return 0;
 }
 
@@ -697,7 +706,7 @@ void erts_lc_fail(char *fmt, ...)
     va_end(args);
     erts_fprintf(stderr, "\n");
     print_curr_locks(get_my_locked_locks());
-    abort();
+    lc_abort();
 }
 
 
@@ -717,7 +726,7 @@ erts_lc_get_lock_order_id(char *name)
 		     "(update erl_lock_check.c)\n",
 		     name);
     }
-    abort();
+    lc_abort();
     return (Sint16) -1;
 }
 
@@ -893,6 +902,25 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len)
     }
 }
 
+void
+erts_lc_check_no_locked_of_type(Uint16 flags)
+{
+    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
+    if (l_lcks) {
+	erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
+	for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) {
+	    if (l_lck->flags & flags) {
+		erts_fprintf(stderr,
+			     "Locked lock of type %s found which isn't "
+			     "allowed here!\n",
+			     lock_type(l_lck->flags));
+		print_curr_locks(l_lcks);
+		lc_abort();
+	    }
+	}
+    }
+}
+
 int
 erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags)
 {
@@ -952,10 +980,10 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags)
 	/* We only force busy if a lock order violation would occur
 	   and when on an even millisecond. */
 	{
-	    erts_thr_timeval_t time;
-	    erts_thr_time_now(&time);
+	    SysTimeval tv;
+	    sys_gettimeofday(&tv);
 
-	    if ((time.tv_nsec / 1000000) & 1)
+	    if ((tv.tv_usec / 1000) & 1)
 		return 0;
 	}
 #endif
@@ -1231,7 +1259,9 @@ void
 erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags)
 {
     lck->id = erts_lc_get_lock_order_id(name);
-    lck->extra = make_boxed(&lck->extra);
+
+    lck->extra = &lck->extra;
+    ASSERT(is_not_immed(lck->extra));
     lck->flags = flags;
     lck->inited = ERTS_LC_INITITALIZED;
 }
@@ -1241,6 +1271,7 @@ erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra)
 {
     lck->id = erts_lc_get_lock_order_id(name);
     lck->extra = extra;
+    ASSERT(is_immed(lck->extra));
     lck->flags = flags;
     lck->inited = ERTS_LC_INITITALIZED;
 }
@@ -1279,13 +1310,8 @@ erts_lc_init(void)
     free_blocks = NULL;
 #endif /* #ifdef ERTS_LC_STATIC_ALLOC */
 
-#ifdef ETHR_HAVE_NATIVE_LOCKS
     if (ethr_spinlock_init(&free_blocks_lock) != 0)
-	abort();
-#else
-    if (ethr_mutex_init(&free_blocks_lock) != 0)
-	abort();
-#endif
+	lc_abort();
 
     erts_tsd_key_create(&locks_key);
 }
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index d5e2ede9ac..b67f36fa06 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -39,7 +39,7 @@ typedef struct {
     int inited;
     Sint16 id;
     Uint16 flags;
-    Eterm extra;
+    UWord extra;
 } erts_lc_lock_t;
 
 #define ERTS_LC_INITITALIZED 0x7f7f7f7f
@@ -77,6 +77,7 @@ void erts_lc_check(erts_lc_lock_t *have, int have_len,
 void erts_lc_check_exact(erts_lc_lock_t *have, int have_len);
 void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len);
 void erts_lc_have_lock_ids(int *resv, int *ids, int len);
+void erts_lc_check_no_locked_of_type(Uint16 flags);
 int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags);
 void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags);
 void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 0d7e1335c1..a36c53560e 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -159,15 +159,15 @@ static char* lock_opt(Uint16 flag) {
 }
 
 static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char *extra) {
-    long int colls, tries, w_state, r_state;
+    erts_aint_t colls, tries, w_state, r_state;
     erts_lcnt_lock_stats_t *stats = NULL;
     
     char *type;
     int i;
     
     type = lcnt_lock_type(lock->flag);
-    ethr_atomic_read(&lock->r_state, &r_state);
-    ethr_atomic_read(&lock->w_state, &w_state);
+    r_state = ethr_atomic_read(&lock->r_state);
+    w_state = ethr_atomic_read(&lock->w_state);
 
     
     if (lock->flag & flag) {
@@ -257,6 +257,10 @@ void erts_lcnt_init() {
     erts_lcnt_clear_counters();
 }
 
+void erts_lcnt_late_init() {
+    erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler);
+}
+
 /* list operations */
 
 /* BEGIN ASSUMPTION: lcnt_data_lock taken */
@@ -381,7 +385,7 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) {
 /* lock */
 
 void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
-    long r_state = 0, w_state = 0;
+    erts_aint_t r_state = 0, w_state = 0;
     erts_lcnt_thread_data_t *eltd;
     
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
@@ -390,10 +394,10 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
 
     ASSERT(eltd);
     
-    ethr_atomic_read(&lock->w_state, &w_state);
+    w_state = ethr_atomic_read(&lock->w_state);
     
     if (option & ERTS_LCNT_LO_WRITE) {
-        ethr_atomic_read(&lock->r_state, &r_state);
+        r_state = ethr_atomic_read(&lock->r_state);
         ethr_atomic_inc( &lock->w_state);
     }
     if (option & ERTS_LCNT_LO_READ) {
@@ -414,12 +418,12 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
 }
 
 void erts_lcnt_lock(erts_lcnt_lock_t *lock) {
-    long w_state;
+    erts_aint_t w_state;
     erts_lcnt_thread_data_t *eltd;
     
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
 
-    ethr_atomic_read(&lock->w_state, &w_state);
+    w_state = ethr_atomic_read(&lock->w_state);
     ethr_atomic_inc( &lock->w_state);
 
     eltd = lcnt_get_thread_data();
@@ -467,14 +471,14 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line
     erts_lcnt_time_t time_wait;
     erts_lcnt_lock_stats_t *stats;
 #ifdef DEBUG
-    long flowstate;
+    erts_aint_t flowstate;
 #endif
 
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
     
 #ifdef DEBUG
     if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) {
-    	ethr_atomic_read(&lock->flowstate, &flowstate);
+	flowstate = ethr_atomic_read(&lock->flowstate);
 	ASSERT(flowstate == 0);
     	ethr_atomic_inc( &lock->flowstate);
     }
@@ -512,18 +516,18 @@ void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
 
 void erts_lcnt_unlock(erts_lcnt_lock_t *lock) {
 #ifdef DEBUG
-    long w_state;  
-    long flowstate;
+    erts_aint_t w_state;  
+    erts_aint_t flowstate;
 #endif
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
 #ifdef DEBUG
     /* flowstate */
-    ethr_atomic_read(&lock->flowstate, &flowstate);
+    flowstate = ethr_atomic_read(&lock->flowstate);
     ASSERT(flowstate == 1);
     ethr_atomic_dec( &lock->flowstate);
     
     /* write state */
-    ethr_atomic_read(&lock->w_state, &w_state);
+    w_state = ethr_atomic_read(&lock->w_state);
     ASSERT(w_state > 0)
 #endif
     ethr_atomic_dec(&lock->w_state);
@@ -548,13 +552,13 @@ void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) {
 void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) {
     /* Determine lock_state via res instead of state */
 #ifdef DEBUG
-    long flowstate;
+    erts_aint_t flowstate;
 #endif 
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
     if (res != EBUSY) {
 	
 #ifdef DEBUG
-    	ethr_atomic_read(&lock->flowstate, &flowstate);
+	flowstate = ethr_atomic_read(&lock->flowstate);
 	ASSERT(flowstate == 0);
     	ethr_atomic_inc( &lock->flowstate);
 #endif
@@ -570,36 +574,26 @@ void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) {
 
 /* thread operations */
 
-static void *lcnt_thr_init(erts_lcnt_thread_data_t *eltd) {
-    void *(*function)(void *);
-    void *argument;
-    void *res;
-    function = eltd->function;
-    argument = eltd->argument;
-    
-    ethr_tsd_set(lcnt_thr_data_key, eltd);
-    
-    res = (void *)function(argument);
-    free(eltd);
-    return (void *)res;
-}
-
-    
-
-int erts_lcnt_thr_create(ethr_tid *tid, void * (*function)(void *), void *arg, ethr_thr_opts *opts) {
+void erts_lcnt_thread_setup(void) {
     erts_lcnt_thread_data_t *eltd;
-    
+
     lcnt_lock();
     /* lock for thread id global update */
     eltd = lcnt_thread_data_alloc();
     lcnt_unlock();
-    
-    eltd->function = function;
-    eltd->argument = arg;
-
-    return ethr_thr_create(tid, (void *)lcnt_thr_init, (void *)eltd, opts);
+    ASSERT(eltd);
+    ethr_tsd_set(lcnt_thr_data_key, eltd);
 }
 
+void erts_lcnt_thread_exit_handler() {
+    erts_lcnt_thread_data_t *eltd;
+
+    eltd = ethr_tsd_get(lcnt_thr_data_key);
+
+    if (eltd) {
+        free(eltd);
+    }
+}
 
 /* bindings for bifs */
 
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index e3044c371f..6306580ae4 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -155,11 +155,6 @@ typedef struct {
     erts_lcnt_time_t timer;	/* timer         */
     int timer_set;		/* bool          */
     int lock_in_conflict;       /* bool          */
-    
-    /* function pointer */
-    void *(*function)(void *);
-    void *argument;
-    
 } erts_lcnt_thread_data_t;
 
 /* globals */
@@ -169,6 +164,11 @@ extern Uint16 erts_lcnt_rt_options;
 /* function declerations */
 
 void erts_lcnt_init(void);
+void erts_lcnt_late_init(void);
+
+/* thread operations */
+void erts_lcnt_thread_setup(void);
+void erts_lcnt_thread_exit_handler(void);
 
 /* list operations (local)  */
 erts_lcnt_lock_list_t *erts_lcnt_list_init(void);
@@ -194,12 +194,7 @@ void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option);
 void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option);
 void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res);
 
-/* thread operations */
-
-int erts_lcnt_thr_create(ethr_tid *tid, void * (*function)(void *), void *arg, ethr_thr_opts *opts);
-
 /* bif interface */
-
 Uint16 erts_lcnt_set_rt_opt(Uint16 opt);
 Uint16 erts_lcnt_clear_rt_opt(Uint16 opt);
 void   erts_lcnt_clear_counters(void);
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index a056fce0c5..82f272d28a 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -30,6 +30,7 @@
 #include "erl_message.h"
 #include "erl_process.h"
 #include "erl_nmgc.h"
+#include "erl_binary.h"
 
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message,
 				 ErlMessage,
@@ -42,6 +43,15 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message,
 #undef HARD_DEBUG
 #endif
 
+
+
+
+static ERTS_INLINE int in_heapfrag(const Eterm* ptr, const ErlHeapFragment *bp)
+{
+    return ((unsigned)(ptr - bp->mem) < bp->used_size);
+}
+
+
 void
 init_message(void)
 {
@@ -81,9 +91,12 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
 #endif
     ErlHeapFragment* nbp;
 
+    /* ToDo: Make use of 'used_size' to avoid realloc
+	when shrinking just a few words */
+
 #ifdef DEBUG
     {
-	Uint off_sz = size < bp->size ? size : bp->size;
+	Uint off_sz = size < bp->used_size ? size : bp->used_size;
 	for (i = 0; i < brefs_size; i++) {
 	    Eterm *ptr;
 	    if (is_immed(brefs[i]))
@@ -95,12 +108,12 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
     }
 #endif
 
-    if (size == bp->size)
+    if (size == bp->used_size)
 	return bp;
 
 #ifdef HARD_DEBUG
     dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size);
-    dbg_bp = new_message_buffer(bp->size);
+    dbg_bp = new_message_buffer(bp->used_size);
     dbg_hp = dbg_bp->mem;
     dbg_tot_size = 0;
     for (i = 0; i < brefs_size; i++) {
@@ -109,15 +122,15 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
 	dbg_brefs[i] = copy_struct(brefs[i], dbg_size, &dbg_hp,
 				   &dbg_bp->off_heap);
     }
-    ASSERT(dbg_tot_size == (size < bp->size ? size : bp->size));
+    ASSERT(dbg_tot_size == (size < bp->used_size ? size : bp->used_size));
 #endif
 
     nbp = (ErlHeapFragment*) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP_FRAG,
 					       (void *) bp,
-					       ERTS_HEAP_FRAG_SIZE(bp->size),
+					       ERTS_HEAP_FRAG_SIZE(bp->alloc_size),
 					       ERTS_HEAP_FRAG_SIZE(size));
     if (bp != nbp) {
-	Uint off_sz = size < nbp->size ? size : nbp->size;
+	Uint off_sz = size < nbp->used_size ? size : nbp->used_size;
 	Eterm *sp = &bp->mem[0];
 	Eterm *ep = sp + off_sz;
 	Sint offs = &nbp->mem[0] - sp;
@@ -135,7 +148,7 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
 	}
 #endif
     }
-    nbp->size = size;
+    nbp->alloc_size = size;
     nbp->used_size = size;
 
 #ifdef HARD_DEBUG
@@ -152,26 +165,40 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
 void
 erts_cleanup_offheap(ErlOffHeap *offheap)
 {
-    if (offheap->mso) {
-	erts_cleanup_mso(offheap->mso);
-    }
-#ifndef HYBRID /* FIND ME! */
-    if (offheap->funs) {
-	erts_cleanup_funs(offheap->funs);
-    }
-#endif
-    if (offheap->externals) {
-	erts_cleanup_externals(offheap->externals);
+    union erl_off_heap_ptr u;
+
+    for (u.hdr = offheap->first; u.hdr; u.hdr = u.hdr->next) {
+	switch (thing_subtag(u.hdr->thing_word)) {
+	case REFC_BINARY_SUBTAG:
+	    if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) {
+		erts_bin_free(u.pb->val);
+	    }
+	    break;
+	case FUN_SUBTAG:
+	    if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) {
+		erts_erase_fun_entry(u.fun->fe);		    
+	    }
+	    break;
+	default:
+	    ASSERT(is_external_header(u.hdr->thing_word));
+	    erts_deref_node_entry(u.ext->node);
+	    break;
+	}
     }
 }
 
 void
 free_message_buffer(ErlHeapFragment* bp)
 {
-    erts_cleanup_offheap(&bp->off_heap);
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG,
-		   (void *) bp,
-		   ERTS_HEAP_FRAG_SIZE(bp->size));
+    ASSERT(bp != NULL);
+    do {
+	ErlHeapFragment* next_bp = bp->next;
+
+	erts_cleanup_offheap(&bp->off_heap);
+	ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
+		       ERTS_HEAP_FRAG_SIZE(bp->size));	
+	bp = next_bp;
+    }while (bp != NULL);
 }
 
 static ERTS_INLINE void
@@ -181,43 +208,19 @@ link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp)
 	/* Link the message buffer */
 	bp->next = MBUF(proc);
 	MBUF(proc) = bp;
-	MBUF_SIZE(proc) += bp->size;
+	MBUF_SIZE(proc) += bp->used_size;
 	FLAGS(proc) |= F_FORCE_GC;
 
-	/* Move any binaries into the process */
-	if (bp->off_heap.mso != NULL) {
-	    ProcBin** next_p = &bp->off_heap.mso;
-	    while (*next_p != NULL) {
-		next_p = &((*next_p)->next);
-	    }
-	    *next_p = MSO(proc).mso;
-	    MSO(proc).mso = bp->off_heap.mso;
-	    bp->off_heap.mso = NULL;
-	    MSO(proc).overhead += bp->off_heap.overhead;
-	}
-
-	/* Move any funs into the process */
-#ifndef HYBRID
-	if (bp->off_heap.funs != NULL) {
-	    ErlFunThing** next_p = &bp->off_heap.funs;
+	/* Move any off_heap's into the process */
+	if (bp->off_heap.first != NULL) {
+	    struct erl_off_heap_header** next_p = &bp->off_heap.first;
 	    while (*next_p != NULL) {
 		next_p = &((*next_p)->next);
 	    }
-	    *next_p = MSO(proc).funs;
-	    MSO(proc).funs = bp->off_heap.funs;
-	    bp->off_heap.funs = NULL;
-	}
-#endif
-
-	/* Move any external things into the process */
-	if (bp->off_heap.externals != NULL) {
-	    ExternalThing** next_p = &bp->off_heap.externals;
-	    while (*next_p != NULL) {
-		next_p = &((*next_p)->next);
-	    }
-	    *next_p = MSO(proc).externals;
-	    MSO(proc).externals = bp->off_heap.externals;
-	    bp->off_heap.externals = NULL;
+	    *next_p = MSO(proc).first;
+	    MSO(proc).first = bp->off_heap.first;
+	    bp->off_heap.first = NULL;
+	    OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
 	}
     }
 }
@@ -242,7 +245,7 @@ erts_msg_distext2heap(Process *pp,
 	goto decode_error;
     if (is_not_nil(*tokenp)) {
 	ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
-	tok_sz = heap_frag->size;
+	tok_sz = heap_frag->used_size;
 	sz += tok_sz;
     }
     if (pp)
@@ -283,12 +286,13 @@ erts_msg_distext2heap(Process *pp,
 	erts_cleanup_offheap(&heap_frag->off_heap);
     }
     erts_free_dist_ext_copy(dist_extp);
-    if (*bpp)
+    if (*bpp) {
 	free_message_buffer(*bpp);
+	*bpp = NULL;
+    }    
     else if (hp) {
 	HRelease(pp, hp_end, hp);
     }
-    *bpp = NULL;
     return THE_NON_VALUE;
  }
 
@@ -436,11 +440,10 @@ erts_queue_message(Process* receiver,
     ERL_MESSAGE_TERM(mp) = message;
     ERL_MESSAGE_TOKEN(mp) = seq_trace_token;
     mp->next = NULL;
+    mp->data.heap_frag = bp;
 
 #ifdef ERTS_SMP
     if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
-	mp->data.heap_frag = bp;
-
 	/*
 	 * We move 'in queue' to 'private queue' and place
 	 * message at the end of 'private queue' in order
@@ -453,11 +456,9 @@ erts_queue_message(Process* receiver,
 	LINK_MESSAGE_PRIVQ(receiver, mp);
     }
     else {
-	mp->data.heap_frag = bp;
 	LINK_MESSAGE(receiver, mp);
     }
 #else
-    mp->data.heap_frag = bp;
     LINK_MESSAGE(receiver, mp);
 #endif
 
@@ -491,19 +492,7 @@ erts_link_mbuf_to_proc(struct process *proc, ErlHeapFragment *bp)
 void
 erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 {
-    /* Unions for typecasts avoids warnings about type-punned pointers and aliasing */
-    union {
-	Uint** upp;
-	ProcBin **pbpp;
-	ErlFunThing **efpp;
-	ExternalThing **etpp;
-    } oh_list_pp, oh_el_next_pp;
-    union {
-	Uint *up;
-	ProcBin *pbp;
-	ErlFunThing *efp;
-	ExternalThing *etp;
-    } oh_el_p;
+    struct erl_off_heap_header* oh;
     Eterm term, token, *fhp, *hp;
     Sint offs;
     Uint sz;
@@ -530,40 +519,33 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 #ifdef HARD_DEBUG
     dbg_term_sz = size_object(term);
     dbg_token_sz = size_object(token);
-    ASSERT(bp->size == dbg_term_sz + dbg_token_sz);
-
-    dbg_bp = new_message_buffer(bp->size);
+    /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg));
+      Copied size may be smaller due to removed SubBins's or garbage.
+      Copied size may be larger due to duplicated shared terms.
+    */
+    dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz);
     dbg_hp = dbg_bp->mem;
     dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap);
     dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap);
     dbg_thp_start = *hpp;
 #endif
 
-    ASSERT(bp);
-    msg->data.attached = NULL;
-
-    off_heap->overhead += bp->off_heap.overhead;
-    sz = bp->size;
-
-#ifdef DEBUG
-    if (is_not_immed(term)) {
-	ASSERT(bp->mem <= ptr_val(term));
-	ASSERT(bp->mem + bp->size > ptr_val(term));
+    if (bp->next != NULL) {
+	move_multi_frags(hpp, off_heap, bp, msg->m, 2);
+	goto copy_done;
     }
 
-    if (is_not_immed(token)) {
-	ASSERT(bp->mem <= ptr_val(token));
-	ASSERT(bp->mem + bp->size > ptr_val(token));
-    }
-#endif
+    OH_OVERHEAD(off_heap, bp->off_heap.overhead);
+    sz = bp->used_size;
+
+    ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp));
+    ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp));
 
     fhp = bp->mem;
     hp = *hpp;
     offs = hp - fhp;
 
-    oh_list_pp.upp = NULL;
-    oh_el_next_pp.upp = NULL; /* Shut up compiler warning */
-    oh_el_p.up = NULL; /* Shut up compiler warning */
+    oh = NULL;
     while (sz--) {
 	Uint cpy_sz;
 	Eterm val = *fhp++;
@@ -574,8 +556,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 	    break;
 	case TAG_PRIMARY_LIST:
 	case TAG_PRIMARY_BOXED:
-	    ASSERT(bp->mem <= ptr_val(val));
-	    ASSERT(bp->mem + bp->size > ptr_val(val));
+	    ASSERT(in_heapfrag(ptr_val(val), bp));
 	    *hp++ = offset_ptr(val, offs);
 	    break;
 	case TAG_PRIMARY_HEADER:
@@ -584,31 +565,18 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 	    case ARITYVAL_SUBTAG:
 		break;
 	    case REFC_BINARY_SUBTAG:
-		oh_list_pp.pbpp = &off_heap->mso;
-		oh_el_p.up = (hp-1);
-		oh_el_next_pp.pbpp = &(oh_el_p.pbp)->next;
-		cpy_sz = thing_arityval(val);
-		goto cpy_words;
 	    case FUN_SUBTAG:
-#ifndef HYBRID
-		oh_list_pp.efpp = &off_heap->funs;
-		oh_el_p.up = (hp-1);
-		oh_el_next_pp.efpp = &(oh_el_p.efp)->next;
-#endif
-		cpy_sz = thing_arityval(val);
-		goto cpy_words;
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
-		oh_list_pp.etpp = &off_heap->externals;
-		oh_el_p.up = (hp-1);
-		oh_el_next_pp.etpp =  &(oh_el_p.etp)->next;
+		oh = (struct erl_off_heap_header*) (hp-1);
 		cpy_sz = thing_arityval(val);
 		goto cpy_words;
 	    default:
 		cpy_sz = header_arity(val);
 
 	    cpy_words:
+		ASSERT(sz >= cpy_sz);
 		sz -= cpy_sz;
 		while (cpy_sz >= 8) {
 		    cpy_sz -= 8;
@@ -631,44 +599,13 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 		case 1: *hp++ = *fhp++;
 		default: break;
 		}
-		if (oh_list_pp.upp) {
-#ifdef HARD_DEBUG
-		    Uint *dbg_old_oh_list_p = *oh_list_pp.upp;
-#endif
+		if (oh) {
 		    /* Add to offheap list */
-		    *oh_el_next_pp.upp = *oh_list_pp.upp;
-		    *oh_list_pp.upp = oh_el_p.up;
-		    ASSERT(*hpp <= oh_el_p.up);
-		    ASSERT(hp > oh_el_p.up);
-#ifdef HARD_DEBUG
-		    switch (val & _HEADER_SUBTAG_MASK) {
-		    case REFC_BINARY_SUBTAG:
-			ASSERT(off_heap->mso == *oh_list_pp.pbpp);
-			ASSERT(off_heap->mso->next
-			       == (ProcBin *) dbg_old_oh_list_p);
-			break;
-#ifndef HYBRID
-		    case FUN_SUBTAG:
-			ASSERT(off_heap->funs == *oh_list_pp.efpp);
-			ASSERT(off_heap->funs->next
-			       == (ErlFunThing *) dbg_old_oh_list_p);
-			break;
-#endif
-		    case EXTERNAL_PID_SUBTAG:
-		    case EXTERNAL_PORT_SUBTAG:
-		    case EXTERNAL_REF_SUBTAG:
-			ASSERT(off_heap->externals
-			       == *oh_list_pp.etpp);
-			ASSERT(off_heap->externals->next
-			       == (ExternalThing *) dbg_old_oh_list_p);
-			break;
-		    default:
-			ASSERT(0);
-		    }
-#endif
-		    oh_list_pp.upp = NULL;
-
-
+		    oh->next = off_heap->first;
+		    off_heap->first = oh;
+		    ASSERT(*hpp <= (Eterm*)oh);
+		    ASSERT(hp > (Eterm*)oh);
+		    oh = NULL;
 		}
 		break;
 	    }
@@ -676,12 +613,11 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 	}
     }
 
-    ASSERT(bp->size == hp - *hpp);
+    ASSERT(bp->used_size == hp - *hpp);
     *hpp = hp;
 
     if (is_not_immed(token)) {
-	ASSERT(bp->mem <= ptr_val(token));
-	ASSERT(bp->mem + bp->size > ptr_val(token));
+	ASSERT(in_heapfrag(ptr_val(token), bp));
 	ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs);
 #ifdef HARD_DEBUG
 	ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg)));
@@ -690,8 +626,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
     }
 
     if (is_not_immed(term)) {
-	ASSERT(bp->mem <= ptr_val(term));
-	ASSERT(bp->mem + bp->size > ptr_val(term));
+	ASSERT(in_heapfrag(ptr_val(term),bp));
 	ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs);
 #ifdef HARD_DEBUG
 	ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg)));
@@ -699,10 +634,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 #endif
     }
 
+copy_done:
 
 #ifdef HARD_DEBUG
     {
 	int i, j;
+	ErlHeapFragment* frag;
 	{
 	    ProcBin *mso = off_heap->mso;
 	    i = j = 0;
@@ -710,10 +647,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 		mso = mso->next;
 		i++;
 	    }
-	    mso = bp->off_heap.mso;
-	    while (mso) {
-		mso = mso->next;
-		j++;
+	    for (frag=bp; frag; frag=frag->next) {
+		mso = frag->off_heap.mso;
+		while (mso) {
+		    mso = mso->next;
+		    j++;
+		}
 	    }
 	    ASSERT(i == j);
 	}
@@ -724,10 +663,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 		fun = fun->next;
 		i++;
 	    }
-	    fun = bp->off_heap.funs;
-	    while (fun) {
-		fun = fun->next;
-		j++;
+	    for (frag=bp; frag; frag=frag->next) {
+		fun = frag->off_heap.funs;
+		while (fun) {
+		    fun = fun->next;
+		    j++;
+		}
 	    }
 	    ASSERT(i == j);
 	}
@@ -738,10 +679,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 		external = external->next;
 		i++;
 	    }
-	    external = bp->off_heap.externals;
-	    while (external) {
-		external = external->next;
-		j++;
+	    for (frag=bp; frag; frag=frag->next) {
+		external = frag->off_heap.externals;
+		while (external) {
+		    external = external->next;
+		    j++;
+		}
 	    }
 	    ASSERT(i == j);
 	}
@@ -749,12 +692,9 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 #endif
 	    
 
-    bp->off_heap.mso = NULL;
-#ifndef HYBRID
-    bp->off_heap.funs = NULL;
-#endif
-    bp->off_heap.externals = NULL;
+    bp->off_heap.first = NULL;
     free_message_buffer(bp);
+    msg->data.heap_frag = NULL;
 
 #ifdef HARD_DEBUG
     ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term));
@@ -764,6 +704,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 
 }
 
+
 Uint
 erts_msg_attached_data_size_aux(ErlMessage *msg)
 {
@@ -789,7 +730,7 @@ erts_msg_attached_data_size_aux(ErlMessage *msg)
     if (is_not_nil(msg->m[1])) {
 	ErlHeapFragment *heap_frag;
 	heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
-	sz += heap_frag->size;
+	sz += heap_frag->used_size;
     }
     return sz;
 }
@@ -805,7 +746,7 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms
 	    ErlHeapFragment *heap_frag;
 	    heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
 	    ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg),
-						 heap_frag->size,
+						 heap_frag->used_size,
 						 hpp,
 						 ohp);
 	    erts_cleanup_offheap(&heap_frag->off_heap);
@@ -1062,3 +1003,4 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 	erts_queue_message(to, to_locksp, bp, save, NIL);
     }
 }
+
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 5cf7c209bd..5aca0db6fe 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -28,13 +28,22 @@ struct external_thing_;
  * but is stored outside of any heap.
  */
 
-typedef struct erl_off_heap {
-    struct proc_bin* mso;	/* List of associated binaries. */
-#ifndef HYBRID /* FIND ME! */
-    struct erl_fun_thing* funs;	/* List of funs. */
+struct erl_off_heap_header {
+    Eterm thing_word;
+    Uint size;
+#if HALFWORD_HEAP
+    void* dummy_ptr_padding__;
 #endif
-    struct external_thing_* externals; /* List of external things. */
-    int overhead;		/* Administrative overhead (used to force GC). */
+    struct erl_off_heap_header* next;
+};
+
+#define OH_OVERHEAD(oh, size) do { \
+    (oh)->overhead += size;        \
+} while(0)
+
+typedef struct erl_off_heap {
+    struct erl_off_heap_header* first;
+    Uint64 overhead;     /* Administrative overhead (used to force GC). */
 } ErlOffHeap;
 
 #include "external.h"
@@ -49,7 +58,7 @@ typedef struct erl_heap_fragment ErlHeapFragment;
 struct erl_heap_fragment {
     ErlHeapFragment* next;	/* Next heap fragment */
     ErlOffHeap off_heap;	/* Offset heap data. */
-    unsigned size;		/* Size in words of mem */
+    unsigned alloc_size;	/* Size in (half)words of mem */
     unsigned used_size;         /* With terms to be moved to heap by GC */
     Eterm mem[1];		/* Data */
 };
@@ -75,6 +84,13 @@ typedef struct {
     ErlMessage** last;  /* point to the last next pointer */
     ErlMessage** save;
     int len;            /* queue length */
+
+    /*
+     * The following two fields are used by the recv_mark/1 and
+     * recv_set/1 instructions.
+     */
+    BeamInstr* mark;		/* address to rec_loop/2 instruction */
+    ErlMessage** saved_last;	/* saved last pointer */
 } ErlMessageQueue;
 
 #ifdef ERTS_SMP
@@ -137,6 +153,7 @@ do {							\
      (p)->msg.len--; \
      if (__mp == NULL) \
          (p)->msg.last = (p)->msg.save; \
+     (p)->msg.mark = 0; \
 } while(0)
 
 /* Reset message save point (after receive match) */
@@ -191,11 +208,9 @@ do {									\
 #define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS)	\
 do {							\
     (HEAP_FRAG_P)->next = NULL;				\
-    (HEAP_FRAG_P)->size = (DATA_WORDS);			\
+    (HEAP_FRAG_P)->alloc_size = (DATA_WORDS);		\
     (HEAP_FRAG_P)->used_size = (DATA_WORDS);            \
-    (HEAP_FRAG_P)->off_heap.mso = NULL;			\
-    (HEAP_FRAG_P)->off_heap.funs = NULL;		\
-    (HEAP_FRAG_P)->off_heap.externals = NULL;		\
+    (HEAP_FRAG_P)->off_heap.first = NULL; 	        \
     (HEAP_FRAG_P)->off_heap.overhead = 0;		\
 } while (0)
 
@@ -219,14 +234,25 @@ void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *);
 Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **,
 			    Eterm *, ErtsDistExternal *);
 
+ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg);
 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg)
+{
+    const ErlHeapFragment *bp;
+    Uint sz = 0;
+    for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) {
+	sz += bp->used_size;
+    }
+    return sz;
+}
+
 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg)
 {
     ASSERT(msg->data.attached);
     if (is_value(ERL_MESSAGE_TERM(msg)))
-	return msg->data.heap_frag->size;
+	return erts_msg_used_frag_sz(msg);
     else if (msg->data.dist_ext->heap_size < 0)
 	return erts_msg_attached_data_size_aux(msg);
     else {
@@ -234,7 +260,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg)
 	if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
 	    ErlHeapFragment *heap_frag;
 	    heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
-	    sz += heap_frag->size;
+	    sz += heap_frag->used_size;
 	}
 	return sz;
     }
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index d873c7a701..9751b5d77c 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -85,7 +85,7 @@ static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2)
     if (is_ref_thing_header(*b2)) {
 	return 1;
     }
-    return cmp(ref1,ref2);
+    return CMP(ref1,ref2);
 }
 	    
 #define CP_LINK_VAL(To, Hp, From)				\
@@ -380,7 +380,7 @@ int erts_add_link(ErtsLink **root, Uint type, Eterm pid)
 	    state = 1;
 	    *this = create_link(type,pid);
 	    break;
-	} else if ((c = cmp(pid,(*this)->pid)) < 0) { 
+	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
 	    /* go left */
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
@@ -415,7 +415,7 @@ erts_add_or_lookup_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid)
 	    state = 1;
 	    res = *this = create_suspend_monitor(pid);
 	    break;
-	} else if ((c = cmp(pid,(*this)->pid)) < 0) { 
+	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
 	    /* go left */
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
@@ -453,7 +453,7 @@ ErtsLink *erts_add_or_lookup_link(ErtsLink **root, Uint type, Eterm pid)
 	    *this = create_link(type,pid);
 	    ret = *this;
 	    break;
-	} else if ((c = cmp(pid,(*this)->pid)) < 0) { 
+	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
 	    /* go left */
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
@@ -663,7 +663,7 @@ ErtsLink *erts_remove_link(ErtsLink **root, Eterm pid)
     for (;;) {
 	if (!*this) { /* Failure */
 	    return NULL;
-	} else if ((c = cmp(pid,(*this)->pid)) < 0) { 
+	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -715,7 +715,7 @@ erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid)
     for (;;) {
 	if (!*this) { /* Nothing found */
 	    return;
-	} else if ((c = cmp(pid,(*this)->pid)) < 0) { 
+	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -771,7 +771,7 @@ ErtsLink *erts_lookup_link(ErtsLink *root, Eterm pid)
     Sint c;
 
     for (;;) {
-	if (root == NULL || (c = cmp(pid,root->pid)) == 0) {
+	if (root == NULL || (c = CMP(pid,root->pid)) == 0) {
 	    return root;
 	} else if (c < 0) { 
 	    root = root->left;
@@ -787,7 +787,7 @@ erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, Eterm pid)
     Sint c;
 
     for (;;) {
-	if (root == NULL || (c = cmp(pid,root->pid)) == 0) {
+	if (root == NULL || (c = CMP(pid,root->pid)) == 0) {
 	    return root;
 	} else if (c < 0) { 
 	    root = root->left;
diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c
index 8b8ac2ec80..b1478758a1 100644
--- a/erts/emulator/beam/erl_mtrace.c
+++ b/erts/emulator/beam/erl_mtrace.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -79,7 +79,7 @@ static erts_mtx_t mtrace_buf_mutex;
 #define UI16_SZ					(2)
 #define UI32_SZ					(4)
 #define UI64_SZ					(8)
-#ifdef ARCH_64
+#ifdef ARCH_64 /* XXX:PaN Halfword? (whole file...) */
 #  define UI_SZ					UI64_SZ
 #else
 #  define UI_SZ					UI32_SZ
@@ -188,7 +188,7 @@ check_alloc_entry(byte *sp, byte *ep,
 		  byte tag,
 		  Uint16 ct_no, int ct_no_n,
 		  Uint16 type, int type_n,
-		  Uint res, int res_n,
+		  UWord res, int res_n,
 		  Uint size, int size_n,
 		  Uint32 ti,int ti_n);
 void
@@ -196,8 +196,8 @@ check_realloc_entry(byte *sp, byte *ep,
 		    byte tag,
 		    Uint16 ct_no, int ct_no_n,
 		    Uint16 type, int type_n,
-		    Uint res, int res_n,
-		    Uint ptr, int ptr_n,
+		    UWord res, int res_n,
+		    UWord ptr, int ptr_n,
 		    Uint size, int size_n,
 		    Uint32 ti,int ti_n);
 void
@@ -205,7 +205,7 @@ check_free_entry(byte *sp, byte *ep,
 		 byte tag,
 		 Uint16 ct_no, int ct_no_n,
 		 Uint16 t_no, int t_no_n,
-		 Uint ptr, int ptr_n,
+		 UWord ptr, int ptr_n,
 		 Uint32 ti,int ti_n);
 void
 check_time_inc_entry(byte *sp, byte *ep,
@@ -585,9 +585,7 @@ void erts_mtrace_init(char *receiver, char *nodename)
 	Uint16 port;
 
 	erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf");
-	erts_mtx_set_forksafe(&mtrace_buf_mutex);
 	erts_mtx_init(&mtrace_op_mutex, "mtrace_op");
-	erts_mtx_set_forksafe(&mtrace_op_mutex);
 
 	socket_desc = erts_sock_open();
 	if (socket_desc == ERTS_SOCK_INVALID_SOCKET) {
@@ -785,7 +783,7 @@ write_alloc_entry(byte tag,
 			      tag,
 			      ct_no, ct_no_n,
 			      t_no, t_no_n,
-			      (Uint) res, res_n,
+			      (UWord) res, res_n,
 			      size, size_n,
 			      ti, ti_n);
 #endif
@@ -865,8 +863,8 @@ write_realloc_entry(byte tag,
 				tag,
 				ct_no, ct_no_n,
 				t_no, t_no_n,
-				(Uint) res, res_n,
-				(Uint) ptr, ptr_n,
+				(UWord) res, res_n,
+				(UWord) ptr, ptr_n,
 				size, size_n,
 				ti, ti_n);
 #endif
@@ -934,7 +932,7 @@ write_free_entry(byte tag,
 			     tag,
 			     ct_no, ct_no_n,
 			     t_no, t_no_n,
-			     (Uint) ptr, ptr_n,
+			     (UWord) ptr, ptr_n,
 			     ti, ti_n);
 #endif
 	}
@@ -1135,7 +1133,7 @@ check_alloc_entry(byte *sp, byte *ep,
 		  byte tag,
 		  Uint16 ct_no, int ct_no_n,
 		  Uint16 t_no, int t_no_n,
-		  Uint res, int res_n,
+		  UWord res, int res_n,
 		  Uint size, int size_n,
 		  Uint32 ti,int ti_n)
 {
@@ -1163,8 +1161,8 @@ check_realloc_entry(byte *sp, byte *ep,
 		    byte tag,
 		    Uint16 ct_no, int ct_no_n,
 		    Uint16 t_no, int t_no_n,
-		    Uint res, int res_n,
-		    Uint ptr, int ptr_n,
+		    UWord res, int res_n,
+		    UWord ptr, int ptr_n,
 		    Uint size, int size_n,
 		    Uint32 ti,int ti_n)
 {
@@ -1193,7 +1191,7 @@ check_free_entry(byte *sp, byte *ep,
 		 byte tag,
 		 Uint16 ct_no, int ct_no_n,
 		 Uint16 t_no, int t_no_n,
-		 Uint ptr, int ptr_n,
+		 UWord ptr, int ptr_n,
 		 Uint32 ti,int ti_n)
 {
     byte *p = sp;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 585a6c1fdf..68421b4387 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -48,9 +48,23 @@ struct erl_module_nif {
     struct enif_entry_t* entry;
     erts_refc_t rt_cnt;       /* number of resource types */
     erts_refc_t rt_dtor_cnt;  /* number of resource types with destructors */
-    int is_orphan;            /* if erlang module has been purged */
+    Module* mod;           /* Can be NULL if orphan with dtor-resources left */ 
 };
 
+#ifdef DEBUG
+#  define READONLY_CHECK
+#endif
+#ifdef READONLY_CHECK
+#  define ADD_READONLY_CHECK(ENV,PTR,SIZE) add_readonly_check(ENV,PTR,SIZE)
+static void add_readonly_check(ErlNifEnv*, unsigned char* ptr, unsigned sz);
+#else
+#  define ADD_READONLY_CHECK(ENV,PTR,SIZE) ((void)0)
+#endif
+
+#ifdef DEBUG
+static int is_offheap(const ErlOffHeap* off_heap);
+#endif
+
 
 #define MIN_HEAP_FRAG_SZ 200
 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp);
@@ -67,23 +81,33 @@ static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need)
 
 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp)
 {    
-    unsigned frag_sz;
     env->hp = hp;
     if (env->heap_frag == NULL) {       
 	ASSERT(HEAP_LIMIT(env->proc) == env->hp_end);
 	HEAP_TOP(env->proc) = env->hp;	
     }
     else {
-	HRelease(env->proc, env->hp_end, env->hp);
+	env->heap_frag->used_size = hp - env->heap_frag->mem;
+	ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
     }
-    frag_sz = need + MIN_HEAP_FRAG_SZ;
-    hp = erts_heap_alloc(env->proc, frag_sz);
-    env->hp = hp + need;
-    env->hp_end = hp + frag_sz;
+    hp = erts_heap_alloc(env->proc, need, MIN_HEAP_FRAG_SZ);
     env->heap_frag = MBUF(env->proc);
+    env->hp = hp + need;
+    env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size;
+
     return hp;
 }
 
+#if SIZEOF_LONG != ERTS_SIZEOF_ETERM
+static ERTS_INLINE void ensure_heap(ErlNifEnv* env, unsigned may_need)
+{
+    if (env->hp + may_need > env->hp_end) {
+	alloc_heap_heavy(env, may_need, env->hp);
+	env->hp -= may_need;
+    }
+}
+#endif
+
 void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif)
 {
     env->mod_nif = mod_nif;
@@ -106,6 +130,13 @@ static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif)
     env->tmp_obj_list = NULL;
 }
 
+/* Temporary object header, auto-deallocated when NIF returns. */
+struct enif_tmp_obj_t {
+    struct enif_tmp_obj_t* next;
+    void (*dtor)(struct enif_tmp_obj_t*);
+    /*char data[];*/
+};
+
 static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env)
 {
     while (env->tmp_obj_list != NULL) {
@@ -126,8 +157,9 @@ void erts_post_nif(ErlNifEnv* env)
     }
     else {
 	ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
-	ASSERT(env->hp_end - env->hp <= env->heap_frag->size);
-	HRelease(env->proc, env->hp_end, env->hp);
+	ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
+	env->heap_frag->used_size = env->hp - env->heap_frag->mem;
+	ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
     }
     free_tmp_objs(env);
 }
@@ -141,7 +173,7 @@ static void post_nif_noproc(ErlNifEnv* env)
 
 /* Flush out our cached heap pointers to allow an ordinary HAlloc
 */
-static void enable_halloc(ErlNifEnv* env)
+static void flush_env(ErlNifEnv* env)
 {
     if (env->heap_frag == NULL) {
 	ASSERT(env->hp_end == HEAP_LIMIT(env->proc));
@@ -151,14 +183,15 @@ static void enable_halloc(ErlNifEnv* env)
     }
     else {
 	ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
-	ASSERT(env->hp_end - env->hp <= env->heap_frag->size);
-	HRelease(env->proc, env->hp_end, env->hp);
+	ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
+	env->heap_frag->used_size = env->hp - env->heap_frag->mem;
+	ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
     }
 }
 
-/* Restore cached heap pointers
+/* Restore cached heap pointers to allow alloc_heap again.
 */
-static void disable_halloc(ErlNifEnv* env)
+static void cache_env(ErlNifEnv* env)
 {
     if (env->heap_frag == NULL) {
 	ASSERT(env->hp_end == HEAP_LIMIT(env->proc));
@@ -168,35 +201,190 @@ static void disable_halloc(ErlNifEnv* env)
     }
     else {
 	ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
-	ASSERT(env->hp_end - env->hp <= env->heap_frag->size);       
+	ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);       
 	env->heap_frag = MBUF(env->proc);
 	ASSERT(env->heap_frag != NULL);
 	env->hp = env->heap_frag->mem + env->heap_frag->used_size;
-	env->hp_end = env->heap_frag->mem + env->heap_frag->size;
+	env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size;
     }
 }
-
-
 void* enif_priv_data(ErlNifEnv* env)
 {
     return env->mod_nif->priv_data;
 }
 
-void* enif_alloc(ErlNifEnv* env, size_t size)
+void* enif_alloc(size_t size)
 {
     return erts_alloc_fnf(ERTS_ALC_T_NIF, (Uint) size);
 }
 
-void* enif_realloc(ErlNifEnv* env, void* ptr, size_t size)
+void* enif_realloc(void* ptr, size_t size)
 {
     return erts_realloc_fnf(ERTS_ALC_T_NIF, ptr, size);
 }
 
-void enif_free(ErlNifEnv* env, void* ptr)
+void enif_free(void* ptr)
 {
     erts_free(ERTS_ALC_T_NIF, ptr);
 }
 
+struct enif_msg_environment_t
+{
+    ErlNifEnv env;
+    Process phony_proc;
+};
+
+ErlNifEnv* enif_alloc_env(void)
+{
+    struct enif_msg_environment_t* msg_env =
+	erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t));
+    Eterm* phony_heap = (Eterm*) msg_env; /* dummy non-NULL ptr */
+	
+    msg_env->env.hp = phony_heap; 
+    msg_env->env.hp_end = phony_heap;
+    msg_env->env.heap_frag = NULL;
+    msg_env->env.mod_nif = NULL;
+    msg_env->env.tmp_obj_list = (struct enif_tmp_obj_t*) 1; /* invalid non-NULL */
+    msg_env->env.proc = &msg_env->phony_proc;
+    memset(&msg_env->phony_proc, 0, sizeof(Process));
+    HEAP_START(&msg_env->phony_proc) = phony_heap;
+    HEAP_TOP(&msg_env->phony_proc) = phony_heap;
+    HEAP_LIMIT(&msg_env->phony_proc) = phony_heap;
+    HEAP_END(&msg_env->phony_proc) = phony_heap;
+    MBUF(&msg_env->phony_proc) = NULL;
+    msg_env->phony_proc.id = ERTS_INVALID_PID;
+#ifdef FORCE_HEAP_FRAGS
+    msg_env->phony_proc.space_verified = 0;
+    msg_env->phony_proc.space_verified_from = NULL;
+#endif
+    return &msg_env->env;
+}
+void enif_free_env(ErlNifEnv* env)
+{
+    enif_clear_env(env);
+    erts_free(ERTS_ALC_T_NIF, env);
+}
+
+static ERTS_INLINE void clear_offheap(ErlOffHeap* oh)
+{
+    oh->first = NULL;
+    oh->overhead = 0;
+}
+
+void enif_clear_env(ErlNifEnv* env)
+{
+    struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env;
+    Process* p = &menv->phony_proc;
+    ASSERT(p == menv->env.proc);
+    ASSERT(p->id == ERTS_INVALID_PID);
+    ASSERT(MBUF(p) == menv->env.heap_frag);
+    if (MBUF(p) != NULL) {
+	erts_cleanup_offheap(&MSO(p));
+	clear_offheap(&MSO(p));
+	free_message_buffer(MBUF(p));
+	MBUF(p) = NULL;
+	menv->env.heap_frag = NULL;
+    }
+    ASSERT(HEAP_TOP(p) == HEAP_END(p));
+    menv->env.hp = menv->env.hp_end = HEAP_TOP(p);
+    
+    ASSERT(!is_offheap(&MSO(p)));
+}
+int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
+	      ErlNifEnv* msg_env, ERL_NIF_TERM msg)
+{
+    struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env;
+    ErtsProcLocks rp_locks = 0;
+    Process* rp;
+    Process* c_p;
+    ErlHeapFragment* frags;
+#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+    ErtsProcLocks rp_had_locks;
+#endif
+    Eterm receiver = to_pid->pid;
+    int flush_me = 0;
+
+    if (env != NULL) {
+	c_p = env->proc;
+	if (receiver == c_p->id) {
+	    rp_locks = ERTS_PROC_LOCK_MAIN;
+	    flush_me = 1;
+	}
+    }
+    else {
+#ifdef ERTS_SMP
+	c_p = NULL;
+#else
+	erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM");
+#endif
+    }    
+
+#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+    rp_had_locks = rp_locks;
+#endif
+    rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
+			   receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC);
+    if (rp == NULL) {
+	ASSERT(env == NULL || receiver != c_p->id);
+	return 0;
+    }
+    flush_env(msg_env);
+    frags = menv->env.heap_frag; 
+    ASSERT(frags == MBUF(&menv->phony_proc));
+    if (frags != NULL) {
+	/* Move all offheap's from phony proc to the first fragment.
+	   Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */
+	ASSERT(!is_offheap(&frags->off_heap));
+	frags->off_heap = MSO(&menv->phony_proc);
+	clear_offheap(&MSO(&menv->phony_proc));
+	menv->env.heap_frag = NULL; 
+	MBUF(&menv->phony_proc) = NULL;
+    }
+    ASSERT(!is_offheap(&MSO(&menv->phony_proc)));
+
+    if (flush_me) {	
+	flush_env(env); /* Needed for ERTS_HOLE_CHECK */ 
+    }
+    erts_queue_message(rp, &rp_locks, frags, msg, am_undefined);
+    if (rp_locks) {	
+	ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ | 
+							ERTS_PROC_LOCK_STATUS)));
+	erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS));
+    }
+    erts_smp_proc_dec_refc(rp);
+    if (flush_me) {
+	cache_env(env);
+    }
+    return 1;
+}
+
+ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)
+{
+    Uint sz;
+    Eterm* hp;
+    sz = size_object(src_term);
+    hp = alloc_heap(dst_env, sz);
+    return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc));
+}
+
+
+#ifdef DEBUG
+static int is_offheap(const ErlOffHeap* oh)
+{
+    return oh->first != NULL;
+}
+#endif
+
+ErlNifPid* enif_self(ErlNifEnv* caller_env, ErlNifPid* pid)
+{
+    pid->pid = caller_env->proc->id;
+    return pid;
+}
+int enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid)
+{
+    return is_internal_pid(term) ? (pid->pid=term, 1) : 0;
+}
+
 int enif_is_atom(ErlNifEnv* env, ERL_NIF_TERM term)
 {
     return is_atom(term);
@@ -232,9 +420,24 @@ int enif_is_ref(ErlNifEnv* env, ERL_NIF_TERM term)
     return is_ref(term);
 }
 
+int enif_is_tuple(ErlNifEnv* env, ERL_NIF_TERM term)
+{
+    return is_tuple(term);
+}
+
+int enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term)
+{
+    return is_list(term) || is_nil(term);
+}
+
+int enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term)
+{
+    return term == THE_NON_VALUE;
+}
+
 static void aligned_binary_dtor(struct enif_tmp_obj_t* obj)
 {
-    erts_free_aligned_binary_bytes((byte*)obj);
+    erts_free_aligned_binary_bytes_extra((byte*)obj,ERTS_ALC_T_TMP);
 }
 
 int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin)
@@ -244,7 +447,7 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin)
 	byte* raw_ptr;
     }u;
     u.tmp = NULL;
-    bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr,
+    bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, ERTS_ALC_T_TMP,
 						    sizeof(struct enif_tmp_obj_t));
     if (bin->data == NULL) {
 	return 0;
@@ -257,6 +460,7 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin)
     bin->bin_term = bin_term;
     bin->size = binary_size(bin_term);
     bin->ref_bin = NULL;
+    ADD_READONLY_CHECK(env, bin->data, bin->size); 
     return 1;
 }
 
@@ -268,7 +472,7 @@ static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj)
 int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
 {
     struct enif_tmp_obj_t* tobj;
-    int sz;
+    Uint sz;
     if (is_binary(term)) {
 	return enif_inspect_binary(env,term,bin);
     }
@@ -279,7 +483,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
 	bin->ref_bin = NULL;
 	return 1;
     }
-    if ((sz = io_list_len(term)) < 0) {
+    if (erts_iolist_size(term, &sz)) {
 	return 0;
     }
     
@@ -293,10 +497,11 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
     bin->bin_term = THE_NON_VALUE;
     bin->ref_bin = NULL;
     io_list_to_buf(term, (char*) bin->data, sz);
+    ADD_READONLY_CHECK(env, bin->data, bin->size); 
     return 1;
 }
 
-int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin)
+int enif_alloc_binary(size_t size, ErlNifBinary* bin)
 {
     Binary* refbin;
 
@@ -315,7 +520,7 @@ int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin)
     return 1;
 }
 
-int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size)
+int enif_realloc_binary(ErlNifBinary* bin, size_t size)
 {
     if (bin->ref_bin != NULL) {
 	Binary* oldbin;
@@ -333,15 +538,15 @@ int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size)
     }
     else {
 	unsigned char* old_data = bin->data;
-	unsigned cpy_sz = (size < bin->size ? size : bin->size);  
-	enif_alloc_binary(env, size, bin);
+	size_t cpy_sz = (size < bin->size ? size : bin->size);  
+	enif_alloc_binary(size, bin);
 	sys_memcpy(bin->data, old_data, cpy_sz); 
     }
     return 1;
 }
 
 
-void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin)
+void enif_release_binary(ErlNifBinary* bin)
 {
     if (bin->ref_bin != NULL) {
 	Binary* refbin = bin->ref_bin;
@@ -357,14 +562,23 @@ void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin)
 #endif
 }
 
-int enif_is_identical(ErlNifEnv* env, Eterm lhs, Eterm rhs)
+unsigned char* enif_make_new_binary(ErlNifEnv* env, size_t size,
+				    ERL_NIF_TERM* termp)
+{
+    flush_env(env);
+    *termp = new_binary(env->proc, NULL, size);
+    cache_env(env);
+    return binary_bytes(*termp);
+}
+
+int enif_is_identical(Eterm lhs, Eterm rhs)
 {
     return EQ(lhs,rhs);
 }
 
-int enif_compare(ErlNifEnv* env, Eterm lhs, Eterm rhs)
+int enif_compare(Eterm lhs, Eterm rhs)
 {
-    return cmp(lhs,rhs);
+    return CMP(lhs,rhs);
 }
 
 int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array)
@@ -425,13 +639,13 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
 	pb = (ProcBin *) alloc_heap(env, PROC_BIN_SIZE);
 	pb->thing_word = HEADER_PROC_BIN;
 	pb->size = bptr->orig_size;
-	pb->next = MSO(env->proc).mso;
-	MSO(env->proc).mso = pb;
+	pb->next = MSO(env->proc).first;
+	MSO(env->proc).first = (struct erl_off_heap_header*) pb;
 	pb->val = bptr;
 	pb->bytes = (byte*) bptr->orig_bytes;
 	pb->flags = 0;
 	
-	MSO(env->proc).overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm));
 	bin_term = make_binary(pb);	
 	if (erts_refc_read(&bptr->refc, 1) == 1) {
 	    /* Total ownership transfer */
@@ -441,15 +655,15 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
 	return bin_term;
     }
     else {
-	enable_halloc(env);
+	flush_env(env);
 	bin->bin_term = new_binary(env->proc, bin->data, bin->size);
-	disable_halloc(env);
+	cache_env(env);
 	return bin->bin_term;
     }
 }
 
 Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term,
-			   unsigned pos, unsigned size)
+			   size_t pos, size_t size)
 {
     ErlSubBin* sb;
     Eterm orig;
@@ -479,9 +693,11 @@ Eterm enif_make_badarg(ErlNifEnv* env)
     BIF_ERROR(env->proc, BADARG);
 }
 
-int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len)
+int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len,
+		  ErlNifCharEncoding encoding)
 {
     Atom* ap;
+    ASSERT(encoding == ERL_NIF_LATIN1);
     if (is_not_atom(atom)) {
 	return 0;
     }
@@ -496,9 +712,9 @@ int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len)
 
 int enif_get_int(ErlNifEnv* env, Eterm term, int* ip)
 {
-#if SIZEOF_INT == SIZEOF_VOID_P
+#if SIZEOF_INT ==  ERTS_SIZEOF_ETERM
     return term_to_Sint(term, (Sint*)ip);
-#elif SIZEOF_LONG == SIZEOF_VOID_P
+#elif SIZEOF_LONG ==  ERTS_SIZEOF_ETERM
     Sint i;
     if (!term_to_Sint(term, &i) || i < INT_MIN || i > INT_MAX) {
 	return 0;
@@ -512,9 +728,9 @@ int enif_get_int(ErlNifEnv* env, Eterm term, int* ip)
 
 int enif_get_uint(ErlNifEnv* env, Eterm term, unsigned* ip)
 {
-#if SIZEOF_INT == SIZEOF_VOID_P
+#if SIZEOF_INT == ERTS_SIZEOF_ETERM
     return term_to_Uint(term, (Uint*)ip);
-#elif SIZEOF_LONG == SIZEOF_VOID_P
+#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM
     Uint i;
     if (!term_to_Uint(term, &i) || i > UINT_MAX) {
 	return 0;
@@ -526,8 +742,10 @@ int enif_get_uint(ErlNifEnv* env, Eterm term, unsigned* ip)
 
 int enif_get_long(ErlNifEnv* env, Eterm term, long* ip)
 {
-#if SIZEOF_LONG == SIZEOF_VOID_P
+#if SIZEOF_LONG == ERTS_SIZEOF_ETERM
     return term_to_Sint(term, ip);
+#elif SIZEOF_LONG == 8
+    return term_to_Sint64(term, ip);
 #else
 #  error Unknown long word size 
 #endif     
@@ -535,14 +753,28 @@ int enif_get_long(ErlNifEnv* env, Eterm term, long* ip)
 
 int enif_get_ulong(ErlNifEnv* env, Eterm term, unsigned long* ip)
 {
-#if SIZEOF_LONG == SIZEOF_VOID_P
+#if SIZEOF_LONG == ERTS_SIZEOF_ETERM
     return term_to_Uint(term, ip);
+#elif SIZEOF_LONG == 8
+    return term_to_Uint64(term, ip);
 #else
 #  error Unknown long word size 
 #endif     
 }
 
-int enif_get_double(ErlNifEnv* env, Eterm term, double* dp)
+#if HAVE_INT64 && SIZEOF_LONG != 8
+int enif_get_int64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifSInt64* ip)
+{
+    return term_to_Sint64(term, ip);
+}
+
+int enif_get_uint64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifUInt64* ip)
+{
+    return term_to_Uint64(term, ip);
+}
+#endif /* HAVE_INT64 && SIZEOF_LONG != 8 */
+
+int enif_get_double(ErlNifEnv* env, ERL_NIF_TERM term, double* dp)
 {
     FloatDef f;
     if (is_not_float(term)) {
@@ -553,6 +785,17 @@ int enif_get_double(ErlNifEnv* env, Eterm term, double* dp)
     return 1;
 }
 
+int enif_get_atom_length(ErlNifEnv* env, Eterm atom, unsigned* len,
+			 ErlNifCharEncoding enc)
+{
+    Atom* ap;
+    ASSERT(enc == ERL_NIF_LATIN1);
+    if (is_not_atom(atom)) return 0;
+    ap = atom_tab(atom_val(atom));
+    *len = ap->len;
+    return 1;
+}
+
 int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail)
 {
     Eterm* val;
@@ -563,37 +806,71 @@ int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail)
     return 1;
 }
 
+int enif_get_list_length(ErlNifEnv* env, Eterm term, unsigned* len)
+{
+    if (is_not_list(term) && is_not_nil(term)) return 0;
+    *len = list_length(term);
+    return 1;
+}
+
 ERL_NIF_TERM enif_make_int(ErlNifEnv* env, int i)
 {
-#if SIZEOF_INT == SIZEOF_VOID_P
+#if SIZEOF_INT == ERTS_SIZEOF_ETERM
     return IS_SSMALL(i) ? make_small(i) : small_to_big(i,alloc_heap(env,2));
-#elif SIZEOF_LONG == SIZEOF_VOID_P
+#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM
     return make_small(i);
 #endif
 }
 
 ERL_NIF_TERM enif_make_uint(ErlNifEnv* env, unsigned i)
 {
-#if SIZEOF_INT == SIZEOF_VOID_P
+#if SIZEOF_INT == ERTS_SIZEOF_ETERM
     return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2));
-#elif SIZEOF_LONG == SIZEOF_VOID_P
+#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM
     return make_small(i);
 #endif
 }
 
 ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i)
 {
-#if SIZEOF_LONG != SIZEOF_VOID_P
-#  error Unknown long word size 
-#endif     
+#if SIZEOF_LONG == ERTS_SIZEOF_ETERM
     return IS_SSMALL(i) ? make_small(i) : small_to_big(i, alloc_heap(env,2));
+#elif SIZEOF_LONG == 8
+    ensure_heap(env,3);
+    return erts_sint64_to_big(i, &env->hp);
+#endif
 }
 
 ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i)
 {
+#if SIZEOF_LONG == ERTS_SIZEOF_ETERM
     return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2));
+#elif SIZEOF_LONG == 8
+    ensure_heap(env,3);
+    return erts_uint64_to_big(i, &env->hp);    
+#endif
+}
+
+#if HAVE_INT64 && SIZEOF_LONG != 8
+ERL_NIF_TERM enif_make_int64(ErlNifEnv* env, ErlNifSInt64 i)
+{
+    Uint* hp;
+    Uint need = 0;
+    erts_bld_sint64(NULL, &need, i);
+    hp = alloc_heap(env, need);
+    return erts_bld_sint64(&hp, NULL, i);
 }
 
+ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i)
+{
+    Uint* hp;
+    Uint need = 0;
+    erts_bld_uint64(NULL, &need, i);
+    hp = alloc_heap(env, need);
+    return erts_bld_uint64(&hp, NULL, i);
+}
+#endif /* HAVE_INT64 && SIZEOF_LONG != 8 */
+
 ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d)
 {
     Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT);
@@ -605,12 +882,25 @@ ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d)
 
 ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name)
 {
-    return am_atom_put(name, sys_strlen(name));
+    return enif_make_atom_len(env, name, sys_strlen(name));
 }
 
-int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom)
+ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len)
 {
-    return erts_atom_get(name, sys_strlen(name), atom);
+    return am_atom_put(name, len);
+}
+
+int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom,
+			    ErlNifCharEncoding enc)
+{
+    return enif_make_existing_atom_len(env, name, sys_strlen(name), atom, enc);
+}
+
+int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len,
+				ERL_NIF_TERM* atom, ErlNifCharEncoding encoding)
+{
+    ASSERT(encoding == ERL_NIF_LATIN1);
+    return erts_atom_get(name, len, atom);
 }
 
 ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...)
@@ -653,21 +943,26 @@ ERL_NIF_TERM enif_make_list_cell(ErlNifEnv* env, Eterm car, Eterm cdr)
 
 ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...)
 {
-    Eterm* hp = alloc_heap(env,cnt*2);
-    Eterm ret = make_list(hp);
-    Eterm* last = &ret;
-    va_list ap;
-
-    va_start(ap,cnt);
-    while (cnt--) {
-	*last = make_list(hp);
-	*hp = va_arg(ap,Eterm);
-	last = ++hp;
-	++hp;
+    if (cnt == 0) {
+	return NIL;
+    }
+    else {
+	Eterm* hp = alloc_heap(env,cnt*2);
+	Eterm ret = make_list(hp);
+	Eterm* last = &ret;
+	va_list ap;
+
+	va_start(ap,cnt);
+	while (cnt--) {
+	    *last = make_list(hp);
+	    *hp = va_arg(ap,Eterm);
+	    last = ++hp;
+	    ++hp;
+	}
+	va_end(ap);
+	*last = NIL;
+	return ret;
     }
-    va_end(ap);
-    *last = NIL;
-    return ret;
 }
 
 ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt)
@@ -689,11 +984,16 @@ ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[],
 
 ERL_NIF_TERM enif_make_string(ErlNifEnv* env, const char* string,
 			      ErlNifCharEncoding encoding)
-{    
-    Sint n = sys_strlen(string);    
-    Eterm* hp = alloc_heap(env,n*2);
+{
+    return enif_make_string_len(env, string, sys_strlen(string), encoding);
+}
+
+ERL_NIF_TERM enif_make_string_len(ErlNifEnv* env, const char* string,
+				  size_t len, ErlNifCharEncoding encoding)
+{
+    Eterm* hp = alloc_heap(env,len*2);
     ASSERT(encoding == ERL_NIF_LATIN1);
-    return erts_bld_string_n(&hp,NULL,string,n); 
+    return erts_bld_string_n(&hp,NULL,string,len);
 }
 
 ERL_NIF_TERM enif_make_ref(ErlNifEnv* env)
@@ -764,7 +1064,8 @@ struct enif_resource_type_t
     ErlNifResourceDtor* dtor;      /* user destructor function */
     erts_refc_t refc;  /* num of resources of this type (HOTSPOT warning)
                           +1 for active erl_module_nif */
-    char name[1];
+    Eterm module;
+    Eterm name;
 };
 
 /* dummy node in circular list */
@@ -782,14 +1083,14 @@ typedef struct enif_resource_t
 #define SIZEOF_ErlNifResource(SIZE) (offsetof(ErlNifResource,data) + (SIZE))
 #define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data)))
 
-static ErlNifResourceType* find_resource_type(const char* name)
+static ErlNifResourceType* find_resource_type(Eterm module, Eterm name)
 {
     ErlNifResourceType* type;
     for (type = resource_type_list.next;
 	 type != &resource_type_list;
 	 type = type->next) {
 
-	if (sys_strcmp(type->name, name) == 0) {
+	if (type->module == module && type->name == name) {
 	    return type;
 	}
     }
@@ -822,33 +1123,42 @@ static void steal_resource_type(ErlNifResourceType* type)
 
     if (type->dtor != NULL
 	&& erts_refc_dectest(&lib->rt_dtor_cnt, 0) == 0
-	&& lib->is_orphan) {
+	&& lib->mod == NULL) {
 	/* last type with destructor gone, close orphan lib */
 
 	close_lib(lib);
     }
     if (erts_refc_dectest(&lib->rt_cnt, 0) == 0
-	&& lib->is_orphan) {
+	&& lib->mod == NULL) {
 	erts_free(ERTS_ALC_T_NIF, lib);
     }
 }
 
 ErlNifResourceType*
-enif_open_resource_type(ErlNifEnv* env, const char* type_name, 
-			  ErlNifResourceDtor* dtor,
-			  enum ErlNifResourceFlags flags,
-			  enum ErlNifResourceFlags* tried)
+enif_open_resource_type(ErlNifEnv* env,
+			const char* module_str, 
+			const char* name_str, 
+			ErlNifResourceDtor* dtor,
+			ErlNifResourceFlags flags,
+			ErlNifResourceFlags* tried)
 {
-    ErlNifResourceType* type = find_resource_type(type_name);
-    enum ErlNifResourceFlags op = flags;
+    ErlNifResourceType* type = NULL;
+    ErlNifResourceFlags op = flags;
+    Eterm module_am, name_am;
+
     ASSERT(erts_smp_is_system_blocked(0));
+    ASSERT(module_str == NULL); /* for now... */
+    module_am = make_atom(env->mod_nif->mod->module);
+    name_am = enif_make_atom(env, name_str);
+
+    type = find_resource_type(module_am, name_am);
     if (type == NULL) {
 	if (flags & ERL_NIF_RT_CREATE) {
 	    type = erts_alloc(ERTS_ALC_T_NIF,
-			      sizeof(struct enif_resource_type_t) 
-			      + sys_strlen(type_name));
+			      sizeof(struct enif_resource_type_t)); 
 	    type->dtor = dtor;
-	    sys_strcpy(type->name, type_name);
+	    type->module = module_am;
+	    type->name = name_am;
 	    erts_refc_init(&type->refc, 1);
 	    type->owner = env->mod_nif;
 	    type->prev = &resource_type_list;
@@ -896,13 +1206,13 @@ static void nif_resource_dtor(Binary* bin)
     if (erts_refc_dectest(&type->refc, 0) == 0) {
 	ASSERT(type->next == NULL);
 	ASSERT(type->owner != NULL);
-	ASSERT(type->owner->is_orphan);
+	ASSERT(type->owner->mod == NULL);
 	steal_resource_type(type);
 	erts_free(ERTS_ALC_T_NIF, type);
     }
 }
 
-void* enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned size)
+void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
 {
     Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor);
     ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin);
@@ -915,7 +1225,7 @@ void* enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned siz
     return resource->data;
 }
 
-void enif_release_resource(ErlNifEnv* env, void* obj)
+void enif_release_resource(void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource);
@@ -929,6 +1239,18 @@ void enif_release_resource(ErlNifEnv* env, void* obj)
     }
 }
 
+void enif_keep_resource(void* obj)
+{
+    ErlNifResource* resource = DATA_TO_RESOURCE(obj);
+    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource);
+
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
+#ifdef DEBUG
+    erts_refc_inc(&resource->nif_refc, 1);
+#endif
+    erts_refc_inc(&bin->binary.refc, 2);
+}
+
 ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
@@ -937,15 +1259,30 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
     return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary);
 }
 
+ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj,
+				       const void* data, size_t size)
+{
+    Eterm bin = enif_make_resource(env, obj);
+    ProcBin* pb = (ProcBin*) binary_val(bin);
+    pb->bytes = (byte*) data;
+    pb->size = size;
+    return bin;
+}
+
 int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type,
 		      void** objp)
 {
+    ProcBin* pb;
     Binary* mbin;
     ErlNifResource* resource;
     if (!ERTS_TERM_IS_MAGIC_BINARY(term)) {
 	return 0;
     }
-    mbin = ((ProcBin*) binary_val(term))->val;
+    pb = (ProcBin*) binary_val(term);
+    /*if (pb->size != 0) {	
+	return 0; / * Or should we allow "resource binaries" as handles? * /
+    }*/
+    mbin = pb->val;
     resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin);
     if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor
 	|| resource->type != type) {	
@@ -955,7 +1292,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
     return 1;
 }
 
-unsigned enif_sizeof_resource(ErlNifEnv* env, void* obj)
+size_t enif_sizeof_resource(void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
     Binary* bin = &ERTS_MAGIC_BIN_FROM_DATA(resource)->binary;
@@ -967,37 +1304,22 @@ unsigned enif_sizeof_resource(ErlNifEnv* env, void* obj)
  ***************************************************************************/
 
 
-static Uint** get_func_pp(Eterm* mod_code, Eterm f_atom, unsigned arity)
+static BeamInstr** get_func_pp(BeamInstr* mod_code, Eterm f_atom, unsigned arity)
 {
     int n = (int) mod_code[MI_NUM_FUNCTIONS];
     int j;
     for (j = 0; j < n; ++j) {
-	Uint* code_ptr = (Uint*) mod_code[MI_FUNCTIONS+j];
-	ASSERT(code_ptr[0] == (Uint) BeamOp(op_i_func_info_IaaI));
+	BeamInstr* code_ptr = (BeamInstr*) mod_code[MI_FUNCTIONS+j];
+	ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
 	if (f_atom == ((Eterm) code_ptr[3])
 	    && arity == ((unsigned) code_ptr[4])) {
 
-	    return (Uint**) &mod_code[MI_FUNCTIONS+j];
+	    return (BeamInstr**) &mod_code[MI_FUNCTIONS+j];
 	}
     }
     return NULL;
 }
 
-/*static void refresh_cached_nif_data(Eterm* mod_code,
-				    struct erl_module_nif* mod_nif)
-{
-    int i;
-    for (i=0; i < mod_nif->entry->num_of_funcs; i++) {
-	Eterm f_atom;
-	ErlNifFunc* func = &mod_nif->entry->funcs[i];	
-	Uint* code_ptr;
-	
-	erts_atom_get(func->name, sys_strlen(func->name), &f_atom); 
-	code_ptr = *get_func_pp(mod_code, f_atom, func->arity);
-	code_ptr[5+2] = (Uint) mod_nif->priv_data;
-    }
-}*/
-
 static Eterm mkatom(const char *str)
 {
     return am_atom_put(str, sys_strlen(str));
@@ -1089,7 +1411,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     static const char bad_lib[] = "bad_lib";
     static const char reload[] = "reload";
     static const char upgrade[] = "upgrade";
-    char lib_name[256]; /* BUGBUG: Max-length? */
+    char* lib_name = NULL;
     void* handle = NULL;
     void* init_func;
     ErlNifEntry* entry = NULL;
@@ -1098,15 +1420,20 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     Module* mod;
     Eterm mod_atom;
     Eterm f_atom;
-    Eterm* caller;
+    BeamInstr* caller;
     ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT;
     Eterm ret = am_ok;
     int veto;
     struct erl_module_nif* lib = NULL;
 
-    len = intlist_to_buf(BIF_ARG_1, lib_name, sizeof(lib_name)-1);
-    if (len < 1) {
-	/*erts_fprintf(stderr, "Invalid library path name '%T'\r\n", BIF_ARG_1);*/
+    len = list_length(BIF_ARG_1);
+    if (len < 0) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+    lib_name = (char *) erts_alloc(ERTS_ALC_T_TMP, len + 1);
+
+    if (intlist_to_buf(BIF_ARG_1, lib_name, len) != len) {
+	erts_free(ERTS_ALC_T_TMP, lib_name);
 	BIF_ERROR(BIF_P, BADARG);
     }
     lib_name[len] = '\0';
@@ -1156,7 +1483,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	
 	ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).",
 			     entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION);
-    }    
+    }   
+    else if (entry->minor >= 1
+	     && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) {
+	ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for "
+			     "this vm variant (%s).",
+			     entry->vm_variant, ERL_NIF_VM_VARIANT);
+    }
     else if (!erts_is_atom_str((char*)entry->name, mod_atom)) {
 	ret = load_nif_error(BIF_P, bad_lib, "Library module name '%s' does not"
 			     " match calling module '%T'", entry->name, mod_atom);
@@ -1165,7 +1498,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	/*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/
     
 	for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) {
-	    Uint** code_pp;
+	    BeamInstr** code_pp;
 	    ErlNifFunc* f = &entry->funcs[i];
 	    if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom)
 		|| (code_pp = get_func_pp(mod->code, f_atom, f->arity))==NULL) { 
@@ -1195,7 +1528,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     lib->entry = entry;
     erts_refc_init(&lib->rt_cnt, 0);
     erts_refc_init(&lib->rt_dtor_cnt, 0);
-    lib->is_orphan = 0;
+    lib->mod = mod;
     env.mod_nif = lib;
     if (mod->nif != NULL) { /* Reload */
 	int k;
@@ -1268,22 +1601,23 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
         mod->nif = lib; 
 	for (i=0; i < entry->num_of_funcs; i++)
 	{
-	    Uint* code_ptr;
+	    BeamInstr* code_ptr;
 	    erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom); 
 	    code_ptr = *get_func_pp(mod->code, f_atom, entry->funcs[i].arity); 
 	    
 	    if (code_ptr[1] == 0) {
-		code_ptr[5+0] = (Uint) BeamOp(op_call_nif);		
+		code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif);
 	    }
 	    else { /* Function traced, patch the original instruction word */
-		BpData* bp = (BpData*) code_ptr[1];
-	        bp->orig_instr = (Uint) BeamOp(op_call_nif); 
+		BpData** bps = (BpData**) code_ptr[1];
+		BpData*  bp  = (BpData*) bps[bp_sched2ix()];
+	        bp->orig_instr = (BeamInstr) BeamOp(op_call_nif);
 	    }	    
-	    code_ptr[5+1] = (Uint) entry->funcs[i].fptr;
-	    code_ptr[5+2] = (Uint) lib;
+	    code_ptr[5+1] = (BeamInstr) entry->funcs[i].fptr;
+	    code_ptr[5+2] = (BeamInstr) lib;
 	}
     }
-    else {    
+    else {
     error:
 	ASSERT(ret != am_ok);
         if (lib != NULL) {
@@ -1297,6 +1631,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 
     erts_smp_release_system();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_free(ERTS_ALC_T_TMP, lib_name);
     BIF_RET(ret);
 }
 
@@ -1308,7 +1643,7 @@ erts_unload_nif(struct erl_module_nif* lib)
     ErlNifResourceType* next;
     ASSERT(erts_smp_is_system_blocked(0));
     ASSERT(lib != NULL);
-    ASSERT(!lib->is_orphan);
+    ASSERT(lib->mod != NULL);
     for (rt = resource_type_list.next;
 	 rt != &resource_type_list;
 	 rt = next) {
@@ -1338,7 +1673,7 @@ erts_unload_nif(struct erl_module_nif* lib)
     else {
 	ASSERT(erts_refc_read(&lib->rt_cnt, 1) > 0);
     }
-    lib->is_orphan = 1;
+    lib->mod = NULL;   /* orphan lib */
 }	
 
 void erl_nif_init()
@@ -1347,6 +1682,53 @@ void erl_nif_init()
     resource_type_list.prev = &resource_type_list;
     resource_type_list.dtor = NULL;
     resource_type_list.owner = NULL;
-    resource_type_list.name[0] = '\0';
+    resource_type_list.module = THE_NON_VALUE;
+    resource_type_list.name = THE_NON_VALUE;
 }
 
+#ifdef READONLY_CHECK
+/* Use checksums to assert that NIFs do not write into inspected binaries
+*/
+static void readonly_check_dtor(struct enif_tmp_obj_t*);
+static unsigned calc_checksum(unsigned char* ptr, unsigned size);
+
+struct readonly_check_t
+{
+    struct enif_tmp_obj_t hdr;
+    unsigned char* ptr;
+    unsigned size;
+    unsigned checksum;
+};
+static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz)
+{
+    struct readonly_check_t* obj = erts_alloc(ERTS_ALC_T_TMP, 
+					      sizeof(struct readonly_check_t));
+    obj->hdr.next = env->tmp_obj_list;
+    env->tmp_obj_list = &obj->hdr;
+    obj->hdr.dtor = &readonly_check_dtor;
+    obj->ptr = ptr;
+    obj->size = sz;
+    obj->checksum = calc_checksum(ptr, sz);    
+}
+static void readonly_check_dtor(struct enif_tmp_obj_t* o)
+{
+    struct readonly_check_t* obj = (struct readonly_check_t*) o;
+    unsigned chksum = calc_checksum(obj->ptr, obj->size);
+    if (chksum != obj->checksum) { 
+	fprintf(stderr, "\r\nReadonly data written by NIF, checksums differ"
+		" %x != %x\r\nABORTING\r\n", chksum, obj->checksum);
+	abort();
+    }
+    erts_free(ERTS_ALC_T_TMP,  obj);
+}
+static unsigned calc_checksum(unsigned char* ptr, unsigned size)
+{
+    unsigned i, sum = 0;
+    for (i=0; i<size; i++) {
+	sum ^= ptr[i] << ((i % 4)*8);
+    }
+    return sum;
+}
+
+#endif /* READONLY_CHECK */
+
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 1ccf00293e..d028567faf 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -23,18 +23,71 @@
 #ifndef __ERL_NIF_H__
 #define __ERL_NIF_H__
 
+
 #include "erl_drv_nif.h"
 
 /* Version history:
 ** 0.1: R13B03
 ** 1.0: R13B04
+** 2.0: R14A
+** 2.1: R14B02 "vm_variant"
+** 2.2: R14B03 enif_is_exception
 */
-#define ERL_NIF_MAJOR_VERSION 1
-#define ERL_NIF_MINOR_VERSION 0
+#define ERL_NIF_MAJOR_VERSION 2
+#define ERL_NIF_MINOR_VERSION 2
 
 #include <stdlib.h>
 
+#ifdef SIZEOF_CHAR
+#  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
+#  undef SIZEOF_CHAR
+#endif
+#ifdef SIZEOF_SHORT
+#  define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT
+#  undef SIZEOF_SHORT
+#endif
+#ifdef SIZEOF_INT
+#  define SIZEOF_INT_SAVED__ SIZEOF_INT
+#  undef SIZEOF_INT
+#endif
+#ifdef SIZEOF_LONG
+#  define SIZEOF_LONG_SAVED__ SIZEOF_LONG
+#  undef SIZEOF_LONG
+#endif
+#ifdef SIZEOF_LONG_LONG
+#  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
+#  undef SIZEOF_LONG_LONG
+#endif
+#ifdef HALFWORD_HEAP_EMULATOR
+#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
+#  undef HALFWORD_HEAP_EMULATOR
+#endif
+#include "erl_int_sizes_config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+typedef unsigned __int64 ErlNifUInt64;
+typedef __int64 ErlNifSInt64;
+#elif SIZEOF_LONG == 8
+typedef unsigned long ErlNifUInt64;
+typedef long ErlNifSInt64;
+#elif SIZEOF_LONG_LONG == 8
+typedef unsigned long long ErlNifUInt64;
+typedef long long ErlNifSInt64;
+#else
+#error No 64-bit integer type
+#endif
+
+#ifdef HALFWORD_HEAP_EMULATOR
+#  define ERL_NIF_VM_VARIANT "beam.halfword" 
+typedef unsigned int ERL_NIF_TERM;
+#else
+#  define ERL_NIF_VM_VARIANT "beam.vanilla" 
 typedef unsigned long ERL_NIF_TERM;
+#endif
 
 struct enif_environment_t;
 typedef struct enif_environment_t ErlNifEnv;
@@ -56,14 +109,15 @@ typedef struct enif_entry_t
     int  (*load)   (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
     int  (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
     int  (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
-    void (*unload) (ErlNifEnv*, void* priv_data);    
+    void (*unload) (ErlNifEnv*, void* priv_data);
+    const char* vm_variant;
 }ErlNifEntry;
 
 
 
 typedef struct
 {
-    unsigned size;
+    size_t size;
     unsigned char* data;
 
     /* Internals (avert your eyes) */
@@ -73,17 +127,22 @@ typedef struct
 
 typedef struct enif_resource_type_t ErlNifResourceType;
 typedef void ErlNifResourceDtor(ErlNifEnv*, void*);
-enum ErlNifResourceFlags
+typedef enum
 {
     ERL_NIF_RT_CREATE = 1,
     ERL_NIF_RT_TAKEOVER = 2
-};
+}ErlNifResourceFlags;
 
 typedef enum
 {
     ERL_NIF_LATIN1 = 1
 }ErlNifCharEncoding;
 
+typedef struct
+{
+    ERL_NIF_TERM pid;  /* internal, may change */
+}ErlNifPid;
+
 typedef ErlDrvSysInfo ErlNifSysInfo;
 
 typedef struct ErlDrvTid_ *ErlNifTid;
@@ -116,8 +175,6 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #endif
 
 
-
-
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks;
 #  define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks)
@@ -133,8 +190,20 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #endif
 
 
+#ifdef __cplusplus
+}
+#  define ERL_NIF_INIT_PROLOGUE extern "C" {
+#  define ERL_NIF_INIT_EPILOGUE }
+#else
+#  define ERL_NIF_INIT_PROLOGUE
+#  define ERL_NIF_INIT_EPILOGUE
+#endif
+
+
 #define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \
+ERL_NIF_INIT_PROLOGUE                   \
 ERL_NIF_INIT_GLOB                       \
+ERL_NIF_INIT_DECL(NAME);		\
 ERL_NIF_INIT_DECL(NAME)			\
 {					\
     static ErlNifEntry entry = 		\
@@ -144,11 +213,14 @@ ERL_NIF_INIT_DECL(NAME)			\
 	#NAME,				\
 	sizeof(FUNCS) / sizeof(*FUNCS),	\
 	FUNCS,				\
-	LOAD, RELOAD, UPGRADE, UNLOAD	\
+	LOAD, RELOAD, UPGRADE, UNLOAD,	\
+	ERL_NIF_VM_VARIANT		\
     };                                  \
     ERL_NIF_INIT_BODY;                  \
     return &entry;			\
-}
+}                                       \
+ERL_NIF_INIT_EPILOGUE
+
 
 #endif /* __ERL_NIF_H__ */
 
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index ec07a976b2..c991b61abe 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,31 +21,38 @@
 #  error This file should not be included directly
 #endif
 
+/*
+** WARNING: add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list
+** to keep compatibility on Windows!!!
+**
+** And don't forget to increase ERL_NIF_MINOR_VERSION in erl_nif.h
+** when adding functions to the API.
+*/
 #ifdef ERL_NIF_API_FUNC_DECL
 ERL_NIF_API_FUNC_DECL(void*,enif_priv_data,(ErlNifEnv*));
-ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(ErlNifEnv*, size_t size));
-ERL_NIF_API_FUNC_DECL(void,enif_free,(ErlNifEnv*, void* ptr));
+ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_free,(void* ptr));
 ERL_NIF_API_FUNC_DECL(int,enif_is_atom,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int,enif_is_binary,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int,enif_is_ref,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int,enif_inspect_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, ErlNifBinary* bin));
-ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(ErlNifEnv*, unsigned size, ErlNifBinary* bin));
-ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifEnv*, ErlNifBinary* bin, unsigned size));
-ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifEnv*, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(size_t size, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifBinary* bin, size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(int,enif_get_int,(ErlNifEnv*, ERL_NIF_TERM term, int* ip));
 ERL_NIF_API_FUNC_DECL(int,enif_get_ulong,(ErlNifEnv*, ERL_NIF_TERM term, unsigned long* ip));
 ERL_NIF_API_FUNC_DECL(int,enif_get_double,(ErlNifEnv*, ERL_NIF_TERM term, double* dp));
 ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM* head, ERL_NIF_TERM* tail));
 ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array));
-ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
-ERL_NIF_API_FUNC_DECL(int,enif_compare,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
+ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
+ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ulong,(ErlNifEnv* env, unsigned long i));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_double,(ErlNifEnv* env, double d));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_atom,(ErlNifEnv* env, const char* name));
-ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom));
+ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple,(ErlNifEnv* env, unsigned cnt, ...));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list,(ErlNifEnv* env, unsigned cnt, ...));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_cell,(ErlNifEnv* env, ERL_NIF_TERM car, ERL_NIF_TERM cdr));
@@ -82,13 +89,13 @@ ERL_NIF_API_FUNC_DECL(int,enif_equal_tids,(ErlNifTid tid1, ErlNifTid tid2));
 ERL_NIF_API_FUNC_DECL(void,enif_thread_exit,(void *resp));
 ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp));
 
-ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(ErlNifEnv*, void* ptr, size_t size));
+ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size));
 ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size));
 ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...));
 ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin));
-ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, unsigned pos, unsigned size));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size));
 ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding));
-ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len));
+ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len, ErlNifCharEncoding));
 ERL_NIF_API_FUNC_DECL(int,enif_is_fun,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int,enif_is_pid,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int,enif_is_port,(ErlNifEnv*, ERL_NIF_TERM term));
@@ -99,17 +106,46 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_long,(ErlNifEnv*, long i));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt));
 ERL_NIF_API_FUNC_DECL(int,enif_is_empty_list,(ErlNifEnv*, ERL_NIF_TERM term));
-ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* type_name, void (*dtor)(ErlNifEnv*,void *), enum ErlNifResourceFlags flags, enum ErlNifResourceFlags* tried));
-ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifEnv*, ErlNifResourceType* type, unsigned size));
-ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(ErlNifEnv*, void* obj));
+ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* module_str, const char* name_str, void (*dtor)(ErlNifEnv*,void *), ErlNifResourceFlags flags, ErlNifResourceFlags* tried));
+ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifResourceType* type, size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(void* obj));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource,(ErlNifEnv*, void* obj));
 ERL_NIF_API_FUNC_DECL(int,enif_get_resource,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp));
-ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj));
+ERL_NIF_API_FUNC_DECL(size_t,enif_sizeof_resource,(void* obj));
+ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,size_t size,ERL_NIF_TERM* termp));
+ERL_NIF_API_FUNC_DECL(int,enif_is_list,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_tuple,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_get_atom_length,(ErlNifEnv*, ERL_NIF_TERM atom, unsigned* len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(int,enif_get_list_length,(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_atom_len,(ErlNifEnv* env, const char* name, size_t len));
+ERL_NIF_API_FUNC_DECL(int, enif_make_existing_atom_len,(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string_len,(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ErlNifEnv*,enif_alloc_env,(void));
+ERL_NIF_API_FUNC_DECL(void,enif_free_env,(ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(void,enif_clear_env,(ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(int,enif_send,(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_copy,(ErlNifEnv* dst_env, ERL_NIF_TERM src_term));
+ERL_NIF_API_FUNC_DECL(ErlNifPid*,enif_self,(ErlNifEnv* caller_env, ErlNifPid* pid));
+ERL_NIF_API_FUNC_DECL(int,enif_get_local_pid,(ErlNifEnv* env, ERL_NIF_TERM, ErlNifPid* pid));
+ERL_NIF_API_FUNC_DECL(void,enif_keep_resource,(void* obj));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource_binary,(ErlNifEnv*,void* obj,const void* data, size_t size));
+#if SIZEOF_LONG != 8
+ERL_NIF_API_FUNC_DECL(int,enif_get_int64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifSInt64* ip));
+ERL_NIF_API_FUNC_DECL(int,enif_get_uint64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifUInt64* ip));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64));
+#endif
+ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term));
+
 /*
-** Add last to keep compatibility on Windows!!!
+** Add new entries here to keep compatibility on Windows!!!
 */
 #endif
 
+/*
+** Please keep the ERL_NIF_API_FUNC_MACRO list below in the same order
+** as the ERL_NIF_API_FUNC_DECL list above
+*/
 #ifdef ERL_NIF_API_FUNC_MACRO
 #  define enif_priv_data ERL_NIF_API_FUNC_MACRO(enif_priv_data)
 #  define enif_alloc ERL_NIF_API_FUNC_MACRO(enif_alloc)
@@ -195,7 +231,35 @@ ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj));
 #  define enif_make_resource ERL_NIF_API_FUNC_MACRO(enif_make_resource)
 #  define enif_get_resource ERL_NIF_API_FUNC_MACRO(enif_get_resource)
 #  define enif_sizeof_resource ERL_NIF_API_FUNC_MACRO(enif_sizeof_resource)
+#  define enif_make_new_binary ERL_NIF_API_FUNC_MACRO(enif_make_new_binary)
+#  define enif_is_list ERL_NIF_API_FUNC_MACRO(enif_is_list)
+#  define enif_is_tuple ERL_NIF_API_FUNC_MACRO(enif_is_tuple)
+#  define enif_get_atom_length ERL_NIF_API_FUNC_MACRO(enif_get_atom_length)
+#  define enif_get_list_length ERL_NIF_API_FUNC_MACRO(enif_get_list_length)
+#  define enif_make_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_atom_len)
+#  define enif_make_existing_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom_len)
+#  define enif_make_string_len ERL_NIF_API_FUNC_MACRO(enif_make_string_len)
+#  define enif_alloc_env ERL_NIF_API_FUNC_MACRO(enif_alloc_env)
+#  define enif_free_env ERL_NIF_API_FUNC_MACRO(enif_free_env)
+#  define enif_clear_env ERL_NIF_API_FUNC_MACRO(enif_clear_env)
+#  define enif_send ERL_NIF_API_FUNC_MACRO(enif_send)
+#  define enif_make_copy ERL_NIF_API_FUNC_MACRO(enif_make_copy)
+#  define enif_self ERL_NIF_API_FUNC_MACRO(enif_self)
+#  define enif_get_local_pid ERL_NIF_API_FUNC_MACRO(enif_get_local_pid)
+#  define enif_keep_resource ERL_NIF_API_FUNC_MACRO(enif_keep_resource)
+#  define enif_make_resource_binary ERL_NIF_API_FUNC_MACRO(enif_make_resource_binary)
+#if  SIZEOF_LONG != 8
+#  define enif_get_int64 ERL_NIF_API_FUNC_MACRO(enif_get_int64)
+#  define enif_get_uint64 ERL_NIF_API_FUNC_MACRO(enif_get_uint64)
+#  define enif_make_int64 ERL_NIF_API_FUNC_MACRO(enif_make_int64)
+#  define enif_make_uint64 ERL_NIF_API_FUNC_MACRO(enif_make_uint64)
+#endif
+
+#  define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception)
 
+/*
+** Add new entries here
+*/
 #endif
 
 #ifndef enif_make_list1
@@ -217,9 +281,15 @@ ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj));
 #  define enif_make_tuple7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_tuple(ENV,7,E1,E2,E3,E4,E5,E6,E7)
 #  define enif_make_tuple8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_tuple(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8)
 #  define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9)
+
+#  define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid))
+
+#if SIZEOF_LONG == 8
+#  define enif_get_int64 enif_get_long
+#  define enif_get_uint64 enif_get_ulong
+#  define enif_make_int64 enif_make_long
+#  define enif_make_uint64 enif_make_ulong
 #endif
 
-#ifndef enif_get_data
-#  define enif_get_data enif_priv_data /* deprecated */
 #endif
 
diff --git a/erts/emulator/beam/erl_nmgc.c b/erts/emulator/beam/erl_nmgc.c
index 626d4e295a..d7bfb2ab12 100644
--- a/erts/emulator/beam/erl_nmgc.c
+++ b/erts/emulator/beam/erl_nmgc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -26,7 +26,6 @@
 #include "erl_nmgc.h"
 #include "erl_debug.h"
 #if HIPE
-#include "hipe_bif0.h" /* for hipe_constants_{start,next} */
 #include "hipe_stack.h"
 #endif
 
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 87dbfc2a04..2c67e781e0 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -251,23 +251,36 @@ extern int erts_use_r9_pids_ports;
  * Refs                                                                    *
 \*                                                                         */
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 
 #define internal_ref_no_of_numbers(x)					\
   (internal_ref_data((x))[0])
+#define internal_thing_ref_no_of_numbers(thing)			        \
+  (internal_thing_ref_data(thing)[0])
 #define internal_ref_numbers(x)						\
   (&internal_ref_data((x))[1])
+#define internal_thing_ref_numbers(thing)				\
+  (&internal_thing_ref_data(thing)[1])
 #define external_ref_no_of_numbers(x)					\
   (external_ref_data((x))[0])
+#define external_thing_ref_no_of_numbers(thing)			        \
+  (external_thing_ref_data(thing)[0])
 #define external_ref_numbers(x)						\
   (&external_ref_data((x))[1])
+#define external_thing_ref_numbers(thing)				\
+  (&external_thing_ref_data(thing)[1])
+
 
 #else
 
 #define internal_ref_no_of_numbers(x)	(internal_ref_data_words((x)))
+#define internal_thing_ref_no_of_numbers(t) (internal_thing_ref_data_words(t))
 #define internal_ref_numbers(x)		(internal_ref_data((x)))
+#define internal_thing_ref_numbers(t)   (internal_thing_ref_data(t))
 #define external_ref_no_of_numbers(x)	(external_ref_data_words((x)))
+#define external_thing_ref_no_of_numbers(t) (external_thing_ref_data_words((t)))
 #define external_ref_numbers(x)		(external_ref_data((x)))
+#define external_thing_ref_numbers(t)   (external_thing_ref_data((t)))
 
 #endif
 
@@ -311,6 +324,8 @@ extern int erts_use_r9_pids_ports;
 					 : external_ref_channel_no((x)))
 #define is_ref(x)			(is_internal_ref((x))		\
 					 || is_external_ref((x)))
+#define is_ref_rel(x,Base)		(is_internal_ref_rel((x),Base)	\
+					 || is_external_ref_rel((x),Base))
 #define is_not_ref(x)			(!is_ref(x))
 
 #endif
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 42b28d987c..6daa127d23 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -80,6 +80,8 @@ dist_table_alloc(void *dep_tmpl)
     Eterm chnl_nr;
     Eterm sysname;
     DistEntry *dep;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 
     if(((DistEntry *) dep_tmpl) == erts_this_dist_entry)
 	return dep_tmpl;
@@ -92,7 +94,7 @@ dist_table_alloc(void *dep_tmpl)
 
     dep->prev				= NULL;
     erts_refc_init(&dep->refc, -1);
-    erts_smp_rwmtx_init_x(&dep->rwmtx, "dist_entry", chnl_nr);
+    erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr);
     dep->sysname			= sysname;
     dep->cid				= NIL;
     dep->connection_id			= 0;
@@ -105,7 +107,7 @@ dist_table_alloc(void *dep_tmpl)
     dep->nlinks				= NULL;
     dep->monitors			= NULL;
 
-    erts_smp_spinlock_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr);
+    erts_smp_mtx_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr);
     dep->qflgs				= 0;
     dep->qsize				= 0;
     dep->out_queue.first		= NULL;
@@ -170,7 +172,7 @@ dist_table_free(void *vdep)
     ASSERT(!dep->cache);
     erts_smp_rwmtx_destroy(&dep->rwmtx);
     erts_smp_mtx_destroy(&dep->lnk_mtx);
-    erts_smp_spinlock_destroy(&dep->qlock);
+    erts_smp_mtx_destroy(&dep->qlock);
 
 #ifdef DEBUG
     sys_memset(vdep, 0x77, sizeof(DistEntry));
@@ -233,7 +235,7 @@ erts_sysname_to_connected_dist_entry(Eterm sysname)
     erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
     res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de);
     if (res_dep) {
-	long refc = erts_refc_inctest(&res_dep->refc, 1);
+	erts_aint_t refc = erts_refc_inctest(&res_dep->refc, 1);
 	if (refc < 2) /* Pending delete */
 	    erts_refc_inc(&res_dep->refc, 1);
     }
@@ -255,7 +257,7 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm sysname)
 {
     DistEntry *res;
     DistEntry de;
-    long refc;
+    erts_aint_t refc;
     res = erts_find_dist_entry(sysname);
     if (res)
 	return res;
@@ -277,7 +279,7 @@ DistEntry *erts_find_dist_entry(Eterm sysname)
     erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
     res = hash_get(&erts_dist_table, (void *) &de);
     if (res) {
-	long refc = erts_refc_inctest(&res->refc, 1);
+	erts_aint_t refc = erts_refc_inctest(&res->refc, 1);
 	if (refc < 2) /* Pending delete */
 	    erts_refc_inc(&res->refc, 1);
     }
@@ -580,11 +582,23 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation)
     ErlNode ne;
     ne.sysname = sysname;
     ne.creation = creation;
+
+    erts_smp_rwmtx_rlock(&erts_node_table_rwmtx);
+    res = hash_get(&erts_node_table, (void *) &ne);
+    if (res && res != erts_this_node) {
+	erts_aint_t refc = erts_refc_inctest(&res->refc, 0);
+	if (refc < 2) /* New or pending delete */
+	    erts_refc_inc(&res->refc, 1);
+    }
+    erts_smp_rwmtx_runlock(&erts_node_table_rwmtx);
+    if (res)
+	return res;
+
     erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx);
     res = hash_put(&erts_node_table, (void *) &ne);
     ASSERT(res);
     if (res != erts_this_node) {
-	long refc = erts_refc_inctest(&res->refc, 0);
+	erts_aint_t refc = erts_refc_inctest(&res->refc, 0);
 	if (refc < 2) /* New or pending delete */
 	    erts_refc_inc(&res->refc, 1);
     }
@@ -696,8 +710,12 @@ erts_set_this_node(Eterm sysname, Uint creation)
 
 void erts_init_node_tables(void)
 {
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     HashFunctions f;
 
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
     f.hash  = (H_FUN)			dist_table_hash;
     f.cmp   = (HCMP_FUN)		dist_table_cmp;
     f.alloc = (HALLOC_FUN)		dist_table_alloc;
@@ -719,9 +737,10 @@ void erts_init_node_tables(void)
     erts_this_dist_entry->prev				= NULL;
     erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */
 
-    erts_smp_rwmtx_init_x(&erts_this_dist_entry->rwmtx,
-			  "dist_entry",
-			  make_small(ERST_INTERNAL_CHANNEL_NO));
+    erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx,
+			      &rwmtx_opt,
+			      "dist_entry",
+			      make_small(ERST_INTERNAL_CHANNEL_NO));
     erts_this_dist_entry->sysname			= am_Noname;
     erts_this_dist_entry->cid				= NIL;
     erts_this_dist_entry->connection_id			= 0;
@@ -736,9 +755,9 @@ void erts_init_node_tables(void)
     erts_this_dist_entry->nlinks			= NULL;
     erts_this_dist_entry->monitors			= NULL;
 
-    erts_smp_spinlock_init_x(&erts_this_dist_entry->qlock,
-			     "dist_entry_out_queue",
-			     make_small(ERST_INTERNAL_CHANNEL_NO));
+    erts_smp_mtx_init_x(&erts_this_dist_entry->qlock,
+			"dist_entry_out_queue",
+			make_small(ERST_INTERNAL_CHANNEL_NO));
     erts_this_dist_entry->qflgs				= 0;
     erts_this_dist_entry->qsize				= 0;
     erts_this_dist_entry->out_queue.first		= NULL;
@@ -772,8 +791,8 @@ void erts_init_node_tables(void)
 
     (void) hash_put(&erts_node_table, (void *) erts_this_node);
 
-    erts_smp_rwmtx_init(&erts_node_table_rwmtx, "node_table");
-    erts_smp_rwmtx_init(&erts_dist_table_rwmtx, "dist_table");
+    erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table");
+    erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table");
 
     references_atoms_need_init = 1;
 }
@@ -1087,49 +1106,62 @@ insert_offheap2(ErlOffHeap *oh, void *arg)
 static void
 insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 {
-    if(oh->externals) {
-	ExternalThing *etp = oh->externals;
-	while (etp) {
-	    insert_node(etp->node, type, id);
-	    etp = etp->next;
-	}
-    }
+    union erl_off_heap_ptr u;
+    struct insert_offheap2_arg a;
+    a.type = BIN_REF;
 
-    if(oh->mso) {
-	ProcBin *pb;
-	struct insert_offheap2_arg a;
-	a.type = BIN_REF;
-	for(pb = oh->mso; pb; pb = pb->next) {
-	    if(IsMatchProgBinary(pb->val)) {
+    for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) {
+	switch (thing_subtag(u.hdr->thing_word)) {
+	case REFC_BINARY_SUBTAG:
+	    if(IsMatchProgBinary(u.pb->val)) {
 		InsertedBin *ib;
 		int insert_bin = 1;
 		for (ib = inserted_bins; ib; ib = ib->next)
-		    if(ib->bin_val == pb->val) {
+		    if(ib->bin_val == u.pb->val) {
 			insert_bin = 0;
 			break;
 		    }
 		if (insert_bin) {
-		    Uint id_heap[BIG_UINT_HEAP_SIZE];
+#if HALFWORD_HEAP
+		    UWord val = (UWord) u.pb->val;
+		    DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE*2); /* extra place allocated */
+#else
+		    DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE);
+#endif
 		    Uint *hp = &id_heap[0];
 		    InsertedBin *nib;
-		    a.id = erts_bld_uint(&hp, NULL, (Uint) pb->val);
-		    erts_match_prog_foreach_offheap(pb->val,
+#if HALFWORD_HEAP
+		    int actual_need = BIG_UWORD_HEAP_SIZE(val);
+		    ASSERT(actual_need <= (BIG_UINT_HEAP_SIZE*2));
+		    UseTmpHeapNoproc(actual_need);
+		    a.id = erts_bld_uword(&hp, NULL, (UWord) val);
+#else
+		    UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
+		    a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val);
+#endif
+		    erts_match_prog_foreach_offheap(u.pb->val,
 						    insert_offheap2,
 						    (void *) &a);
 		    nib = erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(InsertedBin));
-		    nib->bin_val = pb->val;
+		    nib->bin_val = u.pb->val;
 		    nib->next = inserted_bins;
 		    inserted_bins = nib;
+#if HALFWORD_HEAP
+		    UnUseTmpHeapNoproc(actual_need);
+#else
+		    UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
+#endif
 		}
-	    }
+	    }		
+	    break;
+	case FUN_SUBTAG:
+	    break; /* No need to */
+	default:
+	    ASSERT(is_external_header(u.hdr->thing_word));
+	    insert_node(u.ext->node, type, id);
+	    break;
 	}
     }
-
-#if 0
-    if(oh->funs) {
-	/* No need to */
-    }
-#endif
 }
 
 static void doit_insert_monitor(ErtsMonitor *monitor, void *p)
@@ -1190,12 +1222,15 @@ static void
 insert_bif_timer(Eterm receiver, Eterm msg, ErlHeapFragment *bp, void *arg)
 {
     if (bp) {
-	Eterm heap[3];
+	DeclareTmpHeapNoproc(heap,3);
+
+	UseTmpHeapNoproc(3);
 	insert_offheap(&bp->off_heap,
 		       TIMER_REF,
 		       (is_internal_pid(receiver)
 			? receiver
 			: TUPLE2(&heap[0], AM_process, receiver)));
+	UnUseTmpHeapNoproc(3);
     }
 }
 
@@ -1230,7 +1265,7 @@ setup_reference_table(void)
     DistEntry *dep;
     HashInfo hi;
     int i;
-    Eterm heap[3];
+    DeclareTmpHeapNoproc(heap,3);
 
     inserted_bins = NULL;
 
@@ -1251,6 +1286,7 @@ setup_reference_table(void)
     /* Go through the hole system, and build a table of all references
        to ErlNode and DistEntry structures */
 
+    UseTmpHeapNoproc(3);
     insert_node(erts_this_node,
 		SYSTEM_REF,
 		TUPLE2(&heap[0], AM_system, am_undefined));
@@ -1261,11 +1297,13 @@ setup_reference_table(void)
 		   HEAP_REF,
 		   TUPLE2(&heap[0], AM_processes, am_undefined));
 #endif
+    UnUseTmpHeapNoproc(3);
 
     /* Insert all processes */
     for (i = 0; i < erts_max_processes; i++)
 	if (process_tab[i]) {
 	    ErlMessage *msg;
+
 	    /* Insert Heap */
 	    insert_offheap(&(process_tab[i]->off_heap),
 			   HEAP_REF,
@@ -1352,21 +1390,22 @@ setup_reference_table(void)
 
     { /* Add binaries stored elsewhere ... */
 	ErlOffHeap oh;
-	ProcBin pb[2] = {{0},{0}};
-	ProcBin *mso = NULL;
+	ProcBin pb[2];
 	int i = 0;
 	Binary *default_match_spec;
 	Binary *default_meta_match_spec;
 
-	/* Only the ProcBin members val and next will be inspected
+	oh.first = NULL;
+	/* Only the ProcBin members thing_word, val and next will be inspected
 	   (by insert_offheap()) */
 #undef  ADD_BINARY
-#define ADD_BINARY(Bin)					\
-	if ((Bin)) {					\
-	    pb[i].val = (Bin);				\
-	    pb[i].next = mso;				\
-	    mso = &pb[i];				\
-	    i++;					\
+#define ADD_BINARY(Bin)				 	     \
+	if ((Bin)) {					     \
+	    pb[i].thing_word = REFC_BINARY_SUBTAG;           \
+	    pb[i].val = (Bin);				     \
+	    pb[i].next = oh.first;		             \
+	    oh.first = (struct erl_off_heap_header*) &pb[i]; \
+	    i++;				             \
 	}
 
 	erts_get_default_trace_pattern(NULL,
@@ -1378,11 +1417,6 @@ setup_reference_table(void)
 	ADD_BINARY(default_match_spec);
 	ADD_BINARY(default_meta_match_spec);
 
-	oh.mso = mso;
-	oh.externals = NULL;
-#ifndef HYBRID /* FIND ME! */
-	oh.funs = NULL;
-#endif
 	insert_offheap(&oh, BIN_REF, AM_match_spec);
 #undef  ADD_BINARY
     }
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index c48dac6219..b0a63ae035 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -61,7 +61,7 @@
 #define ERTS_DE_QFLGS_ALL			(ERTS_DE_QFLG_BUSY \
 						 | ERTS_DE_QFLG_EXIT)
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL)
 #else
 #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713)
@@ -131,7 +131,7 @@ typedef struct dist_entry_ {
     ErtsLink *nlinks;           /* Link tree with subtrees */
     ErtsMonitor *monitors;      /* Monitor tree */
 
-    erts_smp_spinlock_t qlock;  /* Protects qflgs and out_queue */
+    erts_smp_mtx_t qlock;       /* Protects qflgs and out_queue */
     Uint32 qflgs;
     Sint qsize;
     ErtsDistOutputQueue out_queue;
diff --git a/erts/emulator/beam/erl_obsolete.c b/erts/emulator/beam/erl_obsolete.c
deleted file mode 100644
index 9c5a7c7ff9..0000000000
--- a/erts/emulator/beam/erl_obsolete.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "sys.h"
-#include "erl_driver.h"
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- *                                                                           *
- * ------------------------- OBSOLETE! DO NOT USE! ------------------------- *
- *                                                                           *
-\*                                                                           */
-
-/* cut from ../obsolete/driver.h (since it doesn't mix well with other
- * headers from the emulator).
- */
-#ifdef __WIN32__
-#ifdef CONST
-#  undef CONST
-#endif
-#endif
-
-#if ((defined(__STDC__) || defined(SABER)) && !defined(NO_PROTOTYPE)) || defined(__cplusplus) || defined(USE_PROTOTYPE)
-#   define _USING_PROTOTYPES_ 1
-#   define _ANSI_ARGS_(x)	x
-#   define CONST const
-#else
-#   define _ANSI_ARGS_(x)	()
-#   define CONST
-#endif
-
-typedef void* erl_mutex_t;
-typedef void* erl_cond_t;
-typedef void* erl_thread_t;
-
-EXTERN erl_mutex_t erts_mutex_create _ANSI_ARGS_((void));
-EXTERN int erts_mutex_destroy _ANSI_ARGS_((erl_mutex_t));
-EXTERN int erts_mutex_lock _ANSI_ARGS_((erl_mutex_t));
-EXTERN int erts_mutex_unlock _ANSI_ARGS_((erl_mutex_t));
-
-EXTERN erl_cond_t erts_cond_create _ANSI_ARGS_((void));
-EXTERN int erts_cond_destroy _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_signal _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_broadcast _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_wait _ANSI_ARGS_((erl_cond_t, erl_mutex_t));
-EXTERN int erts_cond_timedwait _ANSI_ARGS_((erl_cond_t, erl_mutex_t, long));
-
-EXTERN int erts_thread_create _ANSI_ARGS_((erl_thread_t*,
-					 void* (*func)(void*),
-					 void* arg,
-					 int detached));
-EXTERN erl_thread_t erts_thread_self _ANSI_ARGS_((void));
-EXTERN void erts_thread_exit _ANSI_ARGS_((void*));
-EXTERN int  erts_thread_join _ANSI_ARGS_((erl_thread_t, void**));
-EXTERN int  erts_thread_kill _ANSI_ARGS_((erl_thread_t));
-
-/*
- * These functions implement the thread interface in ../obsolete/driver.h.
- * Do *not* use this interface! Within the emulator, use the erl_threads.h,
- * erl_smp.h, or ethread.h interface. From a driver use the thread interface
- * in erl_driver.h.
- */
-
-erl_mutex_t
-erts_mutex_create(void)
-{
-    return (erl_mutex_t) erl_drv_mutex_create(NULL);
-}
-
-int
-erts_mutex_destroy(erl_mutex_t mtx)
-{
-    erl_drv_mutex_destroy((ErlDrvMutex *) mtx);
-    return 0;
-}
-
-int
-erts_mutex_lock(erl_mutex_t mtx)
-{
-    erl_drv_mutex_lock((ErlDrvMutex *) mtx);
-    return 0;
-}
-
-int
-erts_mutex_unlock(erl_mutex_t mtx)
-{
-    erl_drv_mutex_unlock((ErlDrvMutex *) mtx);
-    return 0;
-}
-
-erl_cond_t
-erts_cond_create(void)
-{
-    return (erl_cond_t) erl_drv_cond_create(NULL);
-}
-
-int
-erts_cond_destroy(erl_cond_t cnd)
-{
-    erl_drv_cond_destroy((ErlDrvCond *) cnd);
-    return 0;
-}
-
-
-int
-erts_cond_signal(erl_cond_t cnd)
-{
-    erl_drv_cond_signal((ErlDrvCond *) cnd);
-    return 0;
-}
-
-int
-erts_cond_broadcast(erl_cond_t cnd)
-{
-    erl_drv_cond_broadcast((ErlDrvCond *) cnd);
-    return 0;
-}
-
-
-int
-erts_cond_wait(erl_cond_t cnd, erl_mutex_t mtx)
-{
-    erl_drv_cond_wait((ErlDrvCond *) cnd, (ErlDrvMutex *) mtx);
-    return 0;
-}
-
-int
-erts_cond_timedwait(erl_cond_t cnd, erl_mutex_t mtx, long ms)
-{
-    return ENOTSUP;
-}
-
-int
-erts_thread_create(erl_thread_t *tid,
-		   void* (*func)(void*),
-		   void* arg,
-		   int detached)
-{
-    if (detached)
-	return ENOTSUP;
-    return erl_drv_thread_create(NULL, (ErlDrvTid *) tid, func, arg, NULL);
-}
-
-erl_thread_t
-erts_thread_self(void)
-{
-    return (erl_thread_t) erl_drv_thread_self();
-}
-
-void
-erts_thread_exit(void *res)
-{
-    erl_drv_thread_exit(res);
-}
-
-int
-erts_thread_join(erl_thread_t tid, void **respp)
-{
-    return erl_drv_thread_join((ErlDrvTid) tid, respp);
-}
-
-int
-erts_thread_kill(erl_thread_t tid)
-{
-    return ENOTSUP;
-}
-
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 0b6bb0d8e9..e6b55c45e4 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -129,7 +129,7 @@ reset_handle(ErtsPortTask *ptp)
 {
     if (ptp->handle) {
 	ASSERT(ptp == handle2task(ptp->handle));
-	erts_smp_atomic_set(ptp->handle, (long) NULL);
+	erts_smp_atomic_set(ptp->handle, (erts_aint_t) NULL);
     }
 }
 
@@ -138,7 +138,7 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp)
 {
     ptp->handle = pthp;
     if (pthp) {
-	erts_smp_atomic_set(pthp, (long) ptp);
+	erts_smp_atomic_set(pthp, (erts_aint_t) ptp);
 	ASSERT(ptp == handle2task(ptp->handle));
     }
 }
@@ -568,7 +568,7 @@ erts_port_task_schedule(Eterm id,
 	ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
 	if (xrunq) {
 	    /* Port emigrated ... */
-	    erts_smp_atomic_set(&pp->run_queue, (long) xrunq);
+	    erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq);
 	    erts_smp_runq_unlock(runq);
 	    runq = xrunq;
 	}
@@ -625,6 +625,7 @@ erts_port_task_schedule(Eterm id,
 
     if (!enq_port) {
 	ERTS_PT_CHK_PRES_PORTQ(runq, pp);
+	erts_smp_runq_unlock(runq);
     }
     else {
 	enqueue_port(runq, pp);
@@ -634,9 +635,10 @@ erts_port_task_schedule(Eterm id,
 	    profile_runnable_port(pp, am_active);
 	}
 
+	erts_smp_runq_unlock(runq);
+
 	erts_smp_notify_inc_runq(runq);
     }
-    erts_smp_runq_unlock(runq);
     return 0;
 }
 
@@ -656,8 +658,6 @@ erts_port_task_free_port(Port *pp)
 	   when scheduled out... */
 	ErtsPortTask *ptp = port_task_alloc();
 	erts_smp_port_state_lock(pp);
-	ASSERT(erts_smp_atomic_read(&erts_ports_alive) > 0);
-	erts_smp_atomic_dec(&erts_ports_alive);
 	pp->status &= ~ERTS_PORT_SFLG_CLOSING;
 	pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
 	erts_may_save_closed_port(pp);
@@ -679,7 +679,6 @@ erts_port_task_free_port(Port *pp)
 	    port_is_dequeued = 1;
 	}
 	erts_smp_port_state_lock(pp);
-	erts_smp_atomic_dec(&erts_ports_alive);
 	pp->status &= ~ERTS_PORT_SFLG_CLOSING;
 	pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
 	erts_may_save_closed_port(pp);
@@ -725,7 +724,8 @@ resume_after_block(void *vd)
     ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd;
     erts_smp_runq_lock(d->runq);
     if (d->resp)
-	*d->resp = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0;
+	*d->resp = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks)
+		    != (erts_aint_t) 0);
 }
 
 /*
@@ -746,7 +746,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     ErtsPortTask *ptp;
     int res = 0;
     int reds = ERTS_PORT_REDS_EXECUTE;
-    long io_tasks_executed = 0;
+    erts_aint_t io_tasks_executed = 0;
     int fpe_was_unmasked;
     ErtsPortTaskExeBlockData blk_data = {runq, NULL};
 
@@ -940,18 +940,19 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	}
 	else {
 	    /* Port emigrated ... */
-	    erts_smp_atomic_set(&pp->run_queue, (long) xrunq);
+	    erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq);
 	    enqueue_port(xrunq, pp);
 	    ASSERT(pp->sched.exe_taskq);
 	    pp->sched.exe_taskq = NULL;
-	    erts_smp_notify_inc_runq(xrunq);
 	    erts_smp_runq_unlock(xrunq);
+	    erts_smp_notify_inc_runq(xrunq);
 	}
 #endif
 	port_was_enqueued = 1;
     }
 
-    res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0;
+    res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks)
+	   != (erts_aint_t) 0);
 
     ERTS_PT_CHK_PRES_PORTQ(runq, pp);
 
@@ -969,15 +970,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     erts_port_release(pp);
 #else
     {
-	long refc = erts_smp_atomic_dectest(&pp->refc);
+	erts_aint_t refc;
+	erts_smp_mtx_unlock(pp->lock);
+	refc = erts_smp_atomic_dectest(&pp->refc);
 	ASSERT(refc >= 0);
-	if (refc > 0)
-	    erts_smp_mtx_unlock(pp->lock);
-	else {
+	if (refc == 0) {
 	    erts_smp_runq_unlock(runq);
 	    erts_port_cleanup(pp); /* Might aquire runq lock */
 	    erts_smp_runq_lock(runq);
-	    res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0;
+	    res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks)
+		   != (erts_aint_t) 0);
 	}
     }
 #endif
@@ -1110,9 +1112,8 @@ erts_port_migrate(Port *prt, int *prt_locked,
     if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt))
 	return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ;
     dequeue_port(from_rq, prt);
-    erts_smp_atomic_set(&prt->run_queue, (long) to_rq);
+    erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) to_rq);
     enqueue_port(to_rq, prt);
-    erts_smp_notify_inc_runq(to_rq);
     return ERTS_MIGRATE_SUCCESS;
 }
 
@@ -1124,7 +1125,7 @@ erts_port_migrate(Port *prt, int *prt_locked,
 void
 erts_port_task_init(void)
 {
-    erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (long) 0);
+    erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (erts_aint_t) 0);
     init_port_task_alloc();
     init_port_taskq_alloc();
 }
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index f12d02da0c..3e2c5f07ab 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -79,7 +79,7 @@ ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void);
 ERTS_GLB_INLINE void
 erts_port_task_handle_init(ErtsPortTaskHandle *pthp)
 {
-    erts_smp_atomic_init(pthp, (long) NULL);
+    erts_smp_atomic_init(pthp, (erts_aint_t) NULL);
 }
 
 ERTS_GLB_INLINE int
@@ -102,6 +102,7 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp)
 ERTS_GLB_INLINE int
 erts_port_task_have_outstanding_io_tasks(void)
 {
+    ERTS_THR_MEMORY_BARRIER;
     return erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != 0;
 }
 
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 7fe3f3bca5..34da9cab84 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -114,13 +114,13 @@ do {									\
 /* return 0 if list is not a non-empty flat list of printable characters */
 
 static int
-is_printable_string(Eterm list)
+is_printable_string(Eterm list, Eterm* base)
 {
     int len = 0;
     int c;
 
     while(is_list(list)) {
-	Eterm* consp = list_val(list);
+	Eterm* consp = list_val_rel(list, base);
 	Eterm hd = CAR(consp);
 
 	if (!is_byte(hd))
@@ -216,240 +216,289 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount)
 }
 
 
+#define PRT_BAR ((Eterm) 0)
+#define PRT_COMMA ((Eterm) 1)
+#define PRT_CLOSE_LIST ((Eterm) 2)
+#define PRT_CLOSE_TUPLE ((Eterm) 3)
+#define PRT_TERM ((Eterm) 4)
+#define PRT_ONE_CONS ((Eterm) 5)
+#define PRT_PATCH_FUN_SIZE ((Eterm) 6)
+#define PRT_LAST_ARRAY_ELEMENT ((Eterm) 7) /* Note! Must be last... */
 
 static int
-print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount)
+print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
+	   Eterm* obj_base) /* ignored if !HALFWORD_HEAP */
 {
+    DECLARE_WSTACK(s);
     int res;
     int i;
+    Eterm val;
     Uint32 *ref_num;
+    union {
+	UWord word;
+	Eterm* ptr;
+    }popped;
     Eterm* nobj;
+    Wterm wobj;
 
     res = 0;
 
-    if ((*dcount)-- <= 0)
-	return res;
+    goto L_jump_start;
+
+ L_outer_loop:
+    while (!WSTACK_ISEMPTY(s)) {
+	switch (val = WSTACK_POP(s)) {
+	case PRT_COMMA:
+	    PRINT_CHAR(res, fn, arg, ',');
+	    goto L_outer_loop;
+	case PRT_BAR:
+	    PRINT_CHAR(res, fn, arg, '|');
+	    goto L_outer_loop;
+	case PRT_CLOSE_LIST:
+	    PRINT_CHAR(res, fn, arg, ']');
+	    goto L_outer_loop;
+	case PRT_CLOSE_TUPLE:
+	    PRINT_CHAR(res, fn, arg, '}');
+	    goto L_outer_loop;
+	default:
+	    popped.word = WSTACK_POP(s);
 
-#ifdef HYBRID___NOT_ACTIVE
-    /* Color coded output based on memory location */
-    if(ptr_val(obj) >= global_heap && ptr_val(obj) < global_hend)
-        PRINT_STRING(res, fn, arg, "\033[32m");
-#ifdef INCREMENTAL
-    else if(ptr_val(obj) >= inc_fromspc && ptr_val(obj) < inc_fromend)
-        PRINT_STRING(res, fn, arg, "\033[33m");
-#endif
-    else if(IS_CONST(obj))
-        PRINT_STRING(res, fn, arg, "\033[34m");
-    else
-        PRINT_STRING(res, fn, arg, "\033[31m");
-#endif
+	    switch (val) {
+	    case PRT_TERM:
+		obj = (Eterm) popped.word;
+		break;
+	    case PRT_ONE_CONS:
+		obj = (Eterm) popped.word;
+	    L_print_one_cons:
+		{
+		    Eterm* cons = list_val_rel(obj, obj_base);
+		    Eterm tl;
+		    
+		    obj = CAR(cons);
+		    tl = CDR(cons);
+		    if (is_not_nil(tl)) {
+			if (is_list(tl)) {
+			    WSTACK_PUSH(s, tl);
+			    WSTACK_PUSH(s, PRT_ONE_CONS);
+			    WSTACK_PUSH(s, PRT_COMMA);
+			} else {
+			    WSTACK_PUSH(s, tl);
+			    WSTACK_PUSH(s, PRT_TERM);
+			    WSTACK_PUSH(s, PRT_BAR);
+			}
+		    }
+		}
+		break;
+	    case PRT_LAST_ARRAY_ELEMENT:
+		obj = *popped.ptr;
+		break;
+	    default:		/* PRT_LAST_ARRAY_ELEMENT+1 and upwards */
+		obj = *popped.ptr;
+	        WSTACK_PUSH(s, (UWord) (popped.ptr + 1));
+		WSTACK_PUSH(s, val-1);
+		WSTACK_PUSH(s, PRT_COMMA);
+		break;
+	    }
+	    break;
+	}
 
-    if (is_CP(obj)) {
-	PRINT_STRING(res, fn, arg, "<cp/header:");
-	PRINT_POINTER(res, fn, arg, obj);
-	PRINT_CHAR(res, fn, arg, '>');
-	return res;
-    }
+    L_jump_start:
 
-    switch (tag_val_def(obj)) {
-    case NIL_DEF:
-	PRINT_STRING(res, fn, arg, "[]");
-	break;
-    case ATOM_DEF: {
-	int tres = print_atom_name(fn, arg, obj, dcount);
-	if (tres < 0)
-	    return tres;
-	res += tres;
-	if (*dcount <= 0)
-	    return res;
-	break;
-    }
-    case SMALL_DEF:
-	PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj));
-	break;
-    case BIG_DEF: {
-	int print_res;
-	char def_buf[64];
-	char *buf, *big_str;
-	Uint sz = (Uint) big_decimal_estimate(obj);
-	sz++;
-	if (sz <= 64)
-	    buf = &def_buf[0];
-	else
-	    buf = erts_alloc(ERTS_ALC_T_TMP, sz);
-	big_str = erts_big_to_string(obj, buf, sz);
-	print_res = erts_printf_string(fn, arg, big_str);
-	if (buf != &def_buf[0])
-	    erts_free(ERTS_ALC_T_TMP, (void *) buf);
-	if (print_res < 0)
-	    return print_res;
-	res += print_res;
-	break;
-    }
-    case REF_DEF:
-    case EXTERNAL_REF_DEF:
-	PRINT_STRING(res, fn, arg, "#Ref<");
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) ref_channel_no(obj));
-	ref_num = ref_numbers(obj);
-	for (i = ref_no_of_numbers(obj)-1; i >= 0; i--) {
-	    PRINT_CHAR(res, fn, arg, '.');
-	    PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]);
+	if ((*dcount)-- <= 0)
+	    goto L_done;
+
+	if (is_CP(obj)) {
+	    PRINT_STRING(res, fn, arg, "<cp/header:");
+	    PRINT_POINTER(res, fn, arg, cp_val(obj));
+	    PRINT_CHAR(res, fn, arg, '>');
+	    goto L_done;
 	}
-	PRINT_CHAR(res, fn, arg, '>');
-	break;
-    case PID_DEF:
-    case EXTERNAL_PID_DEF:
-	PRINT_CHAR(res, fn, arg, '<');
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) pid_channel_no(obj));
-	PRINT_CHAR(res, fn, arg, '.');
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) pid_number(obj));
-	PRINT_CHAR(res, fn, arg, '.');
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) pid_serial(obj));
-	PRINT_CHAR(res, fn, arg, '>');
-	break;
-    case PORT_DEF:
-    case EXTERNAL_PORT_DEF:
-	PRINT_STRING(res, fn, arg, "#Port<");
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) port_channel_no(obj));
-	PRINT_CHAR(res, fn, arg, '.');
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) port_number(obj));
-	PRINT_CHAR(res, fn, arg, '>');
-	break;
-    case LIST_DEF:
-	if (is_printable_string(obj)) {
-	    int c;
-	    PRINT_CHAR(res, fn, arg, '"');
-	    nobj = list_val(obj);
-	    while (1) {
-		if ((*dcount)-- <= 0)
-		    return res;
-		c = signed_val(*nobj++);
-		if (c == '\n')
-		    PRINT_STRING(res, fn, arg, "\\n");
-		else {
-		    if (c == '"')
-			PRINT_CHAR(res, fn, arg, '\\');
-		    PRINT_CHAR(res, fn, arg, (char) c);
-		}
-		if (is_not_list(*nobj))
-		    break;
-		nobj = list_val(*nobj);
-	    }
-	    PRINT_CHAR(res, fn, arg, '"');
-	} else {
-	    PRINT_CHAR(res, fn, arg, '[');
-	    nobj = list_val(obj);
-	    while (1) {
-		int tres = print_term(fn, arg, *nobj++, dcount);
-		if (tres < 0)
-		    return tres;
-		res += tres;
-		if (*dcount <= 0)
-		    return res;
-		if (is_not_list(*nobj))
-		    break;
-		PRINT_CHAR(res, fn, arg, ',');
-		nobj = list_val(*nobj);
-	    }
-	    if (is_not_nil(*nobj)) {
-		int tres;
-		PRINT_CHAR(res, fn, arg, '|');
-		tres = print_term(fn, arg, *nobj, dcount);
-		if (tres < 0)
-		    return tres;
-		res += tres;
-		if (*dcount <= 0)
-		    return res;
+#if HALFWORD_HEAP
+	wobj = is_immed(obj) ? (Wterm)obj : rterm2wterm(obj, obj_base);
+#else
+	wobj = (Wterm)obj;
+#endif
+	switch (tag_val_def(wobj)) {
+	case NIL_DEF:
+	    PRINT_STRING(res, fn, arg, "[]");
+	    break;
+	case ATOM_DEF: {
+	    int tres = print_atom_name(fn, arg, obj, dcount);
+	    if (tres < 0) {
+		res = tres;
+		goto L_done;
 	    }
-	    PRINT_CHAR(res, fn, arg, ']');
-	}
-	break;
-    case TUPLE_DEF:
-	nobj = tuple_val(obj);	/* pointer to arity */
-	i = arityval(*nobj);	/* arity */
-	PRINT_CHAR(res, fn, arg, '{');
-	while (i--) {
-	    int tres = print_term(fn, arg, *++nobj, dcount);
-	    if (tres < 0)
-		return tres;
 	    res += tres;
 	    if (*dcount <= 0)
-		return res;
-	    if (i >= 1)
-		PRINT_CHAR(res, fn, arg, ',');
-	}
-	PRINT_CHAR(res, fn, arg, '}');
-	break;
-    case FLOAT_DEF: {
-	    FloatDef ff;
-	    GET_DOUBLE(obj, ff);
-	    PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd);
+		goto L_done;
+	    break;
 	}
-	break;
-    case BINARY_DEF:
-	{
-	    ProcBin* pb = (ProcBin *) binary_val(obj);
-	    if (pb->size == 1)
-		PRINT_STRING(res, fn, arg, "<<1 byte>>");
-	    else {
-		PRINT_STRING(res, fn, arg, "<<");
-		PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size);
-		PRINT_STRING(res, fn, arg, " bytes>>");
+	case SMALL_DEF:
+	    PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj));
+	    break;
+	case BIG_DEF: {
+	    int print_res;
+	    char def_buf[64];
+	    char *buf, *big_str;
+	    Uint sz = (Uint) big_decimal_estimate(wobj);
+	    sz++;
+	    if (sz <= 64)
+		buf = &def_buf[0];
+	    else
+		buf = erts_alloc(ERTS_ALC_T_TMP, sz);
+	    big_str = erts_big_to_string(wobj, buf, sz);
+	    print_res = erts_printf_string(fn, arg, big_str);
+	    if (buf != &def_buf[0])
+		erts_free(ERTS_ALC_T_TMP, (void *) buf);
+	    if (print_res < 0) {
+		res = print_res;
+		goto L_done;
 	    }
+	    res += print_res;
+	    break;
 	}
-	break;
-    case EXPORT_DEF:
-	{
-	    Export* ep = (Export *) (export_val(obj))[1];
-	    Atom* module = atom_tab(atom_val(ep->code[0]));
-	    Atom* name = atom_tab(atom_val(ep->code[1]));
-
-	    PRINT_STRING(res, fn, arg, "#Fun<");
-	    PRINT_BUF(res, fn, arg, module->name, module->len);
+	case REF_DEF:
+	case EXTERNAL_REF_DEF:
+	    PRINT_STRING(res, fn, arg, "#Ref<");
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) ref_channel_no(wobj));
+	    ref_num = ref_numbers(wobj);
+	    for (i = ref_no_of_numbers(wobj)-1; i >= 0; i--) {
+		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]);
+	    }
+	    PRINT_CHAR(res, fn, arg, '>');
+	    break;
+	case PID_DEF:
+	case EXTERNAL_PID_DEF:
+	    PRINT_CHAR(res, fn, arg, '<');
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) pid_channel_no(wobj));
 	    PRINT_CHAR(res, fn, arg, '.');
-	    PRINT_BUF(res, fn, arg, name->name, name->len);
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) pid_number(wobj));
 	    PRINT_CHAR(res, fn, arg, '.');
-	    PRINT_SLONG(res, fn, arg, 'd', 0, 1,
-			(signed long) ep->code[2]);
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) pid_serial(wobj));
 	    PRINT_CHAR(res, fn, arg, '>');
-	}
-	break;
-    case FUN_DEF:
-	{
-	    ErlFunThing *funp = (ErlFunThing *) fun_val(obj);
-	    Atom *ap = atom_tab(atom_val(funp->fe->module));
-
-	    PRINT_STRING(res, fn, arg, "#Fun<");
-	    PRINT_BUF(res, fn, arg, ap->name, ap->len);
-	    PRINT_CHAR(res, fn, arg, '.');
-	    PRINT_SLONG(res, fn, arg, 'd', 0, 1,
-			(signed long) funp->fe->old_index);
+	    break;
+	case PORT_DEF:
+	case EXTERNAL_PORT_DEF:
+	    PRINT_STRING(res, fn, arg, "#Port<");
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) port_channel_no(wobj));
 	    PRINT_CHAR(res, fn, arg, '.');
-	    PRINT_SLONG(res, fn, arg, 'd', 0, 1,
-			(signed long) funp->fe->old_uniq);
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) port_number(wobj));
+	    PRINT_CHAR(res, fn, arg, '>');
+	    break;
+	case LIST_DEF:
+	    if (is_printable_string(obj, obj_base)) {
+		int c;
+		PRINT_CHAR(res, fn, arg, '"');
+		nobj = list_val_rel(obj, obj_base);
+		while (1) {
+		    if ((*dcount)-- <= 0)
+			goto L_done;
+		    c = signed_val(*nobj++);
+		    if (c == '\n')
+			PRINT_STRING(res, fn, arg, "\\n");
+		    else {
+			if (c == '"')
+			    PRINT_CHAR(res, fn, arg, '\\');
+			PRINT_CHAR(res, fn, arg, (char) c);
+		    }
+		    if (is_not_list(*nobj))
+			break;
+		    nobj = list_val_rel(*nobj, obj_base);
+		}
+		PRINT_CHAR(res, fn, arg, '"');
+	    } else {
+		PRINT_CHAR(res, fn, arg, '[');
+		WSTACK_PUSH(s,PRT_CLOSE_LIST);
+		goto L_print_one_cons;
+	    }
+	    break;
+	case TUPLE_DEF:
+	    nobj = tuple_val(wobj);	/* pointer to arity */
+	    i = arityval(*nobj);	/* arity */
+	    PRINT_CHAR(res, fn, arg, '{');
+	    WSTACK_PUSH(s,PRT_CLOSE_TUPLE);
+	    ++nobj;
+	    if (i > 0) {
+		WSTACK_PUSH(s, (UWord) nobj);
+		WSTACK_PUSH(s, PRT_LAST_ARRAY_ELEMENT+i-1);
+	    }
+	    break;
+	case FLOAT_DEF: {
+	    FloatDef ff;
+	    GET_DOUBLE(wobj, ff);
+	    PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd);
+	}
+	    break;
+	case BINARY_DEF:
+	    {
+		ProcBin* pb = (ProcBin *) binary_val(wobj);
+		if (pb->size == 1)
+		    PRINT_STRING(res, fn, arg, "<<1 byte>>");
+		else {
+		    PRINT_STRING(res, fn, arg, "<<");
+		    PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size);
+		    PRINT_STRING(res, fn, arg, " bytes>>");
+		}
+	    }
+	    break;
+	case EXPORT_DEF:
+	    {
+		Export* ep = *((Export **) (export_val(wobj) + 1));
+		Atom* module = atom_tab(atom_val(ep->code[0]));
+		Atom* name = atom_tab(atom_val(ep->code[1]));
+
+		PRINT_STRING(res, fn, arg, "#Fun<");
+		PRINT_BUF(res, fn, arg, module->name, module->len);
+		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_BUF(res, fn, arg, name->name, name->len);
+		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_SLONG(res, fn, arg, 'd', 0, 1,
+			    (signed long) ep->code[2]);
+		PRINT_CHAR(res, fn, arg, '>');
+	    }
+	    break;
+	case FUN_DEF:
+	    {
+		ErlFunThing *funp = (ErlFunThing *) fun_val(wobj);
+		Atom *ap = atom_tab(atom_val(funp->fe->module));
+
+		PRINT_STRING(res, fn, arg, "#Fun<");
+		PRINT_BUF(res, fn, arg, ap->name, ap->len);
+		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_SLONG(res, fn, arg, 'd', 0, 1,
+			    (signed long) funp->fe->old_index);
+		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_SLONG(res, fn, arg, 'd', 0, 1,
+			    (signed long) funp->fe->old_uniq);
+		PRINT_CHAR(res, fn, arg, '>');
+	    }
+	    break;
+	default:
+	    PRINT_STRING(res, fn, arg, "<unknown:");
+	    PRINT_POINTER(res, fn, arg, wobj);
 	    PRINT_CHAR(res, fn, arg, '>');
+	    break;
 	}
-	break;
-    default:
-	PRINT_STRING(res, fn, arg, "<unknown:");
-	PRINT_POINTER(res, fn, arg, obj);
-	PRINT_CHAR(res, fn, arg, '>');
-	break;
     }
 
+ L_done:
+    
+    DESTROY_WSTACK(s);
     return res;
 }
 
 int
-erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision)
+erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision,
+		 unsigned long* term_base)
 {
-    int res = print_term(fn, arg, (Uint) term, &precision);
+    int res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base);
     if (res < 0)
 	return res;
     if (precision <= 0)
diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h
index 4f76028396..a48a3de34c 100644
--- a/erts/emulator/beam/erl_printf_term.h
+++ b/erts/emulator/beam/erl_printf_term.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,6 +21,6 @@
 #define ERL_PRINTF_TERM_H__
 
 #include "erl_printf_format.h"
-int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision);
-
+int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision,
+		     unsigned long* term_base);
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 996806fc75..2704359a8f 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,7 +24,6 @@
 #endif
 
 #include <stddef.h> /* offsetof() */
-#include <ctype.h>
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
@@ -38,6 +37,8 @@
 #include "erl_instrument.h"
 #include "erl_threads.h"
 #include "erl_binary.h"
+#include "beam_bp.h"
+#include "erl_cpu_topology.h"
 
 #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS)
 #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \
@@ -45,14 +46,23 @@
 
 #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10)
 
-#define ERTS_SCHED_SLEEP_SPINCOUNT 10000
+#define ERTS_SCHED_SPIN_UNTIL_YIELD 100
+
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT 10
+#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT 1000
+#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT \
+  (ERTS_SCHED_SYS_SLEEP_SPINCOUNT*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT)
+#define ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT 0
+
+#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH (200*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_HIGH (50*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM (10*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_LOW (CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW (CONTEXT_REDS/10)
 
-#define ERTS_WAKEUP_OTHER_LIMIT (100*CONTEXT_REDS/2)
 #define ERTS_WAKEUP_OTHER_DEC 10
 #define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10)
 
-#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff)
-
 #if 0 || defined(DEBUG)
 #define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA
 #endif
@@ -91,9 +101,9 @@ do {								\
 #define ERTS_EMPTY_RUNQ(RQ) \
   ((RQ)->len == 0 && (RQ)->misc.start == NULL)
 
-extern Eterm beam_apply[];
-extern Eterm beam_exit[];
-extern Eterm beam_continue_exit[];
+extern BeamInstr beam_apply[];
+extern BeamInstr beam_exit[];
+extern BeamInstr beam_continue_exit[];
 
 static Sint p_last;
 static Sint p_next;
@@ -105,6 +115,8 @@ Uint erts_no_schedulers;
 Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES;
 Uint erts_process_tab_index_mask;
 
+static int wakeup_other_limit;
+
 int erts_sched_thread_suggested_stack_size = -1;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -115,19 +127,38 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
 
 int erts_disable_proc_not_running_opt;
 
-#define ERTS_SCHED_CHANGING_ONLINE 1
-#define ERTS_SCHED_CHANGING_MULTI_SCHED 2
+#define ERTS_SCHDLR_SSPND_CHNG_WAITER		(((erts_aint32_t) 1) << 0)
+#define ERTS_SCHDLR_SSPND_CHNG_MSB		(((erts_aint32_t) 1) << 1)
+#define ERTS_SCHDLR_SSPND_CHNG_ONLN		(((erts_aint32_t) 1) << 2)
+
+#ifndef DEBUG
+
+#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \
+  erts_smp_atomic32_set(&schdlr_sspnd.changing, (VAL))
+
+#else
+
+#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL)			\
+do {									\
+    erts_aint32_t old_val__;						\
+    old_val__ = erts_smp_atomic32_xchg(&schdlr_sspnd.changing,		\
+				       (VAL));				\
+    ASSERT(old_val__ == (OLD_VAL));					\
+} while (0)
+
+#endif
+
 
 static struct {
     erts_smp_mtx_t mtx;
     erts_smp_cnd_t cnd;
-    int changing;
     int online;
     int curr_online;
     int wait_curr_online;
-    erts_smp_atomic_t active;
+    erts_smp_atomic32_t changing;
+    erts_smp_atomic32_t active;
     struct {
-	erts_smp_atomic_t ongoing;
+	erts_smp_atomic32_t ongoing;
 	long wait_active;
 	ErtsProcList *procs;
     } msb; /* Multi Scheduling Block */
@@ -135,11 +166,10 @@ static struct {
 
 static struct {
     erts_smp_mtx_t update_mtx;
-    erts_smp_atomic_t active_runqs;
+    erts_smp_atomic32_t no_runqs;
     int last_active_runqs;
-    erts_smp_atomic_t used_runqs;
     int forced_check_balance;
-    erts_smp_atomic_t checking_balance;
+    erts_smp_atomic32_t checking_balance;
     int halftime;
     int full_reds_history_index;
     struct {
@@ -159,44 +189,6 @@ do {							\
 
 #endif
 
-/*
- * Cpu topology hierarchy.
- */
-#define ERTS_TOPOLOGY_NODE		0
-#define ERTS_TOPOLOGY_PROCESSOR		1
-#define ERTS_TOPOLOGY_PROCESSOR_NODE	2
-#define ERTS_TOPOLOGY_CORE		3
-#define ERTS_TOPOLOGY_THREAD		4
-#define ERTS_TOPOLOGY_LOGICAL		5
-
-#define ERTS_TOPOLOGY_MAX_DEPTH		6
-
-typedef struct {
-    int bind_id;
-    int bound_id;
-} ErtsCpuBindData;
-
-static ErtsCpuBindData *scheduler2cpu_map;
-erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
-
-typedef enum {
-    ERTS_CPU_BIND_SPREAD,
-    ERTS_CPU_BIND_PROCESSOR_SPREAD,
-    ERTS_CPU_BIND_THREAD_SPREAD,
-    ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD,
-    ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD,
-    ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD,
-    ERTS_CPU_BIND_NO_SPREAD,
-    ERTS_CPU_BIND_NONE
-} ErtsCpuBindOrder;
-
-ErtsCpuBindOrder cpu_bind_order;
-
-static erts_cpu_topology_t *user_cpudata;
-static int user_cpudata_size;
-static erts_cpu_topology_t *system_cpudata;
-static int system_cpudata_size;
-
 erts_sched_stat_t erts_sched_stat;
 
 ErtsRunQueue *erts_common_run_queue;
@@ -207,11 +199,11 @@ static erts_tsd_key_t sched_data_key;
 
 static erts_smp_mtx_t proc_tab_mtx;
 
-static erts_smp_atomic_t function_calls;
+static erts_smp_atomic32_t function_calls;
 
 #ifdef ERTS_SMP
-static erts_smp_atomic_t doing_sys_schedule;
-static erts_smp_atomic_t no_empty_run_queues;
+static erts_smp_atomic32_t doing_sys_schedule;
+static erts_smp_atomic32_t no_empty_run_queues;
 #else /* !ERTS_SMP */
 ErtsSchedulerData *erts_scheduler_data;
 #endif
@@ -219,12 +211,18 @@ ErtsSchedulerData *erts_scheduler_data;
 ErtsAlignedRunQueue *erts_aligned_run_queues;
 Uint erts_no_run_queues;
 
+ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
+
+#ifdef ERTS_SMP
+
 typedef union {
-    ErtsSchedulerData esd;
-    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
-} ErtsAlignedSchedulerData;
+    ErtsSchedulerSleepInfo ssi;
+    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))];
+} ErtsAlignedSchedulerSleepInfo;
 
-ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
+static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info;
+
+#endif
 
 #ifndef BM_COUNTERS
 static int processes_busy;
@@ -249,7 +247,10 @@ Uint erts_num_active_procs;
 Process** erts_active_procs;
 #endif
 
-static erts_smp_atomic_t process_count;
+#if ERTS_MAX_PROCESSES > 0x7fffffff
+#error "Need to store process_count in another type"
+#endif
+static erts_smp_atomic32_t process_count;
 
 typedef struct ErtsTermProcElement_ ErtsTermProcElement;
 struct ErtsTermProcElement_ {
@@ -283,8 +284,9 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist,
 				 200,
 				 ERTS_ALC_T_PROC_LIST)
 
-#define ERTS_RUNQ_IX(IX)	(&erts_aligned_run_queues[(IX)].runq)
-#define ERTS_SCHEDULER_IX(IX)	(&erts_aligned_scheduler_data[(IX)].esd)
+#define ERTS_SCHED_SLEEP_INFO_IX(IX)					\
+  (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers),			\
+   &aligned_sched_sleep_info[(IX)].ssi)
 
 #define ERTS_FOREACH_RUNQ(RQVAR, DO)					\
 do {									\
@@ -334,23 +336,14 @@ do {									\
 static void init_processes_bif(void);
 static void save_terminating_process(Process *p);
 static void exec_misc_ops(ErtsRunQueue *);
-static void print_function_from_pc(int to, void *to_arg, Eterm* x);
+static void print_function_from_pc(int to, void *to_arg, BeamInstr* x);
 static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp,
 			      int yreg);
 #ifdef ERTS_SMP
 static void handle_pending_exiters(ErtsProcList *);
 
-static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
-				int size,
-				ErtsCpuBindOrder bind_order,
-				int mk_seq);
-static void signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size);
-
 #endif
 
-static void early_cpu_bind_init(void);
-static void late_cpu_bind_init(void);
-
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int
 erts_smp_lc_runq_is_locked(ErtsRunQueue *runq)
@@ -388,7 +381,12 @@ erts_pre_init_process(void)
      erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks
 	 = ERTS_PSD_DIST_ENTRY_GET_LOCKS;
      erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks
-	 = ERTS_PSD_DIST_ENTRY_GET_LOCKS;
+	 = ERTS_PSD_DIST_ENTRY_SET_LOCKS;
+
+     erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks
+	 = ERTS_PSD_CALL_TIME_BP_GET_LOCKS;
+     erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks
+	 = ERTS_PSD_CALL_TIME_BP_SET_LOCKS;
 
      /* Check that we have locks for all entries */
      for (ix = 0; ix < ERTS_PSD_SIZE; ix++) {
@@ -401,18 +399,18 @@ erts_pre_init_process(void)
 
 /* initialize the scheduler */
 void
-erts_init_process(void)
+erts_init_process(int ncpu)
 {
     Uint proc_bits = ERTS_PROC_BITS;
 
 #ifdef ERTS_SMP
     erts_disable_proc_not_running_opt = 0;
-    erts_init_proc_lock();
+    erts_init_proc_lock(ncpu);
 #endif
 
     init_proclist_alloc();
 
-    erts_smp_atomic_init(&process_count, 0);
+    erts_smp_atomic32_init(&process_count, 0);
 
     if (erts_use_r9_pids_ports) {
 	proc_bits = ERTS_R9_PROC_BITS;
@@ -572,6 +570,194 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data)
 
 #ifdef ERTS_SMP
 
+void
+erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags)
+{
+    switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) {
+    case ERTS_SSI_FLG_POLL_SLEEPING:
+	erts_sys_schedule_interrupt(1);
+	break;
+    case ERTS_SSI_FLG_TSE_SLEEPING:
+	erts_tse_set(ssi->event);
+	break;
+    case 0:
+	break;
+    default:
+	erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n",
+		 __FILE__, __LINE__);
+	break;
+    }
+}
+
+typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t;
+struct erts_misc_aux_work_t_ {
+    erts_misc_aux_work_t *next;
+    void (*func)(void *);
+    void *arg;
+};
+
+typedef struct {
+    erts_smp_mtx_t mtx;
+    erts_misc_aux_work_t *first;
+    erts_misc_aux_work_t *last;
+} erts_misc_aux_work_q_t;
+
+typedef union {
+    erts_misc_aux_work_q_t data;
+    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_misc_aux_work_q_t))];
+} erts_algnd_misc_aux_work_q_t;
+
+static erts_algnd_misc_aux_work_q_t *misc_aux_work_queues;
+
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work,
+				 erts_misc_aux_work_t,
+				 200,
+				 ERTS_ALC_T_MISC_AUX_WORK)
+
+static void
+init_misc_aux_work(void)
+{
+    int ix;
+
+    init_misc_aux_work_alloc();
+
+    misc_aux_work_queues = 
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q,
+					   erts_no_schedulers *
+					   sizeof(erts_algnd_misc_aux_work_q_t));
+
+    for (ix = 0; ix < erts_no_schedulers; ix++) {
+	erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx,
+			    "misc_aux_work_queue",
+			    make_small(ix + 1));
+	misc_aux_work_queues[ix].data.first = NULL;
+	misc_aux_work_queues[ix].data.last = NULL;
+    }
+}
+
+static void
+handle_misc_aux_work(ErtsSchedulerData *esdp)
+{
+    int ix = (int) esdp->no - 1;
+    erts_misc_aux_work_t *mawp;
+
+    erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx);
+    mawp = misc_aux_work_queues[ix].data.first;
+    misc_aux_work_queues[ix].data.first = NULL;
+    misc_aux_work_queues[ix].data.last = NULL;
+    erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx);
+
+    while (mawp) {
+	erts_misc_aux_work_t *free_mawp;
+	mawp->func(mawp->arg);
+	free_mawp = mawp;
+	mawp = mawp->next;
+	misc_aux_work_free(free_mawp);
+    }
+}
+
+void
+erts_smp_schedule_misc_aux_work(int ignore_self,
+				int max_sched,
+				void (*func)(void *),
+				void *arg)
+{
+    int ix, ignore_ix = -1;
+
+    if (ignore_self) {
+	ErtsSchedulerData *esdp = erts_get_scheduler_data();
+	if (esdp)
+	    ignore_ix = (int) esdp->no - 1;
+    }
+
+    ASSERT(0 <= max_sched && max_sched <= erts_no_schedulers);
+
+    for (ix = 0; ix < max_sched; ix++) {
+	erts_aint32_t aux_work;
+	erts_misc_aux_work_t *mawp;
+	ErtsSchedulerSleepInfo *ssi;
+	if (ix == ignore_ix)
+	    continue;
+
+	mawp = misc_aux_work_alloc();
+
+	mawp->func = func;
+	mawp->arg = arg;
+	mawp->next = NULL;
+
+	erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx);
+	if (!misc_aux_work_queues[ix].data.last)
+	    misc_aux_work_queues[ix].data.first = mawp;
+	else
+	    misc_aux_work_queues[ix].data.last->next = mawp;
+	misc_aux_work_queues[ix].data.last = mawp;
+	erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx);
+
+	ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
+	aux_work = erts_smp_atomic32_bor(&ssi->aux_work,
+					 ERTS_SSI_AUX_WORK_MISC);
+	if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0)
+	    erts_sched_poke(ssi);
+   }
+}
+
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+void
+erts_smp_notify_check_children_needed(void)
+{
+    int i;
+
+    for (i = 0; i < erts_no_schedulers; i++) {
+	erts_aint32_t aux_work;
+	ErtsSchedulerSleepInfo *ssi;
+	ssi = ERTS_SCHED_SLEEP_INFO_IX(i);
+	aux_work = erts_smp_atomic32_bor(&ssi->aux_work,
+					 ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
+	if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN))
+	    erts_sched_poke(ssi);
+    }
+}
+#endif
+
+#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+static ERTS_INLINE erts_aint32_t
+blockable_aux_work(ErtsSchedulerData *esdp,
+		   ErtsSchedulerSleepInfo *ssi,
+		   erts_aint32_t aux_work)
+{
+    if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) {
+	if (aux_work & ERTS_SSI_AUX_WORK_MISC) {
+	    aux_work = erts_smp_atomic32_band(&ssi->aux_work,
+					      ~ERTS_SSI_AUX_WORK_MISC);
+	    aux_work &= ~ERTS_SSI_AUX_WORK_MISC;
+	    handle_misc_aux_work(esdp);
+	}
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+	if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) {
+	    aux_work = erts_smp_atomic32_band(&ssi->aux_work,
+					      ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
+	    aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
+	    erts_check_children();
+	}
+#endif
+    }
+    return aux_work;
+}
+
+#endif
+
+#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK
+static ERTS_INLINE erts_aint32_t
+nonblockable_aux_work(ErtsSchedulerData *esdp,
+		      ErtsSchedulerSleepInfo *ssi,
+		      erts_aint32_t aux_work)
+{
+    if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) {
+
+    }
+}
+#endif
+
 static void
 prepare_for_block(void *vrq)
 {
@@ -627,6 +813,46 @@ erts_active_schedulers(void)
 #ifdef ERTS_SMP
 
 static ERTS_INLINE void
+clear_sys_scheduling(void)
+{
+    erts_smp_atomic32_set_relb(&doing_sys_schedule, 0);
+}
+
+static ERTS_INLINE int
+try_set_sys_scheduling(void)
+{
+    return 0 == erts_smp_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0);
+}
+
+#endif
+
+static ERTS_INLINE int
+prepare_for_sys_schedule(void)
+{
+#ifdef ERTS_SMP
+    while (!erts_port_task_have_outstanding_io_tasks()
+	   && try_set_sys_scheduling()) {
+	if (!erts_port_task_have_outstanding_io_tasks())
+	    return 1;
+	clear_sys_scheduling();
+    }
+    return 0;
+#else
+    return !erts_port_task_have_outstanding_io_tasks();
+#endif
+}
+
+#ifdef ERTS_SMP
+
+static ERTS_INLINE void
+sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq)
+{
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ASSERT(rq->waiting < 0);
+    rq->waiting *= -1;
+}
+
+static ERTS_INLINE void
 sched_waiting(Uint no, ErtsRunQueue *rq)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
@@ -656,287 +882,544 @@ sched_active(Uint no, ErtsRunQueue *rq)
 static int ERTS_INLINE
 ongoing_multi_scheduling_block(void)
 {
-    return erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing) != 0;
+    return erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing) != 0;
 }
 
 static ERTS_INLINE void
 empty_runq(ErtsRunQueue *rq)
 {
-    long oifls = erts_smp_atomic_band(&rq->info_flags, ~ERTS_RUNQ_IFLG_NONEMPTY);
+    erts_aint32_t oifls = erts_smp_atomic32_band(&rq->info_flags,
+						 ~ERTS_RUNQ_IFLG_NONEMPTY);
     if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) {
 #ifdef DEBUG
-	long empty = erts_smp_atomic_read(&no_empty_run_queues);
-	ASSERT(0 <= empty && empty < erts_no_run_queues);
+	erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues);
+	/*
+	 * For a short period of time no_empty_run_queues may have
+	 * been increased twice for a specific run queue.
+	 */
+	ASSERT(0 <= empty && empty < 2*erts_no_run_queues);
 #endif
-	erts_smp_atomic_inc(&no_empty_run_queues);
+	erts_smp_atomic32_inc(&no_empty_run_queues);
     }
 }
 
 static ERTS_INLINE void
 non_empty_runq(ErtsRunQueue *rq)
 {
-    long oifls = erts_smp_atomic_bor(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY);
+    erts_aint32_t oifls = erts_smp_atomic32_bor(&rq->info_flags,
+						ERTS_RUNQ_IFLG_NONEMPTY);
     if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) {
 #ifdef DEBUG
-	long empty = erts_smp_atomic_read(&no_empty_run_queues);
-	ASSERT(0 < empty && empty <= erts_no_run_queues);
+	erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues);
+	/*
+	 * For a short period of time no_empty_run_queues may have
+	 * been increased twice for a specific run queue.
+	 */
+	ASSERT(0 < empty && empty <= 2*erts_no_run_queues);
 #endif
-	erts_smp_atomic_dec(&no_empty_run_queues);
+	erts_smp_atomic32_dec(&no_empty_run_queues);
     }
 }
 
-static ERTS_INLINE int
-sched_spin_wake(ErtsRunQueue *rq)
+static erts_aint32_t
+sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi)
 {
-#if ERTS_SCHED_SLEEP_SPINCOUNT == 0
-    return 0;
-#else
-    long val;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_WAITING);
+    erts_aint32_t xflgs = 0;
 
-    val = erts_smp_atomic_read(&rq->spin_waiter);
-    ASSERT(val >= 0);
-    if (val != 0) {
-	erts_smp_atomic_inc(&rq->spin_wake);
-	return 1;
-    }
-    return 0;
-#endif
+    do {
+	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return nflgs;
+	xflgs = oflgs;
+    } while (!(oflgs & ERTS_SSI_FLG_SUSPENDED));
+    return oflgs;
 }
 
-static ERTS_INLINE int
-sched_spin_wake_all(ErtsRunQueue *rq)
+static erts_aint32_t
+sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi)
 {
-#if ERTS_SCHED_SLEEP_SPINCOUNT == 0
-    return 0;
-#else
-    long val;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_WAITING);
+    erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING;
 
-    val = erts_smp_atomic_read(&rq->spin_waiter);
-    ASSERT(val >= 0);
-    if (val != 0)
-	erts_smp_atomic_add(&rq->spin_wake, val);
-    return val;
-#endif
+    do {
+	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return nflgs;
+	xflgs = oflgs;
+	nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED;
+    } while (oflgs & ERTS_SSI_FLG_WAITING);
+    return oflgs;
+}
+
+static erts_aint32_t
+sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount)
+{
+    int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD;
+    int sc = spincount;
+    erts_aint32_t flgs;
+
+    do {
+	flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
+	    != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) {
+	    break;
+	}
+	ERTS_SPIN_BODY;
+	if (--until_yield == 0) {
+	    until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD;
+	    erts_thr_yield();
+	}
+    } while (--sc > 0);
+    return flgs;
 }
 
+static erts_aint32_t
+sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type)
+{
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING|sleep_type;
+    erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;
+
+    if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING)
+	erts_tse_reset(ssi->event);
+
+    while (1) {
+	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return nflgs;
+	if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
+	    != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) {
+	    return oflgs;
+	}
+	xflgs = oflgs;
+	nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED;
+    }
+}
+
+#define ERTS_SCHED_WAIT_WOKEN(FLGS)				\
+  (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED))	\
+   != ERTS_SSI_FLG_WAITING)
+
 static void
-sched_sys_wait(Uint no, ErtsRunQueue *rq)
+scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 {
-    long dt;
-#if ERTS_SCHED_SLEEP_SPINCOUNT != 0
-    int val;
-    int spincount = ERTS_SCHED_SLEEP_SPINCOUNT;
+    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
+    int spincount;
+    erts_aint32_t flgs;
+#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \
+    || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK)
+    erts_aint32_t aux_work;
+#endif
+
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
+    erts_smp_spin_lock(&rq->sleepers.lock);
+    flgs = sched_prep_spin_wait(ssi);
+    if (flgs & ERTS_SSI_FLG_SUSPENDED) {
+	/* Go suspend instead... */
+	erts_smp_spin_unlock(&rq->sleepers.lock);
+	return;
+    }
+
+    ssi->prev = NULL;
+    ssi->next = rq->sleepers.list;
+    if (rq->sleepers.list)
+	rq->sleepers.list->prev = ssi;
+    rq->sleepers.list = ssi;
+    erts_smp_spin_unlock(&rq->sleepers.lock);
+
+    /*
+     * If all schedulers are waiting, one of them *should*
+     * be waiting in erl_sys_schedule()
+     */
+
+    if (!prepare_for_sys_schedule()) {
+
+	sched_waiting(esdp->no, rq);
+
+	erts_smp_runq_unlock(rq);
+
+	spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT;
+
+    tse_wait:
+
+#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+	aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+    tse_blockable_aux_work:
+	aux_work = blockable_aux_work(esdp, ssi, aux_work);
 #endif
+	erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
 
-    sched_waiting_sys(no, rq);
+	while (1) {
 
-#if ERTS_SCHED_SLEEP_SPINCOUNT != 0
-    erts_smp_atomic_inc(&rq->spin_waiter);
-    erts_smp_runq_unlock(rq);
+#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK
+#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+	    aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+#endif
+	    nonblockable_aux_work(esdp, ssi, aux_work);
+#endif
 
-    erl_sys_schedule(1); /* Might give us something to do */
+	    flgs = sched_spin_wait(ssi, spincount);
+	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+		if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		    int res;
+		    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
+		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+		    do {
+			res = erts_tse_wait(ssi->event);
+		    } while (res == EINTR);
+		}
+	    }
 
-    dt = do_time_read_and_reset();
-    if (dt) bump_timer(dt);
+	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+		break;
+	    }
+
+	    flgs = sched_prep_cont_spin_wait(ssi);
+	    spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT;
+
+	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+		break;
+	    }
+
+#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+	    aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+	    if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) {
+		erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
+		goto tse_blockable_aux_work;
+	    }
+#endif
 
-    while (spincount-- > 0) {
-	val = erts_smp_atomic_read(&rq->spin_wake);
-	ASSERT(val >= 0);
-	if (val != 0) {
-	    erts_smp_runq_lock(rq);
-	    val = erts_smp_atomic_read(&rq->spin_wake);
-	    ASSERT(val >= 0);
-	    if (val != 0)
-		goto woken;
-	    if (spincount == 0)
-		goto sleep;
-	    erts_smp_runq_unlock(rq);
 	}
-    }
 
-    erts_smp_runq_lock(rq);
-    val = erts_smp_atomic_read(&rq->spin_wake);
-    ASSERT(val >= 0);
-    if (val != 0) {
-    woken:
-	erts_smp_atomic_dec(&rq->spin_wake);
-	ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0);
-	erts_smp_atomic_dec(&rq->spin_waiter);
-	ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0);
+	erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
+
+	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
+	    erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+
+	erts_smp_runq_lock(rq);
+	sched_active(esdp->no, rq);
+
     }
     else {
-    sleep:
-	erts_smp_atomic_dec(&rq->spin_waiter);
-	ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0);
+	erts_aint_t dt;
+
+	erts_smp_atomic32_set_relb(&function_calls, 0);
+	*fcalls = 0;
+
+	sched_waiting_sys(esdp->no, rq);
+
+	erts_smp_runq_unlock(rq);
+
+	spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT;
+
+	while (spincount-- > 0) {
+
+	sys_poll_aux_work:
+
+	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
+
+	    erl_sys_schedule(1); /* Might give us something to do */
+
+	    dt = erts_do_time_read_and_reset();
+	    if (dt) erts_bump_timer(dt);
+
+	sys_aux_work:
+
+#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+	    aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+	    aux_work = blockable_aux_work(esdp, ssi, aux_work);
+#endif
+#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK
+#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+	    aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+#endif
+	    nonblockable_aux_work(esdp, ssi, aux_work);
+#endif
+
+	    flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+		goto sys_woken;
+	    }
+	    if (!(flgs & ERTS_SSI_FLG_SLEEPING)) {
+		flgs = sched_prep_cont_spin_wait(ssi);
+		if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+		    ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+		    goto sys_woken;
+		}
+	    }
+
+	    /*
+	     * If we got new I/O tasks we aren't allowed to
+	     * call erl_sys_schedule() until it is handled.
+	     */
+	    if (erts_port_task_have_outstanding_io_tasks()) {
+		clear_sys_scheduling();
+		/*
+		 * Got to check that we still got I/O tasks; otherwise
+		 * we have to continue checking for I/O...
+		 */
+		if (!prepare_for_sys_schedule()) {
+		    spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
+		    goto tse_wait;
+		}
+	    }
+	}
+
+	erts_smp_runq_lock(rq);
+
 	/*
 	 * If we got new I/O tasks we aren't allowed to
 	 * sleep in erl_sys_schedule().
 	 */
-	if (!erts_port_task_have_outstanding_io_tasks()) {
-#endif
+	if (erts_port_task_have_outstanding_io_tasks()) {
+	    clear_sys_scheduling();
+
+	    /*
+	     * Got to check that we still got I/O tasks; otherwise
+	     * we have to wait in erl_sys_schedule() after all...
+	     */
+	    if (prepare_for_sys_schedule())
+		goto do_sys_schedule;
 
+	    /*
+	     * Not allowed to wait in erl_sys_schedule;
+	     * do tse wait instead...
+	     */
+	    sched_change_waiting_sys_to_waiting(esdp->no, rq);
+	    erts_smp_runq_unlock(rq);
+	    spincount = 0;
+	    goto tse_wait;
+	}
+	else {
+	do_sys_schedule:
 	    erts_sys_schedule_interrupt(0);
+	    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
+	    if (!(flgs & ERTS_SSI_FLG_SLEEPING)) {
+		if (!(flgs & ERTS_SSI_FLG_WAITING))
+		    goto sys_locked_woken;
+		erts_smp_runq_unlock(rq);
+		flgs = sched_prep_cont_spin_wait(ssi);
+		if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+		    ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+		    goto sys_woken;
+		}
+		ASSERT(!erts_port_task_have_outstanding_io_tasks());
+		goto sys_poll_aux_work;
+	    }
+
+	    ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
+	    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+
 	    erts_smp_runq_unlock(rq);
 
+	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
+
 	    erl_sys_schedule(0);
 
-	    dt = do_time_read_and_reset();
-	    if (dt) bump_timer(dt);
+	    dt = erts_do_time_read_and_reset();
+	    if (dt) erts_bump_timer(dt);
 
-	    erts_smp_runq_lock(rq);
+	    flgs = sched_prep_cont_spin_wait(ssi);
+	    if (flgs & ERTS_SSI_FLG_WAITING)
+		goto sys_aux_work;
 
-#if ERTS_SCHED_SLEEP_SPINCOUNT != 0
+	sys_woken:
+	    erts_smp_runq_lock(rq);
+	sys_locked_woken:
+	    clear_sys_scheduling();
+	    if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
+		erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+	    sched_active_sys(esdp->no, rq);
 	}
     }
-#endif
 
-    sched_active_sys(no, rq);
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+}
+
+static ERTS_INLINE erts_aint32_t
+ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi)
+{
+    /* reset all flags but suspended */
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = 0;
+    erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;
+    while (1) {
+	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return oflgs;
+	nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED;
+	xflgs = oflgs;
+    }
 }
 
 static void
-sched_cnd_wait(Uint no, ErtsRunQueue *rq)
+wake_scheduler(ErtsRunQueue *rq, int incq, int one)
 {
-#if ERTS_SCHED_SLEEP_SPINCOUNT != 0
-    int val;
-    int spincount = ERTS_SCHED_SLEEP_SPINCOUNT;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
-#endif
+    ErtsSchedulerSleepInfo *ssi;
+    ErtsSchedulerSleepList *sl;
 
-    sched_waiting(no, rq);
-    erts_smp_activity_begin(ERTS_ACTIVITY_WAIT,
-			    prepare_for_block,
-			    resume_after_block,
-			    (void *) rq);
+    /*
+     * The unlocked run queue is not strictly necessary
+     * from a thread safety or deadlock prevention
+     * perspective. It will, however, cost us performance
+     * if it is locked during wakup of another scheduler,
+     * so all code *should* handle this without having
+     * the lock on the run queue.
+     */
+    ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq));
+
+    sl = &rq->sleepers;
+
+    erts_smp_spin_lock(&sl->lock);
+    ssi = sl->list;
+    if (!ssi)
+	erts_smp_spin_unlock(&sl->lock);
+    else if (one) {
+	erts_aint32_t flgs;
+	if (ssi->prev)
+	    ssi->prev->next = ssi->next;
+	else {
+	    ASSERT(sl->list == ssi);
+	    sl->list = ssi->next;
+	}
+	if (ssi->next)
+	    ssi->next->prev = ssi->prev;
 
-#if ERTS_SCHED_SLEEP_SPINCOUNT == 0
-    erts_smp_cnd_wait(&rq->cnd, &rq->mtx);
-#else
-    erts_smp_atomic_inc(&rq->spin_waiter);
-    erts_smp_mtx_unlock(&rq->mtx);
-
-    while (spincount-- > 0) {
-	val = erts_smp_atomic_read(&rq->spin_wake);
-	ASSERT(val >= 0);
-	if (val != 0) {
-	    erts_smp_mtx_lock(&rq->mtx);
-	    val = erts_smp_atomic_read(&rq->spin_wake);
-	    ASSERT(val >= 0);
-	    if (val != 0)
-		goto woken;
-	    if (spincount == 0)
-		goto sleep;
-	    erts_smp_mtx_unlock(&rq->mtx);
-	}
-    }
-
-    erts_smp_mtx_lock(&rq->mtx);
-    val = erts_smp_atomic_read(&rq->spin_wake);
-    ASSERT(val >= 0);
-    if (val == 0) {
-    sleep:
-	erts_smp_atomic_dec(&rq->spin_waiter);
-	ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0);
-	erts_smp_cnd_wait(&rq->cnd, &rq->mtx);
+	erts_smp_spin_unlock(&sl->lock);
+
+	ERTS_THR_MEMORY_BARRIER;
+	flgs = ssi_flags_set_wake(ssi);
+	erts_sched_finish_poke(ssi, flgs);
+
+	if (incq && !erts_common_run_queue && (flgs & ERTS_SSI_FLG_WAITING))
+	    non_empty_runq(rq);
     }
     else {
-    woken:
-	erts_smp_atomic_dec(&rq->spin_wake);
-	ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0);
-	erts_smp_atomic_dec(&rq->spin_waiter);
-	ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0);
-    }
-#endif
-
-    erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
-			  prepare_for_block,
-			  resume_after_block,
-			  (void *) rq);
+	sl->list = NULL;
+	erts_smp_spin_unlock(&sl->lock);
 
-    sched_active(no, rq);
+	ERTS_THR_MEMORY_BARRIER;
+	do {
+	    ErtsSchedulerSleepInfo *wake_ssi = ssi;
+	    ssi = ssi->next;
+	    erts_sched_finish_poke(wake_ssi, ssi_flags_set_wake(wake_ssi));
+	} while (ssi);
+    }
 }
 
 static void
-wake_one_scheduler(void)
-{
-    ASSERT(erts_common_run_queue);
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(erts_common_run_queue));
-    if (erts_common_run_queue->waiting) {
-	if (!sched_spin_wake(erts_common_run_queue)) {
-	    if (erts_common_run_queue->waiting == -1) /* One scheduler waiting
-							 and doing so in
-							 sys_schedule */
-		erts_sys_schedule_interrupt(1);
-	    else
-		erts_smp_cnd_signal(&erts_common_run_queue->cnd);
+wake_all_schedulers(void)
+{
+    if (erts_common_run_queue)
+	wake_scheduler(erts_common_run_queue, 0, 0);
+    else {
+	int ix;
+	for (ix = 0; ix < erts_no_run_queues; ix++) {
+	    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+	    wake_scheduler(rq, 0, 1);
 	}
     }
 }
 
-static void
-wake_scheduler(ErtsRunQueue *rq, int incq)
+#define ERTS_NO_USED_RUNQS_SHIFT 16
+#define ERTS_NO_RUNQS_MASK 0xffff
+
+#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_NO_RUNQS_MASK
+#  error "Too large amount of schedulers allowed"
+#endif
+
+static ERTS_INLINE void
+init_no_runqs(int active, int used)
 {
-    ASSERT(!erts_common_run_queue);
-    ASSERT(-1 <= rq->waiting && rq->waiting <= 1);
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
-    if (rq->waiting && !rq->woken) {
-	if (!sched_spin_wake(rq)) {
-	    if (rq->waiting < 0)
-		erts_sys_schedule_interrupt(1);
-	    else
-		erts_smp_cnd_signal(&rq->cnd);
-	}
-	rq->woken = 1;
-	if (incq)
-	    non_empty_runq(rq);
+    erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK);
+    no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT);
+    erts_smp_atomic32_init(&balance_info.no_runqs, no_runqs);
+}
+
+static ERTS_INLINE void
+get_no_runqs(int *active, int *used)
+{
+    erts_aint32_t no_runqs = erts_smp_atomic32_read(&balance_info.no_runqs);
+    if (active)
+	*active = (int) (no_runqs & ERTS_NO_RUNQS_MASK);
+    if (used)
+	*used = (int) ((no_runqs >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK);
+}
+
+static ERTS_INLINE void
+set_no_used_runqs(int used)
+{
+    erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+    while (1) {
+	erts_aint32_t act, new;
+	new = (used << ERTS_NO_USED_RUNQS_SHIFT) | (exp & ERTS_NO_RUNQS_MASK);
+	act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+	if (act == exp)
+	    break;
+	exp = act;
     }
 }
 
-static void
-wake_all_schedulers(void)
+static ERTS_INLINE void
+set_no_active_runqs(int active)
 {
-    if (erts_common_run_queue) {
-	erts_smp_runq_lock(erts_common_run_queue);
-	if (erts_common_run_queue->waiting) {
-	    if (erts_common_run_queue->waiting < 0)
-		erts_sys_schedule_interrupt(1);
-	    sched_spin_wake_all(erts_common_run_queue);
-	    erts_smp_cnd_broadcast(&erts_common_run_queue->cnd);
-	}
-	erts_smp_runq_unlock(erts_common_run_queue);
+    erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+    while (1) {
+	erts_aint32_t act, new;
+	new = (exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT)) | active;
+	act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+	if (act == exp)
+	    break;
+	exp = act;
     }
-    else {
-	int ix;
-	for (ix = 0; ix < erts_no_run_queues; ix++) {
-	    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
-	    erts_smp_runq_lock(rq);
-	    wake_scheduler(rq, 0);
-	    erts_smp_runq_unlock(rq);
-	}
+}
+
+static ERTS_INLINE int
+try_inc_no_active_runqs(int active)
+{
+    erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+    if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active)
+	return 0;
+    if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) {
+	erts_aint32_t new, act;
+	new = (exp & ~ERTS_NO_RUNQS_MASK) | active;
+	act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+	if (act == exp)
+	    return 1;
     }
+    return 0;
 }
 
+
 static ERTS_INLINE int
 chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
 {
-    long iflgs;
+    erts_aint32_t iflgs;
     ErtsRunQueue *wrq;
     if (crq->ix == ix)
 	return 0;
     wrq = ERTS_RUNQ_IX(ix);
-    iflgs = erts_smp_atomic_read(&wrq->info_flags);
+    iflgs = erts_smp_atomic32_read(&wrq->info_flags);
     if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) {
-	erts_smp_xrunq_lock(crq, wrq);
 	if (activate) {
-	    if (ix == erts_smp_atomic_cmpxchg(&balance_info.active_runqs, ix+1, ix)) {
+	    if (try_inc_no_active_runqs(ix+1)) {
+		erts_smp_xrunq_lock(crq, wrq);
 		wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE;
+		erts_smp_xrunq_unlock(crq, wrq);
 	    }
 	}
-	wake_scheduler(wrq, 0);
-	erts_smp_xrunq_unlock(crq, wrq);
+	wake_scheduler(wrq, 0, 1);
 	return 1;
     }
     return 0;
@@ -947,8 +1430,9 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq)
 {
     int ix = crq->ix;
     int stop_ix = ix;
-    int active_ix = erts_smp_atomic_read(&balance_info.active_runqs);
-    int balance_ix = erts_smp_atomic_read(&balance_info.used_runqs);
+    int active_ix, balance_ix;
+
+    get_no_runqs(&active_ix, &balance_ix);
 
     if (active_ix > balance_ix)
 	active_ix = balance_ix;
@@ -982,19 +1466,42 @@ static ERTS_INLINE void
 smp_notify_inc_runq(ErtsRunQueue *runq)
 {
 #ifdef ERTS_SMP
-    if (erts_common_run_queue)
-	wake_one_scheduler();
-    else
-	wake_scheduler(runq, 1);
+    if (runq)
+	wake_scheduler(runq, 1, 1);
 #endif
 }
 
 void
-erts_smp_notify_inc_runq__(ErtsRunQueue *runq)
+erts_smp_notify_inc_runq(ErtsRunQueue *runq)
 {
     smp_notify_inc_runq(runq);
 }
 
+void
+erts_sched_notify_check_cpu_bind(void)
+{
+#ifdef ERTS_SMP
+    int ix;
+    if (erts_common_run_queue) {
+	for (ix = 0; ix < erts_no_schedulers; ix++)
+	    erts_smp_atomic32_set_relb(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1);
+	wake_all_schedulers();
+    }
+    else {
+	for (ix = 0; ix < erts_no_run_queues; ix++) {
+	    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+	    erts_smp_runq_lock(rq);
+	    rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
+	    erts_smp_runq_unlock(rq);
+	    wake_scheduler(rq, 0, 1);
+	};
+    }
+#else
+    erts_sched_check_cpu_bind(erts_get_scheduler_data());
+#endif
+}
+
+
 #ifdef ERTS_SMP
 
 ErtsRunQueue *
@@ -1136,20 +1643,24 @@ static void
 evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
 {
     Port *prt;
+    int notify_to_rq = 0;
     int prio;
     int prt_locked = 0;
     int rq_locked = 0;
     int evac_rq_locked = 1;
+    ErtsMigrateResult mres;
 
     erts_smp_runq_lock(evac_rq);
 
+    erts_smp_atomic32_bor(&evac_rq->scheduler->ssi->flags,
+			  ERTS_SSI_FLG_SUSPENDED);
+
     evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK;
     evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK
 		       | ERTS_RUNQ_FLGS_EVACUATE_QMASK
 		       | ERTS_RUNQ_FLG_SUSPENDED);
 
-    erts_smp_atomic_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED);
-
+    erts_smp_atomic32_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED);
     /*
      * Need to set up evacuation paths first since we
      * may release the run queue lock on evac_rq
@@ -1177,9 +1688,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
     /* Evacuate scheduled ports */
     prt = evac_rq->ports.start;
     while (prt) {
-	(void) erts_port_migrate(prt, &prt_locked,
+	mres = erts_port_migrate(prt, &prt_locked,
 				 evac_rq, &evac_rq_locked,
 				 rq, &rq_locked);
+	if (mres == ERTS_MIGRATE_SUCCESS)
+	    notify_to_rq = 1;
 	if (prt_locked)
 	    erts_smp_port_unlock(prt);
 	if (!evac_rq_locked) {
@@ -1208,9 +1721,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
 			goto end_of_proc;
 		}
 
-		(void) erts_proc_migrate(proc, &proc_locks,
+		mres = erts_proc_migrate(proc, &proc_locks,
 					 evac_rq, &evac_rq_locked,
 					 rq, &rq_locked);
+		if (mres == ERTS_MIGRATE_SUCCESS)
+		    notify_to_rq = 1;
 		if (proc_locks)
 		    erts_smp_proc_unlock(proc, proc_locks);
 		if (!evac_rq_locked) {
@@ -1242,10 +1757,13 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
     if (rq_locked)
 	erts_smp_runq_unlock(rq);
 
-    if (!evac_rq_locked)
-	erts_smp_runq_lock(evac_rq);
-    wake_scheduler(evac_rq, 0);
-    erts_smp_runq_unlock(evac_rq);
+    if (evac_rq_locked)
+	erts_smp_runq_unlock(evac_rq);
+
+    if (notify_to_rq)
+	smp_notify_inc_runq(rq);
+
+    wake_scheduler(evac_rq, 0, 1);
 }
 
 static int
@@ -1391,7 +1909,7 @@ static ERTS_INLINE int
 check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix)
 {
     ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix);
-    long iflgs = erts_smp_atomic_read(&vrq->info_flags);
+    erts_aint32_t iflgs = erts_smp_atomic32_read(&vrq->info_flags);
     if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY)
 	return try_steal_task_from_victim(rq, rq_lockedp, vrq);
     else
@@ -1421,8 +1939,7 @@ try_steal_task(ErtsRunQueue *rq)
 
     ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked);
 
-    active_rqs = erts_smp_atomic_read(&balance_info.active_runqs);
-    blnc_rqs = erts_smp_atomic_read(&balance_info.used_runqs);
+    get_no_runqs(&active_rqs, &blnc_rqs);
 
     if (active_rqs > blnc_rqs)
 	active_rqs = blnc_rqs;
@@ -1433,7 +1950,7 @@ try_steal_task(ErtsRunQueue *rq)
 	if (active_rqs < blnc_rqs) {
 	    int no = blnc_rqs - active_rqs;
 	    int stop_ix = vix = active_rqs + rq->ix % no;
-	    while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) {
+	    while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
 		res = check_possible_steal_victim(rq, &rq_locked, vix);
 		if (res)
 		    goto done;
@@ -1448,7 +1965,7 @@ try_steal_task(ErtsRunQueue *rq)
 	vix = rq->ix;
 
 	/* ... then try to steal a job from another active queue... */
-	while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) {
+	while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
 	    vix++;
 	    if (vix >= active_rqs)
 		vix = 0;
@@ -1473,31 +1990,6 @@ try_steal_task(ErtsRunQueue *rq)
     return res;
 }
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-void
-erts_smp_notify_check_children_needed(void)
-{
-    int i;
-    for (i = 0; i < erts_no_schedulers; i++) {
-	erts_smp_runq_lock(ERTS_SCHEDULER_IX(i)->run_queue);
-	ERTS_SCHEDULER_IX(i)->check_children = 1;
-	if (!erts_common_run_queue)
-	    wake_scheduler(ERTS_SCHEDULER_IX(i)->run_queue, 0);
-	erts_smp_runq_unlock(ERTS_SCHEDULER_IX(i)->run_queue);
-    }
-    if (ongoing_multi_scheduling_block()) {
-	/* Also blocked schedulers need to check children */
-	erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-	for (i = 0; i < erts_no_schedulers; i++)
-	    ERTS_SCHEDULER_IX(i)->blocked_check_children = 1;
-	erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-	erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-    }
-    if (erts_common_run_queue)
-	wake_all_schedulers();
-}
-#endif
-
 /* Run queue balancing */
 
 typedef struct {
@@ -1561,20 +2053,23 @@ do {									\
 static void
 check_balance(ErtsRunQueue *c_rq)
 {
+#if ERTS_MAX_PROCESSES >= (1 << 27)
+#  error check_balance() assumes ERTS_MAX_PROCESS < (1 << 27)
+#endif
     ErtsRunQueueBalance avg = {0};
     Sint64 scheds_reds, full_scheds_reds;
     int forced, active, current_active, oowc, half_full_scheds, full_scheds,
 	mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix;
 
-    if (erts_smp_atomic_xchg(&balance_info.checking_balance, 1)) {
+    if (erts_smp_atomic32_xchg(&balance_info.checking_balance, 1)) {
 	c_rq->check_balance_reds = INT_MAX;
 	return;
     }
 
-    blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs);
+    get_no_runqs(NULL, &blnc_no_rqs);
     if (blnc_no_rqs == 1) {
 	c_rq->check_balance_reds = INT_MAX;
-	erts_smp_atomic_set(&balance_info.checking_balance, 0);
+	erts_smp_atomic32_set(&balance_info.checking_balance, 0);
 	return;
     }
 
@@ -1582,7 +2077,7 @@ check_balance(ErtsRunQueue *c_rq)
 
     if (balance_info.halftime) {	
 	balance_info.halftime = 0;
-	erts_smp_atomic_set(&balance_info.checking_balance, 0);
+	erts_smp_atomic32_set(&balance_info.checking_balance, 0);
 	ERTS_FOREACH_RUNQ(rq,
 	{
 	    if (rq->waiting)
@@ -1610,12 +2105,13 @@ check_balance(ErtsRunQueue *c_rq)
     forced = balance_info.forced_check_balance;
     balance_info.forced_check_balance = 0;
 
-    blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs);
+    get_no_runqs(&current_active, &blnc_no_rqs);
+
     if (blnc_no_rqs == 1) {
 	erts_smp_mtx_unlock(&balance_info.update_mtx);
 	erts_smp_runq_lock(c_rq);
 	c_rq->check_balance_reds = INT_MAX;
-	erts_smp_atomic_set(&balance_info.checking_balance, 0);
+	erts_smp_atomic32_set(&balance_info.checking_balance, 0);
 	return;
     }
 
@@ -1624,8 +2120,6 @@ check_balance(ErtsRunQueue *c_rq)
     if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE)
 	balance_info.full_reds_history_index = 0;
 
-    current_active = erts_smp_atomic_read(&balance_info.active_runqs);
-
     /* Read balance information for all run queues */
     for (qix = 0; qix < blnc_no_rqs; qix++) {
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(qix);
@@ -1684,12 +2178,14 @@ check_balance(ErtsRunQueue *c_rq)
 		    run_queue_info[qix].prio[pix].avail = 0;
 	    }
 	    else {
-		int xreds = 0;
-		int procreds = treds;
-		procreds -= run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds;
+		Sint64 xreds = 0;
+		Sint64 procreds = treds;
+		procreds -= 
+		    ((Sint64)
+		     run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds);
 
 		for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
-		    int av;
+		    Sint64 av;
 
 		    if (xreds == 0)
 			av = 100;
@@ -1700,9 +2196,10 @@ check_balance(ErtsRunQueue *c_rq)
 			if (av == 0)
 			    av = 1;
 		    }
-		    run_queue_info[qix].prio[pix].avail = av;
+		    run_queue_info[qix].prio[pix].avail = (int) av;
+		    ASSERT(run_queue_info[qix].prio[pix].avail >= 0);
 		    if (pix < PRIORITY_NORMAL) /* ie., max or high */
-			xreds += run_queue_info[qix].prio[pix].reds;
+			xreds += (Sint64) run_queue_info[qix].prio[pix].reds;
 		}
 		run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].avail = 100;
 	    }
@@ -1807,7 +2304,8 @@ check_balance(ErtsRunQueue *c_rq)
 	    if (max_len != 0) {
 		int avail = avg.prio[pix].avail;
 		if (avail != 0) {
-		    max_len = ((100*max_len - 1) / avail) + 1;
+		    max_len = (int) ((100*((Sint64) max_len) - 1)
+				     / ((Sint64) avail)) + 1;
 		    avg.prio[pix].max_len = max_len;
 		    ASSERT(max_len >= 0);
 		}
@@ -1824,9 +2322,10 @@ check_balance(ErtsRunQueue *c_rq)
 		    || run_queue_info[qix].prio[pix].avail == 0)
 		    limit = 0;
 		else
-		    limit = (((avg.prio[pix].max_len
-			       * run_queue_info[qix].prio[pix].avail) - 1)
-			     / 100 + 1);
+		    limit = (int) (((((Sint64) avg.prio[pix].max_len)
+				     * ((Sint64) run_queue_info[qix].prio[pix].avail))
+				    - 1)
+				   / 100 + 1);
 		run_queue_info[qix].prio[pix].migration_limit = limit;
 	    }
 	}
@@ -1954,10 +2453,10 @@ erts_fprintf(stderr, "--------------------------------\n");
     }
 
     balance_info.last_active_runqs = active;
-    erts_smp_atomic_set(&balance_info.active_runqs, active);
+    set_no_active_runqs(active);
 
     balance_info.halftime = 1;
-    erts_smp_atomic_set(&balance_info.checking_balance, 0);
+    erts_smp_atomic32_set(&balance_info.checking_balance, 0);
 
     /* Write migration paths and reset balance statistics in all queues */
     for (qix = 0; qix < blnc_no_rqs; qix++) {
@@ -2054,8 +2553,27 @@ erts_debug_nbalance(void)
 void
 erts_early_init_scheduling(void)
 {
-    early_cpu_bind_init();
+    wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM;
+}
+
+int
+erts_sched_set_wakeup_limit(char *str)
+{
+    if (sys_strcmp(str, "very_high") == 0)
+	wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH;
+    else if (sys_strcmp(str, "high") == 0)
+	wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH;
+    else if (sys_strcmp(str, "medium") == 0)
+	wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM;
+    else if (sys_strcmp(str, "low") == 0)
+	wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_LOW;
+    else if (sys_strcmp(str, "very_low") == 0)
+	wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW;
+    else
+	return EINVAL;
+    return 0;
 }
+	
 
 void
 erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
@@ -2076,24 +2594,21 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 
     n = (int) (mrq ? no_schedulers : 1);
 
-    erts_aligned_run_queues = erts_alloc(ERTS_ALC_T_RUNQS,
-					 (sizeof(ErtsAlignedRunQueue)*(n+1)));
-    if ((((Uint) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0)
-	erts_aligned_run_queues = ((ErtsAlignedRunQueue *)
-				   ((((Uint) erts_aligned_run_queues)
-				     & ~ERTS_CACHE_LINE_MASK)
-				    + ERTS_CACHE_LINE_SIZE));
-
+    erts_aligned_run_queues = 
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS,
+					   sizeof(ErtsAlignedRunQueue) * n);
 #ifdef ERTS_SMP
-    erts_smp_atomic_init(&no_empty_run_queues, 0);
+    erts_smp_atomic32_init(&no_empty_run_queues, 0);
 #endif
 
+    erts_no_run_queues = n;
+
     for (ix = 0; ix < n; ix++) {
 	int pix, rix;
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
 
 	rq->ix = ix;
-	erts_smp_atomic_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY);
+	erts_smp_atomic32_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY);
 
 	/* make sure that the "extra" id correponds to the schedulers
 	 * id if the esdp->no <-> ix+1 mapping change.
@@ -2102,8 +2617,10 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 	erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1));
 	erts_smp_cnd_init(&rq->cnd);
 
-	erts_smp_atomic_init(&rq->spin_waiter, 0);
-	erts_smp_atomic_init(&rq->spin_wake, 0);
+#ifdef ERTS_SMP
+	erts_smp_spinlock_init(&rq->sleepers.lock, "run_queue_sleep_list");
+	rq->sleepers.list = NULL;
+#endif
 
 	rq->waiting = 0;
 	rq->woken = 0;
@@ -2154,7 +2671,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
     }
 
     erts_common_run_queue = !mrq ? ERTS_RUNQ_IX(0) : NULL;
-    erts_no_run_queues = n;
 
 #ifdef ERTS_SMP
 
@@ -2169,23 +2685,48 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 
 #endif
 
+    n = (int) no_schedulers;
+    erts_no_schedulers = n;
+
+#ifdef ERTS_SMP
+    /* Create and initialize scheduler sleep info */
+
+    aligned_sched_sleep_info =
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_SLP_INFO,
+					   n * sizeof(ErtsAlignedSchedulerSleepInfo));
+
+    for (ix = 0; ix < n; ix++) {
+	ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
+#if 0 /* no need to initialize these... */
+	ssi->next = NULL;
+	ssi->prev = NULL;
+#endif
+	erts_smp_atomic32_init(&ssi->flags, 0);
+	ssi->event = NULL; /* initialized in sched_thread_func */
+	erts_smp_atomic32_init(&ssi->aux_work, 0);
+    }
+#endif
+
     /* Create and initialize scheduler specific data */
 
-    n = (int) no_schedulers;
-    erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA,
-					     (sizeof(ErtsAlignedSchedulerData)
-					      *(n+1)));
-    if ((((Uint) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0)
-	erts_aligned_scheduler_data = ((ErtsAlignedSchedulerData *)
-				       ((((Uint) erts_aligned_scheduler_data)
-					 & ~ERTS_CACHE_LINE_MASK)
-					+ ERTS_CACHE_LINE_SIZE));
+    erts_aligned_scheduler_data = 
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
+					   n*sizeof(ErtsAlignedSchedulerData));					   
+
     for (ix = 0; ix < n; ix++) {
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
 #ifdef ERTS_SMP
 	erts_bits_init_state(&esdp->erl_bits_state);
 	esdp->match_pseudo_process = NULL;
+	esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
 	esdp->free_process = NULL;
+#if HALFWORD_HEAP
+	/* Registers need to be heap allocated (correct memory range) for tracing to work */
+	esdp->save_reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm));
+#endif
+#endif
+#if !HEAP_ON_C_STACK
+	esdp->num_tmp_heap_used = 0;
 #endif
 	esdp->no = (Uint) ix+1;
 	esdp->current_process = NULL;
@@ -2206,12 +2747,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 	}
 
 #ifdef ERTS_SMP
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-	esdp->check_children = 0;
-	esdp->blocked_check_children = 0;
-#endif
-	erts_smp_atomic_init(&esdp->suspended, 0);
-	erts_smp_atomic_init(&esdp->chk_cpu_bind, 0);
+	erts_smp_atomic32_init(&esdp->chk_cpu_bind, 0);
 #endif
     }
 
@@ -2219,21 +2755,20 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
     erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd");
     erts_smp_cnd_init(&schdlr_sspnd.cnd);
 
-    schdlr_sspnd.changing = 0;
+    erts_smp_atomic32_init(&schdlr_sspnd.changing, 0);
     schdlr_sspnd.online = no_schedulers_online;
     schdlr_sspnd.curr_online = no_schedulers;
-    erts_smp_atomic_init(&schdlr_sspnd.msb.ongoing, 0);
-    erts_smp_atomic_init(&schdlr_sspnd.active, no_schedulers);
+    erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0);
+    erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers);
     schdlr_sspnd.msb.procs = NULL;
-    erts_smp_atomic_set(&balance_info.used_runqs,
-			erts_common_run_queue ? 1 : no_schedulers_online);
-    erts_smp_atomic_init(&balance_info.active_runqs, no_schedulers);
+    init_no_runqs(no_schedulers,
+		  erts_common_run_queue ? 1 : no_schedulers_online);
     balance_info.last_active_runqs = no_schedulers;
     erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update");
     balance_info.forced_check_balance = 0;
     balance_info.halftime = 1;
     balance_info.full_reds_history_index = 0;
-    erts_smp_atomic_init(&balance_info.checking_balance, 0);
+    erts_smp_atomic32_init(&balance_info.checking_balance, 0);
     balance_info.prev_rise.active_runqs = 0;
     balance_info.prev_rise.max_len = 0;
     balance_info.prev_rise.reds = 0;
@@ -2242,7 +2777,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
     if (no_schedulers_online < no_schedulers) {
 	if (erts_common_run_queue) {
 	    for (ix = no_schedulers_online; ix < no_schedulers; ix++)
-		erts_smp_atomic_set(&(ERTS_SCHEDULER_IX(ix)->suspended), 1);
+		erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags,
+				      ERTS_SSI_FLG_SUSPENDED);
 	}
 	else {
 	    for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++)
@@ -2253,9 +2789,12 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 
     schdlr_sspnd.wait_curr_online = no_schedulers_online;
     schdlr_sspnd.curr_online *= 2; /* Boot strapping... */
-    schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE;
+    ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
+				| ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
 
-    erts_smp_atomic_init(&doing_sys_schedule, 0);
+    erts_smp_atomic32_init(&doing_sys_schedule, 0);
+
+    init_misc_aux_work();
 
 #else /* !ERTS_SMP */
     {
@@ -2269,12 +2808,19 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
     erts_no_schedulers = 1;
 #endif
 
-    erts_smp_atomic_init(&function_calls, 0);
+    erts_smp_atomic32_init(&function_calls, 0);
 
     /* init port tasks */
     erts_port_task_init();
 
-    late_cpu_bind_init();
+#ifndef ERTS_SMP
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+    erts_scheduler_data->verify_unused_temp_alloc
+	= erts_alloc_get_verify_unused_temp_alloc(
+	    &erts_scheduler_data->verify_unused_temp_alloc_data);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
+#endif
+#endif
 }
 
 ErtsRunQueue *
@@ -2374,7 +2920,6 @@ resume_process(Process *p)
 	return;
     switch(p->rstatus) {
     case P_RUNABLE:
-	*statusp = P_WAITING;  /* make erts_add_to_runq work */
 	erts_add_to_runq(p);
 	break;
     case P_WAITING:
@@ -2386,6 +2931,19 @@ resume_process(Process *p)
     p->rstatus = P_FREE;    
 }
 
+int
+erts_get_max_no_executing_schedulers(void)
+{
+#ifdef ERTS_SMP
+    if (erts_smp_atomic32_read(&schdlr_sspnd.changing))
+	return (int) erts_no_schedulers;
+    ERTS_THR_MEMORY_BARRIER;
+    return (int) erts_smp_atomic32_read(&schdlr_sspnd.active);
+#else
+    return 1;
+#endif
+}
+
 #ifdef ERTS_SMP
 
 static void
@@ -2401,13 +2959,113 @@ susp_sched_resume_block(void *unused)
 }
 
 static void
+scheduler_ix_resume_wake(Uint ix)
+{
+    ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
+    erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_TSE_SLEEPING
+			   | ERTS_SSI_FLG_WAITING
+			   | ERTS_SSI_FLG_SUSPENDED);
+    erts_aint32_t oflgs;
+    do {
+	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, 0, xflgs);
+	if (oflgs == xflgs) {
+	    erts_sched_finish_poke(ssi, oflgs);
+	    break;
+	}
+	xflgs = oflgs;
+    } while (oflgs & ERTS_SSI_FLG_SUSPENDED);
+}
+
+static erts_aint32_t
+sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct)
+{
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_WAITING
+			   | ERTS_SSI_FLG_SUSPENDED);
+    erts_aint32_t xflgs = xpct;
+
+    do {
+	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return nflgs;
+	xflgs = oflgs;
+    } while (oflgs & ERTS_SSI_FLG_SUSPENDED);
+
+    return oflgs;
+}
+
+static erts_aint32_t
+sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount)
+{
+    int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD;
+    int sc = spincount;
+    erts_aint32_t flgs;
+
+    do {
+	flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	if ((flgs & (ERTS_SSI_FLG_SLEEPING
+		     | ERTS_SSI_FLG_WAITING
+		     | ERTS_SSI_FLG_SUSPENDED))
+	    != (ERTS_SSI_FLG_SLEEPING
+		| ERTS_SSI_FLG_WAITING
+		| ERTS_SSI_FLG_SUSPENDED)) {
+	    break;
+	}
+	ERTS_SPIN_BODY;
+	if (--until_yield == 0) {
+	    until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD;
+	    erts_thr_yield();
+	}
+    } while (--sc > 0);
+    return flgs;
+}
+
+static erts_aint32_t
+sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi)
+{
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_TSE_SLEEPING
+			   | ERTS_SSI_FLG_WAITING
+			   | ERTS_SSI_FLG_SUSPENDED);
+    erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_WAITING
+			   | ERTS_SSI_FLG_SUSPENDED);
+
+    erts_tse_reset(ssi->event);
+
+    while (1) {
+	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return nflgs;
+	if ((oflgs & (ERTS_SSI_FLG_SLEEPING
+		      | ERTS_SSI_FLG_WAITING
+		      | ERTS_SSI_FLG_SUSPENDED))
+	    != (ERTS_SSI_FLG_SLEEPING
+		| ERTS_SSI_FLG_WAITING
+		| ERTS_SSI_FLG_SUSPENDED)) {
+	    return oflgs;
+	}
+	xflgs = oflgs;
+    }
+}
+
+static void
 suspend_scheduler(ErtsSchedulerData *esdp)
 {
+    erts_aint32_t flgs;
+    erts_aint32_t changing;
     long no = (long) esdp->no;
-    ErtsRunQueue *rq = esdp->run_queue;
+    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
     long active_schedulers;
     int curr_online = 1;
     int wake = 0;
+#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \
+    || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK)
+    erts_aint32_t aux_work;
+#endif
 
     /*
      * Schedulers may be suspended in two different ways:
@@ -2424,126 +3082,145 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
     erts_smp_runq_unlock(esdp->run_queue);
 
-    /* Unbind from cpu */
-    erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx);
-    if (scheduler2cpu_map[esdp->no].bound_id >= 0
-	&& erts_unbind_from_cpu(erts_cpuinfo) == 0) {
-	esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1;
-    }
-    erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx);
+    erts_sched_check_cpu_bind_prep_suspend(esdp);
 
     if (erts_system_profile_flags.scheduler)
     	profile_scheduler(make_small(esdp->no), am_inactive);
 
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
-    active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active);
-    ASSERT(active_schedulers >= 1);
-    if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED) {
-	if (active_schedulers == schdlr_sspnd.msb.wait_active)
-	    wake = 1;
-	if (active_schedulers == 1)
-	    schdlr_sspnd.changing = 0;
-    }
-
-    while (1) {
+    flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
+    if (flgs & ERTS_SSI_FLG_SUSPENDED) {
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-	int check_children;
-	erts_smp_runq_lock(esdp->run_queue);
-	check_children = esdp->check_children;
-	esdp->check_children = 0;
-	erts_smp_runq_unlock(esdp->run_queue);
-	if (check_children) {
-	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-	    erts_check_children();
-	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+	active_schedulers = erts_smp_atomic32_dectest(&schdlr_sspnd.active);
+	ASSERT(active_schedulers >= 1);
+	changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+	if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) {
+	    if (active_schedulers == schdlr_sspnd.msb.wait_active)
+		wake = 1;
+	    if (active_schedulers == 1) {
+		changing = erts_smp_atomic32_band(&schdlr_sspnd.changing,
+						  ~ERTS_SCHDLR_SSPND_CHNG_MSB);
+		changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB;
+	    }
 	}
-#endif
 
-	if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) {
-	    int changed = 0;
-	    if (no > schdlr_sspnd.online && curr_online) {
-		schdlr_sspnd.curr_online--;
-		curr_online = 0;
-		changed = 1;
+	while (1) {
+	    if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
+		int changed = 0;
+		if (no > schdlr_sspnd.online && curr_online) {
+		    schdlr_sspnd.curr_online--;
+		    curr_online = 0;
+		    changed = 1;
+		}
+		else if (no <= schdlr_sspnd.online && !curr_online) {
+		    schdlr_sspnd.curr_online++;
+		    curr_online = 1;
+		    changed = 1;
+		}
+		if (changed
+		    && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
+		    wake = 1;
+		if (schdlr_sspnd.online == schdlr_sspnd.curr_online) {
+		    changing = erts_smp_atomic32_band(&schdlr_sspnd.changing,
+						      ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
+		    changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN;
+		}
 	    }
-	    else if (no <= schdlr_sspnd.online && !curr_online) {
-		schdlr_sspnd.curr_online++;
-		curr_online = 1;
-		changed = 1;
+
+	    if (wake) {
+		erts_smp_cnd_signal(&schdlr_sspnd.cnd);
+		wake = 0;
 	    }
-	    if (changed
-		&& schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
-		wake = 1;
-	    if (schdlr_sspnd.online == schdlr_sspnd.curr_online)
-		schdlr_sspnd.changing = 0;
-	}
 
-	if (wake) {
-	    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-	    wake = 0;
-	}
+	    flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	    if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
+		break;
+	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
 
-	if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ|ERTS_RUNQ_FLG_SUSPENDED)))
-	    break;
-	if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ)
-	    && !erts_smp_atomic_read(&esdp->suspended))
-	    break;
+#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+	    aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+	blockable_aux_work:
+	    blockable_aux_work(esdp, ssi, aux_work);
+#endif
+
+	    erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
+	    while (1) {
+		erts_aint32_t flgs;
+#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK
+#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+		aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+#endif
+		nonblockable_aux_work(esdp, ssi, aux_work);
+#endif
+
+		flgs = sched_spin_suspended(ssi,
+					    ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
+		if (flgs == (ERTS_SSI_FLG_SLEEPING
+			     | ERTS_SSI_FLG_WAITING
+			     | ERTS_SSI_FLG_SUSPENDED)) {
+		    flgs = sched_set_suspended_sleeptype(ssi);
+		    if (flgs == (ERTS_SSI_FLG_SLEEPING
+				 | ERTS_SSI_FLG_TSE_SLEEPING
+				 | ERTS_SSI_FLG_WAITING
+				 | ERTS_SSI_FLG_SUSPENDED)) {
+			int res;
+			do {
+			    res = erts_tse_wait(ssi->event);
+			} while (res == EINTR);
+		    }
+		}
 
-	erts_smp_activity_begin(ERTS_ACTIVITY_WAIT,
-				susp_sched_prep_block,
-				susp_sched_resume_block,
-				NULL);
-	while (1) {
+		flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
+						       | ERTS_SSI_FLG_SUSPENDED));
+		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
+		    break;
+		changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+		if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)
+		    break;
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-	    if (esdp->blocked_check_children)
-		break;
+
+#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+		aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+		if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) {
+		    erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
+		    goto blockable_aux_work;
+		}
 #endif
 
-	    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+	    }
 
-	    if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE)
-		break;
+	    erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
 
-	    if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ
-			       | ERTS_RUNQ_FLG_SUSPENDED)))
-		break;
-	    if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ)
-		&& !erts_smp_atomic_read(&esdp->suspended))
-		break;
+	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+	    changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
 	}
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-	esdp->blocked_check_children = 0;
-#endif
+	active_schedulers = erts_smp_atomic32_inctest(&schdlr_sspnd.active);
+	changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+	if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
+	    && schdlr_sspnd.online == active_schedulers) {
+	    erts_smp_atomic32_band(&schdlr_sspnd.changing,
+				   ~ERTS_SCHDLR_SSPND_CHNG_MSB);
+	}
 
-	erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
-			      susp_sched_prep_block,
-			      susp_sched_resume_block,
-			      NULL);
-    }
+	ASSERT(no <= schdlr_sspnd.online);
+	ASSERT(!erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing));
 
-    active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active);
-    if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED
-	&& schdlr_sspnd.online == active_schedulers) {
-	schdlr_sspnd.changing = 0;
     }
+
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
+    ASSERT(curr_online);
+
     if (erts_system_profile_flags.scheduler)
     	profile_scheduler(make_small(esdp->no), am_active);
 
     erts_smp_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
 
-    /* Make sure we check if we should bind to a cpu or not... */
-    if (rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ)
-	erts_smp_atomic_set(&esdp->chk_cpu_bind, 1);
-    else
-	rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
+    erts_sched_check_cpu_bind_post_suspend(esdp);
 }
 
 #define ERTS_RUNQ_RESET_SUSPEND_INFO(RQ, DBG_ID)			\
@@ -2558,7 +3235,7 @@ do {									\
     (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK				\
                     | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK);		\
     (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;	\
-    erts_smp_atomic_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);	\
+    erts_smp_atomic32_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\
     for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) {	\
 	(RQ)->procs.prio_info[pix__].max_len = 0;			\
 	(RQ)->procs.prio_info[pix__].reds = 0;				\
@@ -2600,8 +3277,10 @@ erts_schedulers_state(Uint *total,
 		      int yield_allowed)
 {
     int res;
+    erts_aint32_t changing;
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    if (yield_allowed && schdlr_sspnd.changing)
+    changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+    if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER))
 	res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
     else {
 	*active = *online = schdlr_sspnd.online;
@@ -2621,6 +3300,7 @@ erts_set_schedulers_online(Process *p,
 			   Sint *old_no)
 {
     int ix, res, no, have_unlocked_plocks;
+    erts_aint32_t changing;
 
     if (new_no < 1 || erts_no_schedulers < new_no)
 	return ERTS_SCHDLR_SSPND_EINVAL;
@@ -2630,7 +3310,8 @@ erts_set_schedulers_online(Process *p,
     have_unlocked_plocks = 0;
     no = (int) new_no;
 
-    if (schdlr_sspnd.changing) {
+    changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+    if (changing) {
 	res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
     }
     else {
@@ -2639,17 +3320,19 @@ erts_set_schedulers_online(Process *p,
 	    res = ERTS_SCHDLR_SSPND_DONE;
 	}
 	else {
-	    schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE;
+	    ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
+					| ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
 	    schdlr_sspnd.online = no;
 	    if (no > online) {
 		int ix;
 		schdlr_sspnd.wait_curr_online = no;
-		if (ongoing_multi_scheduling_block())
-		    /* No schedulers to resume */;
+		if (ongoing_multi_scheduling_block()) {
+		    for (ix = online; ix < no; ix++)
+			erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
+		}
 		else if (erts_common_run_queue) {
 		    for (ix = online; ix < no; ix++)
-			erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended,
-					    0);
+			scheduler_ix_resume_wake(ix);
 		}
 		else {
 		    if (plocks) {
@@ -2663,6 +3346,7 @@ erts_set_schedulers_online(Process *p,
 			erts_smp_runq_lock(rq);
 			ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5);
 			erts_smp_runq_unlock(rq);
+			scheduler_ix_resume_wake(ix);
 		    }
 		    /*
 		     * Spread evacuation paths among all online
@@ -2673,11 +3357,10 @@ erts_set_schedulers_online(Process *p,
 			ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no);
 			evacuate_run_queue(from_rq, to_rq);
 		    }
-		    erts_smp_atomic_set(&balance_info.used_runqs, no);
+		    set_no_used_runqs(no);
 		    erts_smp_mtx_unlock(&balance_info.update_mtx);
 		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 		}
-		erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
 		res = ERTS_SCHDLR_SSPND_DONE;
 	    }
 	    else /* if (no < online) */ {
@@ -2694,12 +3377,17 @@ erts_set_schedulers_online(Process *p,
 		    schdlr_sspnd.wait_curr_online = no+1;
 		}
 
-		if (ongoing_multi_scheduling_block())
-		    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-		else if (erts_common_run_queue) {
+		if (ongoing_multi_scheduling_block()) {
 		    for (ix = no; ix < online; ix++)
-			erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended,
-					    1);
+			erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
+		}
+		else if (erts_common_run_queue) {
+		    for (ix = no; ix < online; ix++) {
+			ErtsSchedulerSleepInfo *ssi;
+			ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
+			erts_smp_atomic32_bor(&ssi->flags,
+					      ERTS_SSI_FLG_SUSPENDED);
+		    }
 		    wake_all_schedulers();
 		}
 		else {
@@ -2723,10 +3411,13 @@ erts_set_schedulers_online(Process *p,
 		    for (ix = erts_no_run_queues-1; ix >= no; ix--)
 			evacuate_run_queue(ERTS_RUNQ_IX(ix),
 					   ERTS_RUNQ_IX(ix % no));
-		    erts_smp_atomic_set(&balance_info.used_runqs, no);
+		    set_no_used_runqs(no);
 		    erts_smp_mtx_unlock(&balance_info.update_mtx);
 		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-		    ERTS_FOREACH_OP_RUNQ(rq, wake_scheduler(rq, 0));
+		    for (ix = no; ix < online; ix++) {
+			ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+			wake_scheduler(rq, 0, 1);
+		    }
 		}
 	    }
 
@@ -2740,6 +3431,13 @@ erts_set_schedulers_online(Process *p,
 				  susp_sched_prep_block,
 				  susp_sched_resume_block,
 				  NULL);
+	    ASSERT(res != ERTS_SCHDLR_SSPND_DONE
+		   ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
+		      & erts_smp_atomic32_read(&schdlr_sspnd.changing))
+		   : (ERTS_SCHDLR_SSPND_CHNG_WAITER
+		      == erts_smp_atomic32_read(&schdlr_sspnd.changing)));
+	    erts_smp_atomic32_band(&schdlr_sspnd.changing,
+				   ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
 	}
     }
 
@@ -2754,37 +3452,41 @@ ErtsSchedSuspendResult
 erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 {
     int ix, res, have_unlocked_plocks = 0;
+    erts_aint32_t changing;
     ErtsProcList *plp;
 
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-
-    if (schdlr_sspnd.changing) {
+    changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+    if (changing) {
 	res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */
     }
     else if (on) { /* ------ BLOCK ------ */
-	if (erts_is_multi_scheduling_blocked()) {
+	if (schdlr_sspnd.msb.procs) {
 	    plp = proclist_create(p);
 	    plp->next = schdlr_sspnd.msb.procs;
 	    schdlr_sspnd.msb.procs = plp;
 	    p->flags |= F_HAVE_BLCKD_MSCHED;
-	    ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1);
+	    ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1);
 	    ASSERT(p->scheduler_data->no == 1);
 	    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
 	}
 	else {
+	    int online = schdlr_sspnd.online;
 	    p->flags |= F_HAVE_BLCKD_MSCHED;
 	    if (plocks) {
 		have_unlocked_plocks = 1;
 		erts_smp_proc_unlock(p, plocks);
 	    }
-	    erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 1);
-	    if (schdlr_sspnd.online == 1) {
+	    ASSERT(0 == erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing));
+	    erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 1);
+	    if (online == 1) {
 		res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
-		ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1);
+		ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1);
 		ASSERT(p->scheduler_data->no == 1);
 	    }
 	    else {
-		schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED;
+		ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
+					    | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
 		if (p->scheduler_data->no == 1) {
 		    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
 		    schdlr_sspnd.msb.wait_active = 1;
@@ -2798,17 +3500,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    schdlr_sspnd.msb.wait_active = 2;
 		}
 		if (erts_common_run_queue) {
-		    for (ix = 1; ix < schdlr_sspnd.online; ix++)
-			erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 1);
+		    for (ix = 1; ix < online; ix++)
+			erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags,
+					      ERTS_SSI_FLG_SUSPENDED);
 		    wake_all_schedulers();
 		}
 		else {
 		    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 		    erts_smp_mtx_lock(&balance_info.update_mtx);
-		    erts_smp_atomic_set(&balance_info.used_runqs, 1);
-		    for (ix = 0; ix < schdlr_sspnd.online; ix++) {
+		    set_no_used_runqs(1);
+		    for (ix = 0; ix < online; ix++) {
 			ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
 			erts_smp_runq_lock(rq);
+			ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED));
 			ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7);
 			erts_smp_runq_unlock(rq);
 		    }
@@ -2826,13 +3530,20 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 					susp_sched_prep_block,
 					susp_sched_resume_block,
 					NULL);
-		while (erts_smp_atomic_read(&schdlr_sspnd.active)
+		while (erts_smp_atomic32_read(&schdlr_sspnd.active)
 		       != schdlr_sspnd.msb.wait_active)
 		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
 		erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
 				      susp_sched_prep_block,
 				      susp_sched_resume_block,
 				      NULL);
+		ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED
+		       ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
+			  & erts_smp_atomic32_read(&schdlr_sspnd.changing))
+		       : (ERTS_SCHDLR_SSPND_CHNG_WAITER
+			  == erts_smp_atomic32_read(&schdlr_sspnd.changing)));
+		erts_smp_atomic32_band(&schdlr_sspnd.changing,
+				       ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
 	    }
 	    plp = proclist_create(p);
 	    plp->next = schdlr_sspnd.msb.procs;
@@ -2876,7 +3587,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 	if (schdlr_sspnd.msb.procs)
 	    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
 	else {
-	    schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED;
+	    ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
 #ifdef DEBUG
 	    ERTS_FOREACH_RUNQ(rq,
 	    {
@@ -2899,17 +3610,17 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 	    });
 #endif
 	    p->flags &= ~F_HAVE_BLCKD_MSCHED;
-	    erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 0);
+	    erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 0);
 	    if (schdlr_sspnd.online == 1) {
 		/* No schedulers to resume */
-		ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1);
-		schdlr_sspnd.changing = 0;
+		ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1);
+		ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB);
 	    }
 	    else if (erts_common_run_queue) {
 		for (ix = 1; ix < schdlr_sspnd.online; ix++)
-		    erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 0);
+		    erts_smp_atomic32_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags,
+					   ~ERTS_SSI_FLG_SUSPENDED);
 		wake_all_schedulers();
-		erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
 	    }
 	    else {
 		int online = schdlr_sspnd.online;
@@ -2926,6 +3637,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    erts_smp_runq_lock(rq);
 		    ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4);
 		    erts_smp_runq_unlock(rq);
+		    scheduler_ix_resume_wake(ix);
 		}
 
 		/* Spread evacuation paths among all online run queues */
@@ -2933,7 +3645,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    evacuate_run_queue(ERTS_RUNQ_IX(ix),
 				       ERTS_RUNQ_IX(ix % online));
 
-		erts_smp_atomic_set(&balance_info.used_runqs, online);
+		set_no_used_runqs(online);
 		/* Make sure that we balance soon... */
 		balance_info.forced_check_balance = 1;
 		erts_smp_runq_lock(ERTS_RUNQ_IX(0));
@@ -2941,7 +3653,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		erts_smp_runq_unlock(ERTS_RUNQ_IX(0));
 		erts_smp_mtx_unlock(&balance_info.update_mtx);
 		erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-		erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
 	    }
 	    res = ERTS_SCHDLR_SSPND_DONE;
 	}
@@ -2958,7 +3669,7 @@ void
 erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value)
 {
     if (return_value == am_blocked) {
-	long active = erts_smp_atomic_read(&schdlr_sspnd.active);
+	erts_aint32_t active = erts_smp_atomic32_read(&schdlr_sspnd.active);
 	ASSERT(1 <= active && active <= 2);
 	ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1);
     }
@@ -2968,8 +3679,11 @@ erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value)
 int
 erts_is_multi_scheduling_blocked(void)
 {
-    return (erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)
-	    && erts_smp_atomic_read(&schdlr_sspnd.active) == 1);
+    int res;
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    res = schdlr_sspnd.msb.procs != NULL;
+    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    return res;
 }
 
 Eterm
@@ -2978,7 +3692,7 @@ erts_multi_scheduling_blockers(Process *p)
     Eterm res = NIL;
 
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    if (erts_is_multi_scheduling_blocked()) {
+    if (schdlr_sspnd.msb.procs) {
 	Eterm *hp, *hp_end;
 	ErtsProcList *plp1, *plp2;
 	Uint max_size;
@@ -3010,18 +3724,26 @@ erts_multi_scheduling_blockers(Process *p)
 static void *
 sched_thread_func(void *vesdp)
 {
+#ifdef ERTS_SMP
+    Uint no = ((ErtsSchedulerData *) vesdp)->no;
+#endif
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
 	char buf[31];
-	Uint no = ((ErtsSchedulerData *) vesdp)->no;
-	erts_snprintf(&buf[0], 31, "scheduler %bpu", no);
+	erts_snprintf(&buf[0], 31, "scheduler %beu", no);
 	erts_lc_set_thread_name(&buf[0]);
     }
 #endif
-    erts_alloc_reg_scheduler_id(((ErtsSchedulerData *) vesdp)->no);
+    erts_alloc_reg_scheduler_id(no);
     erts_tsd_set(sched_data_key, vesdp);
 #ifdef ERTS_SMP
+
+    erts_sched_init_check_cpu_bind((ErtsSchedulerData *) vesdp);
+
     erts_proc_lock_prepare_proc_lock_waiter();
+    ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch();
+
+
 #endif
     erts_register_blockable_thread();
 #ifdef HIPE
@@ -3030,36 +3752,43 @@ sched_thread_func(void *vesdp)
     erts_thread_init_float();
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
-    ASSERT(schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE);
+    ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.changing)
+	   & ERTS_SCHDLR_SSPND_CHNG_ONLN);
 
-    schdlr_sspnd.curr_online--;
+    if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) {
+	erts_smp_atomic32_band(&schdlr_sspnd.changing,
+			       ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
+	if (((ErtsSchedulerData *) vesdp)->no != 1)
+	    erts_smp_cnd_signal(&schdlr_sspnd.cnd);
+    }
 
-    if (((ErtsSchedulerData *) vesdp)->no != 1) {
-	if (schdlr_sspnd.online == schdlr_sspnd.curr_online) {
-	    schdlr_sspnd.changing = 0;
-	    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
+    if (((ErtsSchedulerData *) vesdp)->no == 1) {
+	if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) {
+	    erts_smp_activity_begin(ERTS_ACTIVITY_WAIT,
+				    susp_sched_prep_block,
+				    susp_sched_resume_block,
+				    NULL);
+	    while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
+		erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+	    erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
+				  susp_sched_prep_block,
+				  susp_sched_resume_block,
+				  NULL);
 	}
-    }
-    else if (schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
-	schdlr_sspnd.changing = 0;
-    else {
-	erts_smp_activity_begin(ERTS_ACTIVITY_WAIT,
-				susp_sched_prep_block,
-				susp_sched_resume_block,
-				NULL);
-	while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
-	    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-	erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
-			      susp_sched_prep_block,
-			      susp_sched_resume_block,
-			      NULL);
-	ASSERT(!schdlr_sspnd.changing);
+	ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
     }
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+    ((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc
+	= erts_alloc_get_verify_unused_temp_alloc(
+	    &((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc_data);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
+#endif
+
     process_main();
     /* No schedulers should *ever* terminate */
-    erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %bpu terminated\n",
+    erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n",
 	     ((ErtsSchedulerData *) vesdp)->no);
     return NULL;
 }
@@ -3089,11 +3818,7 @@ erts_start_schedulers(void)
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual);
 	actual++;
 	ASSERT(actual == esdp->no);
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	res = erts_lcnt_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts);
-#else
 	res = ethr_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts);
-#endif
 	if (res != 0) {
 	    actual--;
 	    break;
@@ -3112,8 +3837,8 @@ erts_start_schedulers(void)
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	ASSERT(actual != wanted_no_schedulers);
 	erts_dsprintf(dsbufp,
-		      "Failed to create %bpu scheduler-threads (%s:%d); "
-		      "only %bpu scheduler-thread%s created.\n",
+		      "Failed to create %beu scheduler-threads (%s:%d); "
+		      "only %beu scheduler-thread%s created.\n",
 		      wanted_no_schedulers, erl_errno_id(res), res,
 		      actual, actual == 1 ? " was" : "s were");
 	erts_send_error_to_logger_nogl(dsbufp);
@@ -3122,1351 +3847,6 @@ erts_start_schedulers(void)
 
 #endif /* ERTS_SMP */
 
-static int
-int_cmp(const void *vx, const void *vy)
-{
-    return *((int *) vx) - *((int *) vy);
-}
-
-static int
-cpu_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->processor_node != y->processor_node)
-	return x->processor_node - y->processor_node;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    if (x->node != y->node)
-	return x->node - y->node;
-    return 0;
-}
-
-static int
-cpu_processor_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->processor_node != y->processor_node)
-	return x->processor_node - y->processor_node;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    return 0;
-}
-
-static int
-cpu_thread_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    if (x->processor_node != y->processor_node)
-	return x->processor_node - y->processor_node;
-    if (x->core != y->core)
-	return x->core - y->core;
-    return 0;
-}
-
-static int
-cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    return 0;
-}
-
-static int
-cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    return 0;
-}
-
-static int
-cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    if (x->core != y->core)
-	return x->core - y->core;
-    return 0;
-}
-
-static int
-cpu_no_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    if (x->processor_node != y->processor_node)
-	return x->processor_node - y->processor_node;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    return 0;
-}
-
-static ERTS_INLINE void
-make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node)
-{
-    int ix;
-    int node = -1;
-    int processor = -1;
-    int processor_node = -1;
-    int processor_node_node = -1;
-    int core = -1;
-    int thread = -1;
-    int old_node = -1;
-    int old_processor = -1;
-    int old_processor_node = -1;
-    int old_core = -1;
-    int old_thread = -1;
-
-    for (ix = 0; ix < size; ix++) {
-	if (!no_node || cpudata[ix].node >= 0) {
-	    if (old_node == cpudata[ix].node)
-		cpudata[ix].node = node;
-	    else {
-		old_node = cpudata[ix].node;
-		old_processor = processor = -1;
-		if (!no_node)
-		    old_processor_node = processor_node = -1;
-		old_core = core = -1;
-		old_thread = thread = -1;
-		if (no_node || cpudata[ix].node >= 0)
-		    cpudata[ix].node = ++node;
-	    }
-	}
-	if (old_processor == cpudata[ix].processor)
-	    cpudata[ix].processor = processor;
-	else {
-	    old_processor = cpudata[ix].processor;
-	    if (!no_node)
-		processor_node_node = old_processor_node = processor_node = -1;
-	    old_core = core = -1;
-	    old_thread = thread = -1;
-	    cpudata[ix].processor = ++processor;
-	}
-	if (no_node && cpudata[ix].processor_node < 0)
-	    old_processor_node = -1;
-	else {
-	    if (old_processor_node == cpudata[ix].processor_node) {
-		if (no_node)
-		    cpudata[ix].node = cpudata[ix].processor_node = node;
-		else {
-		    if (processor_node_node >= 0)
-			cpudata[ix].node = processor_node_node;
-		    cpudata[ix].processor_node = processor_node;
-		}
-	    }
-	    else {
-		old_processor_node = cpudata[ix].processor_node;
-		old_core = core = -1;
-		old_thread = thread = -1;
-		if (no_node)
-		    cpudata[ix].node = cpudata[ix].processor_node = ++node;
-		else {
-		    cpudata[ix].node = processor_node_node = ++node;
-		    cpudata[ix].processor_node = ++processor_node;
-		}
-	    }
-	}
-	if (!no_node && cpudata[ix].processor_node < 0)
-	    cpudata[ix].processor_node = 0;
-	if (old_core == cpudata[ix].core)
-	    cpudata[ix].core = core;
-	else {
-	    old_core = cpudata[ix].core;
-	    old_thread = thread = -1;
-	    cpudata[ix].core = ++core;
-	}
-	if (old_thread == cpudata[ix].thread)
-	    cpudata[ix].thread = thread;
-	else
-	    old_thread = cpudata[ix].thread = ++thread;
-    }
-}
-
-static void
-cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
-		    int size,
-		    ErtsCpuBindOrder bind_order,
-		    int mk_seq)
-{
-    if (size > 1) {
-	int no_node = 0;
-	int (*cmp_func)(const void *, const void *);
-	switch (bind_order) {
-	case ERTS_CPU_BIND_SPREAD:
-	    cmp_func = cpu_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_PROCESSOR_SPREAD:
-	    cmp_func = cpu_processor_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_THREAD_SPREAD:
-	    cmp_func = cpu_thread_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD:
-	    no_node = 1;
-	    cmp_func = cpu_thread_no_node_processor_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD:
-	    no_node = 1;
-	    cmp_func = cpu_no_node_processor_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD:
-	    no_node = 1;
-	    cmp_func = cpu_no_node_thread_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_NO_SPREAD:
-	    cmp_func = cpu_no_spread_order_cmp;
-	    break;
-	default:
-	    cmp_func = NULL;
-	    erl_exit(ERTS_ABORT_EXIT,
-		     "Bad cpu bind type: %d\n",
-		     (int) cpu_bind_order);
-	    break;
-	}
-
-	if (mk_seq)
-	    make_cpudata_id_seq(cpudata, size, no_node);
-
-	qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func);
-    }
-}
-
-static int
-processor_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->processor_node != y->processor_node)
-	return x->processor_node - y->processor_node;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    return 0;
-}
-
-static void
-check_cpu_bind(ErtsSchedulerData *esdp)
-{
-    int res;
-    int cpu_id;
-    erts_smp_runq_unlock(esdp->run_queue);
-    erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx);
-    cpu_id = scheduler2cpu_map[esdp->no].bind_id;
-    if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) {
-	res = erts_bind_to_cpu(erts_cpuinfo, cpu_id);
-	if (res == 0)
-	    esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id;
-	else {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n",
-			  (int) esdp->no, cpu_id, erl_errno_id(-res));
-	    erts_send_error_to_logger_nogl(dsbufp);
-	    if (scheduler2cpu_map[esdp->no].bound_id >= 0)
-		goto unbind;
-	}
-    }
-    else if (cpu_id < 0 && scheduler2cpu_map[esdp->no].bound_id >= 0) {
-    unbind:
-	/* Get rid of old binding */
-	res = erts_unbind_from_cpu(erts_cpuinfo);
-	if (res == 0)
-	    esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1;
-	else {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n",
-			  (int) esdp->no, cpu_id, erl_errno_id(-res));
-	    erts_send_error_to_logger_nogl(dsbufp);
-	}
-    }
-    erts_smp_runq_lock(esdp->run_queue);
-#ifdef ERTS_SMP
-    if (erts_common_run_queue)
-	erts_smp_atomic_set(&esdp->chk_cpu_bind, 0);
-    else {
-	esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND;
-    }
-#endif
-    erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx);
-
-}
-
-static void
-signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size)
-{
-    int s_ix = 1;
-    int cpu_ix;
-
-    if (cpu_bind_order != ERTS_CPU_BIND_NONE) {
-
-	cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1);
-
-	for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++)
-	    if (erts_is_cpu_available(erts_cpuinfo, cpudata[cpu_ix].logical))
-		scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical;
-    }
-
-    if (s_ix <= erts_no_schedulers)
-	for (; s_ix <= erts_no_schedulers; s_ix++)
-	    scheduler2cpu_map[s_ix].bind_id = -1;
-
-#ifdef ERTS_SMP
-    if (erts_common_run_queue) {
-	for (s_ix = 0; s_ix < erts_no_schedulers; s_ix++)
-	    erts_smp_atomic_set(&ERTS_SCHEDULER_IX(s_ix)->chk_cpu_bind, 1);
-	wake_all_schedulers();
-    }
-    else {
-	ERTS_FOREACH_RUNQ(rq,
-	{
-	    rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
-	    wake_scheduler(rq, 0);
-	});
-    }
-#else
-    check_cpu_bind(erts_get_scheduler_data());
-#endif
-}
-
-int
-erts_init_scheduler_bind_type(char *how)
-{
-    if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP)
-	return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED;
-
-    if (!system_cpudata && !user_cpudata)
-	return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY;
-
-    if (sys_strcmp(how, "s") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_SPREAD;
-    else if (sys_strcmp(how, "ps") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
-    else if (sys_strcmp(how, "ts") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
-    else if (sys_strcmp(how, "db") == 0
-	     || sys_strcmp(how, "tnnps") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
-    else if (sys_strcmp(how, "nnps") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
-    else if (sys_strcmp(how, "nnts") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
-    else if (sys_strcmp(how, "ns") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
-    else if (sys_strcmp(how, "u") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_NONE;
-    else
-	return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE;
-
-    return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS;
-}
-
-typedef struct {
-    int *id;
-    int used;
-    int size;
-} ErtsCpuTopIdSeq;
-
-typedef struct {
-    ErtsCpuTopIdSeq logical;
-    ErtsCpuTopIdSeq thread;
-    ErtsCpuTopIdSeq core;
-    ErtsCpuTopIdSeq processor_node;
-    ErtsCpuTopIdSeq processor;
-    ErtsCpuTopIdSeq node;
-} ErtsCpuTopEntry;
-
-static void
-init_cpu_top_entry(ErtsCpuTopEntry *cte)
-{
-    int size = 10;
-    cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-				 sizeof(int)*size);
-    cte->logical.size = size;
-    cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-				sizeof(int)*size);
-    cte->thread.size = size;
-    cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-			      sizeof(int)*size);
-    cte->core.size = size;
-    cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-					sizeof(int)*size);
-    cte->processor_node.size = size;
-    cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-				   sizeof(int)*size);
-    cte->processor.size = size;
-    cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-			      sizeof(int)*size);
-    cte->node.size = size;
-}
-
-static void
-destroy_cpu_top_entry(ErtsCpuTopEntry *cte)
-{
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id);
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id);
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id);
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id);
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id);
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id);
-}
-
-static int
-get_cput_value_or_range(int *v, int *vr, char **str)
-{
-    long l;
-    char *c = *str;
-    errno = 0;
-    if (!isdigit((unsigned char)*c))
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID;
-    l = strtol(c, &c, 10);
-    if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID;
-    *v = (int) l;
-    if (*c == '-') {
-	c++;
-	if (!isdigit((unsigned char)*c))
-	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-	l = strtol(c, &c, 10);
-	if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l)
-	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-	*vr = (int) l;
-    }
-    *str = c;
-    return ERTS_INIT_CPU_TOPOLOGY_OK;
-}
-
-static int
-get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str)
-{
-    int ix = 0;
-    int need_size = 0;
-    char *c = *str;
-
-    while (1) {
-	int res;
-	int val;
-	int nids;
-	int val_range = -1;
-	res = get_cput_value_or_range(&val, &val_range, &c);
-	if (res != ERTS_INIT_CPU_TOPOLOGY_OK)
-	    return res;
-	if (val_range < 0 || val_range == val)
-	    nids = 1;
-	else {
-	    if (val_range > val)
-		nids = val_range - val + 1;
-	    else
-		nids = val - val_range + 1;
-	}
-	need_size += nids;
-	if (need_size > idseq->size) {
-	    idseq->size = need_size + 10;
-	    idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS,
-				      idseq->id,
-				      sizeof(int)*idseq->size);
-	}
-	if (nids == 1)
-	    idseq->id[ix++] = val;
-	else if (val_range > val) {
-	    for (; val <= val_range; val++)
-		idseq->id[ix++] = val;
-	}
-	else {
-	    for (; val >= val_range; val--)
-		idseq->id[ix++] = val;
-	}
-	if (*c != ',')
-	    break;
-	c++;
-    }
-    *str = c;
-    idseq->used = ix;
-    return ERTS_INIT_CPU_TOPOLOGY_OK;
-}
-
-static int
-get_cput_entry(ErtsCpuTopEntry *cput, char **str)
-{
-    int h;
-    char *c = *str;
-
-    cput->logical.used = 0;
-    cput->thread.id[0] = 0;
-    cput->thread.used = 1;
-    cput->core.id[0] = 0;
-    cput->core.used = 1;
-    cput->processor_node.id[0] = -1;
-    cput->processor_node.used = 1;
-    cput->processor.id[0] = 0;
-    cput->processor.used = 1;
-    cput->node.id[0] = -1;
-    cput->node.used = 1;
-
-    h = ERTS_TOPOLOGY_MAX_DEPTH;
-    while (*c != ':' && *c != '\0') {
-	int res;
-	ErtsCpuTopIdSeq *idseqp;
-	switch (*c++) {
-	case 'L':
-	    if (h <= ERTS_TOPOLOGY_LOGICAL)
-		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-	    idseqp = &cput->logical;
-	    h = ERTS_TOPOLOGY_LOGICAL;
-	    break;
-	case 't':
-	case 'T':
-	    if (h <= ERTS_TOPOLOGY_THREAD)
-		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-	    idseqp = &cput->thread;
-	    h = ERTS_TOPOLOGY_THREAD;
-	    break;
-	case 'c':
-	case 'C':
-	    if (h <= ERTS_TOPOLOGY_CORE)
-		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-	    idseqp = &cput->core;
-	    h = ERTS_TOPOLOGY_CORE;
-	    break;
-	case 'p':
-	case 'P':
-	    if (h <= ERTS_TOPOLOGY_PROCESSOR)
-		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-	    idseqp = &cput->processor;
-	    h = ERTS_TOPOLOGY_PROCESSOR;
-	    break;
-	case 'n':
-	case 'N':
-	    if (h <= ERTS_TOPOLOGY_PROCESSOR) {
-	    do_node:
-		if (h <= ERTS_TOPOLOGY_NODE)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-		idseqp = &cput->node;
-		h = ERTS_TOPOLOGY_NODE;
-	    }
-	    else {
-		int p_node = 0;
-		char *p_chk = c;
-		while (*p_chk != '\0' && *p_chk != ':') {
-		    if (*p_chk == 'p' || *p_chk == 'P') {
-			p_node = 1;
-			break;
-		    }
-		    p_chk++;
-		}
-		if (!p_node)
-		    goto do_node;
-		if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-		idseqp = &cput->processor_node;
-		h = ERTS_TOPOLOGY_PROCESSOR_NODE;
-	    }
-	    break;
-	default:
-	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE;
-	}
-	res = get_cput_id_seq(idseqp, &c);
-	if (res != ERTS_INIT_CPU_TOPOLOGY_OK)
-		return res;
-    }
-
-    if (cput->logical.used < 1)
-	return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID;
-
-    if (*c == ':') {
-	c++;
-    }
-
-    if (cput->thread.used != 1
-	&& cput->thread.used != cput->logical.used)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-    if (cput->core.used != 1
-	&& cput->core.used != cput->logical.used)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-    if (cput->processor_node.used != 1
-	&& cput->processor_node.used != cput->logical.used)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-    if (cput->processor.used != 1
-	&& cput->processor.used != cput->logical.used)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-    if (cput->node.used != 1
-	&& cput->node.used != cput->logical.used)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-
-    *str = c;
-    return ERTS_INIT_CPU_TOPOLOGY_OK;
-}
-
-static int
-verify_topology(erts_cpu_topology_t *cpudata, int size)
-{
-    if (size > 0) {
-	int *logical;
-	int node, processor, no_nodes, i;
-
-	/* Verify logical ids */
-	logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size);
-
-	for (i = 0; i < user_cpudata_size; i++)
-	    logical[i] = user_cpudata[i].logical;
-
-	qsort(logical, user_cpudata_size, sizeof(int), int_cmp);
-	for (i = 0; i < user_cpudata_size-1; i++) {
-	    if (logical[i] == logical[i+1]) {
-		erts_free(ERTS_ALC_T_TMP, logical);
-		return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS;
-	    }
-	}
-
-	erts_free(ERTS_ALC_T_TMP, logical);
-
-	qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp);
-
-	/* Verify unique entities */
-
-	for (i = 1; i < user_cpudata_size; i++) {
-	    if (user_cpudata[i-1].processor == user_cpudata[i].processor
-		&& user_cpudata[i-1].node == user_cpudata[i].node
-		&& (user_cpudata[i-1].processor_node
-		    == user_cpudata[i].processor_node)
-		&& user_cpudata[i-1].core == user_cpudata[i].core
-		&& user_cpudata[i-1].thread == user_cpudata[i].thread) {
-		return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES;
-	    }
-	}
-
-	/* Verify numa nodes */
-	node = cpudata[0].node;
-	processor = cpudata[0].processor;
-	no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0;
-	for (i = 1; i < size; i++) {
-	    if (no_nodes) {
-		if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
-	    }
-	    else {
-		if (cpudata[i].processor == processor && cpudata[i].node != node)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
-		node = cpudata[i].node;
-		processor = cpudata[i].processor;
-		if (node >= 0 && cpudata[i].processor_node >= 0)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
-		if (node < 0 && cpudata[i].processor_node < 0)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
-	    }
-	}
-    }
-
-    return ERTS_INIT_CPU_TOPOLOGY_OK;
-}
-
-int
-erts_init_cpu_topology(char *topology_str)
-{
-    ErtsCpuTopEntry cput;
-    int need_size;
-    char *c;
-    int ix;
-    int error = ERTS_INIT_CPU_TOPOLOGY_OK;
-
-    if (user_cpudata)
-	erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
-    user_cpudata_size = 10;
-
-    user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
-			      (sizeof(erts_cpu_topology_t)
-			       * user_cpudata_size));
-
-    init_cpu_top_entry(&cput);
-
-    ix = 0;
-    need_size = 0;
-
-    c = topology_str;
-    if (*c == '\0') {
-	error = ERTS_INIT_CPU_TOPOLOGY_MISSING;
-	goto fail;
-    }
-    do {
-	int r;
-	error = get_cput_entry(&cput, &c);
-	if (error != ERTS_INIT_CPU_TOPOLOGY_OK)
-	    goto fail;
-	need_size += cput.logical.used;
-	if (user_cpudata_size < need_size) {
-	    user_cpudata_size = need_size + 10;
-	    user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA,
-					user_cpudata,
-					(sizeof(erts_cpu_topology_t)
-					 * user_cpudata_size));
-	}
-
-	ASSERT(cput.thread.used == 1
-	       || cput.thread.used == cput.logical.used);
-	ASSERT(cput.core.used == 1
-	       || cput.core.used == cput.logical.used);
-	ASSERT(cput.processor_node.used == 1
-	       || cput.processor_node.used == cput.logical.used);
-	ASSERT(cput.processor.used == 1
-	       || cput.processor.used == cput.logical.used);
-	ASSERT(cput.node.used == 1
-	       || cput.node.used == cput.logical.used);
-
-	for (r = 0; r < cput.logical.used; r++) {
-	    user_cpudata[ix].logical = cput.logical.id[r];
-	    user_cpudata[ix].thread =
-		cput.thread.id[cput.thread.used == 1 ? 0 : r];
-	    user_cpudata[ix].core =
-		cput.core.id[cput.core.used == 1 ? 0 : r];
-	    user_cpudata[ix].processor_node =
-		cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r];
-	    user_cpudata[ix].processor =
-		cput.processor.id[cput.processor.used == 1 ? 0 : r];
-	    user_cpudata[ix].node =
-		cput.node.id[cput.node.used == 1 ? 0 : r];
-	    ix++;
-	}
-    } while (*c != '\0');
-
-    if (user_cpudata_size != ix) {
-	user_cpudata_size = ix;
-	user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA,
-				    user_cpudata,
-				    (sizeof(erts_cpu_topology_t)
-				     * user_cpudata_size));
-    }
-
-    error = verify_topology(user_cpudata, user_cpudata_size);
-    if (error == ERTS_INIT_CPU_TOPOLOGY_OK) {
-	destroy_cpu_top_entry(&cput);
-	return ERTS_INIT_CPU_TOPOLOGY_OK;
-    }
-
- fail:
-    if (user_cpudata)
-	erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
-    user_cpudata_size = 0;
-    destroy_cpu_top_entry(&cput);
-    return error;
-}
-
-#define ERTS_GET_CPU_TOPOLOGY_ERROR		-1
-#define ERTS_GET_USED_CPU_TOPOLOGY		0
-#define ERTS_GET_DETECTED_CPU_TOPOLOGY		1
-#define ERTS_GET_DEFINED_CPU_TOPOLOGY		2
-
-static Eterm get_cpu_topology_term(Process *c_p, int type);
-
-Eterm
-erts_set_cpu_topology(Process *c_p, Eterm term)
-{
-    erts_cpu_topology_t *cpudata = NULL;
-    int cpudata_size = 0;
-    Eterm res;
-
-    erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx);
-    res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY);
-    if (term == am_undefined) {
-	if (user_cpudata)
-	    erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
-	user_cpudata = NULL;
-	user_cpudata_size = 0;
-
-	if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) {
-	    cpudata_size = system_cpudata_size;
-	    cpudata = erts_alloc(ERTS_ALC_T_TMP,
-				 (sizeof(erts_cpu_topology_t)
-				  * cpudata_size));
-
-	    sys_memcpy((void *) cpudata,
-		       (void *) system_cpudata,
-		       sizeof(erts_cpu_topology_t)*cpudata_size);
-	}
-    }
-    else if (is_not_list(term)) {
-    error:
-	res = THE_NON_VALUE;
-	goto done;
-    }
-    else {
-	Eterm list = term;
-	int ix = 0;
-
-	cpudata_size = 100;
-	cpudata = erts_alloc(ERTS_ALC_T_TMP,
-			     (sizeof(erts_cpu_topology_t)
-			      * cpudata_size));
-
-	while (is_list(list)) {
-	    Eterm *lp = list_val(list);
-	    Eterm cpu = CAR(lp);
-	    Eterm* tp;
-	    Sint id;
-		
-	    if (is_not_tuple(cpu))
-		goto error;
-
-	    tp = tuple_val(cpu);
-
-	    if (arityval(tp[0]) != 7 || tp[1] != am_cpu)
-		goto error;
-
-	    if (ix >= cpudata_size) {
-		cpudata_size += 100;
-		cpudata = erts_realloc(ERTS_ALC_T_TMP,
-				       cpudata,
-				       (sizeof(erts_cpu_topology_t)
-					* cpudata_size));
-	    }
-
-	    id = signed_val(tp[2]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].node = (int) id;
-
-	    id = signed_val(tp[3]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].processor = (int) id;
-
-	    id = signed_val(tp[4]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].processor_node = (int) id;
-
-	    id = signed_val(tp[5]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].core = (int) id;
-
-	    id = signed_val(tp[6]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].thread = (int) id;
-
-	    id = signed_val(tp[7]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].logical = (int) id;
-
-	    list = CDR(lp);
-	    ix++;
-	}
-
-	if (is_not_nil(list))
-	    goto error;
-	
-	cpudata_size = ix;
-
-	if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size))
-	    goto error;
-
-	if (user_cpudata_size != cpudata_size) {
-	    if (user_cpudata)
-		erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
-	    user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
-				      sizeof(erts_cpu_topology_t)*cpudata_size);
-	    user_cpudata_size = cpudata_size;
-	}
-
-	sys_memcpy((void *) user_cpudata,
-		   (void *) cpudata,
-		   sizeof(erts_cpu_topology_t)*cpudata_size);
-    }
-
-    signal_schedulers_bind_change(cpudata, cpudata_size);
-
- done:
-    erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx);
-
-    if (cpudata)
-	erts_free(ERTS_ALC_T_TMP, cpudata);
-
-    return res;
-}
-
-static Eterm
-bound_schedulers_term(ErtsCpuBindOrder order)
-{
-    switch (order) {
-    case ERTS_CPU_BIND_SPREAD: {
-	ERTS_DECL_AM(spread);
-	return AM_spread;
-    }
-    case ERTS_CPU_BIND_PROCESSOR_SPREAD: {
-	ERTS_DECL_AM(processor_spread);
-	return AM_processor_spread;
-    }
-    case ERTS_CPU_BIND_THREAD_SPREAD: {
-	ERTS_DECL_AM(thread_spread);
-	return AM_thread_spread;
-    }
-    case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: {
-	ERTS_DECL_AM(thread_no_node_processor_spread);
-	return AM_thread_no_node_processor_spread;
-    }
-    case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: {
-	ERTS_DECL_AM(no_node_processor_spread);
-	return AM_no_node_processor_spread;
-    }
-    case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: {
-	ERTS_DECL_AM(no_node_thread_spread);
-	return AM_no_node_thread_spread;
-    }
-    case ERTS_CPU_BIND_NO_SPREAD: {
-	ERTS_DECL_AM(no_spread);
-	return AM_no_spread;
-    }
-    case ERTS_CPU_BIND_NONE: {
-	ERTS_DECL_AM(unbound);
-	return AM_unbound;
-    }
-    default:
-	ASSERT(0);
-	return THE_NON_VALUE;
-    }
-}
-
-Eterm
-erts_bound_schedulers_term(Process *c_p)
-{
-    ErtsCpuBindOrder order;
-    erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
-    order = cpu_bind_order;
-    erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
-    return bound_schedulers_term(order);
-}
-
-static void
-create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size)
-{
-    if (user_cpudata) {
-	*cpudata_size = user_cpudata_size;
-	*cpudata = erts_alloc(ERTS_ALC_T_TMP,
-			      (sizeof(erts_cpu_topology_t)
-			       * (*cpudata_size)));
-	sys_memcpy((void *) *cpudata,
-		   (void *) user_cpudata,
-		   sizeof(erts_cpu_topology_t)*(*cpudata_size));
-    }
-    else if (system_cpudata) {
-	*cpudata_size = system_cpudata_size;
-	*cpudata = erts_alloc(ERTS_ALC_T_TMP,
-			      (sizeof(erts_cpu_topology_t)
-			       * (*cpudata_size)));
-	sys_memcpy((void *) *cpudata,
-		   (void *) system_cpudata,
-		   sizeof(erts_cpu_topology_t)*(*cpudata_size));
-    }
-    else {
-	*cpudata = NULL;
-	*cpudata_size = 0;
-    }
-}
-
-static void
-destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata)
-{
-    if (cpudata)
-	erts_free(ERTS_ALC_T_TMP, cpudata);
-}
-
-Eterm
-erts_bind_schedulers(Process *c_p, Eterm how)
-{
-    Eterm res;
-    erts_cpu_topology_t *cpudata;
-    int cpudata_size;
-    ErtsCpuBindOrder old_cpu_bind_order;
-
-    erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx);
-
-    if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP) {
-	ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP);
-    }
-    else {
-
-	old_cpu_bind_order = cpu_bind_order;
-
-	if (ERTS_IS_ATOM_STR("spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_SPREAD;
-	else if (ERTS_IS_ATOM_STR("processor_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
-	else if (ERTS_IS_ATOM_STR("thread_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
-	else if (ERTS_IS_ATOM_STR("default_bind", how)
-		 || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
-	else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
-	else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
-	else if (ERTS_IS_ATOM_STR("no_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
-	else if (ERTS_IS_ATOM_STR("unbound", how))
-	    cpu_bind_order = ERTS_CPU_BIND_NONE;
-	else {
-	    cpu_bind_order = old_cpu_bind_order;
-	    ERTS_BIF_PREP_ERROR(res, c_p, BADARG);
-	    goto done;
-	}
-
-	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
-
-	if (!cpudata) {
-	    cpu_bind_order = old_cpu_bind_order;
-	    ERTS_BIF_PREP_ERROR(res, c_p, BADARG);
-	    goto done;
-	}
-
-	signal_schedulers_bind_change(cpudata, cpudata_size);
-
-	destroy_tmp_cpu_topology_copy(cpudata);
-    
-	res = bound_schedulers_term(old_cpu_bind_order);
-    }
-
- done:
-
-    erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx);
-
-    return res;
-}
-
-Eterm
-erts_fake_scheduler_bindings(Process *p, Eterm how)
-{
-    ErtsCpuBindOrder fake_cpu_bind_order;
-    erts_cpu_topology_t *cpudata;
-    int cpudata_size;
-    Eterm res;
-
-    if (ERTS_IS_ATOM_STR("spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD;
-    else if (ERTS_IS_ATOM_STR("processor_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
-    else if (ERTS_IS_ATOM_STR("thread_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
-    else if (ERTS_IS_ATOM_STR("default_bind", how)
-	     || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
-    else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
-    else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
-    else if (ERTS_IS_ATOM_STR("no_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
-    else if (ERTS_IS_ATOM_STR("unbound", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_NONE;
-    else {
-	ERTS_BIF_PREP_ERROR(res, p, BADARG);
-	return res;
-    }
-
-    erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
-    create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
-    erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
-
-    if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE)
-	ERTS_BIF_PREP_RET(res, am_false);
-    else {
-	int i;
-	Eterm *hp;
-	
-	cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1);
-
-#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA
-
-	erts_fprintf(stderr, "node:          ");
-	for (i = 0; i < cpudata_size; i++)
-	    erts_fprintf(stderr, " %2d", cpudata[i].node);
-	erts_fprintf(stderr, "\n");
-	erts_fprintf(stderr, "processor:     ");
-	for (i = 0; i < cpudata_size; i++)
-	    erts_fprintf(stderr, " %2d", cpudata[i].processor);
-	erts_fprintf(stderr, "\n");
-	if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD
-	    && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD
-	    && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) {
-	    erts_fprintf(stderr, "processor_node:");
-	    for (i = 0; i < cpudata_size; i++)
-		erts_fprintf(stderr, " %2d", cpudata[i].processor_node);
-	    erts_fprintf(stderr, "\n");
-	}
-	erts_fprintf(stderr, "core:          ");
-	for (i = 0; i < cpudata_size; i++)
-	    erts_fprintf(stderr, " %2d", cpudata[i].core);
-	erts_fprintf(stderr, "\n");
-	erts_fprintf(stderr, "thread:        ");
-	for (i = 0; i < cpudata_size; i++)
-	    erts_fprintf(stderr, " %2d", cpudata[i].thread);
-	erts_fprintf(stderr, "\n");
-	erts_fprintf(stderr, "logical:       ");
-	for (i = 0; i < cpudata_size; i++)
-	    erts_fprintf(stderr, " %2d", cpudata[i].logical);
-	erts_fprintf(stderr, "\n");
-#endif
-
-	hp = HAlloc(p, cpudata_size+1);
-	ERTS_BIF_PREP_RET(res, make_tuple(hp));
-	*hp++ = make_arityval((Uint) cpudata_size);
-	for (i = 0; i < cpudata_size; i++)
-	    *hp++ = make_small((Uint) cpudata[i].logical);
-    }
-
-    destroy_tmp_cpu_topology_copy(cpudata);
-
-    return res;
-}
-
-Eterm
-erts_get_schedulers_binds(Process *c_p)
-{
-    int ix;
-    ERTS_DECL_AM(unbound);
-    Eterm *hp = HAlloc(c_p, erts_no_schedulers+1);
-    Eterm res = make_tuple(hp);
-
-    *(hp++) = make_arityval(erts_no_schedulers);
-    erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
-    for (ix = 1; ix <= erts_no_schedulers; ix++)
-	*(hp++) = (scheduler2cpu_map[ix].bound_id >= 0
-		   ? make_small(scheduler2cpu_map[ix].bound_id)
-		   : AM_unbound);
-    erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
-    return res;
-}
-
-static Eterm
-bld_topology_term(Eterm **hpp,
-		  Uint *hszp,
-		  erts_cpu_topology_t *cpudata,
-		  int size)
-{
-    Eterm res = NIL;
-    int i;
-
-    if (size == 0)
-	return am_undefined;
-
-    for (i = size-1; i >= 0; i--) {
-	res = erts_bld_cons(hpp,
-			    hszp,
-			    erts_bld_tuple(hpp,
-					   hszp,
-					   7,
-					   am_cpu,
-					   make_small(cpudata[i].node),
-					   make_small(cpudata[i].processor),
-					   make_small(cpudata[i].processor_node),
-					   make_small(cpudata[i].core),
-					   make_small(cpudata[i].thread),
-					   make_small(cpudata[i].logical)),
-			    res);
-    }
-    return res;
-}
-
-static Eterm
-get_cpu_topology_term(Process *c_p, int type)
-{
-#ifdef DEBUG
-    Eterm *hp_end;
-#endif
-    Eterm *hp;
-    Uint hsz;
-    Eterm res = THE_NON_VALUE;
-    erts_cpu_topology_t *cpudata = NULL;
-    int size = 0;
-
-    switch (type) {
-    case ERTS_GET_USED_CPU_TOPOLOGY:
-	if (user_cpudata)
-	    goto defined;
-	else
-	    goto detected;
-    case ERTS_GET_DETECTED_CPU_TOPOLOGY:
-    detected:
-	if (!system_cpudata)
-	    res = am_undefined;
-	else {
-	    size = system_cpudata_size;
-	    cpudata = erts_alloc(ERTS_ALC_T_TMP,
-				 (sizeof(erts_cpu_topology_t)
-				  * size));
-	    sys_memcpy((void *) cpudata,
-		       (void *) system_cpudata,
-		       sizeof(erts_cpu_topology_t)*size);
-	}
-	break;
-    case ERTS_GET_DEFINED_CPU_TOPOLOGY:
-    defined:
-	if (!user_cpudata)
-	    res = am_undefined;
-	else {
-	    size = user_cpudata_size;
-	    cpudata = user_cpudata;
-	}
-	break;
-    default:
-	erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type);
-	break;
-    }
-
-    if (res == am_undefined) {
-	ASSERT(!cpudata);
-	return res;
-    }
-
-    hsz = 0;
-
-    bld_topology_term(NULL, &hsz,
-		      cpudata, size);
-
-    hp = HAlloc(c_p, hsz);
-
-#ifdef DEBUG
-    hp_end = hp + hsz;
-#endif
-
-    res = bld_topology_term(&hp, NULL,
-			    cpudata, size);
-
-    ASSERT(hp_end == hp);
-
-    if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata)
-	erts_free(ERTS_ALC_T_TMP, cpudata);
-
-    return res;
-}
-
-Eterm
-erts_get_cpu_topology_term(Process *c_p, Eterm which)
-{
-    Eterm res;
-    int type;
-    erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
-    if (ERTS_IS_ATOM_STR("used", which))
-	type = ERTS_GET_USED_CPU_TOPOLOGY;
-    else if (ERTS_IS_ATOM_STR("detected", which))
-	type = ERTS_GET_DETECTED_CPU_TOPOLOGY;
-    else if (ERTS_IS_ATOM_STR("defined", which))
-	type = ERTS_GET_DEFINED_CPU_TOPOLOGY;
-    else
-	type = ERTS_GET_CPU_TOPOLOGY_ERROR;
-    if (type == ERTS_GET_CPU_TOPOLOGY_ERROR)
-	res = THE_NON_VALUE;
-    else
-	res = get_cpu_topology_term(c_p, type);
-    erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
-    return res;
-}
-
-static void
-early_cpu_bind_init(void)
-{
-    user_cpudata = NULL;
-    user_cpudata_size = 0;
-
-    system_cpudata_size = erts_get_cpu_topology_size(erts_cpuinfo);
-    system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
-				(sizeof(erts_cpu_topology_t)
-				 * system_cpudata_size));
-
-    cpu_bind_order = ERTS_CPU_BIND_NONE;
-
-    if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata)
-	|| ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata,
-							system_cpudata_size)) {
-	erts_free(ERTS_ALC_T_CPUDATA, system_cpudata);
-	system_cpudata = NULL;
-	system_cpudata_size = 0;
-    }
-}
-
-static void
-late_cpu_bind_init(void)
-{
-    int ix;
-
-    erts_smp_rwmtx_init(&erts_cpu_bind_rwmtx, "cpu_bind");
-
-    scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA,
-				   (sizeof(ErtsCpuBindData)
-				    * (erts_no_schedulers+1)));
-    for (ix = 1; ix <= erts_no_schedulers; ix++) {
-	scheduler2cpu_map[ix].bind_id = -1;
-	scheduler2cpu_map[ix].bound_id = -1;
-    }
-
-    if (cpu_bind_order != ERTS_CPU_BIND_NONE) {
-	erts_cpu_topology_t *cpudata;
-	int cpudata_size;
-	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
-	ASSERT(cpudata);
-	signal_schedulers_bind_change(cpudata, cpudata_size);
-	destroy_tmp_cpu_topology_copy(cpudata);
-    }
-}
-
 #ifdef ERTS_SMP
 
 static void
@@ -4481,7 +3861,7 @@ add_pend_suspend(Process *suspendee,
 					 sizeof(ErtsPendingSuspend));
     psp->next = NULL;
 #ifdef DEBUG
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead;
 #else
     psp->end = (ErtsPendingSuspend *) 0xdeaddead;
@@ -4572,21 +3952,9 @@ handle_pend_sync_suspend(Process *suspendee,
     }
 }
 
-/*
- * Like erts_pid2proc() but:
- *
- * * At least ERTS_PROC_LOCK_MAIN have to be held on c_p.
- * * At least ERTS_PROC_LOCK_MAIN have to be taken on pid.
- * * It also waits for proc to be in a state != running and garbing.
- * * If ERTS_PROC_LOCK_BUSY is returned, the calling process has to
- *   yield (ERTS_BIF_YIELD[0-3]()). c_p might in this case have been
- *   suspended.
- */
-
-
-Process *
-erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
-			  Eterm pid, ErtsProcLocks pid_locks)
+static Process *
+pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
+		     Eterm pid, ErtsProcLocks pid_locks, int suspend)
 {
     Process *rp;
     int unlock_c_p_status;
@@ -4613,7 +3981,7 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	c_p->suspendee = NIL;
 	ASSERT(c_p->flags & F_P2PNR_RESCHED);
 	c_p->flags &= ~F_P2PNR_RESCHED;
-	if (rp)
+	if (!suspend && rp)
 	    resume_process(rp);
     }
     else {
@@ -4677,6 +4045,8 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	    }
 
 	    /* rp is not running and we got the locks we want... */
+	    if (suspend)
+		suspend_process(rp_rq, rp);
 	}
 	erts_smp_runqs_unlock(cp_rq, rp_rq);
     }
@@ -4689,6 +4059,35 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
     return rp;
 }
 
+
+/*
+ * Like erts_pid2proc() but:
+ *
+ * * At least ERTS_PROC_LOCK_MAIN have to be held on c_p.
+ * * At least ERTS_PROC_LOCK_MAIN have to be taken on pid.
+ * * It also waits for proc to be in a state != running and garbing.
+ * * If ERTS_PROC_LOCK_BUSY is returned, the calling process has to
+ *   yield (ERTS_BIF_YIELD[0-3]()). c_p might in this case have been
+ *   suspended.
+ */
+Process *
+erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
+			  Eterm pid, ErtsProcLocks pid_locks)
+{
+    return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 0);
+}
+
+/*
+ * Like erts_pid2proc_not_running(), but hands over the process
+ * in a suspended state unless (c_p is looked up).
+ */
+Process *
+erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks,
+		      Eterm pid, ErtsProcLocks pid_locks)
+{
+    return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 1);
+}
+
 /*
  * erts_pid2proc_nropt() is normally the same as
  * erts_pid2proc_not_running(). However it is only
@@ -4802,6 +4201,21 @@ handle_pend_bif_async_suspend(Process *suspendee,
     }
 }
 
+#else
+
+/*
+ * Non-smp version of erts_pid2proc_suspend().
+ */
+Process *
+erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks,
+		      Eterm pid, ErtsProcLocks pid_locks)
+{
+    Process *rp = erts_pid2proc(c_p, c_p_locks, pid, pid_locks);
+    if (rp)
+	erts_suspend(rp, pid_locks, NULL);
+    return rp;
+}
+
 #endif /* ERTS_SMP */
 
 /*
@@ -5322,7 +4736,7 @@ dequeue_process(ErtsRunQueue *runq, Process *p)
 }
 
 /* schedule a process */
-static ERTS_INLINE void
+static ERTS_INLINE ErtsRunQueue *
 internal_add_to_runq(ErtsRunQueue *runq, Process *p)
 {
     Uint32 prev_status = p->status;
@@ -5333,12 +4747,12 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p)
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
 
     if (p->status_flags & ERTS_PROC_SFLG_INRUNQ)
-	return;
+	return NULL;
     else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) {
-	ASSERT(p->status != P_SUSPENDED);
+	ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0);
 	ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p);
 	p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ;
-	return;
+	return NULL;
     }
     ASSERT(!p->scheduler_data);
 #endif
@@ -5346,9 +4760,8 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p)
     ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p);
 #ifndef ERTS_SMP
     /* Never schedule a suspended process (ok in smp case) */
-    ASSERT(p->status != P_SUSPENDED);
+    ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0);
     add_runq = runq;
-
 #else
     ASSERT(!p->bound_runq || p->bound_runq == p->run_queue);
     if (p->bound_runq) {
@@ -5377,20 +4790,23 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p)
     	profile_runnable_proc(p, am_active);
     }
 
-    smp_notify_inc_runq(add_runq);
-
     if (add_runq != runq)
 	erts_smp_runq_unlock(add_runq);
+
+    return add_runq;
 }
 
 
 void
 erts_add_to_runq(Process *p)
 {
+    ErtsRunQueue *notify_runq;
     ErtsRunQueue *runq = erts_get_runq_proc(p);
     erts_smp_runq_lock(runq);
-    internal_add_to_runq(runq, p);
+    notify_runq = internal_add_to_runq(runq, p);
     erts_smp_runq_unlock(runq);
+    smp_notify_inc_runq(notify_runq);
+
 }
 
 /* Possibly remove a scheduled process we need to suspend */
@@ -5529,8 +4945,6 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks,
     p->run_queue = to_rq;
     enqueue_process(to_rq, p);
 
-    smp_notify_inc_runq(to_rq);
-
     return ERTS_MIGRATE_SUCCESS;
 }
 #endif /* ERTS_SMP */
@@ -5727,30 +5141,6 @@ erts_set_process_priority(Process *p, Eterm new_value)
     return old_value;
 }
 
-#ifdef ERTS_SMP
-
-static ERTS_INLINE int
-prepare_for_sys_schedule(void)
-{
-    while (!erts_port_task_have_outstanding_io_tasks()
-	   && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) {
-	if (!erts_port_task_have_outstanding_io_tasks())
-	    return 1;
-	erts_smp_atomic_set(&doing_sys_schedule, 0);
-    }
-    return 0;
-}
-
-#else
-
-static ERTS_INLINE int
-prepare_for_sys_schedule(void)
-{
-    return !erts_port_task_have_outstanding_io_tasks();
-}
-
-#endif
-
 /* note that P_RUNNING is only set so that we don't try to remove
 ** running processes from the schedule queue if they exit - a running
 ** process not being in the schedule queue!! 
@@ -5780,10 +5170,10 @@ Process *schedule(Process *p, int calls)
 {
     ErtsRunQueue *rq;
     ErtsRunPrioQueue *rpq;
-    long dt;
+    erts_aint_t dt;
     ErtsSchedulerData *esdp;
     int context_reds;
-    long fcalls;
+    int fcalls;
     int input_reductions;
     int actual_reds;
     int reds;
@@ -5806,7 +5196,7 @@ Process *schedule(Process *p, int calls)
 	esdp = erts_get_scheduler_data();
 	rq = erts_get_runq_current(esdp);
 	ASSERT(esdp);
-	fcalls = erts_smp_atomic_read(&function_calls);
+	fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls);
 	actual_reds = reds = 0;
 	erts_smp_runq_lock(rq);
     } else {
@@ -5824,7 +5214,7 @@ Process *schedule(Process *p, int calls)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
 	esdp->virtual_reds = 0;
 
-	fcalls = erts_smp_atomic_addtest(&function_calls, reds);
+	fcalls = (int) erts_smp_atomic32_addtest(&function_calls, reds);
 	ASSERT(esdp && esdp == erts_get_scheduler_data());
 
 	rq = erts_get_runq_current(esdp);
@@ -5839,6 +5229,9 @@ Process *schedule(Process *p, int calls)
 	}
 
 	if (IS_TRACED(p)) {
+	    if (IS_TRACED_FL(p, F_TRACE_CALLS) &&  p->status != P_FREE) {
+		erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT);
+	    }
 	    switch (p->status) {
 	    case P_EXITING:
 		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
@@ -5868,7 +5261,7 @@ Process *schedule(Process *p, int calls)
 	    handle_pending_suspend(p,
 				   ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
 	    ASSERT(!(p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ)
-		   || p->status != P_SUSPENDED);
+		   || p->rcount == 0);
 	}
 #endif
 	erts_smp_runq_lock(rq);
@@ -5882,8 +5275,11 @@ Process *schedule(Process *p, int calls)
 	p->status_flags &= ~ERTS_PROC_SFLG_RUNNING;
 
 	if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) {
+	    ErtsRunQueue *notify_runq;
 	    p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ;
-	    internal_add_to_runq(rq, p);
+	    notify_runq = internal_add_to_runq(rq, p);
+	    if (notify_runq != rq)
+		smp_notify_inc_runq(notify_runq);
 	}
 #endif
 
@@ -5919,10 +5315,10 @@ Process *schedule(Process *p, int calls)
 
 	ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-	dt = do_time_read_and_reset();
+	dt = erts_do_time_read_and_reset();
 	if (dt) {
 	    erts_smp_runq_unlock(rq);
-	    bump_timer(dt);
+	    erts_bump_timer(dt);
 	    erts_smp_runq_lock(rq);
 	}
 	BM_STOP_TIMER(system);
@@ -5951,21 +5347,33 @@ Process *schedule(Process *p, int calls)
 			 | ERTS_RUNQ_FLG_CHK_CPU_BIND
 			 | ERTS_RUNQ_FLG_SUSPENDED)) {
 	    if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
-		|| erts_smp_atomic_read(&esdp->suspended)) {
+		|| (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
+		    & ERTS_SSI_FLG_SUSPENDED)) {
+		ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags)
+		       & ERTS_SSI_FLG_SUSPENDED);
 		suspend_scheduler(esdp);
 	    }
 	    if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND)
-		|| erts_smp_atomic_read(&esdp->chk_cpu_bind)) {
-		check_cpu_bind(esdp);
+		|| erts_smp_atomic32_read_acqb(&esdp->chk_cpu_bind)) {
+		erts_sched_check_cpu_bind(esdp);
 	    }
 	}
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-	if (esdp->check_children) {
-	    esdp->check_children = 0;
-	    erts_smp_runq_unlock(rq);
-	    erts_check_children();
-	    erts_smp_runq_lock(rq);
+#if defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \
+	|| defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK)
+	{
+	    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
+	    erts_aint32_t aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+	    if (aux_work) {
+		erts_smp_runq_unlock(rq);
+#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+		aux_work = blockable_aux_work(esdp, ssi, aux_work);
+#endif
+#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK
+		nonblockable_aux_work(esdp, ssi, aux_work);
+#endif
+		erts_smp_runq_lock(rq);
+	    }
 	}
 #endif
 
@@ -5997,7 +5405,10 @@ Process *schedule(Process *p, int calls)
 	    if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ
 			     | ERTS_RUNQ_FLG_SUSPENDED)) {
 		if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
-		    || erts_smp_atomic_read(&esdp->suspended)) {
+		    || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
+			& ERTS_SSI_FLG_SUSPENDED)) {
+		    ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags)
+			   & ERTS_SSI_FLG_SUSPENDED);
 		    non_empty_runq(rq);
 		    goto continue_check_activities_to_run;
 		}
@@ -6014,17 +5425,7 @@ Process *schedule(Process *p, int calls)
 		}
 	    }
 
-	    if (prepare_for_sys_schedule()) {
-		erts_smp_atomic_set(&function_calls, 0);
-		fcalls = 0;
-		sched_sys_wait(esdp->no, rq);
-		erts_smp_atomic_set(&doing_sys_schedule, 0);
-	    }
-	    else {
-		/* If all schedulers are waiting, one of them *should*
-		   be waiting in erl_sys_schedule() */
-		sched_cnd_wait(esdp->no, rq);
-	    }
+	    scheduler_wait(&fcalls, esdp, rq);
 
 	    non_empty_runq(rq);
 
@@ -6048,19 +5449,21 @@ Process *schedule(Process *p, int calls)
 	     * Schedule system-level activities.
 	     */
 
-	    erts_smp_atomic_set(&function_calls, 0);
+	    erts_smp_atomic32_set_relb(&function_calls, 0);
 	    fcalls = 0;
+
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
+
 #ifdef ERTS_SMP
 	    /* erts_sys_schedule_interrupt(0); */
 #endif
 	    erts_smp_runq_unlock(rq);
 	    erl_sys_schedule(runnable);
-	    dt = do_time_read_and_reset();
-	    if (dt) bump_timer(dt);
+	    dt = erts_do_time_read_and_reset();
+	    if (dt) erts_bump_timer(dt);
 #ifdef ERTS_SMP
 	    erts_smp_runq_lock(rq);
-	    erts_smp_atomic_set(&doing_sys_schedule, 0);
+	    clear_sys_scheduling();
 	    goto continue_check_activities_to_run;
 #else
 	    if (!runnable)
@@ -6081,14 +5484,14 @@ Process *schedule(Process *p, int calls)
 		    if (rq->wakeup_other < 0)
 			rq->wakeup_other = 0;
 		}
-		else if (rq->wakeup_other < ERTS_WAKEUP_OTHER_LIMIT)
+		else if (rq->wakeup_other < wakeup_other_limit)
 		    rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC;
 		else {
 		    if (erts_common_run_queue) {
 			if (erts_common_run_queue->waiting)
-			    wake_one_scheduler();
+			    wake_scheduler(erts_common_run_queue, 0, 1);
 		    }
-		    else if (erts_smp_atomic_read(&no_empty_run_queues) != 0) {
+		    else if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) {
 			wake_scheduler_on_empty_runq(rq);
 			rq->wakeup_other = 0;
 		    }
@@ -6231,10 +5634,10 @@ Process *schedule(Process *p, int calls)
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
 
 	if (erts_sched_stat.enabled) {
-	    Uint old = ERTS_PROC_SCHED_ID(p,
+	    UWord old = ERTS_PROC_SCHED_ID(p,
 					  (ERTS_PROC_LOCK_MAIN
 					   | ERTS_PROC_LOCK_STATUS),
-					  esdp->no);
+					  (UWord) esdp->no);
 	    int migrated = old && old != esdp->no;
 
 	    erts_smp_spin_lock(&erts_sched_stat.lock);
@@ -6275,7 +5678,11 @@ Process *schedule(Process *p, int calls)
 		    trace_virtual_sched(p, am_in);
 		break;
 	    }
+	    if (IS_TRACED_FL(p, F_TRACE_CALLS)) {
+		erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN);
+	    }
 	}
+
 	if (p->status != P_EXITING)
 	    p->status = P_RUNNING;
 
@@ -6378,6 +5785,15 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
     ErtsRunQueue *rq = erts_get_runq_current(NULL);
     ErtsMiscOpList *molp = misc_op_list_alloc();
 
+    if (!rq) {
+	/*
+	 * This can only happen when the sys msg dispatcher
+	 * thread schedules misc ops (this happens *very*
+	 * seldom; only when trace drivers are unloaded).
+	 */
+	rq =  ERTS_RUNQ_IX(0);
+    }
+
     erts_smp_runq_lock(rq);
 
     while (rq->misc.evac_runq) {
@@ -6397,8 +5813,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
     else
 	rq->misc.start = molp;
     rq->misc.end = molp;
-    smp_notify_inc_runq(rq);
     erts_smp_runq_unlock(rq);
+    smp_notify_inc_runq(rq);
 }
 
 static void
@@ -6532,7 +5948,7 @@ erts_test_next_pid(int set, Uint next)
 
 Uint erts_process_count(void)
 {
-    long res = erts_smp_atomic_read(&process_count);
+    erts_aint32_t res = erts_smp_atomic32_read(&process_count);
     ASSERT(res >= 0);
     return (Uint) res;
 }
@@ -6581,7 +5997,7 @@ alloc_process(void)
     ASSERT(!process_tab[p_next]);
 
     process_tab[p_next] = p;
-    erts_smp_atomic_inc(&process_count);
+    erts_smp_atomic32_inc(&process_count);
     p->id = make_internal_pid(p_serial << p_serial_shift | p_next);
     if (p->id == ERTS_INVALID_PID) {
 	/* Do not use the invalid pid; change serial */
@@ -6640,7 +6056,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 		   Eterm args,	/* Arguments for function (must be well-formed list). */
 		   ErlSpawnOpts* so) /* Options for spawn. */
 {
-    ErtsRunQueue *rq;
+    ErtsRunQueue *rq, *notify_runq;
     Process *p;
     Sint arity;			/* Number of arguments. */
 #ifndef HYBRID
@@ -6707,7 +6123,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	p->min_heap_size  = H_MIN_SIZE;
 	p->min_vheap_size = BIN_VH_MIN_SIZE;
 	p->prio           = PRIORITY_NORMAL;
-	p->max_gen_gcs    = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs);
+	p->max_gen_gcs    = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs);
     }
     p->skipped = 0;
     ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0));
@@ -6719,11 +6135,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     /*
      * Must initialize binary lists here before copying binaries to process.
      */
-    p->off_heap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    p->off_heap.funs = NULL;
-#endif
-    p->off_heap.externals = NULL;
+    p->off_heap.first = NULL;
     p->off_heap.overhead = 0;
 
     heap_need +=
@@ -6757,13 +6169,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->bin_vheap_sz     = p->min_vheap_size;
     p->bin_old_vheap_sz = p->min_vheap_size;
     p->bin_old_vheap    = 0;
+    p->bin_vheap_mature = 0;
 
     /* No need to initialize p->fcalls. */
 
     p->current = p->initial+INITIAL_MOD;
 
-    p->i = (Eterm *) beam_apply;
-    p->cp = (Eterm *) beam_apply+1;
+    p->i = (BeamInstr *) beam_apply;
+    p->cp = (BeamInstr *) beam_apply+1;
 
     p->arg_reg = p->def_arg_reg;
     p->max_arg_reg = sizeof(p->def_arg_reg)/sizeof(p->def_arg_reg[0]);
@@ -6813,7 +6226,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	p->group_leader =
 	    IS_CONST(parent->group_leader)
 	    ? parent->group_leader
-	    : STORE_NC(&p->htop, &p->off_heap.externals, parent->group_leader);
+	    : STORE_NC(&p->htop, &p->off_heap, parent->group_leader);
     }
 
     erts_get_default_tracing(&p->trace_flags, &p->tracer_proc);
@@ -6957,10 +6370,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 #endif
 
     p->status = P_WAITING;
-    internal_add_to_runq(rq, p);
+    notify_runq = internal_add_to_runq(rq, p);
 
     erts_smp_runq_unlock(rq);
 
+    smp_notify_inc_runq(notify_runq);
+
     res = p->id;
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
 
@@ -7007,6 +6422,7 @@ void erts_init_empty_process(Process *p)
     p->bin_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap = 0;
+    p->bin_vheap_mature = 0;
 #ifdef ERTS_SMP
     p->u.ptimer = NULL;
     p->bound_runq = NULL;
@@ -7014,11 +6430,7 @@ void erts_init_empty_process(Process *p)
     memset(&(p->u.tm), 0, sizeof(ErlTimer));
 #endif
     p->next = NULL;
-    p->off_heap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    p->off_heap.funs = NULL;
-#endif
-    p->off_heap.externals = NULL;
+    p->off_heap.first = NULL;
     p->off_heap.overhead = 0;
     p->reg = NULL;
     p->heap_sz = 0;
@@ -7165,11 +6577,7 @@ erts_debug_verify_clean_empty_process(Process* p)
 
     /* Thing that erts_cleanup_empty_process() cleans up */
 
-    ASSERT(p->off_heap.mso == NULL);
-#ifndef HYBRID /* FIND ME! */
-    ASSERT(p->off_heap.funs == NULL);
-#endif
-    ASSERT(p->off_heap.externals == NULL);
+    ASSERT(p->off_heap.first == NULL);
     ASSERT(p->off_heap.overhead == 0);
 
     ASSERT(p->mbuf == NULL);
@@ -7180,25 +6588,16 @@ erts_debug_verify_clean_empty_process(Process* p)
 void
 erts_cleanup_empty_process(Process* p)
 {
-    ErlHeapFragment* mbufp;
-
     /* We only check fields that are known to be used... */
 
     erts_cleanup_offheap(&p->off_heap);
-    p->off_heap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    p->off_heap.funs = NULL;
-#endif
-    p->off_heap.externals = NULL;
+    p->off_heap.first = NULL;
     p->off_heap.overhead = 0;
 
-    mbufp = p->mbuf;
-    while (mbufp) {
-	ErlHeapFragment *next = mbufp->next;
-	free_message_buffer(mbufp);
-	mbufp = next;
+    if (p->mbuf != NULL) {
+	free_message_buffer(p->mbuf);
+	p->mbuf = NULL;
     }
-    p->mbuf = NULL;
 #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
     erts_lcnt_proc_lock_destroy(p);
 #endif
@@ -7214,7 +6613,6 @@ static void
 delete_process(Process* p)
 {
     ErlMessage* mp;
-    ErlHeapFragment* bp;
 
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->id));
 
@@ -7230,7 +6628,7 @@ delete_process(Process* p)
      * The mso list should not be used anymore, but if it is, make sure that
      * we'll notice.
      */
-    p->off_heap.mso = (void *) 0x8DEFFACD;
+    p->off_heap.first = (void *) 0x8DEFFACD;
 
     if (p->arg_reg != p->def_arg_reg) {
 	erts_free(ERTS_ALC_T_ARG_REG, p->arg_reg);
@@ -7264,11 +6662,8 @@ delete_process(Process* p)
     /*
      * Free all pending message buffers.
      */
-    bp = p->mbuf;
-    while (bp != NULL) {
-	ErlHeapFragment* next_bp = bp->next;
-	free_message_buffer(bp);
-	bp = next_bp;
+    if (p->mbuf != NULL) {	
+	free_message_buffer(p->mbuf);
     }
 
     erts_erase_dicts(p);
@@ -7348,7 +6743,7 @@ set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp)
     p->freason = EXTAG_EXIT;
     KILL_CATCHES(p);
     cancel_timer(p);
-    p->i = (Eterm *) beam_exit;
+    p->i = (BeamInstr *) beam_exit;
 }
 
 
@@ -7778,13 +7173,14 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 	    erts_port_release(prt); 
 	} else if (is_internal_pid(mon->pid)) {/* local by name or pid */
 	    Eterm watched;
-	    Eterm lhp[3];
+	    DeclareTmpHeapNoproc(lhp,3);
 	    ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK
 				      | ERTS_PROC_LOCKS_MSG_SEND);
 	    rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks);
 	    if (rp == NULL) {
 		goto done;
 	    }
+	    UseTmpHeapNoproc(3);
 	    rmon = erts_remove_monitor(&(rp->monitors),mon->ref);
 	    if (rmon) {
 		erts_destroy_monitor(rmon);
@@ -7795,6 +7191,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, 
 					   watched, pcontext->reason);
 	    }
+	    UnUseTmpHeapNoproc(3);
 	    /* else: demonitor while we exited, i.e. do nothing... */
 	    erts_smp_proc_unlock(rp, rp_locks);
 	} else { /* external by pid or name */
@@ -8025,8 +7422,13 @@ erts_do_exit_process(Process* p, Eterm reason)
     ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
 #endif
 
-    if (IS_TRACED_FL(p,F_TRACE_PROCS))
-	trace_proc(p, p, am_exit, reason);
+    if (IS_TRACED(p)) {
+	if (IS_TRACED_FL(p, F_TRACE_CALLS))
+	    erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING);
+
+	if (IS_TRACED_FL(p,F_TRACE_PROCS))
+	    trace_proc(p, p, am_exit, reason);
+    }
 
     erts_trace_check_exiting(p->id);
 
@@ -8075,6 +7477,8 @@ continue_exit_process(Process *p
     Eterm reason = p->fvalue;
     DistEntry *dep;
     struct saved_calls *scb;
+    process_breakpoint_time_t *pbt;
+
 #ifdef DEBUG
     int yield_allowed = 1;
 #endif
@@ -8176,8 +7580,8 @@ continue_exit_process(Process *p
 	p->status_flags = 0;
 #endif
 	process_tab[pix] = NULL; /* Time of death! */
-	ASSERT(erts_smp_atomic_read(&process_count) > 0);
-	erts_smp_atomic_dec(&process_count);
+	ASSERT(erts_smp_atomic32_read(&process_count) > 0);
+	erts_smp_atomic32_dec(&process_count);
 
 #ifdef ERTS_SMP
 	erts_pix_unlock(pix_lock);
@@ -8214,6 +7618,7 @@ continue_exit_process(Process *p
 	   ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL)
 	   : NULL);
     scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL);
+    pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL);
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
     processes_busy--;
@@ -8228,11 +7633,12 @@ continue_exit_process(Process *p
      * Pre-build the EXIT tuple if there are any links.
      */
     if (lnk) {
-	Eterm tmp_heap[4];
+	DeclareTmpHeap(tmp_heap,4,p);
 	Eterm exit_tuple;
 	Uint exit_tuple_sz;
 	Eterm* hp;
 
+	UseTmpHeap(4,p);
 	hp = &tmp_heap[0];
 
 	exit_tuple = TUPLE3(hp, am_EXIT, p->id, reason);
@@ -8243,16 +7649,21 @@ continue_exit_process(Process *p
 	    ExitLinkContext context = {p, reason, exit_tuple, exit_tuple_sz};
 	    erts_sweep_links(lnk, &doit_exit_link, &context);
 	}
+	UnUseTmpHeap(4,p);
     }
 
     {
 	ExitMonitorContext context = {reason, p};
-	erts_sweep_monitors(mon,&doit_exit_monitor,&context);
+	erts_sweep_monitors(mon,&doit_exit_monitor,&context); /* Allocates TmpHeap, but we
+								 have none here */
     }
 
     if (scb)
         erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb);
 
+    if (pbt)
+        erts_free(ERTS_ALC_T_BPD, (void *) pbt);
+
     delete_process(p);
 
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
@@ -8271,7 +7682,7 @@ continue_exit_process(Process *p
 
     ASSERT(p->status == P_EXITING);
 
-    p->i = (Eterm *) beam_continue_exit;
+    p->i = (BeamInstr *) beam_continue_exit;
 
     if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) {
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
@@ -8291,14 +7702,33 @@ continue_exit_process(Process *p
 static void
 timeout_proc(Process* p)
 {
-    p->i = (Eterm *) p->def_arg_reg[0];
+    BeamInstr** pi = (BeamInstr **) p->def_arg_reg;
+    p->i = *pi;
     p->flags |= F_TIMO;
     p->flags &= ~F_INSLPQUEUE;
 
-    if (p->status == P_WAITING)
-	erts_add_to_runq(p); 
-    if (p->status == P_SUSPENDED)
+    switch (p->status) {
+    case P_GARBING:
+	switch (p->gcstatus) {
+	case P_SUSPENDED:
+	    goto suspended;
+	case P_WAITING:
+	    goto waiting;
+	default:
+	    break;
+	}
+	break;
+    case P_WAITING:
+    waiting:
+	erts_add_to_runq(p);
+	break;
+    case P_SUSPENDED:
+    suspended:
 	p->rstatus = P_RUNABLE;   /* MUST set resume status to runnable */
+	break;
+    default:
+	break;
+    }
 }
 
 
@@ -8310,7 +7740,7 @@ cancel_timer(Process* p)
 #ifdef ERTS_SMP
     erts_cancel_smp_ptimer(p->u.ptimer);
 #else
-    erl_cancel_timer(&p->u.tm);
+    erts_cancel_timer(&p->u.tm);
 #endif
 }
 
@@ -8336,7 +7766,7 @@ set_timer(Process* p, Uint timeout)
 			   (ErlTimeoutProc) timeout_proc,
 			   timeout);
 #else
-    erl_set_timer(&p->u.tm,
+    erts_set_timer(&p->u.tm,
 		  (ErlTimeoutProc) timeout_proc,
 		  NULL,
 		  (void*) p,
@@ -8384,15 +7814,15 @@ erts_program_counter_info(int to, void *to_arg, Process *p)
 	     * only cause problems.
 	     */
 	    for (i = 0; i < p->arity; i++)
-		erts_print(to, to_arg, "   %T\n", p->arg_reg[i]);
+		erts_print(to, to_arg, "   %.*T\n", INT_MAX, p->arg_reg[i]);
 	}
     }
 }
 
 static void
-print_function_from_pc(int to, void *to_arg, Eterm* x)
+print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 {
-    Eterm* addr = find_function_from_pc(x);
+    BeamInstr* addr = find_function_from_pc(x);
     if (addr == NULL) {
         if (x == beam_exit) {
             erts_print(to, to_arg, "<terminate process>");
@@ -8426,7 +7856,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg)
     }
 
     if (is_CP(x)) {
-        erts_print(to, to_arg, "Return addr %p (", (Eterm *) x);
+        erts_print(to, to_arg, "Return addr %p (", (Eterm *) EXPAND_POINTER(x));
         print_function_from_pc(to, to_arg, cp_val(x));
         erts_print(to, to_arg, ")\n");
         yreg = 0;
@@ -9255,8 +8685,8 @@ init_processes_bif(void)
     processes_trap_export.code[0] = am_erlang;
     processes_trap_export.code[1] = am_processes_trap;
     processes_trap_export.code[2] = 2;
-    processes_trap_export.code[3] = (Eterm) em_apply_bif;
-    processes_trap_export.code[4] = (Eterm) &processes_trap;
+    processes_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    processes_trap_export.code[4] = (BeamInstr) &processes_trap;
 
 #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
     erts_get_emu_time(&debug_tv_start);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index f58b6932b3..296acc7367 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -28,6 +28,12 @@
 #define ERTS_INCLUDE_SCHEDULER_INTERNALS
 #endif
 
+/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */
+
+#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
+#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+#endif
+
 typedef struct process Process;
 
 #include "sys.h"
@@ -174,12 +180,12 @@ extern int erts_sched_thread_suggested_stack_size;
 #define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
   ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))
 
-#define ERTS_RUNQ_IFLG_SUSPENDED		(((long) 1) << 0)
-#define ERTS_RUNQ_IFLG_NONEMPTY			(((long) 1) << 1)
+#define ERTS_RUNQ_IFLG_SUSPENDED		(((erts_aint32_t) 1) << 0)
+#define ERTS_RUNQ_IFLG_NONEMPTY			(((erts_aint32_t) 1) << 1)
 
 
 #ifdef DEBUG
-#  ifdef ARCH_64
+#  if defined(ARCH_64) && !HALFWORD_HEAP
 #    define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \
        (*((char **) &(RQP)) = (char *) (0xdeadbeefdead0003 | ((N) << 4)))
 #  define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
@@ -194,8 +200,8 @@ do { \
 #  define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
 do { \
     ASSERT((RQP) != NULL); \
-    ASSERT(((((Uint) (RQP)) & ((Uint) 1))) == ((Uint) 0)); \
-    ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdead0000)); \
+    ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \
+    ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \
 } while (0)
 #  endif
 #else
@@ -219,6 +225,49 @@ typedef enum {
     ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
 } ErtsMigrateResult;
 
+#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
+#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
+#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
+#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
+#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)
+
+#define ERTS_SSI_FLGS_SLEEP_TYPE			\
+ (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)
+
+#define ERTS_SSI_FLGS_SLEEP				\
+ (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)
+
+#define ERTS_SSI_FLGS_ALL				\
+ (ERTS_SSI_FLGS_SLEEP					\
+  | ERTS_SSI_FLG_WAITING				\
+  | ERTS_SSI_FLG_SUSPENDED)
+
+#define ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
+
+#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN	(((erts_aint32_t) 1) << 0)
+#define ERTS_SSI_AUX_WORK_MISC			(((erts_aint32_t) 1) << 1)
+
+#define ERTS_SSI_BLOCKABLE_AUX_WORK_MASK \
+  (ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
+   | ERTS_SSI_AUX_WORK_MISC)
+#define ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK \
+  (0)
+
+typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
+
+typedef struct {
+    erts_smp_spinlock_t lock;
+    ErtsSchedulerSleepInfo *list;
+} ErtsSchedulerSleepList;
+
+struct ErtsSchedulerSleepInfo_ {
+    ErtsSchedulerSleepInfo *next;
+    ErtsSchedulerSleepInfo *prev;
+    erts_smp_atomic32_t flags;
+    erts_tse_t *event;
+    erts_smp_atomic32_t aux_work;
+};
+
 /* times to reschedule low prio process before running */
 #define RESCHEDULE_LOW        8
 
@@ -266,13 +315,14 @@ typedef struct {
 
 struct ErtsRunQueue_ {
     int ix;
-    erts_smp_atomic_t info_flags;
+    erts_smp_atomic32_t info_flags;
 
     erts_smp_mtx_t mtx;
     erts_smp_cnd_t cnd;
 
-    erts_smp_atomic_t spin_waiter;
-    erts_smp_atomic_t spin_wake;
+#ifdef ERTS_SMP
+    ErtsSchedulerSleepList sleepers;
+#endif
 
     ErtsSchedulerData *scheduler;
     int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
@@ -344,13 +394,25 @@ struct ErtsSchedulerData_ {
      * numbered registers as possible in the same cache
      * line).
      */
+#if !HALFWORD_HEAP
     Eterm save_reg[ERTS_X_REGS_ALLOCATED]; /* X registers */
+#else
+    Eterm *save_reg;
+#endif
     FloatDef freg[MAX_REG];	/* Floating point registers. */
     ethr_tid tid;		/* Thread id */
     struct erl_bits_state erl_bits_state; /* erl_bits.c state */
     void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
+    ErtsSchedulerSleepInfo *ssi;
     Process *free_process;
 #endif
+#if !HEAP_ON_C_STACK
+    Eterm tmp_heap[TMP_HEAP_SIZE];
+    int num_tmp_heap_used;
+    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
+    Eterm cmp_tmp_heap[CMP_TMP_HEAP_SIZE];
+    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
+#endif
 
     Process *current_process;
     Uint no;			/* Scheduler number */
@@ -363,15 +425,22 @@ struct ErtsSchedulerData_ {
 
 #ifdef ERTS_SMP
     /* NOTE: These fields are modified under held mutexes by other threads */
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-    int check_children; /* run queue mutex */
-    int blocked_check_children; /* schdlr_sspnd mutex */
+    erts_smp_atomic32_t chk_cpu_bind; /* Only used when common run queue */
 #endif
-    erts_smp_atomic_t suspended; /* Only used when common run queue */
-    erts_smp_atomic_t chk_cpu_bind; /* Only used when common run queue */
+
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+    erts_alloc_verify_func_t verify_unused_temp_alloc;
+    Allctr_t *verify_unused_temp_alloc_data;
 #endif
 };
 
+typedef union {
+    ErtsSchedulerData esd;
+    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
+} ErtsAlignedSchedulerData;
+
+extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
+
 #ifndef ERTS_SMP
 extern ErtsSchedulerData *erts_scheduler_data;
 #endif
@@ -386,8 +455,9 @@ extern ErtsSchedulerData *erts_scheduler_data;
 #define ERTS_PSD_SAVED_CALLS_BUF		1
 #define ERTS_PSD_SCHED_ID			2
 #define ERTS_PSD_DIST_ENTRY			3
+#define ERTS_PSD_CALL_TIME_BP			4
 
-#define ERTS_PSD_SIZE				4
+#define ERTS_PSD_SIZE				5
 
 typedef struct {
     void *data[ERTS_PSD_SIZE];
@@ -408,6 +478,9 @@ typedef struct {
 #define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
+#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN
+
 typedef struct {
     ErtsProcLocks get_locks;
     ErtsProcLocks set_locks;
@@ -479,6 +552,7 @@ struct ErtsPendingSuspend_ {
 
 #  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
 #  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
+#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
 #  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
 #  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap
 
@@ -518,8 +592,8 @@ struct process {
     unsigned max_arg_reg;	/* Maximum number of argument registers available. */
     Eterm def_arg_reg[6];	/* Default array for argument registers. */
 
-    Eterm* cp;			/* Continuation pointer (for threaded code). */
-    Eterm* i;			/* Program counter for threaded code. */
+    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
+    BeamInstr* i;		/* Program counter for threaded code. */
     Sint catches;		/* Number of catches on stack */
     Sint fcalls;		/* 
 				 * Number of reductions left to execute.
@@ -566,11 +640,12 @@ struct process {
     Uint seq_trace_lastcnt;
     Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */
 
-    Eterm initial[3];		/* Initial module(0), function(1), arity(2) */
-    Eterm* current;		/* Current Erlang function:
+    BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
+				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
+    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
 				 * module(0), function(1), arity(2)
 				 * (module and functions are tagged atoms;
-				 * arity an untagged integer).
+				 * arity an untagged integer). BeamInstr * because it references code
 				 */
     
     /*
@@ -595,9 +670,10 @@ struct process {
     Uint mbuf_sz;		/* Size of all message buffers */
     ErtsPSD *psd;		/* Rarely used process specific data */
 
-    Uint bin_vheap_sz;		/* Virtual heap block size for binaries */
-    Uint bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
-    Uint bin_old_vheap;		/* Virtual old heap size for binaries */
+    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
+    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
+    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
+    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
 
     union {
 #ifdef ERTS_SMP
@@ -753,13 +829,13 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
 {
     ErlHeapFragment* hf = MBUF(p);
 
-    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->size));
+    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));
 
     hf->used_size = hp - hf->mem;
 }	
 #endif /* inline */
 
-Eterm* erts_heap_alloc(Process* p, Uint need);
+Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
 #ifdef CHECK_FOR_HOLES
 Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
 #endif
@@ -812,13 +888,14 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
 #define F_TIMO               (1 <<  2) /* Set if timeout */
 #define F_HEAP_GROW          (1 <<  3)
-#define F_NEED_FULLSWEEP     (1 <<  4) /* If process has old binaries & funs. */
+#define F_NEED_FULLSWEEP     (1 <<  4)
 #define F_USING_DB           (1 <<  5) /* If have created tables */
 #define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
 #define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
 #define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
 #define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
+#define F_HIBERNATE_SCHED    (1 << 11) /* Schedule out after hibernate op */
 
 /* process trace_flags */
 #define F_SENSITIVE          (1 << 0)
@@ -946,26 +1023,12 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 	    (p)->flags &= ~F_TIMO; \
     } while (0)
 
-
-#define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS		0
-#define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED		1
-#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY	2
-#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE	3
-
-int erts_init_scheduler_bind_type(char *how);
-
-#define ERTS_INIT_CPU_TOPOLOGY_OK			0
-#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID		1
-#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE		2
-#define ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY	3
-#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE		4
-#define ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES		5
-#define ERTS_INIT_CPU_TOPOLOGY_MISSING_LID		6
-#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS		7
-#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES	8
-#define ERTS_INIT_CPU_TOPOLOGY_MISSING			9
-
-int erts_init_cpu_topology(char *topology_str);
+#define ERTS_RUNQ_IX(IX)						\
+  (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues),			\
+   &erts_aligned_run_queues[(IX)].runq)
+#define ERTS_SCHEDULER_IX(IX)						\
+  (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers),			\
+   &erts_aligned_scheduler_data[(IX)].esd)
 
 void erts_pre_init_process(void);
 void erts_late_init_process(void);
@@ -976,9 +1039,12 @@ ErtsProcList *erts_proclist_create(Process *);
 void erts_proclist_destroy(ErtsProcList *);
 int erts_proclist_same(ErtsProcList *, Process *);
 
+int erts_sched_set_wakeup_limit(char *str);
+
 #ifdef DEBUG
 void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
 #endif
+int erts_get_max_no_executing_schedulers(void);
 #ifdef ERTS_SMP
 ErtsSchedSuspendResult
 erts_schedulers_state(Uint *, Uint *, Uint *, int);
@@ -993,9 +1059,15 @@ int erts_is_multi_scheduling_blocked(void);
 Eterm erts_multi_scheduling_blockers(Process *);
 void erts_start_schedulers(void);
 void erts_smp_notify_check_children_needed(void);
+void
+erts_smp_schedule_misc_aux_work(int ignore_self,
+				int max_sched,
+				void (*func)(void *),
+				void *arg);
 #endif
+void erts_sched_notify_check_cpu_bind(void);
 Uint erts_active_schedulers(void);
-void erts_init_process(void);
+void erts_init_process(int);
 Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
 Uint erts_run_queues_len(Uint *);
 void erts_add_to_runq(Process *);
@@ -1069,6 +1141,9 @@ void erts_handle_pending_exit(Process *, ErtsProcLocks);
 
 void erts_deep_process_dump(int, void *);
 
+Eterm erts_get_reader_groups_map(Process *c_p);
+Eterm erts_debug_reader_groups_map(Process *c_p, int groups);
+
 Sint erts_test_next_pid(int, Uint);
 Eterm erts_debug_processes(Process *c_p);
 Eterm erts_debug_processes_bif_info(Process *c_p);
@@ -1082,6 +1157,20 @@ Uint erts_debug_nbalance(void);
 #  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
 #endif
 
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
+do {										\
+    ErtsSchedulerData *esdp__ = ((P)						\
+				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
+				 : erts_get_scheduler_data());			\
+    if (esdp__)									\
+	esdp__->verify_unused_temp_alloc(					\
+	    esdp__->verify_unused_temp_alloc_data);				\
+} while (0)
+#else
+#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
+#endif
+
 #if defined(ERTS_SMP) || defined(USE_THREADS)
 ErtsSchedulerData *erts_get_scheduler_data(void);
 #else
@@ -1184,7 +1273,7 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
 #endif
 
 #define ERTS_PROC_SCHED_ID(P, L, ID) \
-  ((Uint) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))
+  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))
 
 #define ERTS_PROC_GET_DIST_ENTRY(P) \
   ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY))
@@ -1196,6 +1285,12 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
 #define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
   ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))
 
+#define ERTS_PROC_GET_CALL_TIME(P) \
+  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
+#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
+  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))
+
+
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
 ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
 						  ErtsProcLocks plocks,
@@ -1209,8 +1304,8 @@ erts_proc_get_error_handler(Process *p)
     if (!val)
 	return am_error_handler;
     else {
-	ASSERT(is_atom(((Eterm) val)));
-	return (Eterm) val;
+	ASSERT(is_atom(((Eterm) (UWord) val)));
+	return (Eterm) (UWord) val;
     }
 }
 
@@ -1220,13 +1315,13 @@ erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
     void *old_val;
     void *new_val;
     ASSERT(is_atom(handler));
-    new_val = handler == am_error_handler ? NULL : (void *) handler;
+    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
     old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
     if (!old_val)
 	return am_error_handler;
     else {
-	ASSERT(is_atom(((Eterm) old_val)));
-	return (Eterm) old_val;
+	ASSERT(is_atom(((Eterm) (UWord) old_val)));
+	return (Eterm) (UWord) old_val;
     }
 }
 
@@ -1440,6 +1535,10 @@ erts_get_atom_cache_map(Process *c_p)
 }
 #endif
 
+Process *erts_pid2proc_suspend(Process *,
+			       ErtsProcLocks,
+			       Eterm,
+			       ErtsProcLocks);
 #ifdef ERTS_SMP
 
 Process *erts_pid2proc_not_running(Process *,
@@ -1487,29 +1586,32 @@ extern int erts_disable_proc_not_running_opt;
 #define ERTS_MIN_PROCESSES		16
 #endif
 
-#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
-ERTS_GLB_INLINE void erts_smp_notify_inc_runq(ErtsRunQueue *runq);
-void erts_smp_notify_inc_runq__(ErtsRunQueue *runq);
-#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
+void erts_smp_notify_inc_runq(ErtsRunQueue *runq);
 
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+#ifdef ERTS_SMP
+void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
+ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);
 
-#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
-erts_smp_notify_inc_runq(ErtsRunQueue *runq)
+erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
 {
-#ifdef ERTS_SMP
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
-    if (runq->waiting)
-	erts_smp_notify_inc_runq__(runq);
-#endif
+    erts_aint32_t flags;
+    ERTS_THR_MEMORY_BARRIER;
+    flags = erts_smp_atomic32_read(&ssi->flags);
+    ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING)
+	   || (flags & ERTS_SSI_FLG_WAITING));
+    if (flags & ERTS_SSI_FLG_SLEEPING) {
+	flags = erts_smp_atomic32_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
+	erts_sched_finish_poke(ssi, flags);
+    }
 }
 
-#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
-
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
+#endif /* #ifdef ERTS_SMP */
+
 #include "erl_process_lock.h"
 
 #undef ERTS_INCLUDE_SCHEDULER_INTERNALS
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 1666509c72..5410bcd495 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -45,16 +45,16 @@ static void dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep);
 static void dump_element_nl(int to, void *to_arg, Eterm x);
 static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp,
 			      int yreg);
-static void print_function_from_pc(int to, void *to_arg, Eterm* x);
+static void print_function_from_pc(int to, void *to_arg, BeamInstr* x);
 static void heap_dump(int to, void *to_arg, Eterm x);
 static void dump_binaries(int to, void *to_arg, Binary* root);
 static void dump_externally(int to, void *to_arg, Eterm term);
 
 static Binary* all_binaries;
 
-extern Eterm beam_apply[];
-extern Eterm beam_exit[];
-extern Eterm beam_continue_exit[];
+extern BeamInstr beam_apply[];
+extern BeamInstr beam_exit[];
+extern BeamInstr beam_continue_exit[];
 
 
 void
@@ -194,7 +194,7 @@ dump_element(int to, void *to_arg, Eterm x)
 	} else if (is_pid(x)) {
 	    erts_print(to, to_arg, "P%T", x);
 	} else if (is_port(x)) {
-	    erts_print(to, to_arg, "p<%bpu.%bpu>",
+	    erts_print(to, to_arg, "p<%beu.%beu>",
 		       port_channel_no(x), port_number(x));
 	} else if (is_nil(x)) {
 	    erts_putc(to, to_arg, 'N');
@@ -223,7 +223,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg)
     }
 
     if (is_CP(x)) {
-        erts_print(to, to_arg, "SReturn addr 0x%X (", (Eterm *) x);
+        erts_print(to, to_arg, "SReturn addr 0x%X (", cp_val(x));
         print_function_from_pc(to, to_arg, cp_val(x));
         erts_print(to, to_arg, ")\n");
         yreg = 0;
@@ -239,9 +239,9 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg)
 }
 
 static void
-print_function_from_pc(int to, void *to_arg, Eterm* x)
+print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 {
-    Eterm* addr = find_function_from_pc(x);
+    BeamInstr* addr = find_function_from_pc(x);
     if (addr == NULL) {
         if (x == beam_exit) {
             erts_print(to, to_arg, "<terminate process>");
@@ -261,139 +261,139 @@ print_function_from_pc(int to, void *to_arg, Eterm* x)
 static void
 heap_dump(int to, void *to_arg, Eterm x)
 {
+    DeclareTmpHeapNoproc(last,1);
+    Eterm* next = last;
     Eterm* ptr;
-    Eterm last = OUR_NIL;
-    Eterm* next = &last;
 
     if (is_immed(x) || is_CP(x)) {
 	return;
     }
-
- again:
-    if (x == OUR_NIL) {	/* We are done. */
-	return;
-    } if (is_CP(x)) {
-	next = (Eterm *) x;
-    } else if (is_list(x)) {
-	ptr = list_val(x);
-	if (ptr[0] != OUR_NIL) {
-	    erts_print(to, to_arg, ADDR_FMT ":l", ptr);
-	    dump_element(to, to_arg, ptr[0]);
-	    erts_putc(to, to_arg, '|');
-	    dump_element(to, to_arg, ptr[1]);
-	    erts_putc(to, to_arg, '\n');
-	    if (is_immed(ptr[1])) {
-		ptr[1] = make_small(0);
-	    }
-	    x = ptr[0];
-	    ptr[0] = (Eterm) next;
-	    next = ptr + 1;
-	    goto again;
-	}
-    } else if (is_boxed(x)) {
-	Eterm hdr;
-	
-	ptr = boxed_val(x);
-	hdr = *ptr;
-	if (hdr != OUR_NIL) {	/* If not visited */
-	    erts_print(to, to_arg, ADDR_FMT ":", ptr);
-	    if (is_arity_value(hdr)) {
-		Uint i;
-		Uint arity = arityval(hdr);
-
-		erts_print(to, to_arg, "t" WORD_FMT ":", arity);
-		for (i = 1; i <= arity; i++) {
-		    dump_element(to, to_arg, ptr[i]);
-		    if (is_immed(ptr[i])) {
-			ptr[i] = make_small(0);
-		    }
-		    if (i < arity) {
-			erts_putc(to, to_arg, ',');
-		    }
-		}
+    UseTmpHeapNoproc(1);
+    *last = OUR_NIL;
+
+    while (x != OUR_NIL) {
+	if (is_CP(x)) {
+	    next = (Eterm *) EXPAND_POINTER(x);
+	} else if (is_list(x)) {
+	    ptr = list_val(x);
+	    if (ptr[0] != OUR_NIL) {
+		erts_print(to, to_arg, ADDR_FMT ":l", ptr);
+		dump_element(to, to_arg, ptr[0]);
+		erts_putc(to, to_arg, '|');
+		dump_element(to, to_arg, ptr[1]);
 		erts_putc(to, to_arg, '\n');
-		if (arity == 0) {
-		    ptr[0] = OUR_NIL;
-		} else {
-		    x = ptr[arity];
-		    ptr[0] = (Eterm) next;
-		    next = ptr + arity - 1;
-		    goto again;
+		if (is_immed(ptr[1])) {
+		    ptr[1] = make_small(0);
 		}
-	    } else if (hdr == HEADER_FLONUM) {
-		FloatDef f;
-		char sbuf[31];
-		int i;
-
-		GET_DOUBLE_DATA((ptr+1), f);
-		i = sys_double_to_chars(f.fd, (char*) sbuf);
-		sys_memset(sbuf+i, 0, 31-i);
-		erts_print(to, to_arg, "F%X:%s\n", i, sbuf);
-		*ptr = OUR_NIL;
-	    } else if (_is_bignum_header(hdr)) {
-		erts_print(to, to_arg, "B%T\n", x);
-		*ptr = OUR_NIL;
-	    } else if (is_binary_header(hdr)) {
-		Uint tag = thing_subtag(hdr);
-		Uint size = binary_size(x);
-		Uint i;
-
-		if (tag == HEAP_BINARY_SUBTAG) {
-		    byte* p;
-
-		    erts_print(to, to_arg, "Yh%X:", size);
-		    p = binary_bytes(x);
-		    for (i = 0; i < size; i++) {
-			erts_print(to, to_arg, "%02X", p[i]);
+		x = ptr[0];
+		ptr[0] = (Eterm) COMPRESS_POINTER(next);
+		next = ptr + 1;
+		continue;
+	    }
+	} else if (is_boxed(x)) {
+	    Eterm hdr;
+
+	    ptr = boxed_val(x);
+	    hdr = *ptr;
+	    if (hdr != OUR_NIL) {	/* If not visited */
+		erts_print(to, to_arg, ADDR_FMT ":", ptr);
+	        if (is_arity_value(hdr)) {
+		    Uint i;
+		    Uint arity = arityval(hdr);
+
+		    erts_print(to, to_arg, "t" WORD_FMT ":", arity);
+		    for (i = 1; i <= arity; i++) {
+			dump_element(to, to_arg, ptr[i]);
+			if (is_immed(ptr[i])) {
+			    ptr[i] = make_small(0);
+			}
+			if (i < arity) {
+			    erts_putc(to, to_arg, ',');
+			}
 		    }
-		} else if (tag == REFC_BINARY_SUBTAG) {
-		    ProcBin* pb = (ProcBin *) binary_val(x);
-		    Binary* val = pb->val;
-
-		    if (erts_smp_atomic_xchg(&val->refc, 0) != 0) {
-			val->flags = (Uint) all_binaries;
-			all_binaries = val;
+		    erts_putc(to, to_arg, '\n');
+		    if (arity == 0) {
+			ptr[0] = OUR_NIL;
+		    } else {
+			x = ptr[arity];
+			ptr[0] = (Eterm) COMPRESS_POINTER(next);
+			next = ptr + arity - 1;
+			continue;
 		    }
-		    erts_print(to, to_arg, "Yc%X:%X:%X", val,
-			       pb->bytes - (byte *)val->orig_bytes,
-			       size);
-		} else if (tag == SUB_BINARY_SUBTAG) {
-		    ErlSubBin* Sb = (ErlSubBin *) binary_val(x);
-		    Eterm* real_bin = binary_val(Sb->orig);
-		    void* val;
-
-		    if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) {
-			ProcBin* pb = (ProcBin *) real_bin;
-			val = pb->val;
-		    } else {	/* Heap binary */
-			val = real_bin;
+		} else if (hdr == HEADER_FLONUM) {
+		    FloatDef f;
+		    char sbuf[31];
+		    int i;
+
+		    GET_DOUBLE_DATA((ptr+1), f);
+		    i = sys_double_to_chars(f.fd, (char*) sbuf);
+		    sys_memset(sbuf+i, 0, 31-i);
+		    erts_print(to, to_arg, "F%X:%s\n", i, sbuf);
+		    *ptr = OUR_NIL;
+		} else if (_is_bignum_header(hdr)) {
+		    erts_print(to, to_arg, "B%T\n", x);
+		    *ptr = OUR_NIL;
+		} else if (is_binary_header(hdr)) {
+		    Uint tag = thing_subtag(hdr);
+		    Uint size = binary_size(x);
+		    Uint i;
+
+		    if (tag == HEAP_BINARY_SUBTAG) {
+			byte* p;
+
+			erts_print(to, to_arg, "Yh%X:", size);
+			p = binary_bytes(x);
+			for (i = 0; i < size; i++) {
+			    erts_print(to, to_arg, "%02X", p[i]);
+			}
+		    } else if (tag == REFC_BINARY_SUBTAG) {
+			ProcBin* pb = (ProcBin *) binary_val(x);
+			Binary* val = pb->val;
+
+			if (erts_smp_atomic_xchg(&val->refc, 0) != 0) {
+			    val->flags = (UWord) all_binaries;
+			    all_binaries = val;
+			}
+			erts_print(to, to_arg, "Yc%X:%X:%X", val,
+				   pb->bytes - (byte *)val->orig_bytes,
+				   size);
+		    } else if (tag == SUB_BINARY_SUBTAG) {
+			ErlSubBin* Sb = (ErlSubBin *) binary_val(x);
+			Eterm* real_bin = binary_val(Sb->orig);
+			void* val;
+
+			if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) {
+			    ProcBin* pb = (ProcBin *) real_bin;
+			    val = pb->val;
+			} else {	/* Heap binary */
+			    val = real_bin;
+			}
+			erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size);
 		    }
-		    erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size);
+		    erts_putc(to, to_arg, '\n');
+		    *ptr = OUR_NIL;
+		} else if (is_external_pid_header(hdr)) {
+		    erts_print(to, to_arg, "P%T\n", x);
+		    *ptr = OUR_NIL;
+		} else if (is_external_port_header(hdr)) {
+		    erts_print(to, to_arg, "p<%beu.%beu>\n",
+			       port_channel_no(x), port_number(x));
+		    *ptr = OUR_NIL;
+		} else {
+		    /*
+		     * All other we dump in the external term format.
+		     */
+			dump_externally(to, to_arg, x);
+		    erts_putc(to, to_arg, '\n');
+		    *ptr = OUR_NIL;
 		}
-		erts_putc(to, to_arg, '\n');
-		*ptr = OUR_NIL;
-	    } else if (is_external_pid_header(hdr)) {
-		erts_print(to, to_arg, "P%T\n", x);
-		*ptr = OUR_NIL;
-	    } else if (is_external_port_header(hdr)) {
-		erts_print(to, to_arg, "p<%bpu.%bpu>\n",
-			   port_channel_no(x), port_number(x));
-		*ptr = OUR_NIL;
-	    } else {
-		/*
-		 * All other we dump in the external term format.
-		 */
-		dump_externally(to, to_arg, x);
-		erts_putc(to, to_arg, '\n');
-		*ptr = OUR_NIL;
 	    }
 	}
+	x = *next;
+	*next = OUR_NIL;
+	next--;
     }
-
-    x = *next;
-    *next = OUR_NIL;
-    next--;
-    goto again;
+    UnUseTmpHeapNoproc(1);
 }
 
 static void
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 52440fb635..72560aa124 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2007-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -71,9 +71,12 @@ const Process erts_proc_lock_busy;
 
 #ifdef ERTS_SMP
 
-/*#define ERTS_PROC_LOCK_SPIN_ON_GATE*/
-#define ERTS_PROC_LOCK_SPIN_COUNT_MAX  16000
+#define ERTS_PROC_LOCK_SPIN_COUNT_MAX  2000
+#define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32
 #define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000
+#define ERTS_PROC_LOCK_AUX_SPIN_COUNT 50
+
+#define ERTS_PROC_LOCK_SPIN_UNTIL_YIELD 25
 
 #ifdef ERTS_PROC_LOCK_DEBUG
 #define ERTS_PROC_LOCK_HARD_DEBUG
@@ -83,32 +86,19 @@ const Process erts_proc_lock_busy;
 static void check_queue(erts_proc_lock_t *lck);
 #endif
 
-
-typedef struct erts_proc_lock_waiter_t_ erts_proc_lock_waiter_t;
-struct erts_proc_lock_waiter_t_ {
-    erts_proc_lock_waiter_t *next;
-    erts_proc_lock_waiter_t *prev;
-    ErtsProcLocks wait_locks;
-    erts_smp_gate_t gate;
-    erts_proc_lock_queues_t *queues;
-};
+#if SIZEOF_INT < 4
+#error "The size of the 'uflgs' field of the erts_tse_t type is too small"
+#endif
 
 struct erts_proc_lock_queues_t_ {
     erts_proc_lock_queues_t *next;
-    erts_proc_lock_waiter_t *queue[ERTS_PROC_LOCK_MAX_BIT+1];
-};
-
-struct erts_proc_lock_thr_spec_data_t_ {
-    erts_proc_lock_queues_t *qs;
-    erts_proc_lock_waiter_t *wtr;
+    erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1];
 };
 
 static erts_proc_lock_queues_t zeroqs = {0};
 
-static erts_smp_spinlock_t wtr_lock;
-static erts_proc_lock_waiter_t *waiter_free_list;
+static erts_smp_spinlock_t qs_lock;
 static erts_proc_lock_queues_t *queue_free_list;
-static erts_tsd_key_t waiter_key;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 static struct {
@@ -122,122 +112,131 @@ static struct {
 erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];
 
 static int proc_lock_spin_count;
-static int proc_lock_trans_spin_cost;
+static int aux_thr_proc_lock_spin_count;
 
-static void cleanup_waiter(void);
+static void cleanup_tse(void);
 
 void
-erts_init_proc_lock(void)
+erts_init_proc_lock(int cpus)
 {
     int i;
-    int cpus;
-    erts_smp_spinlock_init(&wtr_lock, "proc_lck_wtr_alloc");
+    erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc");
     for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) {
-#if ERTS_PROC_LOCK_MUTEX_IMPL
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	erts_smp_mtx_init_x(&erts_pix_locks[i].u.mtx, "pix_lock", make_small(i));
-#else
-	erts_smp_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock");
-#endif
-#else
 #ifdef ERTS_ENABLE_LOCK_COUNT
-	erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, "pix_lock", make_small(i));
+	erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck,
+				 "pix_lock", make_small(i));
 #else
 	erts_smp_spinlock_init(&erts_pix_locks[i].u.spnlck, "pix_lock");
 #endif
-#endif
     }
-    waiter_free_list = NULL;
     queue_free_list = NULL;
-    erts_tsd_key_create(&waiter_key);
-    erts_thr_install_exit_handler(cleanup_waiter);
+    erts_thr_install_exit_handler(cleanup_tse);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     lc_id.proc_lock_main	= erts_lc_get_lock_order_id("proc_main");
     lc_id.proc_lock_link	= erts_lc_get_lock_order_id("proc_link");
     lc_id.proc_lock_msgq	= erts_lc_get_lock_order_id("proc_msgq");
     lc_id.proc_lock_status	= erts_lc_get_lock_order_id("proc_status");
 #endif
-    cpus = erts_get_cpu_configured(erts_cpuinfo);
-    if (cpus > 1)
-	proc_lock_spin_count = (ERTS_PROC_LOCK_SPIN_COUNT_BASE
-				* ((int) erts_no_schedulers));
-    else if (cpus == 1)
-	proc_lock_spin_count = 0;
-    else /* No of cpus unknown. Assume multi proc, but be conservative. */
+    if (cpus > 1) {
 	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE;
-    if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX)
-	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX;
-    proc_lock_trans_spin_cost = proc_lock_spin_count/20;
-}
-
-static ERTS_INLINE erts_proc_lock_waiter_t *
-alloc_wtr(void)
-{
-    erts_proc_lock_waiter_t *wtr;
-    erts_smp_spin_lock(&wtr_lock);
-    wtr = waiter_free_list;
-    if (wtr) {
-	waiter_free_list = wtr->next;
-	ERTS_LC_ASSERT(queue_free_list);
-	wtr->queues = queue_free_list;
-	queue_free_list = wtr->queues->next;
-	erts_smp_spin_unlock(&wtr_lock);
+	proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC
+				 * ((int) erts_no_schedulers));
+	aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT;
     }
-    else {
-	erts_smp_spin_unlock(&wtr_lock);
-	wtr = erts_alloc(ERTS_ALC_T_PROC_LCK_WTR,
-			 sizeof(erts_proc_lock_waiter_t));
-	erts_smp_gate_init(&wtr->gate);
-	wtr->wait_locks = (ErtsProcLocks) 0;
-	wtr->queues = erts_alloc(ERTS_ALC_T_PROC_LCK_QS,
-				 sizeof(erts_proc_lock_queues_t));
-	sys_memcpy((void *) wtr->queues,
-		   (void *) &zeroqs,
-		   sizeof(erts_proc_lock_queues_t));
+    else if (cpus == 1) {
+	proc_lock_spin_count = 0;
+	aux_thr_proc_lock_spin_count = 0;
     }
-    return wtr;
+    else { /* No of cpus unknown. Assume multi proc, but be conservative. */
+	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE/2;
+	aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT/2;
+    }
+    if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX)
+	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX;
 }
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 static void
-check_unused_waiter(erts_proc_lock_waiter_t *wtr)
+check_unused_tse(erts_tse_t *wtr)
 {
     int i;
-    ERTS_LC_ASSERT(wtr->wait_locks == 0);
+    erts_proc_lock_queues_t *queues = wtr->udata;
+    ERTS_LC_ASSERT(wtr->uflgs == 0);
     for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
-	ERTS_LC_ASSERT(!wtr->queues->queue[i]);
+	ERTS_LC_ASSERT(!queues->queue[i]);
 }
-#define CHECK_UNUSED_WAITER(W) check_unused_waiter((W))
+#define CHECK_UNUSED_TSE(W) check_unused_tse((W))
 #else
-#define CHECK_UNUSED_WAITER(W)
+#define CHECK_UNUSED_TSE(W)
 #endif
 
+static ERTS_INLINE erts_tse_t *
+tse_fetch(erts_pix_lock_t *pix_lock)
+{
+    erts_tse_t *tse = erts_tse_fetch();
+    if (!tse->udata) {
+	erts_proc_lock_queues_t *qs;
+#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL
+	if (pix_lock)
+	    erts_pix_unlock(pix_lock);
+#endif
+	erts_smp_spin_lock(&qs_lock);
+	qs = queue_free_list;
+	if (qs) {
+	    queue_free_list = queue_free_list->next;
+	    erts_smp_spin_unlock(&qs_lock);
+	}
+	else {
+	    erts_smp_spin_unlock(&qs_lock);
+	    qs = erts_alloc(ERTS_ALC_T_PROC_LCK_QS,
+			    sizeof(erts_proc_lock_queues_t));
+	    sys_memcpy((void *) qs,
+		       (void *) &zeroqs,
+		       sizeof(erts_proc_lock_queues_t));
+	}
+	tse->udata = qs;
+#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL
+	if (pix_lock)
+	    erts_pix_lock(pix_lock);
+#endif
+    }
+    tse->uflgs = 0;
+    return tse;
+}
 
 static ERTS_INLINE void
-free_wtr(erts_proc_lock_waiter_t *wtr)
+tse_return(erts_tse_t *tse, int force_free_q)
 {
-    CHECK_UNUSED_WAITER(wtr);
-    erts_smp_spin_lock(&wtr_lock);
-    wtr->next = waiter_free_list;
-    waiter_free_list = wtr;
-    wtr->queues->next = queue_free_list;
-    queue_free_list = wtr->queues;
-    erts_smp_spin_unlock(&wtr_lock);
+    CHECK_UNUSED_TSE(tse);
+    if (force_free_q || erts_tse_is_tmp(tse)) {
+	erts_proc_lock_queues_t *qs = tse->udata;
+	ASSERT(qs);
+	erts_smp_spin_lock(&qs_lock);
+	qs->next = queue_free_list;
+	queue_free_list = qs;
+	erts_smp_spin_unlock(&qs_lock);
+	tse->udata = NULL;
+    }
+    erts_tse_return(tse);
 }
 
 void
 erts_proc_lock_prepare_proc_lock_waiter(void)
 {
-    erts_tsd_set(waiter_key, (void *) alloc_wtr());
+    tse_return(tse_fetch(NULL), 0);
 }
 
 
 static void
-cleanup_waiter(void)
+cleanup_tse(void)
 {
-    erts_proc_lock_waiter_t *wtr = erts_tsd_get(waiter_key);
-    if (wtr)
-	free_wtr(wtr);
+    erts_tse_t *tse = erts_tse_fetch();
+    if (tse) {
+	if (tse->udata)
+	    tse_return(tse, 1);
+	else
+	    erts_tse_return(tse);
+    }
 }
 
 
@@ -250,7 +249,7 @@ cleanup_waiter(void)
 static ERTS_INLINE void
 enqueue_waiter(erts_proc_lock_queues_t *qs,
 	       int ix,
-	       erts_proc_lock_waiter_t *wtr)
+	       erts_tse_t *wtr)
 {
     if (!qs->queue[ix]) {
 	qs->queue[ix] = wtr;
@@ -266,10 +265,10 @@ enqueue_waiter(erts_proc_lock_queues_t *qs,
     }
 }
 
-static erts_proc_lock_waiter_t *
+static erts_tse_t *
 dequeue_waiter(erts_proc_lock_queues_t *qs, int ix)
 {
-    erts_proc_lock_waiter_t *wtr = qs->queue[ix];
+    erts_tse_t *wtr = qs->queue[ix];
     ERTS_LC_ASSERT(qs->queue[ix]);
     if (wtr->next == wtr) {
 	ERTS_LC_ASSERT(qs->queue[ix]->prev == wtr);
@@ -295,10 +294,10 @@ dequeue_waiter(erts_proc_lock_queues_t *qs, int ix)
  *       lock.
  */
 static ERTS_INLINE void
-try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr)
+try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr)
 {
     ErtsProcLocks got_locks = (ErtsProcLocks) 0;
-    ErtsProcLocks locks = wtr->wait_locks;
+    ErtsProcLocks locks = wtr->uflgs;
     int lock_no;
 
     ERTS_LC_ASSERT(lck->queues);
@@ -334,7 +333,7 @@ try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr)
 	}
     }
 
-    wtr->wait_locks &= ~got_locks;
+    wtr->uflgs &= ~got_locks;
 }
 
 /*
@@ -350,8 +349,8 @@ transfer_locks(Process *p,
 	       int unlock)
 {
     int transferred = 0;
-    erts_proc_lock_waiter_t *wake = NULL;
-    erts_proc_lock_waiter_t *wtr;
+    erts_tse_t *wake = NULL;
+    erts_tse_t *wtr;
     ErtsProcLocks unset_waiter = 0;
     ErtsProcLocks tlocks = trnsfr_lcks;
     int lock_no;
@@ -377,11 +376,11 @@ transfer_locks(Process *p,
 	    ERTS_LC_ASSERT(wtr);
 	    if (!qs->queue[lock_no])
 		unset_waiter |= lock;
-	    ERTS_LC_ASSERT(wtr->wait_locks & lock);
-	    wtr->wait_locks &= ~lock;
-	    if (wtr->wait_locks)
+	    ERTS_LC_ASSERT(wtr->uflgs & lock);
+	    wtr->uflgs &= ~lock;
+	    if (wtr->uflgs)
 		try_aquire(&p->lock, wtr);
-	    if (!wtr->wait_locks) {
+	    if (!wtr->uflgs) {
 		/*
 		 * The other thread got all locks it needs;
 		 * need to wake it up.
@@ -412,9 +411,10 @@ transfer_locks(Process *p,
 	erts_pix_unlock(pix_lock);
     
 	do {
-	    erts_proc_lock_waiter_t *tmp = wake;
+	    erts_tse_t *tmp = wake;
 	    wake = wake->next;
-	    erts_smp_gate_let_through(&tmp->gate, 1);
+	    erts_atomic32_set(&tmp->uaflgs, 0);
+	    erts_tse_set(tmp);
 	} while (wake);
 
 	if (!unlock)
@@ -462,26 +462,16 @@ wait_for_locks(Process *p,
                ErtsProcLocks olflgs)
 {
     erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id);
-    int tsd;
-    erts_proc_lock_waiter_t *wtr;
+    erts_tse_t *wtr;
+    erts_proc_lock_queues_t *qs;
 
     /* Acquire a waiter object on which this thread can wait. */
-    wtr = erts_tsd_get(waiter_key);
-    if (wtr)
-	tsd = 1;
-    else {
-#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL
-	erts_pix_unlock(pix_lock);
-#endif
-	wtr = alloc_wtr();
-	tsd = 0;
-#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL
-	erts_pix_lock(pix_lock);
-#endif
-    }
+    wtr = tse_fetch(pix_lock);
     
     /* Record which locks this waiter needs. */
-    wtr->wait_locks = need_locks;
+    wtr->uflgs = need_locks;
+
+    ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
 
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
     erts_pix_lock(pix_lock);
@@ -489,14 +479,16 @@ wait_for_locks(Process *p,
 
     ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock));
 
+    qs = wtr->udata;
+    ASSERT(qs);
     /* Provide the process with waiter queues, if it doesn't have one. */
     if (!p->lock.queues) {
-	wtr->queues->next = NULL;
-	p->lock.queues = wtr->queues;
+	qs->next = NULL;
+	p->lock.queues = qs;
     }
     else {
-	wtr->queues->next = p->lock.queues->next;
-	p->lock.queues->next = wtr->queues;
+	qs->next = p->lock.queues->next;
+	p->lock.queues->next = qs;
     }
 
 #ifdef ERTS_PROC_LOCK_HARD_DEBUG
@@ -506,46 +498,59 @@ wait_for_locks(Process *p,
     /* Try to aquire locks one at a time in lock order and set wait flag */
     try_aquire(&p->lock, wtr);
 
+    ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
+
 #ifdef ERTS_PROC_LOCK_HARD_DEBUG
     check_queue(&p->lock);
 #endif
 
-    if (wtr->wait_locks) { /* We didn't get them all; need to wait... */
-	/* Got to wait for locks... */
+    if (wtr->uflgs) {
+	/* We didn't get them all; need to wait... */
+
+	ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
+
+	erts_atomic32_set(&wtr->uaflgs, 1);
 	erts_pix_unlock(pix_lock);
 
-	/*
-	 * Wait for needed locks. When we return all needed locks have
-	 * have been acquired by other threads and transfered to us.
-	 */
-#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE
-	erts_smp_gate_swait(&wtr->gate, proc_lock_spin_count);
-#else
-	erts_smp_gate_wait(&wtr->gate);
-#endif
+	while (1) {
+	    int res;
+	    erts_tse_reset(wtr);
+
+	    if (erts_atomic32_read(&wtr->uaflgs) == 0)
+		break;
+
+	    /*
+	     * Wait for needed locks. When we are woken all needed locks have
+	     * have been acquired by other threads and transfered to us.
+	     * However, we need to be prepared for spurious wakeups.
+	     */
+	    do {
+		res = erts_tse_wait(wtr); /* might return EINTR */
+	    } while (res != 0);
+	}
 
 	erts_pix_lock(pix_lock);
+
+	ASSERT(wtr->uflgs == 0);
     }
 
     /* Recover some queues to store in the waiter. */
     ERTS_LC_ASSERT(p->lock.queues);
     if (p->lock.queues->next) {
-	wtr->queues = p->lock.queues->next;
-	p->lock.queues->next = wtr->queues->next;
+	qs = p->lock.queues->next;
+	p->lock.queues->next = qs->next;
     }
     else {
-	wtr->queues = p->lock.queues;
+	qs = p->lock.queues;
 	p->lock.queues = NULL;
     }
+    wtr->udata = qs;
 
     erts_pix_unlock(pix_lock);
 
     ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));
 
-    if (tsd)
-	CHECK_UNUSED_WAITER(wtr);
-    else
-	free_wtr(wtr);
+    tse_return(wtr, 0);
 }
 
 /*
@@ -563,52 +568,57 @@ erts_proc_lock_failed(Process *p,
 		      ErtsProcLocks locks,
 		      ErtsProcLocks old_lflgs)
 {
-#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE
-    int spin_count = 0;
-#else
-    int spin_count = proc_lock_spin_count;
-#endif
-
+    int until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD;
+    int thr_spin_count;
+    int spin_count;
     ErtsProcLocks need_locks = locks;
     ErtsProcLocks olflgs = old_lflgs;
 
-    while (need_locks != 0)
-    {
-        ErtsProcLocks can_grab = in_order_locks(olflgs, need_locks);
+    if (erts_thr_get_main_status())
+	thr_spin_count = proc_lock_spin_count;
+    else
+	thr_spin_count = aux_thr_proc_lock_spin_count;
+
+    spin_count = thr_spin_count;
+
+    while (need_locks != 0) {
+        ErtsProcLocks can_grab;
+
+	can_grab = in_order_locks(olflgs, need_locks);
 
-        if (can_grab == 0)
-        {
+        if (can_grab == 0) {
             /* Someone already has the lowest-numbered lock we want. */
 
-            if (spin_count-- <= 0)
-            {
+            if (spin_count-- <= 0) {
                 /* Too many retries, give up and sleep for the lock. */
                 wait_for_locks(p, pixlck, locks, need_locks, olflgs);
                 return;
             }
 
+	    ERTS_SPIN_BODY;
+
+	    if (--until_yield == 0) {
+		until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD;
+		erts_thr_yield();
+	    }
+
             olflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock);
         }
-        else
-        {
+        else {
             /* Try to grab all of the grabbable locks at once with cmpxchg. */
             ErtsProcLocks grabbed = olflgs | can_grab;
             ErtsProcLocks nflgs =
-                ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, grabbed, olflgs);
+                ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, grabbed, olflgs);
 
-            if (nflgs == olflgs)
-            {
+            if (nflgs == olflgs) {
                 /* Success! We grabbed the 'can_grab' locks. */
                 olflgs = grabbed;
                 need_locks &= ~can_grab;
 
-#ifndef ERTS_PROC_LOCK_SPIN_ON_GATE
                 /* Since we made progress, reset the spin count. */
-                spin_count = proc_lock_spin_count;
-#endif
+                spin_count = thr_spin_count;
             }
-            else
-            {
+            else {
                 /* Compare-and-exchange failed, try again. */
                 olflgs = nflgs;
             }
@@ -945,7 +955,7 @@ erts_proc_lock_init(Process *p)
 {
     /* We always start with all locks locked */
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
-    erts_smp_atomic_init(&p->lock.flags, (long) ERTS_PROC_LOCKS_ALL);
+    erts_smp_atomic32_init(&p->lock.flags, (erts_aint32_t) ERTS_PROC_LOCKS_ALL);
 #else
     p->lock.flags = ERTS_PROC_LOCKS_ALL;
 #endif
@@ -964,7 +974,7 @@ erts_proc_lock_init(Process *p)
     {
 	int i;
 	for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
-	    erts_smp_atomic_init(&p->lock.locked[i], (long) 1);
+	    erts_smp_atomic32_init(&p->lock.locked[i], (erts_aint32_t) 1);
     }
 #endif
 }
@@ -1407,7 +1417,7 @@ check_queue(erts_proc_lock_t *lck)
 	wtr = (((ErtsProcLocks) 1) << lock_no) << ERTS_PROC_LOCK_WAITER_SHIFT;
 	if (lflgs & wtr) {
 	    int n;
-	    erts_proc_lock_waiter_t *wtr;
+	    erts_tse_t *wtr;
 	    ERTS_LC_ASSERT(lck->queues && lck->queues->queue[lock_no]);
 	    wtr = lck->queues->queue[lock_no];
 	    n = 0;
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index d71e5a0a6e..355179f084 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -54,20 +54,20 @@
 
 #define ERTS_PROC_LOCK_MAX_BIT 3
 
-typedef Uint32 ErtsProcLocks;
+typedef erts_aint32_t ErtsProcLocks;
 
 typedef struct erts_proc_lock_queues_t_ erts_proc_lock_queues_t;
 
 typedef struct erts_proc_lock_t_ {
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
-    erts_smp_atomic_t flags;
+    erts_smp_atomic32_t flags;
 #else
     ErtsProcLocks flags;
 #endif
     erts_proc_lock_queues_t *queues;
-    long refc;
+    Sint32 refc;
 #ifdef ERTS_PROC_LOCK_DEBUG
-    erts_smp_atomic_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
+    erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_t lcnt_main;
@@ -255,11 +255,7 @@ void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks);
 
 typedef struct {
     union {
-#if ERTS_PROC_LOCK_MUTEX_IMPL
-	erts_smp_mtx_t mtx;
-#else
 	erts_smp_spinlock_t spnlck;
-#endif
 	char buf[64]; /* Try to get locks in different cache lines */
     } u;
 } erts_pix_lock_t;
@@ -274,14 +270,19 @@ typedef struct {
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 
 #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \
-  ((ErtsProcLocks) erts_smp_atomic_band(&(L)->flags, (long) (MSK)))
+  ((ErtsProcLocks) erts_smp_atomic32_band(&(L)->flags, (erts_aint32_t) (MSK)))
 #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \
-  ((ErtsProcLocks) erts_smp_atomic_bor(&(L)->flags, (long) (MSK)))
-#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \
-  ((ErtsProcLocks) erts_smp_atomic_cmpxchg(&(L)->flags, \
-                                           (long) (NEW), (long) (EXPECTED)))
+  ((ErtsProcLocks) erts_smp_atomic32_bor(&(L)->flags, (erts_aint32_t) (MSK)))
+#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
+  ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \
+						  (erts_aint32_t) (NEW), \
+						  (erts_aint32_t) (EXPECTED)))
+#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \
+  ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_relb(&(L)->flags, \
+						  (erts_aint32_t) (NEW), \
+						  (erts_aint32_t) (EXPECTED)))
 #define ERTS_PROC_LOCK_FLGS_READ_(L) \
-  ((ErtsProcLocks) erts_smp_atomic_read(&(L)->flags))
+  ((ErtsProcLocks) erts_smp_atomic32_read(&(L)->flags))
 
 #else /* no opt atomic ops */
 
@@ -289,6 +290,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_band(erts_proc_lock_t *,
 							ErtsProcLocks);
 ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_bor(erts_proc_lock_t *,
 						       ErtsProcLocks);
+ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *,
+							   ErtsProcLocks,
+							   ErtsProcLocks);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -322,7 +326,9 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new,
 
 #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK))
 #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) erts_proc_lock_flags_bor((L), (MSK))
-#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \
+#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
+  erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED))
+#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \
   erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED))
 #define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags)
 
@@ -330,7 +336,7 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new,
 
 extern erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];
 
-void erts_init_proc_lock(void);
+void erts_init_proc_lock(int cpus);
 void erts_proc_lock_prepare_proc_lock_waiter(void);
 void erts_proc_lock_failed(Process *,
 			   erts_pix_lock_t *,
@@ -348,9 +354,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p,
 							  ErtsProcLocks locks);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *,
-					  erts_pix_lock_t *,
-					  ErtsProcLocks,
-					  char *file, unsigned int line);
+					    erts_pix_lock_t *,
+					    ErtsProcLocks,
+					    char *file, unsigned int line);
 #else
 ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *,
 					  erts_pix_lock_t *,
@@ -372,30 +378,18 @@ ERTS_GLB_INLINE void erts_proc_lock_op_debug(Process *, ErtsProcLocks, int);
 ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *pixlck)
 {
     ERTS_LC_ASSERT(pixlck);
-#if ERTS_PROC_LOCK_MUTEX_IMPL
-    erts_smp_mtx_lock(&pixlck->u.mtx);
-#else
     erts_smp_spin_lock(&pixlck->u.spnlck);
-#endif
 }
 
 ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *pixlck)
 {
     ERTS_LC_ASSERT(pixlck);
-#if ERTS_PROC_LOCK_MUTEX_IMPL
-    erts_smp_mtx_unlock(&pixlck->u.mtx);
-#else
     erts_smp_spin_unlock(&pixlck->u.spnlck);
-#endif
 }
 
 ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck)
 {
-#if ERTS_PROC_LOCK_MUTEX_IMPL
-    return erts_smp_lc_mtx_is_locked(&pixlck->u.mtx);
-#else
     return erts_smp_lc_spinlock_is_locked(&pixlck->u.spnlck);
-#endif
 }
 
 /*
@@ -417,9 +411,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
     ErtsProcLocks expct_lflgs = 0;
 
     while (1) {
-        ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock,
-							   expct_lflgs | locks,
-							   expct_lflgs);
+        ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock,
+								expct_lflgs | locks,
+								expct_lflgs);
         if (ERTS_LIKELY(lflgs == expct_lflgs)) {
             /* We successfully grabbed all locks. */
             return 0;
@@ -535,7 +529,7 @@ erts_smp_proc_unlock__(Process *p,
 
         if (want_lflgs != old_lflgs) {
             ErtsProcLocks new_lflgs =
-                ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, want_lflgs, old_lflgs);
+                ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(&p->lock, want_lflgs, old_lflgs);
 
             if (new_lflgs != old_lflgs) {
                 /* cmpxchg failed, try again. */
@@ -627,13 +621,13 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked)
     for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) {
 	ErtsProcLocks lock = ((ErtsProcLocks) 1) << i;
 	if (locks & lock) {
-	    long lock_count;
+	    erts_aint32_t lock_count;
 	    if (locked) {
-		lock_count = erts_smp_atomic_inctest(&p->lock.locked[i]);
+		lock_count = erts_smp_atomic32_inctest(&p->lock.locked[i]);
 		ERTS_LC_ASSERT(lock_count == 1);
 	    }
 	    else {
-		lock_count = erts_smp_atomic_dectest(&p->lock.locked[i]);
+		lock_count = erts_smp_atomic32_dectest(&p->lock.locked[i]);
 		ERTS_LC_ASSERT(lock_count == 0);
 	    }
 	}
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 03d2a586e3..287327bfe1 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -43,27 +43,48 @@ typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
 typedef erts_tid_t erts_smp_tid_t;
 typedef erts_mtx_t erts_smp_mtx_t;
 typedef erts_cnd_t erts_smp_cnd_t;
+#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER
+#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL
+#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ
+#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
+  ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ
+#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED
+#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED
+#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED
+typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
 typedef erts_rwmtx_t erts_smp_rwmtx_t;
 typedef erts_tsd_key_t erts_smp_tsd_key_t;
-typedef erts_gate_t erts_smp_gate_t;
-typedef ethr_atomic_t erts_smp_atomic_t;
+typedef erts_atomic_t erts_smp_atomic_t;
+typedef erts_atomic32_t erts_smp_atomic32_t;
 typedef erts_spinlock_t erts_smp_spinlock_t;
 typedef erts_rwlock_t erts_smp_rwlock_t;
-typedef erts_thr_timeval_t erts_smp_thr_timeval_t;
 void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
 
 #else /* #ifdef ERTS_SMP */
 
-#define ERTS_SMP_THR_OPTS_DEFAULT_INITER 0
+#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
 typedef int erts_smp_thr_opts_t;
 typedef int erts_smp_thr_init_data_t;
 typedef int erts_smp_tid_t;
 typedef int erts_smp_mtx_t;
 typedef int erts_smp_cnd_t;
+#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0}
+#define ERTS_SMP_RWMTX_TYPE_NORMAL 0
+#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0
+#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
+#define ERTS_SMP_RWMTX_LONG_LIVED 0
+#define ERTS_SMP_RWMTX_SHORT_LIVED 0
+#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0
+typedef struct {
+    char type;
+    char lived;
+    int main_spincount;
+    int aux_spincount;
+} erts_smp_rwmtx_opt_t;
 typedef int erts_smp_rwmtx_t;
 typedef int erts_smp_tsd_key_t;
-typedef int erts_smp_gate_t;
-typedef long erts_smp_atomic_t;
+typedef SWord erts_smp_atomic_t;
+typedef Uint32 erts_smp_atomic32_t;
 #if __GNUC__ > 2
 typedef struct { } erts_smp_spinlock_t;
 typedef struct { } erts_smp_rwlock_t;
@@ -72,11 +93,6 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
 typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
 #endif
 
-typedef struct {
-    long tv_sec;
-    long tv_nsec;
-} erts_smp_thr_timeval_t;
-
 #endif /* #ifdef ERTS_SMP */
 
 ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
@@ -103,8 +119,6 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
 ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx);
 ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line);
@@ -119,9 +133,17 @@ ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
 					   erts_smp_mtx_t *mtx);
 ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
 ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
+ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
+ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
+					       erts_smp_rwmtx_opt_t *opt,
+					       char *name,
+					       Eterm extra);
 ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx,
 					   char *name,
 					   Eterm extra);
+ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
+					     erts_smp_rwmtx_opt_t *opt,
+					     char *name);
 ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
 					 char *name);
 ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
@@ -138,23 +160,82 @@ ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
 ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var, long i);
-ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, long i);
-ERTS_GLB_INLINE long erts_smp_atomic_read(erts_smp_atomic_t *var);
-ERTS_GLB_INLINE long erts_smp_atomic_inctest(erts_smp_atomic_t *incp);
-ERTS_GLB_INLINE long erts_smp_atomic_dectest(erts_smp_atomic_t *decp);
+ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var,
+					  erts_aint_t i);
+ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp);
 ERTS_GLB_INLINE void erts_smp_atomic_inc(erts_smp_atomic_t *incp);
 ERTS_GLB_INLINE void erts_smp_atomic_dec(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE long erts_smp_atomic_addtest(erts_smp_atomic_t *addp,
-					     long i);
-ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp, long i);
-ERTS_GLB_INLINE long erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp,
-					  long new);
-ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp,
-					     long new,
-					     long expected);
-ERTS_GLB_INLINE long erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask);
-ERTS_GLB_INLINE long erts_smp_atomic_band(erts_smp_atomic_t *var, long mask);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_addtest(erts_smp_atomic_t *addp,
+						    erts_aint_t i);
+ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp,
+					 erts_aint_t i);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp,
+						 erts_aint_t new);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp,
+						    erts_aint_t new,
+						    erts_aint_t expected);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_bor(erts_smp_atomic_t *var,
+						erts_aint_t mask);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_band(erts_smp_atomic_t *var,
+						 erts_aint_t mask);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read_acqb(erts_smp_atomic_t *var);
+ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var,
+					      erts_aint_t i);
+ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp,
+							 erts_aint_t new,
+							 erts_aint_t exp);
+ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp,
+							 erts_aint_t new,
+							 erts_aint_t exp);
+ERTS_GLB_INLINE void
+erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i);
+ERTS_GLB_INLINE void
+erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_read(erts_smp_atomic32_t *var);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp);
+ERTS_GLB_INLINE void
+erts_smp_atomic32_inc(erts_smp_atomic32_t *incp);
+ERTS_GLB_INLINE void
+erts_smp_atomic32_dec(erts_smp_atomic32_t *decp);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i);
+ERTS_GLB_INLINE void
+erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp,
+			  erts_aint32_t new,
+			  erts_aint32_t expected);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var);
+ERTS_GLB_INLINE void
+erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i);
+ERTS_GLB_INLINE void
+erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp,
+			       erts_aint32_t new,
+			       erts_aint32_t exp);
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp,
+			       erts_aint32_t new,
+			       erts_aint32_t exp);
 ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
 					      char *name,
 					      Eterm extra);
@@ -185,17 +266,10 @@ ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
-ERTS_GLB_INLINE void erts_smp_thr_time_now(erts_smp_thr_timeval_t *time);
 ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp);
 ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
 ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
 ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);
-ERTS_GLB_INLINE void erts_smp_gate_init(erts_smp_gate_t *gp);
-ERTS_GLB_INLINE void erts_smp_gate_destroy(erts_smp_gate_t *gp);
-ERTS_GLB_INLINE void erts_smp_gate_close(erts_smp_gate_t *gp);
-ERTS_GLB_INLINE void erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no);
-ERTS_GLB_INLINE void erts_smp_gate_wait(erts_smp_gate_t *gp);
-ERTS_GLB_INLINE void erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount);
 
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
 #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
@@ -331,22 +405,6 @@ erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx)
-{
-#ifdef ERTS_SMP
-    erts_mtx_set_forksafe(mtx);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx)
-{
-#ifdef ERTS_SMP
-    erts_mtx_unset_forksafe(mtx);
-#endif
-}
-
 ERTS_GLB_INLINE int
 erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
 {
@@ -433,6 +491,25 @@ erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
 }
 
 ERTS_GLB_INLINE void
+erts_smp_rwmtx_set_reader_group(int no)
+{
+#ifdef ERTS_SMP
+    erts_rwmtx_set_reader_group(no);
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
+			  erts_smp_rwmtx_opt_t *opt,
+			  char *name,
+			  Eterm extra)
+{
+#ifdef ERTS_SMP
+    erts_rwmtx_init_opt_x(rwmtx, opt, name, extra);
+#endif
+}
+
+ERTS_GLB_INLINE void
 erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
 {
 #ifdef ERTS_SMP
@@ -441,6 +518,16 @@ erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
 }
 
 ERTS_GLB_INLINE void
+erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
+			erts_smp_rwmtx_opt_t *opt,
+			char *name)
+{
+#ifdef ERTS_SMP
+    erts_rwmtx_init_opt(rwmtx, opt, name);
+#endif
+}
+
+ERTS_GLB_INLINE void
 erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name)
 {
 #ifdef ERTS_SMP
@@ -568,7 +655,7 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_atomic_init(erts_smp_atomic_t *var, long i)
+erts_smp_atomic_init(erts_smp_atomic_t *var, erts_aint_t i)
 {
 #ifdef ERTS_SMP
     erts_atomic_init(var, i);
@@ -578,7 +665,7 @@ erts_smp_atomic_init(erts_smp_atomic_t *var, long i)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_atomic_set(erts_smp_atomic_t *var, long i)
+erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i)
 {
 #ifdef ERTS_SMP
     erts_atomic_set(var, i);
@@ -587,7 +674,7 @@ erts_smp_atomic_set(erts_smp_atomic_t *var, long i)
 #endif
 }
 
-ERTS_GLB_INLINE long
+ERTS_GLB_INLINE erts_aint_t
 erts_smp_atomic_read(erts_smp_atomic_t *var)
 {
 #ifdef ERTS_SMP
@@ -597,7 +684,7 @@ erts_smp_atomic_read(erts_smp_atomic_t *var)
 #endif
 }
 
-ERTS_GLB_INLINE long
+ERTS_GLB_INLINE erts_aint_t
 erts_smp_atomic_inctest(erts_smp_atomic_t *incp)
 {
 #ifdef ERTS_SMP
@@ -607,7 +694,7 @@ erts_smp_atomic_inctest(erts_smp_atomic_t *incp)
 #endif
 }
 
-ERTS_GLB_INLINE long
+ERTS_GLB_INLINE erts_aint_t
 erts_smp_atomic_dectest(erts_smp_atomic_t *decp)
 {
 #ifdef ERTS_SMP
@@ -637,8 +724,8 @@ erts_smp_atomic_dec(erts_smp_atomic_t *decp)
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_smp_atomic_addtest(erts_smp_atomic_t *addp, long i)
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_atomic_addtest(erts_smp_atomic_t *addp, erts_aint_t i)
 {
 #ifdef ERTS_SMP
     return erts_atomic_addtest(addp, i);
@@ -648,7 +735,7 @@ erts_smp_atomic_addtest(erts_smp_atomic_t *addp, long i)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_atomic_add(erts_smp_atomic_t *addp, long i)
+erts_smp_atomic_add(erts_smp_atomic_t *addp, erts_aint_t i)
 {
 #ifdef ERTS_SMP
     erts_atomic_add(addp, i);
@@ -657,58 +744,344 @@ erts_smp_atomic_add(erts_smp_atomic_t *addp, long i)
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, long new)
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, erts_aint_t new)
 {
 #ifdef ERTS_SMP
     return erts_atomic_xchg(xchgp, new);
 #else
-    long old;
+    erts_aint_t old;
     old = *xchgp;
     *xchgp = new;
     return old;
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, long new, long expected)
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp,
+			erts_aint_t new,
+			erts_aint_t expected)
 {
 #ifdef ERTS_SMP
     return erts_atomic_cmpxchg(xchgp, new, expected);
 #else
-    long old = *xchgp;
+    erts_aint_t old = *xchgp;
     if (old == expected)
         *xchgp = new;
     return old;
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask)
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_atomic_bor(erts_smp_atomic_t *var, erts_aint_t mask)
 {
 #ifdef ERTS_SMP
     return erts_atomic_bor(var, mask);
 #else
-    long old;
+    erts_aint_t old;
     old = *var;
     *var |= mask;
     return old;
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_smp_atomic_band(erts_smp_atomic_t *var, long mask)
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_atomic_band(erts_smp_atomic_t *var, erts_aint_t mask)
 {
 #ifdef ERTS_SMP
     return erts_atomic_band(var, mask);
 #else
-    long old;
+    erts_aint_t old;
+    old = *var;
+    *var &= mask;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_atomic_read_acqb(erts_smp_atomic_t *var)
+{
+#ifdef ERTS_SMP
+    return erts_atomic_read_acqb(var);
+#else
+    return *var;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_atomic_set_relb(erts_smp_atomic_t *var, erts_aint_t i)
+{
+#ifdef ERTS_SMP
+    erts_atomic_set_relb(var, i);
+#else
+    *var = i;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp)
+{
+#ifdef ERTS_SMP
+    erts_atomic_dec_relb(decp);
+#else
+    --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp)
+{
+#ifdef ERTS_SMP
+    return erts_atomic_dectest_relb(decp);
+#else
+    return --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp,
+			     erts_aint_t new,
+			     erts_aint_t exp)
+{
+#ifdef ERTS_SMP
+    return erts_atomic_cmpxchg_acqb(xchgp, new, exp);
+#else
+    erts_aint_t old = *xchgp;
+    if (old == exp)
+        *xchgp = new;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp,
+			     erts_aint_t new,
+			     erts_aint_t exp)
+{
+#ifdef ERTS_SMP
+    return erts_atomic_cmpxchg_relb(xchgp, new, exp);
+#else
+    erts_aint_t old = *xchgp;
+    if (old == exp)
+        *xchgp = new;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i)
+{
+#ifdef ERTS_SMP
+    erts_atomic32_init(var, i);
+#else
+    *var = i;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i)
+{
+#ifdef ERTS_SMP
+    erts_atomic32_set(var, i);
+#else
+    *var = i;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_read(erts_smp_atomic32_t *var)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_read(var);
+#else
+    return *var;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_inctest(incp);
+#else
+    return ++(*incp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_dectest(decp);
+#else
+    return --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_atomic32_inc(erts_smp_atomic32_t *incp)
+{
+#ifdef ERTS_SMP
+    erts_atomic32_inc(incp);
+#else
+    ++(*incp);
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_atomic32_dec(erts_smp_atomic32_t *decp)
+{
+#ifdef ERTS_SMP
+    erts_atomic32_dec(decp);
+#else
+    --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_addtest(addp, i);
+#else
+    return *addp += i;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i)
+{
+#ifdef ERTS_SMP
+    erts_atomic32_add(addp, i);
+#else
+    *addp += i;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_xchg(xchgp, new);
+#else
+    erts_aint32_t old;
+    old = *xchgp;
+    *xchgp = new;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp,
+			  erts_aint32_t new,
+			  erts_aint32_t expected)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_cmpxchg(xchgp, new, expected);
+#else
+    erts_aint32_t old = *xchgp;
+    if (old == expected)
+        *xchgp = new;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_bor(var, mask);
+#else
+    erts_aint32_t old;
+    old = *var;
+    *var |= mask;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_band(var, mask);
+#else
+    erts_aint32_t old;
     old = *var;
     *var &= mask;
     return old;
 #endif
 }
 
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_read_acqb(var);
+#else
+    return *var;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i)
+{
+#ifdef ERTS_SMP
+    erts_atomic32_set_relb(var, i);
+#else
+    *var = i;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp)
+{
+#ifdef ERTS_SMP
+    erts_atomic32_dec_relb(decp);
+#else
+    --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_dectest_relb(decp);
+#else
+    return --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp,
+			       erts_aint32_t new,
+			       erts_aint32_t exp)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_cmpxchg_acqb(xchgp, new, exp);
+#else
+    erts_aint32_t old = *xchgp;
+    if (old == exp)
+        *xchgp = new;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp,
+			       erts_aint32_t new,
+			       erts_aint32_t exp)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_cmpxchg_relb(xchgp, new, exp);
+#else
+    erts_aint32_t old = *xchgp;
+    if (old == exp)
+        *xchgp = new;
+    return old;
+#endif
+}
+
 ERTS_GLB_INLINE void
 erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
 {
@@ -878,14 +1251,6 @@ erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_thr_time_now(erts_smp_thr_timeval_t *time)
-{
-#ifdef ERTS_SMP
-    erts_thr_time_now(time);
-#endif
-}
-
-ERTS_GLB_INLINE void
 erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp)
 {
 #ifdef ERTS_SMP
@@ -919,54 +1284,6 @@ erts_smp_tsd_get(erts_smp_tsd_key_t key)
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_smp_gate_init(erts_smp_gate_t *gp)
-{
-#ifdef ERTS_SMP
-    erts_gate_init((erts_gate_t *) gp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_gate_destroy(erts_smp_gate_t *gp)
-{
-#ifdef ERTS_SMP
-    erts_gate_destroy((erts_gate_t *) gp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_gate_close(erts_smp_gate_t *gp)
-{
-#ifdef ERTS_SMP
-    erts_gate_close((erts_gate_t *) gp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no)
-{
-#ifdef ERTS_SMP
-    erts_gate_let_through((erts_gate_t *) gp, no);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_gate_wait(erts_smp_gate_t *gp)
-{
-#ifdef ERTS_SMP
-    erts_gate_wait((erts_gate_t *) gp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount)
-{
-#ifdef ERTS_SMP
-    erts_gate_swait((erts_gate_t *) gp, spincount);
-#endif
-}
-
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
 #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
 
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 2924abbd51..f77e8b798f 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -58,9 +58,9 @@ do { \
 #endif
 
 #if ET_DEBUG
-unsigned tag_val_def_debug(Eterm x, const char *file, unsigned line)
+unsigned tag_val_def_debug(Wterm x, const char *file, unsigned line)
 #else
-unsigned tag_val_def(Eterm x)
+unsigned tag_val_def(Wterm x)
 #define file __FILE__
 #define line __LINE__
 #endif
@@ -68,7 +68,9 @@ unsigned tag_val_def(Eterm x)
     static char msg[32];
 
     switch (x & _TAG_PRIMARY_MASK) {
-      case TAG_PRIMARY_LIST:					return LIST_DEF;
+    case TAG_PRIMARY_LIST:
+	ET_ASSERT(_list_precond(x),file,line);
+	return LIST_DEF;
       case TAG_PRIMARY_BOXED: {
 	  Eterm hdr = *boxed_val(x);
 	  ET_ASSERT(is_header(hdr),file,line);
@@ -103,7 +105,7 @@ unsigned tag_val_def(Eterm x)
 	  break;
       }
     }
-    sprintf(msg, "tag_val_def: %#lx", x);
+    sprintf(msg, "tag_val_def: %#lx", (unsigned long) x);
     et_abort(msg, file, line);
 #undef file
 #undef line
@@ -121,12 +123,12 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \
     return _unchecked_##FUN(x); \
 }
 
-ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_aligned);
+ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer);
 ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header);
-ET_DEFINE_CHECKED(Eterm*,boxed_val,Eterm,is_boxed);
-ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_aligned);
+ET_DEFINE_CHECKED(Eterm*,boxed_val,Wterm,_boxed_precond);
+ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer);
 ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header);
-ET_DEFINE_CHECKED(Eterm*,list_val,Eterm,is_list);
+ET_DEFINE_CHECKED(Eterm*,list_val,Wterm,_list_precond);
 ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small);
 ET_DEFINE_CHECKED(Sint,signed_val,Eterm,is_small);
 ET_DEFINE_CHECKED(Uint,atom_val,Eterm,is_atom);
@@ -134,37 +136,38 @@ ET_DEFINE_CHECKED(Uint,header_arity,Eterm,is_header);
 ET_DEFINE_CHECKED(Uint,arityval,Eterm,is_arity_value);
 ET_DEFINE_CHECKED(Uint,thing_arityval,Eterm,is_thing);
 ET_DEFINE_CHECKED(Uint,thing_subtag,Eterm,is_thing);
-ET_DEFINE_CHECKED(Eterm*,binary_val,Eterm,is_binary);
-ET_DEFINE_CHECKED(Eterm*,fun_val,Eterm,is_fun);
+ET_DEFINE_CHECKED(Eterm*,binary_val,Wterm,is_binary);
+ET_DEFINE_CHECKED(Eterm*,fun_val,Wterm,is_fun);
 ET_DEFINE_CHECKED(int,bignum_header_is_neg,Eterm,_is_bignum_header);
 ET_DEFINE_CHECKED(Eterm,bignum_header_neg,Eterm,_is_bignum_header);
 ET_DEFINE_CHECKED(Uint,bignum_header_arity,Eterm,_is_bignum_header);
-ET_DEFINE_CHECKED(Eterm*,big_val,Eterm,is_big);
-ET_DEFINE_CHECKED(Eterm*,float_val,Eterm,is_float);
-ET_DEFINE_CHECKED(Eterm*,tuple_val,Eterm,is_tuple);
+ET_DEFINE_CHECKED(Eterm*,big_val,Wterm,is_big);
+ET_DEFINE_CHECKED(Eterm*,float_val,Wterm,is_float);
+ET_DEFINE_CHECKED(Eterm*,tuple_val,Wterm,is_tuple);
 ET_DEFINE_CHECKED(Uint,internal_pid_data,Eterm,is_internal_pid);
 ET_DEFINE_CHECKED(struct erl_node_*,internal_pid_node,Eterm,is_internal_pid);
 ET_DEFINE_CHECKED(Uint,internal_port_data,Eterm,is_internal_port);
 ET_DEFINE_CHECKED(struct erl_node_*,internal_port_node,Eterm,is_internal_port);
-ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Eterm,is_internal_ref);
-ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Eterm,is_internal_ref);
-ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Eterm,is_internal_ref);
+ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Wterm,is_internal_ref);
+ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Wterm,is_internal_ref);
+ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Wterm,is_internal_ref);
 ET_DEFINE_CHECKED(struct erl_node_*,internal_ref_node,Eterm,is_internal_ref);
-ET_DEFINE_CHECKED(Eterm*,external_val,Eterm,is_external);
-ET_DEFINE_CHECKED(Uint,external_data_words,Eterm,is_external);
-ET_DEFINE_CHECKED(Uint,external_pid_data_words,Eterm,is_external_pid);
-ET_DEFINE_CHECKED(Uint,external_pid_data,Eterm,is_external_pid);
-ET_DEFINE_CHECKED(struct erl_node_*,external_pid_node,Eterm,is_external_pid);
-ET_DEFINE_CHECKED(Uint,external_port_data_words,Eterm,is_external_port);
-ET_DEFINE_CHECKED(Uint,external_port_data,Eterm,is_external_port);
-ET_DEFINE_CHECKED(struct erl_node_*,external_port_node,Eterm,is_external_port);
-ET_DEFINE_CHECKED(Uint,external_ref_data_words,Eterm,is_external_ref);
-ET_DEFINE_CHECKED(Uint32*,external_ref_data,Eterm,is_external_ref);
+ET_DEFINE_CHECKED(Eterm*,external_val,Wterm,is_external);
+ET_DEFINE_CHECKED(Uint,external_data_words,Wterm,is_external);
+ET_DEFINE_CHECKED(Uint,external_pid_data_words,Wterm,is_external_pid);
+ET_DEFINE_CHECKED(Uint,external_pid_data,Wterm,is_external_pid);
+ET_DEFINE_CHECKED(struct erl_node_*,external_pid_node,Wterm,is_external_pid);
+ET_DEFINE_CHECKED(Uint,external_port_data_words,Wterm,is_external_port);
+ET_DEFINE_CHECKED(Uint,external_port_data,Wterm,is_external_port);
+ET_DEFINE_CHECKED(struct erl_node_*,external_port_node,Wterm,is_external_port);
+ET_DEFINE_CHECKED(Uint,external_ref_data_words,Wterm,is_external_ref);
+ET_DEFINE_CHECKED(Uint32*,external_ref_data,Wterm,is_external_ref);
 ET_DEFINE_CHECKED(struct erl_node_*,external_ref_node,Eterm,is_external_ref);
-ET_DEFINE_CHECKED(Eterm*,export_val,Eterm,is_export);
+ET_DEFINE_CHECKED(Eterm*,export_val,Wterm,is_export);
+ET_DEFINE_CHECKED(Uint,external_thing_data_words,ExternalThing*,is_thing_ptr);
 
-ET_DEFINE_CHECKED(Eterm,make_cp,Uint*,_is_aligned);
-ET_DEFINE_CHECKED(Uint*,cp_val,Eterm,is_CP);
+ET_DEFINE_CHECKED(Eterm,make_cp,UWord *,_is_taggable_pointer);
+ET_DEFINE_CHECKED(UWord *,cp_val,Eterm,is_CP);
 ET_DEFINE_CHECKED(Uint,catch_val,Eterm,is_catch);
 ET_DEFINE_CHECKED(Uint,x_reg_offset,Uint,_is_xreg);
 ET_DEFINE_CHECKED(Uint,y_reg_offset,Uint,_is_yreg);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index a6596558fa..1d75fa313c 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -20,6 +20,34 @@
 #ifndef __ERL_TERM_H
 #define __ERL_TERM_H
 
+#include "sys.h" /* defines HALFWORD_HEAP */
+
+typedef UWord Wterm;  /* Full word terms */
+
+#if HALFWORD_HEAP
+#  define HEAP_ON_C_STACK 0
+#  if HALFWORD_ASSERT
+#    ifdef ET_DEBUG
+#      undef ET_DEBUG
+#    endif
+#    define ET_DEBUG 1
+#  endif
+#  if 1
+#    define CHECK_POINTER_MASK 0xFFFFFFFF00000000UL
+#    define COMPRESS_POINTER(APointer) ((Eterm) (UWord) (APointer))
+#    define EXPAND_POINTER(AnEterm) ((UWord) (AnEterm))
+#  else
+#    define CHECK_POINTER_MASK 0x0UL
+#    define COMPRESS_POINTER(AnUint) (AnUint)
+#    define EXPAND_POINTER(APointer) (APointer)
+#  endif
+#else
+#  define HEAP_ON_C_STACK 1
+#  define CHECK_POINTER_MASK 0x0UL
+#  define COMPRESS_POINTER(AnUint) (AnUint)
+#  define EXPAND_POINTER(APointer) (APointer)
+#endif
+
 struct erl_node_; /* Declared in erl_node_tables.h */
 
 /*
@@ -158,8 +186,15 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 
 
 /* boxed object access methods */
+#if HALFWORD_HEAP
+#define _is_taggable_pointer(x)	 (((UWord)(x) & (CHECK_POINTER_MASK | 0x3)) == 0)
+#define _boxed_precond(x)        (is_boxed(x))
+#else
+#define _is_taggable_pointer(x)	 (((Uint)(x) & 0x3) == 0)
+#define  _boxed_precond(x)       (is_boxed(x))
+#endif
 #define _is_aligned(x)		(((Uint)(x) & 0x3) == 0)
-#define _unchecked_make_boxed(x)	((Uint)(x) + TAG_PRIMARY_BOXED)
+#define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED)
 _ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*)
 #define make_boxed(x)		_ET_APPLY(make_boxed,(x))
 #if 1
@@ -170,12 +205,12 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm)
 #else
 #define is_boxed(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED)
 #endif
-#define _unchecked_boxed_val(x) ((Eterm*)((x) - TAG_PRIMARY_BOXED))
-_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm)
+#define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED)))
+_ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm)
 #define boxed_val(x)		_ET_APPLY(boxed_val,(x))
 
 /* cons cell ("list") access methods */
-#define _unchecked_make_list(x)	((Uint)(x) + TAG_PRIMARY_LIST)
+#define _unchecked_make_list(x)	((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST)
 _ET_DECLARE_CHECKED(Eterm,make_list,Eterm*)
 #define make_list(x)		_ET_APPLY(make_list,(x))
 #if 1
@@ -187,8 +222,13 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm)
 #define is_list(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_LIST)
 #define is_not_list(x)		(!is_list((x)))
 #endif
-#define _unchecked_list_val(x)	((Eterm*)((x) - TAG_PRIMARY_LIST))
-_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm)
+#if HALFWORD_HEAP
+#define _list_precond(x)        (is_list(x))
+#else
+#define _list_precond(x)       (is_list(x))
+#endif
+#define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST))
+_ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define list_val(x)		_ET_APPLY(list_val,(x))
 
 #define CONS(hp, car, cdr) \
@@ -198,13 +238,15 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Eterm)
 #define CDR(x)  ((x)[1])
 
 /* generic tagged pointer (boxed or list) access methods */
-#define _unchecked_ptr_val(x)	((Eterm*)((x) & ~((Uint) 0x3)))
+#define _unchecked_ptr_val(x)	((Eterm*) EXPAND_POINTER((x) & ~((Uint) 0x3)))
 #define ptr_val(x)		_unchecked_ptr_val((x))	/*XXX*/
 #define _unchecked_offset_ptr(x,offs)	((x)+((offs)*sizeof(Eterm)))
 #define offset_ptr(x,offs)	_unchecked_offset_ptr(x,offs)	/*XXX*/
+#define _unchecked_byte_offset_ptr(x,byte_offs)	((x)+(offs))
+#define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs)  /*XXX*/
 
 /* fixnum ("small") access methods */
-#if defined(ARCH_64)
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define SMALL_BITS	(64-4)
 #define SMALL_DIGITS	(17)
 #else
@@ -267,6 +309,7 @@ _ET_DECLARE_CHECKED(Uint,arityval,Eterm)
 
 /* thing access methods */
 #define is_thing(x)	(is_header((x)) && header_is_thing((x)))
+#define is_thing_ptr(t) (is_thing((t)->header))
 #define _unchecked_thing_arityval(x)	_unchecked_header_arity((x))
 _ET_DECLARE_CHECKED(Uint,thing_arityval,Eterm)
 #define thing_arityval(x)	_ET_APPLY(thing_arityval,(x))
@@ -301,7 +344,7 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm)
 #define is_binary(x)	(is_boxed((x)) && is_binary_header(*boxed_val((x))))
 #define is_not_binary(x) (!is_binary((x)))
 #define _unchecked_binary_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,binary_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,binary_val,Wterm)
 #define binary_val(x)	_ET_APPLY(binary_val,(x))
 
 /* process binaries stuff (special case of binaries) */
@@ -318,7 +361,7 @@ _ET_DECLARE_CHECKED(Eterm*,binary_val,Eterm)
 #define is_fun(x)		(is_boxed((x)) && is_fun_header(*boxed_val((x))))
 #define is_not_fun(x)		(!is_fun((x)))
 #define _unchecked_fun_val(x)   _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,fun_val,Wterm)
 #define fun_val(x)		_ET_APPLY(fun_val,(x))
 
 /* export access methods */
@@ -326,10 +369,14 @@ _ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm)
 #define is_export(x)     (is_boxed((x)) && is_export_header(*boxed_val((x))))
 #define is_not_export(x) (!is_export((x)))
 #define _unchecked_export_val(x)   _unchecked_boxed_val(x)
-_ET_DECLARE_CHECKED(Eterm*,export_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,export_val,Wterm)
 #define export_val(x)	_ET_APPLY(export_val,(x))
 #define is_export_header(x)	((x) == HEADER_EXPORT)
+#if HALFWORD_HEAP
+#define HEADER_EXPORT   _make_header(2,_TAG_HEADER_EXPORT)
+#else
 #define HEADER_EXPORT   _make_header(1,_TAG_HEADER_EXPORT)
+#endif
 
 /* bignum access methods */
 #define make_pos_bignum_header(sz)	_make_header((sz),_TAG_HEADER_POS_BIG)
@@ -349,11 +396,11 @@ _ET_DECLARE_CHECKED(Uint,bignum_header_arity,Eterm)
 #define is_big(x)	(is_boxed((x)) && _is_bignum_header(*boxed_val((x))))
 #define is_not_big(x)	(!is_big((x)))
 #define _unchecked_big_val(x)	_unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,big_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,big_val,Wterm)
 #define big_val(x)		_ET_APPLY(big_val,(x))
 
 /* flonum ("float") access methods */
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define HEADER_FLONUM   _make_header(1,_TAG_HEADER_FLOAT)
 #else
 #define HEADER_FLONUM	_make_header(2,_TAG_HEADER_FLOAT)
@@ -362,7 +409,7 @@ _ET_DECLARE_CHECKED(Eterm*,big_val,Eterm)
 #define is_float(x)	(is_boxed((x)) && *boxed_val((x)) == HEADER_FLONUM)
 #define is_not_float(x)	(!is_float(x))
 #define _unchecked_float_val(x)	_unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,float_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,float_val,Wterm)
 #define float_val(x)	_ET_APPLY(float_val,(x))
 
 /* Float definition for byte and word access */
@@ -374,21 +421,22 @@ typedef union float_def
     byte   fb[sizeof(ieee754_8)];
     Uint16 fs[sizeof(ieee754_8) / sizeof(Uint16)];
     Uint32 fw[sizeof(ieee754_8) / sizeof(Uint32)];
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     Uint   fdw;
 #endif
 } FloatDef;
 
-#ifdef ARCH_64
-#define GET_DOUBLE(x, f) (f).fdw = *(float_val(x)+1)
+#if defined(ARCH_64) && !HALFWORD_HEAP
+
+#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fdw = *((fval)+1)
 
 #define PUT_DOUBLE(f, x)  *(x) = HEADER_FLONUM, \
                           *((x)+1) = (f).fdw
 #define GET_DOUBLE_DATA(p, f) (f).fdw = *((Uint *) (p))
 #define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fdw
 #else
-#define GET_DOUBLE(x, f) (f).fw[0] = *(float_val(x)+1), \
-                         (f).fw[1] = *(float_val(x)+2)
+#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fw[0] = *((fval)+1), \
+				      (f).fw[1] = *((fval)+2)
 
 #define PUT_DOUBLE(f, x)  *(x) = HEADER_FLONUM, \
                           *((x)+1) = (f).fw[0], \
@@ -398,6 +446,9 @@ typedef union float_def
 #define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fw[0],\
                              *(((Uint *) (p))+1) = (f).fw[1]
 #endif
+
+#define GET_DOUBLE(x, f) FLOAT_VAL_GET_DOUBLE(float_val(x), f)
+
 #define DOUBLE_DATA_WORDS (sizeof(ieee754_8)/sizeof(Eterm))
 #define FLOAT_SIZE_OBJECT (DOUBLE_DATA_WORDS+1)
 
@@ -409,7 +460,7 @@ typedef union float_def
    (is_boxed((x)) && *boxed_val((x)) == make_arityval((a)))
 #define is_not_tuple_arity(x, a) (!is_tuple_arity((x),(a)))
 #define _unchecked_tuple_val(x)	_unchecked_boxed_val(x)
-_ET_DECLARE_CHECKED(Eterm*,tuple_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm)
 #define tuple_val(x)	_ET_APPLY(tuple_val,(x))
 
 #define TUPLE0(t) \
@@ -679,7 +730,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm)
 #define ERTS_MAX_REF_NUMBERS	3
 #define ERTS_REF_NUMBERS	ERTS_MAX_REF_NUMBERS
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #  define ERTS_REF_WORDS	(ERTS_REF_NUMBERS/2 + 1)
 #  define ERTS_REF_32BIT_WORDS  (ERTS_REF_NUMBERS+1)
 #else
@@ -701,7 +752,7 @@ typedef struct {
 #define make_ref_thing_header(DW) \
   _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF)
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 
 /*
  * Ref layout on a 64-bit little endian machine:
@@ -748,21 +799,24 @@ do {									\
   ((RefThing*) internal_ref_val(x))
 
 #define is_internal_ref(x) \
-  (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x))))
+    (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x))))
+
 #define is_not_internal_ref(x) \
   (!is_internal_ref((x)))
 
 #define _unchecked_internal_ref_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Wterm)
 #define internal_ref_val(x) _ET_APPLY(internal_ref_val,(x))
 
+#define internal_thing_ref_data_words(t) (thing_arityval(*(Eterm*)(t)))
 #define _unchecked_internal_ref_data_words(x) \
  (_unchecked_thing_arityval(*_unchecked_internal_ref_val(x)))
-_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Wterm)
 #define internal_ref_data_words(x) _ET_APPLY(internal_ref_data_words,(x))
 
-#define _unchecked_internal_ref_data(x) (_unchecked_ref_thing_ptr(x)->data.ui32)
-_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Eterm)
+#define internal_thing_ref_data(thing) ((thing)->data.ui32)
+#define _unchecked_internal_ref_data(x) (internal_thing_ref_data(_unchecked_ref_thing_ptr(x)))
+_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Wterm)
 #define internal_ref_data(x) _ET_APPLY(internal_ref_data,(x))
 
 #define _unchecked_internal_ref_node(x) erts_this_node
@@ -779,10 +833,10 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm)
  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *  |A A A A A A A A A A A A A A A A A A A A A A A A A A|t t t t|0 0| Thing
  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *  |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next
- *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *  |E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E| ErlNode
  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *  |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next
+ *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *  |X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X| Data 0
  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *  .                                                               .   .
@@ -793,7 +847,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm)
  *  t : External pid thing tag  (1100)
  *  t : External port thing tag (1101)
  *  t : External ref thing tag  (1110)
- *  N : Next (external thing) pointer
+ *  N : Next (off_heap) pointer
  *  E : ErlNode pointer
  *  X : Type specific data
  *
@@ -807,11 +861,19 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm)
  *
  */
 
+/* XXX:PaN - this structure is not perfect for halfword heap, it takes
+   a lot of memory due to padding, and the array will not begin at the end of the
+   structure, as otherwise expected. Be sure to access data.ui32 array and not try
+   to do pointer manipulation on an Eterm * to reach the actual data...
+
+   XXX:Sverk - Problem made worse by "one off-heap list" when 'next' pointer
+     must align with 'next' in ProcBin, erl_fun_thing and erl_off_heap_header.
+*/
 typedef struct external_thing_ {
     /*                                 ----+                        */
     Eterm                   header;     /* |                        */
-    struct external_thing_ *next;       /*  > External thing head   */
-    struct erl_node_       *node;       /* |                        */
+    struct erl_node_*       node;       /*  > External thing head   */
+    struct erl_off_heap_header* next;   /* |                        */
     /*                                 ----+                        */
     union {
 	Uint32              ui32[1];
@@ -839,14 +901,14 @@ typedef struct external_thing_ {
 #define is_external_header(x) \
   (((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_EXTERNAL_PID)
 
-#define is_external(x) \
-  (is_boxed((x)) && is_external_header(*boxed_val((x))))
+#define is_external(x) (is_boxed((x)) && is_external_header(*boxed_val((x))))
+
 #define is_external_pid(x) \
   (is_boxed((x)) && is_external_pid_header(*boxed_val((x))))
 #define is_external_port(x) \
-  (is_boxed((x)) && is_external_port_header(*boxed_val((x))))
-#define is_external_ref(x) \
-  (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x))))
+    (is_boxed((x)) && is_external_port_header(*boxed_val((x))))
+
+#define is_external_ref(x) (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x))))
 
 #define _unchecked_is_external(x) \
   (_unchecked_is_boxed((x)) && is_external_header(*_unchecked_boxed_val((x))))
@@ -864,17 +926,21 @@ typedef struct external_thing_ {
 #define make_external_ref		make_external
 
 #define _unchecked_external_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,external_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,external_val,Wterm)
 #define external_val(x) _ET_APPLY(external_val,(x))
 
 #define external_thing_ptr(x) ((ExternalThing *) external_val((x)))
 #define _unchecked_external_thing_ptr(x) \
   ((ExternalThing *) _unchecked_external_val((x)))
 
+#define _unchecked_external_thing_data_words(thing) \
+    (_unchecked_thing_arityval((thing)->header) + (1 - EXTERNAL_THING_HEAD_SIZE))
+_ET_DECLARE_CHECKED(Uint,external_thing_data_words,ExternalThing*)
+#define external_thing_data_words(thing) _ET_APPLY(external_thing_data_words,(thing))
+
 #define _unchecked_external_data_words(x) \
-  (_unchecked_thing_arityval(_unchecked_external_thing_ptr((x))->header) \
-   + (1 - EXTERNAL_THING_HEAD_SIZE))
-_ET_DECLARE_CHECKED(Uint,external_data_words,Eterm)
+    _unchecked_external_thing_data_words(_unchecked_external_thing_ptr((x)))
+_ET_DECLARE_CHECKED(Uint,external_data_words,Wterm)
 #define external_data_words(x) _ET_APPLY(external_data_words,(x))
 
 #define _unchecked_external_data(x) (_unchecked_external_thing_ptr((x))->data.ui)
@@ -885,15 +951,15 @@ _ET_DECLARE_CHECKED(Uint,external_data_words,Eterm)
 
 #define _unchecked_external_pid_data_words(x) \
   _unchecked_external_data_words((x))
-_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Wterm)
 #define external_pid_data_words(x) _ET_APPLY(external_pid_data_words,(x))
 
 #define _unchecked_external_pid_data(x) _unchecked_external_data((x))[0]
-_ET_DECLARE_CHECKED(Uint,external_pid_data,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_pid_data,Wterm)
 #define external_pid_data(x) _ET_APPLY(external_pid_data,(x))
 
 #define _unchecked_external_pid_node(x) _unchecked_external_node((x))
-_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Eterm)
+_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Wterm)
 #define external_pid_node(x) _ET_APPLY(external_pid_node,(x))
 
 #define external_pid_number(x) _GET_PID_NUM(external_pid_data((x)))
@@ -901,27 +967,29 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Eterm)
 
 #define _unchecked_external_port_data_words(x) \
   _unchecked_external_data_words((x))
-_ET_DECLARE_CHECKED(Uint,external_port_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_port_data_words,Wterm)
 #define external_port_data_words(x) _ET_APPLY(external_port_data_words,(x))
 
 #define _unchecked_external_port_data(x) _unchecked_external_data((x))[0]
-_ET_DECLARE_CHECKED(Uint,external_port_data,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_port_data,Wterm)
 #define external_port_data(x) _ET_APPLY(external_port_data,(x))
 
 #define _unchecked_external_port_node(x) _unchecked_external_node((x))
-_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Eterm)
+_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Wterm)
 #define external_port_node(x) _ET_APPLY(external_port_node,(x))
 
 #define external_port_number(x) _GET_PORT_NUM(external_port_data((x)))
 
 #define _unchecked_external_ref_data_words(x) \
   _unchecked_external_data_words((x))
-_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Wterm)
 #define external_ref_data_words(x) _ET_APPLY(external_ref_data_words,(x))
+#define external_thing_ref_data_words(thing) external_thing_data_words(thing)
 
 #define _unchecked_external_ref_data(x) (_unchecked_external_thing_ptr((x))->data.ui32)
-_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Eterm)
+_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Wterm)
 #define external_ref_data(x) _ET_APPLY(external_ref_data,(x))
+#define external_thing_ref_data(thing) ((thing)->data.ui32)
 
 #define _unchecked_external_ref_node(x) _unchecked_external_node((x))
 _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
@@ -944,15 +1012,15 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 #error "fix yer arch, like"
 #endif
 
-#define _unchecked_make_cp(x)	((Eterm)(x))
-_ET_DECLARE_CHECKED(Eterm,make_cp,Uint*)
+#define _unchecked_make_cp(x)	((Eterm) COMPRESS_POINTER(x))
+_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*)
 #define make_cp(x)	_ET_APPLY(make_cp,(x))
 
 #define is_not_CP(x)	((x) & _CPMASK)
 #define is_CP(x)	(!is_not_CP(x))
 
-#define _unchecked_cp_val(x)	((Uint*)(x))
-_ET_DECLARE_CHECKED(Uint*,cp_val,Eterm)
+#define _unchecked_cp_val(x)	((BeamInstr*) EXPAND_POINTER(x))
+_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm)
 #define cp_val(x)	_ET_APPLY(cp_val,(x))
 
 #define make_catch(x)	(((x) << _TAG_IMMED2_SIZE) | _TAG_IMMED2_CATCH)
@@ -1033,10 +1101,10 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
 #define SMALL_DEF		0xf
 
 #if ET_DEBUG
-extern unsigned tag_val_def_debug(Eterm, const char*, unsigned);
+extern unsigned tag_val_def_debug(Wterm, const char*, unsigned);
 #define tag_val_def(x)	tag_val_def_debug((x),__FILE__,__LINE__)
 #else
-extern unsigned tag_val_def(Eterm);
+extern unsigned tag_val_def(Wterm);
 #endif
 #define not_eq_tags(X,Y)	(tag_val_def((X)) ^ tag_val_def((Y)))
 
@@ -1052,5 +1120,81 @@ extern unsigned tag_val_def(Eterm);
 #define FLOAT_BIG 	_NUMBER_CODE(FLOAT_DEF,BIG_DEF)
 #define FLOAT_FLOAT	_NUMBER_CODE(FLOAT_DEF,FLOAT_DEF)
 
+#if HALFWORD_HEAP
+#define ptr2rel(PTR,BASE) ((Eterm*)((char*)(PTR) - (char*)(BASE)))
+#define rterm2wterm(REL,BASE) ((Wterm)(REL) + (Wterm)(BASE))
+
+#else /* HALFWORD_HEAP */
+
+#define ptr2rel(PTR,BASE) (PTR)
+#define rterm2wterm(REL,BASE) (REL)
+
+#endif /* !HALFWORD_HEAP */
+
+#define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE))
+#define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE))
+#define make_fun_rel make_boxed_rel
+#define make_binary_rel make_boxed_rel
+#define make_tuple_rel make_boxed_rel
+#define make_external_rel make_boxed_rel
+#define make_internal_ref_rel make_boxed_rel
+
+#define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE))
+#define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE))
+#define boxed_val_rel(RTERM, BASE) boxed_val(rterm2wterm(RTERM, BASE))
+#define tuple_val_rel(RTERM, BASE) tuple_val(rterm2wterm(RTERM, BASE))
+#define export_val_rel(RTERM, BASE) export_val(rterm2wterm(RTERM, BASE))
+#define fun_val_rel(RTERM, BASE) fun_val(rterm2wterm(RTERM, BASE))
+#define big_val_rel(RTERM,BASE)	big_val(rterm2wterm(RTERM,BASE))
+#define float_val_rel(RTERM,BASE) float_val(rterm2wterm(RTERM,BASE))
+#define internal_ref_val_rel(RTERM,BASE) internal_ref_val(rterm2wterm(RTERM,BASE))
+
+#define external_thing_ptr_rel(RTERM, BASE) external_thing_ptr(rterm2wterm(RTERM, BASE))
+#define external_data_words_rel(RTERM,BASE) external_data_words(rterm2wterm(RTERM,BASE))
+
+#define external_port_node_rel(RTERM,BASE) external_port_node(rterm2wterm(RTERM,BASE))
+#define external_port_data_rel(RTERM,BASE) external_port_data(rterm2wterm(RTERM,BASE))
+
+#define is_external_pid_rel(RTERM,BASE) is_external_pid(rterm2wterm(RTERM,BASE))
+#define external_pid_node_rel(RTERM,BASE) external_pid_node(rterm2wterm(RTERM,BASE))
+#define external_pid_data_rel(RTERM,BASE) external_pid_data(rterm2wterm(RTERM,BASE))
+
+#define is_binary_rel(RTERM,BASE) is_binary(rterm2wterm(RTERM,BASE))
+#define is_float_rel(RTERM,BASE) is_float(rterm2wterm(RTERM,BASE))
+#define is_fun_rel(RTERM,BASE) is_fun(rterm2wterm(RTERM,BASE))
+#define is_big_rel(RTERM,BASE) is_big(rterm2wterm(RTERM,BASE))
+#define is_export_rel(RTERM,BASE) is_export(rterm2wterm(RTERM,BASE))
+#define is_tuple_rel(RTERM,BASE) is_tuple(rterm2wterm(RTERM,BASE))
+
+#define GET_DOUBLE_REL(RTERM, f, BASE) GET_DOUBLE(rterm2wterm(RTERM,BASE), f)
+
+#define ref_thing_ptr_rel(RTERM,BASE) ref_thing_ptr(rterm2wterm(RTERM,BASE))
+#define is_internal_ref_rel(RTERM,BASE) is_internal_ref(rterm2wterm(RTERM,BASE))
+#define is_external_rel(RTERM,BASE) is_external(rterm2wterm(RTERM,BASE))
+#define is_external_port_rel(RTERM,BASE) is_external_port(rterm2wterm(RTERM,BASE))
+#define is_external_ref_rel(RTERM,BASE) is_external_ref(rterm2wterm(RTERM,BASE))
+
+#define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE))
+
+
+#if HALFWORD_HEAP
+ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
+{
+    /* If bases differ, assume a and b are on different "heaps",
+       ie can only be same if immed */
+    ASSERT(a_base == b_base || is_immed(a) || is_immed(b)
+	   || rterm2wterm(a,a_base) != rterm2wterm(b,b_base));
+
+    return a == b && (a_base == b_base || is_immed(a));
+}
+#endif
+
+#else /* !HALFWORD_HEAP */
+#define is_same(A,A_BASE,B,B_BASE) ((A)==(B))
+#endif
+
 #endif	/* __ERL_TERM_H */
 
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index d635916dd8..8c9cace0c5 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -25,6 +25,8 @@
 #ifndef ERL_THREAD_H__
 #define ERL_THREAD_H__
 
+#define ERTS_SPIN_BODY ETHR_SPIN_BODY
+
 #include "sys.h"
 #ifdef USE_THREADS
 
@@ -34,6 +36,18 @@
 #include "erl_lock_count.h"
 #include "erl_term.h"
 
+#if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4
+/*
+ * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have
+ * only seen this bug in glibc versions before 2.4. Note that condition
+ * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by
+ * this bug since these implementations may use mutexes internally.
+ */
+#  define ERTS_THR_HAVE_BUSY_DESTROY_BUG
+#endif
+
+#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+
 #ifdef ERTS_ENABLE_LOCK_COUNT
 #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
 #define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__)
@@ -46,6 +60,7 @@
 #define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER
 typedef ethr_thr_opts erts_thr_opts_t;
 typedef ethr_init_data erts_thr_init_data_t;
+typedef ethr_late_init_data erts_thr_late_init_data_t;
 typedef ethr_tid erts_tid_t;
 
 /* mutex */
@@ -71,9 +86,23 @@ typedef struct {
     erts_lcnt_lock_t lcnt;
 #endif
 } erts_rwmtx_t;
+
+#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER
+#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL
+#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ
+#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
+  ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
+#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED
+#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED
+#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED
+typedef ethr_rwmutex_opt erts_rwmtx_opt_t;
+
 typedef ethr_tsd_key erts_tsd_key_t;
-typedef ethr_gate erts_gate_t;
+typedef ethr_ts_event erts_tse_t;
+typedef ethr_sint_t erts_aint_t;
 typedef ethr_atomic_t erts_atomic_t;
+typedef ethr_sint32_t erts_aint32_t;
+typedef ethr_atomic32_t erts_atomic32_t;
 
 /* spinlock */
 typedef struct {
@@ -97,41 +126,48 @@ typedef struct {
 #endif
 } erts_rwlock_t;
 
-typedef ethr_timeval erts_thr_timeval_t;
 __decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *); 
                                  /* implemented in erl_init.c */
 
-#ifdef ERTS_ENABLE_LOCK_CHECK
-#define ERTS_REC_MTX_INITER \
-  {ETHR_REC_MUTEX_INITER, \
-   ERTS_LC_LOCK_INIT(-1,THE_NON_VALUE,ERTS_LC_FLG_LT_MUTEX)}
-#define ERTS_MTX_INITER \
-  {ETHR_MUTEX_INITER, \
-   ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_MUTEX)}
-#else
-#define ERTS_REC_MTX_INITER		{ETHR_REC_MUTEX_INITER}
-#define ERTS_MTX_INITER			{ETHR_MUTEX_INITER}
-#endif
-#define ERTS_CND_INITER			ETHR_COND_INITER
 #define ERTS_THR_INIT_DATA_DEF_INITER	ETHR_INIT_DATA_DEFAULT_INITER
+#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \
+                                        ETHR_LATE_INIT_DATA_DEFAULT_INITER
 
 #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
 #  define ERTS_HAVE_REC_MTX_INIT	ETHR_HAVE_ETHR_REC_MUTEX_INIT
 #endif
 
-
 #else /* #ifdef USE_THREADS */
 
+#define ERTS_THR_MEMORY_BARRIER
+
 #define ERTS_THR_OPTS_DEFAULT_INITER 0
 typedef int erts_thr_opts_t;
 typedef int erts_thr_init_data_t;
+typedef int erts_thr_late_init_data_t;
 typedef int erts_tid_t;
 typedef int erts_mtx_t;
 typedef int erts_cnd_t;
+#define ERTS_RWMTX_OPT_DEFAULT_INITER {0}
+#define ERTS_RWMTX_TYPE_NORMAL 0
+#define ERTS_RWMTX_TYPE_FREQUENT_READ 0
+#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
+#define ERTS_RWMTX_LONG_LIVED 0
+#define ERTS_RWMTX_SHORT_LIVED 0
+#define ERTS_RWMTX_UNKNOWN_LIVED 0
+typedef struct {
+    char type;
+    char lived;
+    int main_spincount;
+    int aux_spincount;
+} erts_rwmtx_opt_t;
 typedef int erts_rwmtx_t;
 typedef int erts_tsd_key_t;
-typedef int erts_gate_t;
-typedef long erts_atomic_t;
+typedef int erts_tse_t;
+typedef SWord erts_aint_t;
+typedef SWord erts_atomic_t;
+typedef SWord erts_aint32_t;
+typedef SWord erts_atomic32_t;
 #if __GNUC__ > 2
 typedef struct { } erts_spinlock_t;
 typedef struct { } erts_rwlock_t;
@@ -139,12 +175,7 @@ typedef struct { } erts_rwlock_t;
 typedef struct { int gcc_is_buggy; } erts_spinlock_t;
 typedef struct { int gcc_is_buggy; } erts_rwlock_t;
 #endif
-typedef struct {
-    long tv_sec;
-    long tv_nsec;
-} erts_thr_timeval_t;
 
-#define ERTS_REC_MTX_INITER		0
 #define ERTS_MTX_INITER			0
 #define ERTS_CND_INITER			0
 #define ERTS_THR_INIT_DATA_DEF_INITER	0
@@ -153,7 +184,13 @@ typedef struct {
 
 #endif /* #ifdef USE_THREADS */
 
+#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
+#define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
+#define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
+#define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
+
 ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id);
+ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id);
 ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *),
 				     void *arg, erts_thr_opts_t *opts);
 ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res);
@@ -162,9 +199,6 @@ ERTS_GLB_INLINE void erts_thr_exit(void *res);
 ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
 ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
 ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
-#ifdef ERTS_HAVE_REC_MTX_INIT
-ERTS_GLB_INLINE void erts_rec_mtx_init(erts_mtx_t *mtx);
-#endif
 ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra);
 ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt);
 ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
@@ -173,8 +207,6 @@ ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
 ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_mtx_set_forksafe(erts_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_mtx_unset_forksafe(erts_mtx_t *mtx);
 ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line);
@@ -188,9 +220,17 @@ ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd);
 ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx);
 ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd);
 ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd);
+ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no);
+ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
+					   erts_rwmtx_opt_t *opt,
+					   char *name,
+					   Eterm extra);
 ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
 				       char *name,
 				       Eterm extra);
+ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
+					 erts_rwmtx_opt_t *opt,
+					 char *name);
 ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
 					 char *name);
 ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
@@ -207,23 +247,69 @@ ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);
-ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, long i);
-ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, long i);
-ERTS_GLB_INLINE long erts_atomic_read(erts_atomic_t *var);
-ERTS_GLB_INLINE long erts_atomic_inctest(erts_atomic_t *incp);
-ERTS_GLB_INLINE long erts_atomic_dectest(erts_atomic_t *decp);
+ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, erts_aint_t i);
+ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, erts_aint_t i);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp);
 ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp);
 ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp);
-ERTS_GLB_INLINE long erts_atomic_addtest(erts_atomic_t *addp,
-					 long i);
-ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, long i);
-ERTS_GLB_INLINE long erts_atomic_xchg(erts_atomic_t *xchgp,
-				      long new);
-ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp,
-					 long new,
-					 long expected);
-ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask);
-ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_addtest(erts_atomic_t *addp,
+						erts_aint_t i);
+ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, erts_aint_t i);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_xchg(erts_atomic_t *xchgp,
+					     erts_aint_t new);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg(erts_atomic_t *xchgp,
+						erts_aint_t new,
+						erts_aint_t expected);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_bor(erts_atomic_t *var,
+					    erts_aint_t mask);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_band(erts_atomic_t *var,
+					     erts_aint_t mask);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_read_acqb(erts_atomic_t *var);
+ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i);
+ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest_relb(erts_atomic_t *decp);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp,
+						     erts_aint_t new,
+						     erts_aint_t exp);
+ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp,
+						     erts_aint_t new,
+						     erts_aint_t exp);
+ERTS_GLB_INLINE void erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i);
+ERTS_GLB_INLINE void erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read(erts_atomic32_t *var);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_inctest(erts_atomic32_t *incp);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest(erts_atomic32_t *decp);
+ERTS_GLB_INLINE void erts_atomic32_inc(erts_atomic32_t *incp);
+ERTS_GLB_INLINE void erts_atomic32_dec(erts_atomic32_t *decp);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_addtest(erts_atomic32_t *addp,
+						    erts_aint32_t i);
+ERTS_GLB_INLINE void erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_xchg(erts_atomic32_t *xchgp,
+						 erts_aint32_t new);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg(erts_atomic32_t *xchgp,
+						    erts_aint32_t new,
+						    erts_aint32_t expected);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_bor(erts_atomic32_t *var,
+						erts_aint32_t mask);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_band(erts_atomic32_t *var,
+						 erts_aint32_t mask);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_acqb(erts_atomic32_t *var);
+ERTS_GLB_INLINE void erts_atomic32_set_relb(erts_atomic32_t *var,
+					    erts_aint32_t i);
+ERTS_GLB_INLINE void erts_atomic32_dec_relb(erts_atomic32_t *decp);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest_relb(erts_atomic32_t *decp);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp,
+							 erts_aint32_t new,
+							 erts_aint32_t exp);
+ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp,
+							 erts_aint32_t new,
+							 erts_aint32_t exp);
+ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
+					      char *name,
+					      Eterm extra,
+					      Uint16 opt);
 ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock,
 					  char *name,
 					  Eterm extra);
@@ -254,17 +340,20 @@ ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock);
 ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock);
-ERTS_GLB_INLINE void erts_thr_time_now(erts_thr_timeval_t *time);
 ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp);
 ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key);
 ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value);
 ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key);
-ERTS_GLB_INLINE void erts_gate_init(erts_gate_t *gp);
-ERTS_GLB_INLINE void erts_gate_destroy(erts_gate_t *gp);
-ERTS_GLB_INLINE void erts_gate_close(erts_gate_t *gp);
-ERTS_GLB_INLINE void erts_gate_let_through(erts_gate_t *gp, unsigned no);
-ERTS_GLB_INLINE void erts_gate_wait(erts_gate_t *gp);
-ERTS_GLB_INLINE void erts_gate_swait(erts_gate_t *gp, int spincount);
+ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void);
+ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep);
+ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep);
+ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep);
+ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep);
+ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount);
+ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep);
+ERTS_GLB_INLINE void erts_thr_set_main_status(int, int);
+ERTS_GLB_INLINE int erts_thr_get_main_status(void);
+ERTS_GLB_INLINE void erts_thr_yield(void);
 
 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
 #define ERTS_THR_HAVE_SIG_FUNCS 1
@@ -286,15 +375,21 @@ erts_thr_init(erts_thr_init_data_t *id)
 }
 
 ERTS_GLB_INLINE void
+erts_thr_late_init(erts_thr_late_init_data_t *id)
+{
+#ifdef USE_THREADS
+    int res = ethr_late_init(id);
+    if (res)
+	erts_thr_fatal_error(res, "complete initialization of thread library");
+#endif
+}
+
+ERTS_GLB_INLINE void
 erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg,
 		erts_thr_opts_t *opts)
 {
 #ifdef USE_THREADS
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    int res = erts_lcnt_thr_create(tid, func, arg, opts);
-#else
     int res = ethr_thr_create(tid, func, arg, opts);
-#endif
     if (res)
 	erts_thr_fatal_error(res, "create thread");
 #endif
@@ -362,20 +457,6 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y)
 #endif
 }
 
-
-#ifdef ERTS_HAVE_REC_MTX_INIT
-ERTS_GLB_INLINE void
-erts_rec_mtx_init(erts_mtx_t *mtx)
-{
-#ifdef USE_THREADS
-    int res = ethr_rec_mutex_init(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "initialize recursive mutex");
-#endif
-}
-#endif
-
-
 ERTS_GLB_INLINE void
 erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra)
 {
@@ -422,9 +503,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
 #endif
-    res = ethr_mutex_lock(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "lock mutex");
+    ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &mtx->lc);
 #endif
@@ -463,9 +542,7 @@ erts_mtx_init_locked(erts_mtx_t *mtx, char *name)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
 #endif
-    res = ethr_mutex_lock(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "lock mutex");
+    ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &mtx->lc);
 #endif
@@ -487,28 +564,16 @@ erts_mtx_destroy(erts_mtx_t *mtx)
     erts_lcnt_destroy_lock(&mtx->lcnt);
 #endif
     res = ethr_mutex_destroy(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "destroy mutex");
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_mtx_set_forksafe(erts_mtx_t *mtx)
-{
-#ifdef USE_THREADS
-    int res = ethr_mutex_set_forksafe(&mtx->mtx);
-    if (res != 0 && res != ENOTSUP)
-	erts_thr_fatal_error(res, "set mutex forksafe");
+    if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+	if (res == EBUSY) {
+	    char *warn = "Ignoring busy mutex destroy. "
+		"Most likely a bug in pthread implementation.";
+	    erts_send_warning_to_logger_str_nogl(warn);
+	}
 #endif
-}
-
-ERTS_GLB_INLINE void
-erts_mtx_unset_forksafe(erts_mtx_t *mtx)
-{
-#ifdef USE_THREADS
-    int res = ethr_mutex_unset_forksafe(&mtx->mtx);
-    if (res != 0 && res != ENOTSUP)
-	erts_thr_fatal_error(res, "unset mutex forksafe");
+	erts_thr_fatal_error(res, "destroy mutex");
+    }
 #endif
 }
 
@@ -531,11 +596,7 @@ erts_mtx_trylock(erts_mtx_t *mtx)
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_trylock(&mtx->lcnt, res);
-#endif
-
-    if (res != 0 && res != EBUSY)
-	erts_thr_fatal_error(res, "try lock mutex");
-    
+#endif    
     return res;
 #else
     return 0;
@@ -551,19 +612,16 @@ erts_mtx_lock(erts_mtx_t *mtx)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock(&mtx->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock(&mtx->lcnt);
 #endif
-    res = ethr_mutex_lock(&mtx->mtx);
+    ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&mtx->lcnt, file, line);
 #endif
-    if (res)
-	erts_thr_fatal_error(res, "lock mutex");
 #endif
 }
 
@@ -571,16 +629,13 @@ ERTS_GLB_INLINE void
 erts_mtx_unlock(erts_mtx_t *mtx)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock(&mtx->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock(&mtx->lcnt);
 #endif
-    res = ethr_mutex_unlock(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "unlock mutex");
+    ethr_mutex_unlock(&mtx->mtx);
 #endif
 }
 
@@ -613,8 +668,16 @@ erts_cnd_destroy(erts_cnd_t *cnd)
 {
 #ifdef USE_THREADS
     int res = ethr_cond_destroy(cnd);
-    if (res)
+    if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+	if (res == EBUSY) {
+	    char *warn = "Ignoring busy cond destroy. "
+		"Most likely a bug in pthread implementation.";
+	    erts_send_warning_to_logger_str_nogl(warn);
+	}
+#endif
 	erts_thr_fatal_error(res, "destroy condition variable");
+    }
 #endif
 }
 
@@ -648,9 +711,7 @@ ERTS_GLB_INLINE void
 erts_cnd_signal(erts_cnd_t *cnd)
 {
 #ifdef USE_THREADS
-    int res = ethr_cond_signal(cnd);
-    if (res)
-	erts_thr_fatal_error(res, "signal on condition variable");
+    ethr_cond_signal(cnd);
 #endif
 }
 
@@ -659,19 +720,34 @@ ERTS_GLB_INLINE void
 erts_cnd_broadcast(erts_cnd_t *cnd)
 {
 #ifdef USE_THREADS
-    int res = ethr_cond_broadcast(cnd);
-    if (res)
-	erts_thr_fatal_error(res, "broadcast on condition variable");
+    ethr_cond_broadcast(cnd);
 #endif
 }
 
 /* rwmutex */
 
 ERTS_GLB_INLINE void
-erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra)
+erts_rwmtx_set_reader_group(int no)
+{
+#ifdef USE_THREADS
+    int res;
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX);
+#endif
+    res = ethr_rwmutex_set_reader_group(no);
+    if (res != 0)
+	erts_thr_fatal_error(res, "set reader group");
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
+		      erts_rwmtx_opt_t *opt,
+		      char *name,
+		      Eterm extra)
 {
 #ifdef USE_THREADS
-    int res = ethr_rwmutex_init(&rwmtx->rwmtx);
+    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
     if (res != 0)
 	erts_thr_fatal_error(res, "initialize rwmutex");
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -684,10 +760,20 @@ erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra)
 }
 
 ERTS_GLB_INLINE void
-erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
+erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
+		  char *name,
+		  Eterm extra)
+{
+    erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra);
+}
+
+ERTS_GLB_INLINE void
+erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
+		    erts_rwmtx_opt_t *opt,
+		    char *name)
 {
 #ifdef USE_THREADS
-    int res = ethr_rwmutex_init(&rwmtx->rwmtx);
+    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
     if (res != 0)
 	erts_thr_fatal_error(res, "initialize rwmutex");
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -700,6 +786,12 @@ erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
 }
 
 ERTS_GLB_INLINE void
+erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
+{
+    erts_rwmtx_init_opt(rwmtx, NULL, name);
+}
+
+ERTS_GLB_INLINE void
 erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
@@ -711,8 +803,16 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
     erts_lcnt_destroy_lock(&rwmtx->lcnt);
 #endif
     res = ethr_rwmutex_destroy(&rwmtx->rwmtx);
-    if (res != 0)
+    if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+	if (res == EBUSY) {
+	    char *warn = "Ignoring busy rwmutex destroy. "
+		"Most likely a bug in pthread implementation.";
+	    erts_send_warning_to_logger_str_nogl(warn);
+	}
+#endif
 	erts_thr_fatal_error(res, "destroy rwmutex");
+    }
 #endif
 }
 
@@ -736,9 +836,6 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ);
 #endif
-
-    if (res != 0 && res != EBUSY)
-	erts_thr_fatal_error(res, "try read lock rwmutex");
     
     return res;
 #else
@@ -754,19 +851,16 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
 #endif
-    res = ethr_rwmutex_rlock(&rwmtx->rwmtx);
+    ethr_rwmutex_rlock(&rwmtx->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
 #endif
-    if (res != 0)
-	erts_thr_fatal_error(res, "read lock rwmutex");
 #endif
 }
 
@@ -774,16 +868,13 @@ ERTS_GLB_INLINE void
 erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
 #endif
-    res = ethr_rwmutex_runlock(&rwmtx->rwmtx);
-    if (res != 0)
-	erts_thr_fatal_error(res, "read unlock rwmutex");
+    ethr_rwmutex_runlock(&rwmtx->rwmtx);
 #endif
 }
 
@@ -808,9 +899,6 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
 #endif
-
-    if (res != 0 && res != EBUSY)
-	erts_thr_fatal_error(res, "try write lock rwmutex");
     
     return res;
 #else
@@ -826,19 +914,16 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
 #endif
-    res = ethr_rwmutex_rwlock(&rwmtx->rwmtx);
+    ethr_rwmutex_rwlock(&rwmtx->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
 #endif
-    if (res != 0)
-	erts_thr_fatal_error(res, "write lock rwmutex");
 #endif
 }
 
@@ -846,16 +931,13 @@ ERTS_GLB_INLINE void
 erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
 #endif
-    res = ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
-    if (res != 0)
-	erts_thr_fatal_error(res, "write unlock rwmutex");
+    ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
 #endif
 }
 
@@ -914,66 +996,50 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
 }
 
 ERTS_GLB_INLINE void
-erts_atomic_init(erts_atomic_t *var, long i)
+erts_atomic_init(erts_atomic_t *var, erts_aint_t i)
 {
 #ifdef USE_THREADS
-    int res = ethr_atomic_init(var, i);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic init");
+    ethr_atomic_init(var, i);
 #else
     *var = i;
 #endif
 }
 
 ERTS_GLB_INLINE void
-erts_atomic_set(erts_atomic_t *var, long i)
+erts_atomic_set(erts_atomic_t *var, erts_aint_t i)
 {
 #ifdef USE_THREADS
-    int res = ethr_atomic_set(var, i);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic set");
+    ethr_atomic_set(var, i);
 #else
     *var = i;
 #endif
 }
 
-ERTS_GLB_INLINE long
+ERTS_GLB_INLINE erts_aint_t
 erts_atomic_read(erts_atomic_t *var)
 {
 #ifdef USE_THREADS
-    long i;
-    int res = ethr_atomic_read(var, &i);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic read");
-    return i;
+    return ethr_atomic_read(var);
 #else
     return *var;
 #endif
 }
 
-ERTS_GLB_INLINE long
+ERTS_GLB_INLINE erts_aint_t
 erts_atomic_inctest(erts_atomic_t *incp)
 {
 #ifdef USE_THREADS
-    long test;
-    int res = ethr_atomic_inctest(incp, &test);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic increment and test");
-    return test;
+    return ethr_atomic_inc_read(incp);
 #else
     return ++(*incp);
 #endif
 }
 
-ERTS_GLB_INLINE long
+ERTS_GLB_INLINE erts_aint_t
 erts_atomic_dectest(erts_atomic_t *decp)
 {
 #ifdef USE_THREADS
-    long test;
-    int res = ethr_atomic_dectest(decp, &test);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic decrement and test");
-    return test;
+    return ethr_atomic_dec_read(decp);
 #else
     return --(*decp);
 #endif
@@ -983,9 +1049,7 @@ ERTS_GLB_INLINE void
 erts_atomic_inc(erts_atomic_t *incp)
 {
 #ifdef USE_THREADS
-    int res = ethr_atomic_inc(incp);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic increment");
+    ethr_atomic_inc(incp);
 #else
     ++(*incp);
 #endif
@@ -995,100 +1059,364 @@ ERTS_GLB_INLINE void
 erts_atomic_dec(erts_atomic_t *decp)
 {
 #ifdef USE_THREADS
-    int res = ethr_atomic_dec(decp);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic decrement");
+    ethr_atomic_dec(decp);
 #else
     --(*decp);
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_addtest(erts_atomic_t *addp, long i)
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_addtest(erts_atomic_t *addp, erts_aint_t i)
 {
 #ifdef USE_THREADS
-    long test;
-    int res = ethr_atomic_addtest(addp, i, &test);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic addition and test");
-    return test;
+    return ethr_atomic_add_read(addp, i);
 #else
     return *addp += i;
 #endif
 }
 
 ERTS_GLB_INLINE void
-erts_atomic_add(erts_atomic_t *addp, long i)
+erts_atomic_add(erts_atomic_t *addp, erts_aint_t i)
 {
 #ifdef USE_THREADS
-    int res = ethr_atomic_add(addp, i);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic addition");
+    ethr_atomic_add(addp, i);
 #else
     *addp += i;
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_xchg(erts_atomic_t *xchgp, long new)
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_xchg(erts_atomic_t *xchgp, erts_aint_t new)
 {
-    long old;
 #ifdef USE_THREADS
-    int res = ethr_atomic_xchg(xchgp, new, &old);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic exchange");
+    return ethr_atomic_xchg(xchgp, new);
 #else
-    old = *xchgp;
+    erts_aint_t old = *xchgp;
     *xchgp = new;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_cmpxchg(erts_atomic_t *xchgp, erts_aint_t new, erts_aint_t expected)
+{
+#ifdef USE_THREADS
+    return ethr_atomic_cmpxchg(xchgp, new, expected);
+#else
+    erts_aint_t old = *xchgp;
+    if (old == expected)
+        *xchgp = new;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_bor(erts_atomic_t *var, erts_aint_t mask)
+{
+#ifdef USE_THREADS
+    return ethr_atomic_read_bor(var, mask);
+#else
+    erts_aint_t old;
+    old = *var;
+    *var |= mask;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_band(erts_atomic_t *var, erts_aint_t mask)
+{
+#ifdef USE_THREADS
+    return ethr_atomic_read_band(var, mask);
+#else
+    erts_aint_t old;
+    old = *var;
+    *var &= mask;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_acqb(erts_atomic_t *var)
+{
+#ifdef USE_THREADS
+    return ethr_atomic_read_acqb(var);
+#else
+    return *var;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i)
+{
+#ifdef USE_THREADS
+    ethr_atomic_set_relb(var, i);
+#else
+    *var = i;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_atomic_dec_relb(erts_atomic_t *decp)
+{
+#ifdef USE_THREADS
+    ethr_atomic_dec_relb(decp);
+#else
+    --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_dectest_relb(erts_atomic_t *decp)
+{
+#ifdef USE_THREADS
+    return ethr_atomic_dec_read_relb(decp);
+#else
+    return --(*decp);
 #endif
+}
+
+ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp,
+						     erts_aint_t new,
+						     erts_aint_t exp)
+{
+#ifdef USE_THREADS
+    return ethr_atomic_cmpxchg_acqb(xchgp, new, exp);
+#else
+    erts_aint_t old = *xchgp;
+    if (old == exp)
+        *xchgp = new;
     return old;
+#endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected)
+ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp,
+						     erts_aint_t new,
+						     erts_aint_t exp)
 {
 #ifdef USE_THREADS
-    long old;
-    int res = ethr_atomic_cmpxchg(xchgp, new, expected, &old);
-    if (ERTS_UNLIKELY(res != 0))
-	erts_thr_fatal_error(res, "perform atomic exchange");
+    return ethr_atomic_cmpxchg_relb(xchgp, new, exp);
+#else
+    erts_aint_t old = *xchgp;
+    if (old == exp)
+        *xchgp = new;
     return old;
+#endif
+}
+
+/* atomic32 */
+
+ERTS_GLB_INLINE void
+erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i)
+{
+#ifdef USE_THREADS
+    ethr_atomic32_init(var, i);
 #else
-    long old = *xchgp;
+    *var = i;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i)
+{
+#ifdef USE_THREADS
+    ethr_atomic32_set(var, i);
+#else
+    *var = i;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read(erts_atomic32_t *var)
+{
+#ifdef USE_THREADS
+    return ethr_atomic32_read(var);
+#else
+    return *var;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_inctest(erts_atomic32_t *incp)
+{
+#ifdef USE_THREADS
+    return ethr_atomic32_inc_read(incp);
+#else
+    return ++(*incp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_dectest(erts_atomic32_t *decp)
+{
+#ifdef USE_THREADS
+    return ethr_atomic32_dec_read(decp);
+#else
+    return --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_atomic32_inc(erts_atomic32_t *incp)
+{
+#ifdef USE_THREADS
+    ethr_atomic32_inc(incp);
+#else
+    ++(*incp);
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_atomic32_dec(erts_atomic32_t *decp)
+{
+#ifdef USE_THREADS
+    ethr_atomic32_dec(decp);
+#else
+    --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_addtest(erts_atomic32_t *addp, erts_aint32_t i)
+{
+#ifdef USE_THREADS
+    return ethr_atomic32_add_read(addp, i);
+#else
+    return *addp += i;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i)
+{
+#ifdef USE_THREADS
+    ethr_atomic32_add(addp, i);
+#else
+    *addp += i;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_xchg(erts_atomic32_t *xchgp, erts_aint32_t new)
+{
+#ifdef USE_THREADS
+    return ethr_atomic32_xchg(xchgp, new);
+#else
+    erts_aint32_t old = *xchgp;
+    *xchgp = new;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_cmpxchg(erts_atomic32_t *xchgp,
+		      erts_aint32_t new,
+		      erts_aint32_t expected)
+{
+#ifdef USE_THREADS
+    return ethr_atomic32_cmpxchg(xchgp, new, expected);
+#else
+    erts_aint32_t old = *xchgp;
     if (old == expected)
         *xchgp = new;
     return old;
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_bor(erts_atomic_t *var, long mask)
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_bor(erts_atomic32_t *var, erts_aint32_t mask)
 {
-    long old;
 #ifdef USE_THREADS
-    int res = ethr_atomic_or_old(var, mask, &old);
-    if (res != 0)
-	erts_thr_fatal_error(res, "perform atomic bitwise or");
+    return ethr_atomic32_read_bor(var, mask);
 #else
+    erts_aint32_t old;
     old = *var;
     *var |= mask;
-#endif
     return old;
+#endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_band(erts_atomic_t *var, long mask)
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_band(erts_atomic32_t *var, erts_aint32_t mask)
 {
-    long old;
 #ifdef USE_THREADS
-    int res = ethr_atomic_and_old(var, mask, &old);
-    if (res != 0)
-	erts_thr_fatal_error(res, "perform atomic bitwise and");
+    return ethr_atomic32_read_band(var, mask);
 #else
+    erts_aint32_t old;
     old = *var;
     *var &= mask;
+    return old;
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_acqb(erts_atomic32_t *var)
+{
+#ifdef USE_THREADS
+    return ethr_atomic32_read_acqb(var);
+#else
+    return *var;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_atomic32_set_relb(erts_atomic32_t *var, erts_aint32_t i)
+{
+#ifdef USE_THREADS
+    ethr_atomic32_set_relb(var, i);
+#else
+    *var = i;
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_atomic32_dec_relb(erts_atomic32_t *decp)
+{
+#ifdef USE_THREADS
+    ethr_atomic32_dec_relb(decp);
+#else
+    --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_dectest_relb(erts_atomic32_t *decp)
+{
+#ifdef USE_THREADS
+    return ethr_atomic32_dec_read_relb(decp);
+#else
+    return --(*decp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp,
+			   erts_aint32_t new,
+			   erts_aint32_t exp)
+{
+#ifdef USE_THREADS
+    return ethr_atomic32_cmpxchg_acqb(xchgp, new, exp);
+#else
+    erts_aint32_t old = *xchgp;
+    if (old == exp)
+        *xchgp = new;
+    return old;
 #endif
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp,
+			   erts_aint32_t new,
+			   erts_aint32_t exp)
+{
+#ifdef USE_THREADS
+    return ethr_atomic32_cmpxchg_relb(xchgp, new, exp);
+#else
+    erts_aint32_t old = *xchgp;
+    if (old == exp)
+        *xchgp = new;
     return old;
+#endif
 }
 
 /* spinlock */
@@ -1112,6 +1440,26 @@ erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra)
 }
 
 ERTS_GLB_INLINE void
+erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra,
+			 Uint16 opt)
+{
+#ifdef USE_THREADS
+    int res = ethr_spinlock_init(&lock->slck);
+    if (res)
+	erts_thr_fatal_error(res, "init spinlock");
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
+#endif
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra);
+#endif
+#else
+    (void)lock;
+#endif
+}
+
+
+ERTS_GLB_INLINE void
 erts_spinlock_init(erts_spinlock_t *lock, char *name)
 {
 #ifdef USE_THREADS
@@ -1141,8 +1489,16 @@ erts_spinlock_destroy(erts_spinlock_t *lock)
     erts_lcnt_destroy_lock(&lock->lcnt);
 #endif
     res = ethr_spinlock_destroy(&lock->slck);
-    if (res)
-	erts_thr_fatal_error(res, "destroy spinlock");
+    if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+	if (res == EBUSY) {
+	    char *warn = "Ignoring busy spinlock destroy. "
+		"Most likely a bug in pthread implementation.";
+	    erts_send_warning_to_logger_str_nogl(warn);
+	}
+#endif
+	erts_thr_fatal_error(res, "destroy rwlock");
+    }
 #else
     (void)lock;
 #endif
@@ -1152,16 +1508,13 @@ ERTS_GLB_INLINE void
 erts_spin_unlock(erts_spinlock_t *lock)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock(&lock->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock(&lock->lcnt);
 #endif
-    res = ethr_spin_unlock(&lock->slck);
-    if (res)
-	erts_thr_fatal_error(res, "release spin lock");
+    ethr_spin_unlock(&lock->slck);
 #else
     (void)lock;
 #endif
@@ -1175,19 +1528,16 @@ erts_spin_lock(erts_spinlock_t *lock)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock(&lock->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock(&lock->lcnt);
 #endif
-    res = ethr_spin_lock(&lock->slck);
+    ethr_spin_lock(&lock->slck);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&lock->lcnt, file, line);
 #endif
-    if (res)
-	erts_thr_fatal_error(res, "take spin lock");
 #else
     (void)lock;
 #endif
@@ -1257,8 +1607,16 @@ erts_rwlock_destroy(erts_rwlock_t *lock)
     erts_lcnt_destroy_lock(&lock->lcnt);
 #endif
     res = ethr_rwlock_destroy(&lock->rwlck);
-    if (res)
+    if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+	if (res == EBUSY) {
+	    char *warn = "Ignoring busy rwlock destroy. "
+		"Most likely a bug in pthread implementation.";
+	    erts_send_warning_to_logger_str_nogl(warn);
+	}
+#endif
 	erts_thr_fatal_error(res, "destroy rwlock");
+    }
 #else
     (void)lock;
 #endif
@@ -1268,16 +1626,13 @@ ERTS_GLB_INLINE void
 erts_read_unlock(erts_rwlock_t *lock)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
 #endif
-    res = ethr_read_unlock(&lock->rwlck);
-    if (res)
-	erts_thr_fatal_error(res, "release read lock");
+    ethr_read_unlock(&lock->rwlck);
 #else
     (void)lock;
 #endif
@@ -1291,19 +1646,16 @@ erts_read_lock(erts_rwlock_t *lock)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
 #endif
-    res = ethr_read_lock(&lock->rwlck);
+    ethr_read_lock(&lock->rwlck);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&lock->lcnt, file, line);
 #endif
-    if (res)
-	erts_thr_fatal_error(res, "take read lock");
 #else
     (void)lock;
 #endif
@@ -1313,16 +1665,13 @@ ERTS_GLB_INLINE void
 erts_write_unlock(erts_rwlock_t *lock)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
 #endif
-    res = ethr_write_unlock(&lock->rwlck);
-    if (res)
-	erts_thr_fatal_error(res, "release write lock");
+    ethr_write_unlock(&lock->rwlck);
 #else
     (void)lock;
 #endif
@@ -1336,19 +1685,16 @@ erts_write_lock(erts_rwlock_t *lock)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
 #endif
-    res = ethr_write_lock(&lock->rwlck);
+    ethr_write_lock(&lock->rwlck);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&lock->lcnt, file, line);
 #endif
-    if (res)
-	erts_thr_fatal_error(res, "take write lock");
 #else
     (void)lock;
 #endif
@@ -1383,16 +1729,6 @@ erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-erts_thr_time_now(erts_thr_timeval_t *time)
-{
-#ifdef USE_THREADS
-    int res = ethr_time_now(time);
-    if (res)
-	erts_thr_fatal_error(res, "get current time");
-#endif
-}
-
-ERTS_GLB_INLINE void
 erts_tsd_key_create(erts_tsd_key_t *keyp)
 {
 #ifdef USE_THREADS
@@ -1432,66 +1768,95 @@ erts_tsd_get(erts_tsd_key_t key)
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_init(erts_gate_t *gp)
+ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_init((ethr_gate *) gp);
-    if (res != 0)
-	erts_thr_fatal_error(res, "initialize gate");
+    return (erts_tse_t *) ethr_get_ts_event();
+#else
+    return (erts_tse_t *) NULL;
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_destroy(erts_gate_t *gp)
+ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_destroy((ethr_gate *) gp);
-    if (res != 0)
-	erts_thr_fatal_error(res, "destroy gate");
+    ethr_leave_ts_event(ep);
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_close(erts_gate_t *gp)
+ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_close((ethr_gate *) gp);
-    if (res != 0)
-	erts_thr_fatal_error(res, "close gate");
+    ethr_event_set(&((ethr_ts_event *) ep)->event);
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_let_through(erts_gate_t *gp, unsigned no)
+ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_let_through((ethr_gate *) gp, no);
-    if (res != 0)
-	erts_thr_fatal_error(res, "let through gate");
+    ethr_event_reset(&((ethr_ts_event *) ep)->event);
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_wait(erts_gate_t *gp)
+ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep)
+{
+#ifdef USE_THREADS
+    return ethr_event_wait(&((ethr_ts_event *) ep)->event);
+#else
+    return ENOTSUP;
+#endif
+}
+
+ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
+{
+#ifdef USE_THREADS
+    return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount);
+#else
+    return ENOTSUP;
+#endif
+}
+
+ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep)
+{
+#ifdef USE_THREADS
+    return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP;
+#else
+    return 0;
+#endif
+}
+
+ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_wait((ethr_gate *) gp);
+    int res = ethr_set_main_thr_status(on, no);
     if (res != 0)
-	erts_thr_fatal_error(res, "wait on gate");
+	erts_thr_fatal_error(res, "set thread main status");
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_swait(erts_gate_t *gp, int spincount)
+ERTS_GLB_INLINE int erts_thr_get_main_status(void)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_swait((ethr_gate *) gp, spincount);
+    int main_status;
+    int res = ethr_get_main_thr_status(&main_status);
     if (res != 0)
-	erts_thr_fatal_error(res, "swait on gate");
+	erts_thr_fatal_error(res, "get thread main status");
+    return main_status;
+#else
+    return 1;
 #endif
 }
 
+ERTS_GLB_INLINE void erts_thr_yield(void)
+{
+#ifdef USE_THREADS
+    int res = ETHR_YIELD();
+    if (res != 0)
+	erts_thr_fatal_error(res, "yield");
+#endif    
+}
+
+
 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
 
 ERTS_GLB_INLINE void
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 6f6b971d34..d0ad73cd81 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -20,11 +20,15 @@
 #ifndef ERL_TIME_H__
 #define ERL_TIME_H__
 
+extern erts_smp_atomic_t do_time;	/* set at clock interrupt */
+extern SysTimeval erts_first_emu_time;
+
 /*
 ** Timer entry:
 */
 typedef struct erl_timer {
     struct erl_timer* next;	/* next entry tiw slot or chain */
+    struct erl_timer* prev;	/* prev entry tiw slot or chain */
     Uint slot;			/* slot in timer wheel */
     Uint count;			/* number of loops remaining */
     int    active;		/* 1=activated, 0=deactivated */
@@ -39,7 +43,6 @@ typedef void (*ErlTimeoutProc)(void*);
 typedef void (*ErlCancelProc)(void*);
 
 #ifdef ERTS_SMP
-
 /*
  * Process and port timer
  */
@@ -61,7 +64,66 @@ void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
 			    ErlTimeoutProc timeout_func,
 			    Uint timeout);
 void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer);
+#endif
+
+/* timer-wheel api */
 
+void erts_init_time(void);
+void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint);
+void erts_cancel_timer(ErlTimer*);
+void erts_bump_timer(erts_aint_t);
+Uint erts_timer_wheel_memory_size(void);
+Uint erts_time_left(ErlTimer *);
+erts_aint_t erts_next_time(void);
+
+#ifdef DEBUG
+void erts_p_slpq(void);
 #endif
 
+ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void);
+ERTS_GLB_INLINE void erts_do_time_add(long);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg(&do_time, 0L); }
+ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add(&do_time, elapsed); }
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+
+/* time_sup */
+
+#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME))
+#  ifndef HAVE_ERTS_NOW_CPU
+#    define HAVE_ERTS_NOW_CPU
+#    ifdef HAVE_GETHRVTIME
+#      define erts_start_now_cpu() sys_start_hrvtime()
+#      define erts_stop_now_cpu()  sys_stop_hrvtime()
+#    endif
+#  endif
+void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
 #endif
+
+void erts_get_timeval(SysTimeval *tv);
+long erts_get_time(void);
+void erts_get_emu_time(SysTimeval *);
+
+ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int
+erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p)
+{
+    if (t1p->tv_sec == t2p->tv_sec) {
+	if (t1p->tv_usec < t2p->tv_usec)
+	    return -1;
+	else if (t1p->tv_usec > t2p->tv_usec)
+	    return 1;
+	return 0;
+    }
+    return t1p->tv_sec < t2p->tv_sec ? -1 : 1;
+}
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+#endif /* ERL_TIME_H__ */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index c15f85f8f1..ca4b54188e 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -358,10 +358,6 @@ static int clock_resolution;
 ** instead of something like select.
 */
 
-#if defined(ERTS_TIMER_THREAD)
-static ERTS_INLINE void init_erts_deliver_time(const SysTimeval *inittv) { }
-static ERTS_INLINE void do_erts_deliver_time(const SysTimeval *current) { }
-#else
 static SysTimeval last_delivered; 
 
 static void init_erts_deliver_time(const SysTimeval *inittv)
@@ -389,11 +385,10 @@ static void do_erts_deliver_time(const SysTimeval *current)
        this by simply pretend as if the time stood still. :) */
 
     if (elapsed > 0) {
-	do_time_add(elapsed);
+	erts_do_time_add(elapsed);
 	last_delivered = cur_time;
     }
 }
-#endif
 
 int 
 erts_init_time_sup(void)
@@ -650,6 +645,22 @@ local_to_univ(Sint *year, Sint *month, Sint *day,
     t.tm_sec = *second;
     t.tm_isdst = isdst;
     the_clock = mktime(&t);
+    if (the_clock == -1) {
+	if (isdst) {
+	    /* If this is a timezone without DST and the OS (correctly)
+	       refuses to give us a DST time, we simulate the Linux/Solaris
+	       behaviour of giving the same data as if is_dst was not set. */
+	    t.tm_isdst = 0;
+	    the_clock = mktime(&t);
+	    if (the_clock == -1) {
+		/* Failed anyway, something else is bad - will be a badarg */
+		return 0;
+	    }
+	} else {
+	    /* Something else is the matter, badarg. */
+	    return 0;
+	}
+    }
 #ifdef HAVE_GMTIME_R
     gmtime_r(&the_clock, (tm = &tmbuf));
 #else
@@ -663,6 +674,10 @@ local_to_univ(Sint *year, Sint *month, Sint *day,
     *second = tm->tm_sec;
     return 1;
 }
+#if defined(HAVE_POSIX2TIME) && defined(HAVE_DECL_POSIX2TIME) && \
+    !HAVE_DECL_POSIX2TIME
+extern time_t posix2time(time_t);
+#endif
 
 int 
 univ_to_local(Sint *year, Sint *month, Sint *day, 
@@ -766,7 +781,6 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec)
    to a struct timeval representing current time (to save
    a gettimeofday() where possible) or NULL */
 
-#if !defined(ERTS_TIMER_THREAD)
 void erts_deliver_time(void) {
     SysTimeval now;
     
@@ -777,7 +791,6 @@ void erts_deliver_time(void) {
     
     erts_smp_mtx_unlock(&erts_timeofday_mtx);
 }
-#endif
 
 /* get *real* time (not ticks) remaining until next timeout - if there
    isn't one, give a "long" time, that is guaranteed
@@ -786,14 +799,12 @@ void erts_deliver_time(void) {
 void erts_time_remaining(SysTimeval *rem_time)
 {
     int ticks;
-#if !defined(ERTS_TIMER_THREAD)
     SysTimeval cur_time;
-#endif
     long elapsed;
 
-    /* next_time() returns no of ticks to next timeout or -1 if none */
+    /* erts_next_time() returns no of ticks to next timeout or -1 if none */
 
-    if ((ticks = next_time()) == -1) {
+    if ((ticks = erts_next_time()) == -1) {
 	/* timer queue empty */
 	/* this will cause at most 100000000 ticks */
 	rem_time->tv_sec = 100000;
@@ -802,9 +813,6 @@ void erts_time_remaining(SysTimeval *rem_time)
 	/* next timeout after ticks ticks */
 	ticks *= CLOCK_RESOLUTION;
 	
-#if defined(ERTS_TIMER_THREAD)
-	elapsed = 0;
-#else
 	erts_smp_mtx_lock(&erts_timeofday_mtx);
 	
 	get_tolerant_timeofday(&cur_time);
@@ -819,7 +827,6 @@ void erts_time_remaining(SysTimeval *rem_time)
 	    rem_time->tv_sec = rem_time->tv_usec = 0;
 	    return;
 	}
-#endif
 	rem_time->tv_sec = (ticks - elapsed) / 1000;
 	rem_time->tv_usec = 1000 * ((ticks - elapsed) % 1000);
     }
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 2842c2361a..8833137112 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -43,8 +43,9 @@
 #undef DEBUG_PRINTOUTS
 #endif
 
-extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */
-extern Eterm beam_return_trace[1];    /* OpCode(i_return_trace) */
+extern Eterm beam_return_to_trace[1];   /* OpCode(i_return_to_trace) */
+extern Eterm beam_return_trace[1];      /* OpCode(i_return_trace) */
+extern Eterm beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
 
 /* Pseudo export entries. Never filled in with data, only used to
    yield unique pointers of the correct type. */
@@ -397,11 +398,13 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to,
  */
 static void 
 do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) {
-    Eterm local_heap[4+5+5];
+#define LOCAL_HEAP_SIZE (4+5+5)
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
     Eterm message;
     Eterm *hp;
     Eterm mfarity;
 
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
     ASSERT(is_pid(pid));
     ASSERT(is_tuple(timestamp));
     ASSERT(*tuple_val(timestamp) == make_arityval(3));
@@ -426,6 +429,8 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) {
 		    pid,
 		    SYS_MSG_TYPE_UNDEFINED,
 		    message);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 }
 #endif
 
@@ -442,7 +447,9 @@ send_to_port(Process *c_p, Eterm message,
 	     Eterm *tracer_pid, Uint *tracee_flags) {
     Port* trace_port;
 #ifndef ERTS_SMP
-    Eterm ts, local_heap[4], *hp;
+#define LOCAL_HEAP_SIZE (4)
+    Eterm ts, *hp;
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 #endif
 
     ASSERT(is_internal_port(*tracer_pid));
@@ -486,6 +493,8 @@ send_to_port(Process *c_p, Eterm message,
      * (e.g. getting_linked) need not be the current process. That other
      * process might not have timestamps enabled.
      */
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
     if (*tracee_flags & F_TIMESTAMP) {
 	ASSERT(is_tuple(message));
 	hp = tuple_val(message);
@@ -522,6 +531,8 @@ send_to_port(Process *c_p, Eterm message,
 	 */
 	do_send_schedfix_to_port(trace_port, c_p->id, ts);
     }
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #endif
 }
 
@@ -589,7 +600,10 @@ seq_trace_send_to_port(Process *c_p,
 {
     Port* trace_port;
 #ifndef ERTS_SMP
-    Eterm ts, local_heap[4], *hp;
+    Eterm ts, *hp;
+#define LOCAL_HEAP_SIZE (4)
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #endif
 
     ASSERT(is_internal_port(seq_tracer));
@@ -607,6 +621,9 @@ seq_trace_send_to_port(Process *c_p,
     if (INVALID_TRACER_PORT(trace_port, seq_tracer)) {
     invalid_tracer_port:
 	system_seq_tracer = am_false;
+#ifndef ERTS_SMP
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#endif
 	return;
     }
 
@@ -620,6 +637,7 @@ seq_trace_send_to_port(Process *c_p,
 			message);
 
 #ifndef ERTS_SMP
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 	return;
     }
     /* Make a fake schedule only if the current process is traced
@@ -660,6 +678,8 @@ seq_trace_send_to_port(Process *c_p,
 	 */
 	do_send_schedfix_to_port(trace_port, c_p->id, ts);
     }
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #endif
 }
 
@@ -719,7 +739,8 @@ send_to_tracer(Process *tracee,
 static void
 trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
 {
-    Eterm local_heap[5+4+1+TS_HEAP_WORDS];
+#define LOCAL_HEAP_SIZE (5+4+1+TS_HEAP_WORDS)
+    DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
     Eterm tmp, mess, *hp;
     ErlHeapFragment *bp = NULL;
     ErlOffHeap *off_heap;
@@ -768,8 +789,10 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
 	curr_func = p->current != NULL;
     }
 
+    UseTmpHeap(LOCAL_HEAP_SIZE,p);
+
     if (to_port)
-	hp = &local_heap[0];
+	hp = local_heap;
     else {
 	Uint size = 5;
 	if (curr_func)
@@ -802,6 +825,8 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
     }
 
     send_to_tracer(p, tracer_ref, mess, &hp, bp, no_fake_sched);
+    UnUseTmpHeap(LOCAL_HEAP_SIZE,p);
+#undef LOCAL_HEAP_SIZE
 }
 
 /* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
@@ -848,7 +873,10 @@ trace_send(Process *p, Eterm to, Eterm msg)
     }
 
     if (is_internal_port(p->tracer_proc)) {
-	Eterm local_heap[11];
+#define LOCAL_HEAP_SIZE (11)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	mess = TUPLE5(hp, am_trace, p->id, operation, msg, to);
 	hp += 6;
@@ -857,6 +885,8 @@ trace_send(Process *p, Eterm to, Eterm msg)
 	    hp = patch_ts(mess, hp);
 	}
 	send_to_port(p, mess, &p->tracer_proc, &p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	Uint need;
@@ -908,7 +938,10 @@ trace_receive(Process *rp, Eterm msg)
     Eterm* hp;
 
     if (is_internal_port(rp->tracer_proc)) {
-	Eterm local_heap[10];
+#define LOCAL_HEAP_SIZE (10)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	mess = TUPLE4(hp, am_trace, rp->id, am_receive, msg);
 	hp += 5;
@@ -917,6 +950,8 @@ trace_receive(Process *rp, Eterm msg)
 	    hp = patch_ts(mess, hp);
 	}
 	send_to_port(rp, mess, &rp->tracer_proc, &rp->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	Uint hsz;
@@ -1018,7 +1053,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
     }
 
     if (is_internal_port(seq_tracer)) {
-	Eterm local_heap[64];
+#define LOCAL_HEAP_SIZE (64)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	label = SEQ_TRACE_T_LABEL(token);
 	lastcnt_serial = TUPLE2(hp, SEQ_TRACE_T_LASTCNT(token),
@@ -1043,6 +1081,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 	    mess = TUPLE4(hp, am_seq_trace, label, mess, ts);
 	    seq_trace_send_to_port(process, seq_tracer, mess, ts);
 	}
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 #ifndef ERTS_SMP
@@ -1143,14 +1183,18 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
  * or   {trace, Pid, return_to, {Mod, Func, Arity}}
  */
 void 
-erts_trace_return_to(Process *p, Uint *pc)
+erts_trace_return_to(Process *p, BeamInstr *pc)
 {
+#define LOCAL_HEAP_SIZE (4+5+5)
     Eterm* hp;
     Eterm mfa;
     Eterm mess;
-    Eterm local_heap[4+5+5];
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+
+    BeamInstr *code_ptr = find_function_from_pc(pc);
 
-    Eterm *code_ptr = find_function_from_pc(pc);
+
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
     hp = local_heap;
 
@@ -1196,6 +1240,8 @@ erts_trace_return_to(Process *p, Uint *pc)
 	mess = copy_struct(mess, size, &hp, off_heap);
 	ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp);
     }
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
     erts_smp_mtx_unlock(&smq_mtx);
 }
 
@@ -1204,7 +1250,7 @@ erts_trace_return_to(Process *p, Uint *pc)
  * or   {trace, Pid, return_from, {Mod, Name, Arity}, Retval}
  */
 void
-erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
+erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid)
 {
     Eterm* hp;
     Eterm mfa;
@@ -1262,7 +1308,9 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
     arity = fi[2];
     
     if (is_internal_port(*tracer_pid)) {
-	Eterm local_heap[4+6+5];
+#define LOCAL_HEAP_SIZE (4+6+5)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 	hp = local_heap;
 	mfa = TUPLE3(hp, mod, name, make_small(arity));
 	hp += 4;
@@ -1273,6 +1321,8 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
 	    hp = patch_ts(mess, hp);
 	}
 	send_to_port(p, mess, tracer_pid, tracee_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	ErlHeapFragment *bp;
@@ -1331,7 +1381,7 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
  * Where Class is atomic but Value is any term.
  */
 void
-erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value, 
+erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
 		     Eterm *tracer_pid)
 {
     Eterm* hp;
@@ -1385,21 +1435,26 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value,
     }
     
     if (is_internal_port(*tracer_pid)) {
-	Eterm local_heap[4+3+6+5];
+#define LOCAL_HEAP_SIZE (4+3+6+5)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
-	mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2]));
+	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2]));
 	hp += 4;
 	cv = TUPLE2(hp, class, value);
 	hp += 3;
 	mess = TUPLE5(hp, am_trace, p->id, am_exception_from, mfa_tuple, cv);
 	hp += 6;
-	ASSERT((hp - local_heap)*sizeof(*hp) <= sizeof(local_heap));
+	ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE);
 	erts_smp_mtx_lock(&smq_mtx);
 	if (*tracee_flags & F_TIMESTAMP) {
 	    hp = patch_ts(mess, hp); /* hp += 5 */
-	    ASSERT((hp - local_heap)*sizeof(*hp) == sizeof(local_heap));
+	    ASSERT((hp - local_heap) == LOCAL_HEAP_SIZE);
 	}
 	send_to_port(p, mess, tracer_pid, tracee_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	ErlHeapFragment *bp;
@@ -1431,7 +1486,7 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value,
 	 * Build the trace tuple and put it into receive queue of the tracer process.
 	 */
 	
-	mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2]));
+	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm) mfa[2]));
 	hp += 4;
 	value = copy_struct(value, value_size, &hp, off_heap);
 	cv = TUPLE2(hp, class, value);
@@ -1468,7 +1523,7 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value,
  * if it is a pid or port we do a meta trace.
  */
 Uint32
-erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, 
+erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 		Eterm* args, int local, Eterm *tracer_pid)
 {
     Eterm* hp;
@@ -1483,7 +1538,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
     Eterm tracee;
 #endif
     Eterm transformed_args[MAX_ARG];
-    ErlSubBin sub_bin_heap;
+    DeclareTypedTmpHeap(ErlSubBin,sub_bin_heap,p);
 
     ASSERT(tracer_pid);
     if (*tracer_pid == am_true) {
@@ -1534,37 +1589,42 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
      * such as size_object() and copy_struct(), we must make sure that we
      * temporarily convert any match contexts to sub binaries.
      */
-    arity = mfa[2];
+    arity = (Eterm) mfa[2];
+    UseTmpHeap(ERL_SUB_BIN_SIZE,p);
 #ifdef DEBUG
-    sub_bin_heap.thing_word = 0;
+    sub_bin_heap->thing_word = 0;
 #endif
     for (i = 0; i < arity; i++) {
 	Eterm arg = args[i];
 	if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) {
 	    ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg);
 	    ErlBinMatchBuffer* mb = &ms->mb;
-	    ErlSubBin* sb = &sub_bin_heap;
 	    Uint bit_size;
 
-	    ASSERT(sub_bin_heap.thing_word == 0); /* At most one of match context */
+	    ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */
 
 	    bit_size = mb->size - mb->offset;
-	    sb->thing_word = HEADER_SUB_BIN;
-	    sb->size = BYTE_OFFSET(bit_size);
-	    sb->bitsize = BIT_OFFSET(bit_size);
-	    sb->offs = BYTE_OFFSET(mb->offset);
-	    sb->bitoffs = BIT_OFFSET(mb->offset);
-	    sb->is_writable = 0;
-	    sb->orig = mb->orig;
-
-	    arg = make_binary(sb);
+	    sub_bin_heap->thing_word = HEADER_SUB_BIN;
+	    sub_bin_heap->size = BYTE_OFFSET(bit_size);
+	    sub_bin_heap->bitsize = BIT_OFFSET(bit_size);
+	    sub_bin_heap->offs = BYTE_OFFSET(mb->offset);
+	    sub_bin_heap->bitoffs = BIT_OFFSET(mb->offset);
+	    sub_bin_heap->is_writable = 0;
+	    sub_bin_heap->orig = mb->orig;
+
+	    arg = make_binary(sub_bin_heap);
 	}
 	transformed_args[i] = arg;
     }
     args = transformed_args;
 
     if (is_internal_port(*tracer_pid)) {
+#if HEAP_ON_C_STACK
 	Eterm local_heap[64+MAX_ARG];
+#else
+	Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM,
+				       sizeof(Eterm)*(64+MAX_ARG));
+#endif
 	hp = local_heap;
 
 	if (!erts_is_valid_tracer_port(*tracer_pid)) {
@@ -1579,6 +1639,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	    if (is_not_nil(tracee))
 		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 #endif
+#if !HEAP_ON_C_STACK
+	    erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+	    UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 	    return 0;
 	}
 	
@@ -1602,9 +1666,13 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	return_flags = 0;
 	if (match_spec) {
 	    pam_result = erts_match_set_run(p, match_spec, args, arity,
-					    &return_flags);
+					    ERTS_PAM_TMP_RESULT, &return_flags);
 	    if (is_non_value(pam_result)) {
 		erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return 0;
 	    }
 	}
@@ -1612,16 +1680,28 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	    /* Meta trace */
 	    if (pam_result == am_false) {
 		erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return return_flags;
 	    }
 	} else {
 	    /* Non-meta trace */
 	    if (*tracee_flags & F_TRACE_SILENT) { 
 		erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return 0;
 	    }
 	    if (pam_result == am_false) {
 		erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return return_flags;
 	    }
 	    if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
@@ -1644,7 +1724,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 		hp += 2;
 	    }
 	}
-	mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], mfa_tuple);
+	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple);
 	hp += 4;
 	
 	/*
@@ -1664,6 +1744,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	send_to_port(p, mess, tracer_pid, tracee_flags);
 	erts_smp_mtx_unlock(&smq_mtx);
 	erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+	erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+	UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 	return *tracer_pid == NIL ? 0 : return_flags;
 
     } else {
@@ -1706,6 +1790,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	    if (is_not_nil(tracee))
 		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 #endif
+	    UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 	    return 0;
 	}
 
@@ -1728,9 +1813,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	return_flags = 0;
 	if (match_spec) {
 	    pam_result = erts_match_set_run(p, match_spec, args, arity,
-					    &return_flags);
+					    ERTS_PAM_TMP_RESULT, &return_flags);
 	    if (is_non_value(pam_result)) {
 		erts_match_set_release_result(p);
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return 0;
 	    }
 	}
@@ -1738,16 +1824,19 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	    /* Meta trace */
 	    if (pam_result == am_false) {
 		erts_match_set_release_result(p);
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return return_flags;
 	    }
 	} else {
 	    /* Non-meta trace */
 	    if (*tracee_flags & F_TRACE_SILENT) { 
 		erts_match_set_release_result(p);
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return 0;
 	    }
 	    if (pam_result == am_false) {
 		erts_match_set_release_result(p);
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return return_flags;
 	    }
 	    if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
@@ -1798,7 +1887,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 		hp += 2;
 	    }
 	}
-	mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], mfa_tuple);
+	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple);
 	hp += 4;
 	
 	/*
@@ -1831,6 +1920,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	ASSERT(hp == limit);
 	ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
+	UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 	return return_flags;
     }
 }
@@ -1850,8 +1940,13 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
     Eterm* hp;
     int need;
 
+    ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) || erts_is_system_blocked(0));
     if (is_internal_port(t_p->tracer_proc)) {
-	Eterm local_heap[5+5];
+#define LOCAL_HEAP_SIZE (5+5)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
+
 	hp = local_heap;
 	mess = TUPLE4(hp, am_trace, t_p->id, what, data);
 	hp += 5;
@@ -1868,6 +1963,8 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
 	    c_p,
 #endif
 	    mess, &t_p->tracer_proc, &t_p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	Eterm tmp;
@@ -1919,7 +2016,10 @@ trace_proc_spawn(Process *p, Eterm pid,
     Eterm* hp;
 
     if (is_internal_port(p->tracer_proc)) {
-	Eterm local_heap[4+6+5];
+#define LOCAL_HEAP_SIZE (4+6+5)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	mfa = TUPLE3(hp, mod, func, args);
 	hp += 4;
@@ -1930,6 +2030,8 @@ trace_proc_spawn(Process *p, Eterm pid,
 	    hp = patch_ts(mess, hp);
 	}
 	send_to_port(p, mess, &p->tracer_proc, &p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	Eterm tmp;
@@ -1991,7 +2093,7 @@ void save_calls(Process *p, Export *e)
  */
 Eterm
 erts_bif_trace(int bif_index, Process* p, 
-	       Eterm arg1, Eterm arg2, Eterm arg3, Uint *I)
+	       Eterm arg1, Eterm arg2, Eterm arg3, BeamInstr *I)
 {
     Eterm result;
     int meta = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_META);
@@ -2005,28 +2107,22 @@ erts_bif_trace(int bif_index, Process* p,
 	 * no tracing will occur. Doing the whole else branch will 
 	 * also do nothing, only slower.
 	 */
-	Eterm (*func)(Process*, Eterm, Eterm, Eterm, Uint*) = bif_table[bif_index].f;
+	Eterm (*func)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = bif_table[bif_index].f;
 	result = func(p, arg1, arg2, arg3, I);
     } else {
-	Eterm (*func)(Process*, Eterm, Eterm, Eterm, Uint*);
+	Eterm (*func)(Process*, Eterm, Eterm, Eterm, BeamInstr*);
 	Export* ep = bif_export[bif_index];
 	Uint32 flags = 0, flags_meta = 0;
 	int global = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_GLOBAL);
 	int local  = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_LOCAL);
+	int time   = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_CALL_TIME);
 	Eterm meta_tracer_pid = NIL;
 	int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif
 					       * is actually in the 
 					       * export entry */
-	Eterm *cp = p->cp;
+	BeamInstr *cp = p->cp;
 	
-#ifndef _OSE_
 	Eterm args[3] = {arg1, arg2, arg3};
-#else
-	Eterm args[3];
-	args[0] = arg1;
-	args[1] = arg2;
-	args[2] = arg3;
-#endif
 	
 	/* 
 	 * Make continuation pointer OK, it is not during direct BIF calls,
@@ -2043,6 +2139,17 @@ erts_bif_trace(int bif_index, Process* p,
 	    flags_meta = erts_bif_mtrace(p, ep->code+3, args, local, 
 					 &meta_tracer_pid);
 	}
+	if (time) {
+	    BpDataTime *bdt = NULL;
+	    BeamInstr *pc = (BeamInstr *)ep->code+3;
+
+	    bdt = (BpDataTime *) erts_get_time_break(p, pc);
+	    ASSERT(bdt);
+
+	    if (!bdt->pause) {
+		erts_trace_time_break(p, pc, bdt, ERTS_BP_CALL_TIME_CALL);
+	    }
+	}
 	/* Restore original continuation pointer (if changed). */
 	p->cp = cp;
 	
@@ -2051,17 +2158,21 @@ erts_bif_trace(int bif_index, Process* p,
 	result = func(p, arg1, arg2, arg3, I);
 	
 	if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) {
-	    Uint i_return_trace = beam_return_trace[0];
-	    Uint i_return_to_trace = beam_return_to_trace[0];
+	    BeamInstr i_return_trace      = beam_return_trace[0];
+	    BeamInstr i_return_to_trace   = beam_return_to_trace[0];
+	    BeamInstr i_return_time_trace = beam_return_time_trace[0];
 	    Eterm *cpp;
 	    /* Maybe advance cp to skip trace stack frames */
 	    for (cpp = p->stop;  ;  cp = cp_val(*cpp++)) {
-		ASSERT(is_CP((Eterm) cp));
-		if (*cp_val((Eterm) cp) == i_return_trace) {
+		if (*cp == i_return_trace) {
 		    /* Skip stack frame variables */
 		    while (is_not_CP(*cpp)) cpp++;
 		    cpp += 2; /* Skip return_trace parameters */
-		} else if (*cp_val((Eterm) cp) == i_return_to_trace) {
+		} else if (*cp == i_return_time_trace) {
+		    /* Skip stack frame variables */
+		    while (is_not_CP(*cpp)) cpp++;
+		    cpp += 1; /* Skip return_time_trace parameters */
+		} else if (*cp == i_return_to_trace) {
 		    /* A return_to trace message is going to be generated
 		     * by normal means, so we do not have to.
 		     */
@@ -2078,7 +2189,8 @@ erts_bif_trace(int bif_index, Process* p,
 	    if (reason != TRAP) {
 		Eterm class;
 		Eterm value = p->fvalue;
-		Eterm nocatch[3];
+		DeclareTmpHeapNoproc(nocatch,3);
+		UseTmpHeapNoproc(3);
 		/* Expand error value like in handle_error() */
 		if (reason & EXF_ARGLIST) {
 		    Eterm *tp;
@@ -2126,6 +2238,7 @@ erts_bif_trace(int bif_index, Process* p,
 			}
 		    }
 		}
+		UnUseTmpHeapNoproc(3);
 		if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) {
 		    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 		    p->trace_flags |= F_EXCEPTION_TRACE;
@@ -2213,15 +2326,19 @@ trace_gc(Process *p, Eterm what)
 	BIN_OLD_VHEAP(p),
 	BIN_OLD_VHEAP_SZ(p)
     };
-    Eterm local_heap[(sizeof(values)/sizeof(Uint))
-		     *(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE)
-		     + 5/*4-tuple */ + TS_HEAP_WORDS];
+#define LOCAL_HEAP_SIZE						\
+    (sizeof(values)/sizeof(Eterm)) *				\
+	(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) +	\
+	5/*4-tuple */ + TS_HEAP_WORDS
+    DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
 #ifdef DEBUG
     Eterm* limit;
 #endif
 
     ASSERT(sizeof(values)/sizeof(Uint) == sizeof(tags)/sizeof(Eterm));
 
+    UseTmpHeap(LOCAL_HEAP_SIZE,p);
+
     if (is_internal_port(p->tracer_proc)) {
 	hp = local_heap;
 #ifdef DEBUG
@@ -2252,7 +2369,7 @@ trace_gc(Process *p, Eterm what)
 
 #ifdef DEBUG
     limit = hp + size;
-    ASSERT(size <= sizeof(local_heap)/sizeof(Eterm));
+    ASSERT(size <= LOCAL_HEAP_SIZE);
 #endif
 
     msg = erts_bld_atom_uint_2tup_list(&hp,
@@ -2275,6 +2392,8 @@ trace_gc(Process *p, Eterm what)
     else
 	ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, msg, bp);
     erts_smp_mtx_unlock(&smq_mtx);
+    UnUseTmpHeap(LOCAL_HEAP_SIZE,p);
+#undef LOCAL_HEAP_SIZE
 }
 
 
@@ -2465,7 +2584,9 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
     Uint Ms, s, us;
 
 #ifndef ERTS_SMP
-    Eterm local_heap[4 + 7];
+#define LOCAL_HEAP_SIZE (4 + 7)
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
     hp = local_heap;
 #else
     ErlHeapFragment *bp;
@@ -2498,6 +2619,8 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
 
 #ifndef ERTS_SMP
     profile_send(msg);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #else
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
 #endif
@@ -2510,7 +2633,10 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M
     Eterm *hp, msg, timestamp;
     
 #ifndef ERTS_SMP	
-    Eterm local_heap[4 + 7];
+#define LOCAL_HEAP_SIZE (4 + 7)
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
     hp = local_heap;
 #else    
     ErlHeapFragment *bp;
@@ -2528,6 +2654,8 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M
     msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7;
 #ifndef ERTS_SMP
     profile_send(msg);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #else
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
 #endif
@@ -2558,7 +2686,10 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
     Eterm* hp;
 
     if (is_internal_port(p->tracer_proc)) {
-	Eterm local_heap[5+6];
+#define LOCAL_HEAP_SIZE (5+6)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 
 	mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->id, drv_name);
@@ -2569,6 +2700,8 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
 	}
 	/* No fake schedule */
 	send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	ErlHeapFragment *bp;
@@ -2612,8 +2745,13 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
     Eterm mess;
     Eterm* hp;
 
+    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_is_system_blocked(0));
+
     if (is_internal_port(t_p->tracer_proc)) {
-	Eterm local_heap[5+5];
+#define LOCAL_HEAP_SIZE (5+5)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	mess = TUPLE4(hp, am_trace, t_p->id, what, data);
 	hp += 5;
@@ -2623,6 +2761,8 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
 	}
 	/* No fake schedule */
 	send_to_port(NULL, mess, &t_p->tracer_proc, &t_p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	ErlHeapFragment *bp;
@@ -2674,7 +2814,10 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
     Eterm sched_id = am_undefined;
     
     if (is_internal_port(p->tracer_proc)) {
-	Eterm local_heap[5+6];
+#define LOCAL_HEAP_SIZE (5+6)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	
 	if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) {
@@ -2700,6 +2843,8 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
 	
 	/* No fake scheduling */
 	send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else  {
 	ErlHeapFragment *bp;
@@ -2750,7 +2895,11 @@ profile_runnable_port(Port *p, Eterm status) {
     Eterm count = make_small(0);
 
 #ifndef ERTS_SMP
-    Eterm local_heap[4 + 6];
+#define LOCAL_HEAP_SIZE (4 + 6)
+
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
     hp = local_heap;
     
 #else
@@ -2771,6 +2920,8 @@ profile_runnable_port(Port *p, Eterm status) {
 
 #ifndef ERTS_SMP
     profile_send(msg);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #else
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
 #endif
@@ -2785,7 +2936,11 @@ profile_runnable_proc(Process *p, Eterm status){
     Eterm where = am_undefined;
 
 #ifndef ERTS_SMP
-    Eterm local_heap[4 + 6 + 4];
+#define LOCAL_HEAP_SIZE (4 + 6 + 4)
+
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
     hp = local_heap;
 #else
     ErlHeapFragment *bp;
@@ -2818,6 +2973,8 @@ profile_runnable_proc(Process *p, Eterm status){
     msg = TUPLE5(hp, am_profile, p->id, status, where, timestamp); hp += 6;
 #ifndef ERTS_SMP
     profile_send(msg);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #else
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
 #endif
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index ab5811c70f..158eb361a4 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -30,6 +30,8 @@
 #include "big.h"
 
 #include "erl_unicode.h"
+#include "erl_unicode_normalize.h"
+
 
 typedef struct _restart_context {
     byte *bytes;
@@ -54,13 +56,6 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
 					 Uint num_resulting_chars, 
 					 int state, int left,
 					 Eterm tail);
-static int analyze_utf8(byte *source, Uint size, 
-			byte **err_pos, Uint *num_chars, int *left);
-#define UTF8_OK 0
-#define UTF8_INCOMPLETE 1
-#define UTF8_ERROR 2
-#define UTF8_ANALYZE_MORE 3
-
 static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3);
 static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3);
 static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3);
@@ -90,9 +85,9 @@ void erts_init_unicode(void)
 	am_atom_put("characters_to_utf8_trap",23);
     characters_to_utf8_trap_exp.code[2] = 3;
     characters_to_utf8_trap_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     characters_to_utf8_trap_exp.code[4] = 
-	(Eterm) &characters_to_utf8_trap;
+	(BeamInstr) &characters_to_utf8_trap;
 
     memset(&characters_to_list_trap_1_exp, 0, sizeof(Export));
     characters_to_list_trap_1_exp.address = 
@@ -102,9 +97,9 @@ void erts_init_unicode(void)
 	am_atom_put("characters_to_list_trap_1",25);
     characters_to_list_trap_1_exp.code[2] = 3;
     characters_to_list_trap_1_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     characters_to_list_trap_1_exp.code[4] = 
-	(Eterm) &characters_to_list_trap_1;
+	(BeamInstr) &characters_to_list_trap_1;
 
     memset(&characters_to_list_trap_2_exp, 0, sizeof(Export));
     characters_to_list_trap_2_exp.address = 
@@ -114,9 +109,9 @@ void erts_init_unicode(void)
 	am_atom_put("characters_to_list_trap_2",25);
     characters_to_list_trap_2_exp.code[2] = 3;
     characters_to_list_trap_2_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     characters_to_list_trap_2_exp.code[4] = 
-	(Eterm) &characters_to_list_trap_2;
+	(BeamInstr) &characters_to_list_trap_2;
 
 
     memset(&characters_to_list_trap_3_exp, 0, sizeof(Export));
@@ -127,9 +122,9 @@ void erts_init_unicode(void)
 	am_atom_put("characters_to_list_trap_3",25);
     characters_to_list_trap_3_exp.code[2] = 3;
     characters_to_list_trap_3_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     characters_to_list_trap_3_exp.code[4] = 
-	(Eterm) &characters_to_list_trap_3;
+	(BeamInstr) &characters_to_list_trap_3;
 
     memset(&characters_to_list_trap_4_exp, 0, sizeof(Export));
     characters_to_list_trap_4_exp.address = 
@@ -139,9 +134,9 @@ void erts_init_unicode(void)
 	am_atom_put("characters_to_list_trap_4",25);
     characters_to_list_trap_4_exp.code[2] = 1;
     characters_to_list_trap_4_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     characters_to_list_trap_4_exp.code[4] = 
-	(Eterm) &characters_to_list_trap_4;
+	(BeamInstr) &characters_to_list_trap_4;
 
     c_to_b_int_trap_exportp =  erts_export_put(am_unicode,am_characters_to_binary_int,2);
     c_to_l_int_trap_exportp =  erts_export_put(am_unicode,am_characters_to_list_int,2);
@@ -463,7 +458,7 @@ L_Again:   /* Restart with sublist, old listend was pushed on stack */
 			}
 			objp = list_val(ioterm);
 			obj = CAR(objp);
-			if (!is_byte(obj))
+			if (!is_small(obj))
 			    break;
 		    }
 		} else if (is_nil(obj)) {
@@ -907,7 +902,6 @@ static BIF_RETTYPE build_utf8_return(Process *p,Eterm bin,int pos,
 static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3)
 {
     Eterm *real_bin;
-    Sint need;
     byte* bytes;
     Eterm rest_term;
     int left, sleft;
@@ -923,7 +917,6 @@ static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3)
     ASSERT(is_binary(BIF_ARG_1));
     real_bin = binary_val(BIF_ARG_1);
     ASSERT(*real_bin == HEADER_PROC_BIN);
-    need = ((ProcBin *) real_bin)->val->orig_size;
     pos = (int) binary_size(BIF_ARG_1);
     bytes = binary_bytes(BIF_ARG_1);
     sleft = left = allowed_iterations(BIF_P);
@@ -970,11 +963,11 @@ static int is_valid_utf8(Eterm orig_bin)
 	bytes = erts_get_aligned_binary_bytes(orig_bin, &temp_alloc);
     }
     size = binary_size(orig_bin);
-    ret = analyze_utf8(bytes,
+    ret = erts_analyze_utf8(bytes,
 		       size,
 		       &endpos,&numchar,NULL);
     erts_free_aligned_binary_bytes(temp_alloc);
-    return (ret == UTF8_OK);
+    return (ret == ERTS_UTF8_OK);
 }
 
 BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2)
@@ -1084,14 +1077,14 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char
 	    hp += 2;
 	    rest_term = CONS(hp,leftover_bin,rest_term);
 	}
-	BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, UTF8_ERROR, left, NIL));
+	BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL));
     } else if (rest_term == NIL && num_leftovers != 0) {
 	Eterm leftover_bin = new_binary(p, leftover, num_leftovers);
 	if (check_leftovers(leftover,num_leftovers) != 0) {
-	    BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_ERROR, 
+	    BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_ERROR, 
 					  left, NIL));
 	} else {
-	    BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_INCOMPLETE, 
+	    BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_INCOMPLETE, 
 					  left, NIL));
 	}
     } else { /* All OK */	    
@@ -1107,11 +1100,11 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char
 	    rc.num_processed_bytes = 0; /* not used */
 	    rc.num_bytes_to_process = pos;
 	    rc.num_resulting_chars = characters;
-	    rc.state = UTF8_OK; /* not used */
+	    rc.state = ERTS_UTF8_OK; /* not used */
 	    BIF_TRAP3(&characters_to_list_trap_1_exp, p, make_magic_bin_for_restart(p,&rc), 
 		      rest_term, latin1);
 	} else { /* Success */
-	    BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, UTF8_OK, left, NIL));
+	    BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, ERTS_UTF8_OK, left, NIL));
 	}
     }
 }
@@ -1205,7 +1198,7 @@ BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2)
  * When input to characters_to_list is a plain binary and the format is 'unicode', we do
  * a faster analyze and size count with this function.
  */
-static int analyze_utf8(byte *source, Uint size, 
+int erts_analyze_utf8(byte *source, Uint size, 
 			byte **err_pos, Uint *num_chars, int *left)
 {
     *err_pos = source;
@@ -1216,60 +1209,60 @@ static int analyze_utf8(byte *source, Uint size,
 	    --size; 
 	} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
 	    if (size < 2) {
-		return UTF8_INCOMPLETE;
+		return ERTS_UTF8_INCOMPLETE;
 	    }
 	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
 		((*source) < 0xC2) /* overlong */) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    source += 2;
 	    size -= 2;
 	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
 	    if (size < 3) {
-		return UTF8_INCOMPLETE;
+		return ERTS_UTF8_INCOMPLETE;
 	    }
 	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
 		((source[2] & ((byte) 0xC0)) != 0x80) ||
 		(((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    if ((((*source) & ((byte) 0xF)) == 0xD) && 
 		((source[1] & 0x20) != 0)) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    if (((*source) == 0xEF) && (source[1] == 0xBF) &&
 		((source[2] == 0xBE) || (source[2] == 0xBF))) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    source += 3;
 	    size -= 3;
 	} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
 	    if (size < 4) {
-		return UTF8_INCOMPLETE;
+		return ERTS_UTF8_INCOMPLETE;
 	    }
 	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
 		((source[2] & ((byte) 0xC0)) != 0x80) ||
 		((source[3] & ((byte) 0xC0)) != 0x80) ||
 		(((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    if ((((*source) & ((byte)0x7)) > 0x4U) ||
 		((((*source) & ((byte)0x7)) == 0x4U) && 
 		 ((source[1] & ((byte)0x3F)) > 0xFU))) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    source += 4;
 	    size -= 4; 
 	} else {
-	    return UTF8_ERROR;
+	    return ERTS_UTF8_ERROR;
 	}
 	++(*num_chars);
 	*err_pos = source;
 	if (left && --(*left) <= 0) {
-	    return UTF8_ANALYZE_MORE;
+	    return ERTS_UTF8_ANALYZE_MORE;
 	}
     }
-    return UTF8_OK;
+    return ERTS_UTF8_OK;
 }
 
 /*
@@ -1304,7 +1297,7 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
 	} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
 	    unipoint = 
 		(((Uint) ((*source) & ((byte) 0x1F))) << 6) |
-		((Uint) (source[1] & ((byte) 0x3F))); 	
+		((Uint) (source[1] & ((byte) 0x3F)));
 	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
 	    unipoint = 
 		(((Uint) ((*source) & ((byte) 0xF))) << 12) |
@@ -1330,6 +1323,216 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
     return ret;
 }
 
+static int is_candidate(Uint cp)
+{
+    int index,pos;
+    if (cp < 768) return 0;
+    if (cp > 4023) {
+	if (cp == 12441 || cp == 12442) return 1;
+	return 0;
+    }
+    index = cp / 32 - COMP_CANDIDATE_MAP_OFFSET;
+    pos = cp % 32;
+    return !!(comp_candidate_map[index] & (1UL << pos));
+}
+
+static int hashsearch(int *htab, int htab_size, CompEntry *cv, Uint16 c)
+{
+	int bucket = c % htab_size;
+	while (htab[bucket] != -1 && cv[htab[bucket]].c != c)
+	    bucket = (bucket + 1) % htab_size;
+	return htab[bucket];
+}
+
+#define TRANSLATE_NO 0
+#define TRANSLATE_MAYBE -1
+
+/* The s array is reversed */
+static int translate(Uint16 *s, int slen, Uint16 *res)
+{
+    /* Go backwards through buffer and match against tree */
+    int pos = 0;
+    CompEntry *cv = compose_tab;
+    int *hc = hash_compose_tab;
+    int cvs = compose_tab_size;
+    int x;
+    while (pos < slen) {
+	x = hashsearch(hc,cvs*HASH_SIZE_FACTOR,cv,s[pos]);
+	if (x < 0) {
+	    return TRANSLATE_NO;
+	} 
+	if (cv[x].res) {
+	    *res = cv[x].res;
+	    return pos;
+	}
+	cvs = cv[x].num_subs;
+	hc = cv[x].hash;
+	cv = cv[x].subs;
+	++pos;
+    }
+    return TRANSLATE_MAYBE;
+}
+
+static void handle_first_norm(Uint16 *savepoints, int *numpointsp, Uint unipoint)
+{
+    /*erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) *numpointsp);*/
+    *numpointsp = 1;
+    savepoints[0] = (Uint16) unipoint;
+}
+
+static void cleanup_norm(Eterm **hpp, Uint16 *savepoints, int numpoints, Eterm *retp)
+{
+    Eterm *hp = *hpp;
+    int res,i;
+    Uint16 newpoint;
+    Eterm ret = *retp;
+    
+    ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+    hp += 2;
+    
+    for (i = 1;i < numpoints;) {
+	if(!is_candidate(savepoints[i]) || 
+	   ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) {
+	    ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+	    hp += 2;
+	    ++i;
+	} else {
+	    ret = CONS(hp,make_small((Uint) newpoint),ret);
+	    hp += 2;
+	    i += res;
+	}
+    }
+    *retp = ret;
+}
+    
+static void handle_potential_norm(Eterm **hpp, Uint16 *savepoints, int *numpointsp, Uint unipoint, Eterm *retp)
+{
+    Eterm *hp = *hpp;
+    int numpoints = *numpointsp;
+    int res,i;
+    Uint16 newpoint;
+    Eterm ret = *retp;
+
+    /* erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) numpoints);*/
+    if ((unipoint >> 16) == 0) { /* otherwise we're done here */ 
+	savepoints[numpoints++] = (Uint16) unipoint;
+	res = translate(savepoints,numpoints,&newpoint);
+	if (res == TRANSLATE_NO) {
+	    ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+	    hp += 2;
+	    for (i = 1;i < numpoints;) {
+		if(!is_candidate(savepoints[i]) ||
+		   ((res = translate(savepoints+i,numpoints - i, &newpoint)) == 0)) {
+		    ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+		    hp += 2;
+		    ++i;
+		} else if (res > 0) {
+		    ret = CONS(hp,make_small((Uint) newpoint),ret);
+		    hp += 2;
+		    i += res;
+		} else { /* res < 0 */
+		    /* A "maybe", means we are not done yet */
+		    int j = 0;
+		    while (i < numpoints) {
+			savepoints[j++] = savepoints[i++];
+		    }
+		    numpoints = j;
+		    goto breakaway;
+		}
+	    }
+	    numpoints = 0;
+	breakaway:
+	    ;
+	} else if (res > 0) {
+	    numpoints = 0;
+	    ret = CONS(hp,make_small((Uint) newpoint),ret);
+	    hp += 2;
+	} /* < 0 means go on */
+    } else {
+	/* Unconditional rollup, this character is larger than 16 bit */
+	ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+	hp += 2;
+	
+	for (i = 1;i < numpoints;) {
+	    if(!is_candidate(savepoints[i]) || 
+	       ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) {
+		ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+		hp += 2;
+		++i;
+	    } else {
+		ret = CONS(hp,make_small((Uint) newpoint),ret);
+		hp += 2;
+		i += res;
+	    }
+	}
+	ret = CONS(hp,make_small(unipoint),ret);
+	hp += 2;
+	numpoints = 0;
+    }	
+    *hpp = hp;
+    *numpointsp = numpoints;
+    *retp = ret;
+} 
+
+static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint sz)
+{
+    Eterm *hp,*hp_end;
+    Eterm ret;
+    byte *source;
+    Uint unipoint;
+    Uint16 savepoints[4];
+    int numpoints = 0;
+
+    ASSERT(num > 0);
+
+    hp = HAlloc(p,num * 2); /* May be to much */
+    hp_end = hp + num * 2;
+    ret = NIL;
+    source = bytes + sz;
+    while(--source >= bytes) {
+	if (((*source) & ((byte) 0x80)) == 0) {
+	    unipoint = (Uint) *source;
+	} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
+	    unipoint = 
+		(((Uint) ((*source) & ((byte) 0x1F))) << 6) |
+		((Uint) (source[1] & ((byte) 0x3F))); 	
+	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
+	    unipoint = 
+		(((Uint) ((*source) & ((byte) 0xF))) << 12) |
+		(((Uint) (source[1] & ((byte) 0x3F))) << 6) |
+		((Uint) (source[2] & ((byte) 0x3F))); 	 	
+	} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
+	    unipoint = 
+		(((Uint) ((*source) & ((byte) 0x7))) << 18) |
+		(((Uint) (source[1] & ((byte) 0x3F))) << 12) |
+		(((Uint) (source[2] & ((byte) 0x3F))) << 6) |
+		((Uint) (source[3] & ((byte) 0x3F))); 	 	
+	} else {
+	    /* ignore 2#10XXXXXX */
+	    continue;
+	}
+	if (numpoints) {
+	    handle_potential_norm(&hp,savepoints,&numpoints,unipoint,&ret);
+	    continue;
+	}
+	/* We are not building up any normalizations yet, look that we shouldn't start... */
+	if (is_candidate(unipoint)) {
+	    handle_first_norm(savepoints,&numpoints,unipoint);
+	    continue;
+	} 
+	ret = CONS(hp,make_small(unipoint),ret);
+	hp += 2;
+    }
+    /* so, we'we looped to the beginning, do we have anything saved? */
+    if (numpoints) {
+	cleanup_norm(&hp,savepoints,numpoints,&ret);
+    }
+    if (hp_end != hp) {
+	HRelease(p,hp_end,hp);
+    }
+    return ret;
+}
+
 /*
  * The last step of characters_to_list, build a list from the buffer 'bytes' (created in the same way
  * as for characters_to_utf8). All sizes are known in advance and most data will be held in a 
@@ -1378,10 +1581,10 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
      */
 
     free_restart(bytes);
-    if (state == UTF8_INCOMPLETE) {
+    if (state == ERTS_UTF8_INCOMPLETE) {
 	hp = HAlloc(p,4);
 	ret = TUPLE3(hp,am_incomplete,converted,rest);
-    } else if (state == UTF8_ERROR) {
+    } else if (state == ERTS_UTF8_ERROR) {
 	hp = HAlloc(p,4);
 	ret = TUPLE3(hp,am_error,converted,rest);
     } else {
@@ -1408,7 +1611,7 @@ static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3)
 
 /*
  * Hooks into the process of decoding a binary depending on state.
- * If last_state is UTF8_ANALYZE_MORE, num_bytes_to_process 
+ * If last_state is ERTS_UTF8_ANALYZE_MORE, num_bytes_to_process 
  * and num_resulting_chars will grow
  * until we're done analyzing the binary. Then we'll eat 
  * the bytes to process, lowering num_bytes_to_process and num_resulting_chars,
@@ -1465,14 +1668,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
 
     left = allowed_iterations(p);
     
-    if (state == UTF8_ANALYZE_MORE) {
-	state = analyze_utf8(bytes + num_bytes_to_process,
+    if (state == ERTS_UTF8_ANALYZE_MORE) {
+	state = erts_analyze_utf8(bytes + num_bytes_to_process,
 			     size - num_bytes_to_process,
 			     &endpos,&numchar,&left);
 	cost_to_proc(p,numchar);
 	num_resulting_chars += numchar;
 	num_bytes_to_process = endpos - bytes;
-	if (state == UTF8_ANALYZE_MORE) {
+	if (state == ERTS_UTF8_ANALYZE_MORE) {
 	    Eterm epos = erts_make_integer(num_bytes_to_process,p);
 	    Eterm enumchar = erts_make_integer(num_resulting_chars,p);
 	    erts_free_aligned_binary_bytes(temp_alloc);
@@ -1528,7 +1731,7 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
 	ErlSubBin *sb;
 	Eterm orig;
 	Uint offset;
-	ASSERT(state != UTF8_OK);
+	ASSERT(state != ERTS_UTF8_OK);
 	hp = HAlloc(p, ERL_SUB_BIN_SIZE);
 	sb = (ErlSubBin *) hp;
 	ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize);
@@ -1544,14 +1747,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
 
     /* Done */
 
-    if (state == UTF8_INCOMPLETE) {
+    if (state == ERTS_UTF8_INCOMPLETE) {
 	if (check_leftovers(bytes + num_bytes_to_process + num_processed_bytes,
 			    b_sz) != 0) {
 	    goto error_return;
 	}
 	hp = HAlloc(p,4);
 	ret = TUPLE3(hp,am_incomplete,converted,rest);
-    } else if (state == UTF8_ERROR) {
+    } else if (state == ERTS_UTF8_ERROR) {
  error_return:
 	hp = HAlloc(p,4);
 	ret = TUPLE3(hp,am_error,converted,rest);
@@ -1589,7 +1792,7 @@ static BIF_RETTYPE characters_to_list_trap_3(BIF_ALIST_3)
 			       0U, /* nothing processed yet */
 			       num_bytes_to_process, 
 			       num_resulting_chars,
-			       UTF8_ANALYZE_MORE, /* always this state here */
+			       ERTS_UTF8_ANALYZE_MORE, /* always this state here */
 			       NIL); /* Nothing built -> no tail yet */
 	
 }
@@ -1642,7 +1845,7 @@ static BIF_RETTYPE utf8_to_list(BIF_ALIST_1)
 	BIF_ERROR(BIF_P,BADARG);
     }
     return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U, 
-			       UTF8_ANALYZE_MORE,NIL);
+			       ERTS_UTF8_ANALYZE_MORE,NIL);
 }
 
 
@@ -1728,8 +1931,8 @@ binary_to_atom(Process* p, Eterm bin, Eterm enc, int must_exist)
 	    Uint n;
 	    int reds_left = bin_size+1; /* Number of reductions left. */
 
-	    if (analyze_utf8(bytes, bin_size, &err_pos,
-			     &n, &reds_left) == UTF8_OK) {
+	    if (erts_analyze_utf8(bytes, bin_size, &err_pos,
+			     &n, &reds_left) == ERTS_UTF8_OK) {
 		/* 
 		 * Correct UTF-8 encoding, but too many characters to
 		 * fit in an atom.
@@ -1813,3 +2016,616 @@ BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2)
 {
     return binary_to_atom(BIF_P, BIF_ARG_1, BIF_ARG_2, 1);
 }
+
+/**********************************************************
+ * Simpler non-interruptable routines for UTF-8 and 
+ * Windowish UTF-16 (restricted)
+ **********************************************************/
+/*
+ * This function is the heart of the Unicode support for 
+ * open_port - spawn_executable. It converts both the name
+ * of the executable and the arguments according to the same rules
+ * as for filename conversion. That means as if your arguments are
+ * to be raw, you supply binaries, else unicode characters are allowed up to
+ * the encoding maximum (256 of the unicode max).
+ * Depending on the filename encoding standard, the vector is then
+ * converted to whatever is used, which might mean win_utf16 if on windows.
+ * Do not peek into the argument vector or filenam with ordinary
+ * string routines, that will certainly fail on some OS.
+ */
+
+char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty)
+{
+    int encoding = erts_get_native_filename_encoding();
+    char* name_buf = NULL;
+
+    if (is_atom(name) || is_list(name) || (allow_empty && is_nil(name))) {
+	Sint need;
+	if ((need = erts_native_filename_need(name,encoding)) < 0) {
+	    return NULL;
+	}
+	if (encoding == ERL_FILENAME_WIN_WCHAR) {
+	    need += 2;
+	} else {
+	    ++need;
+	}
+	name_buf = (char *) erts_alloc(alloc_type, need);
+	erts_native_filename_put(name,encoding,(byte *)name_buf); 
+	name_buf[need-1] = 0;
+	if (encoding == ERL_FILENAME_WIN_WCHAR) {
+	    name_buf[need-2] = 0;
+	}
+    } else if (is_binary(name)) {
+	byte *temp_alloc = NULL;
+	byte *bytes;
+	byte *err_pos;
+	Uint size,num_chars;
+	
+	size = binary_size(name);
+	bytes = erts_get_aligned_binary_bytes(name, &temp_alloc);
+	if (encoding != ERL_FILENAME_WIN_WCHAR) {
+	    /*Add 0 termination only*/
+	    name_buf = (char *) erts_alloc(alloc_type, size+1);
+	    memcpy(name_buf,bytes,size);
+	    name_buf[size]=0;
+	} else if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || 
+		   erts_get_user_requested_filename_encoding() ==  ERL_FILENAME_LATIN1) {
+	    byte *p;
+	    /* What to do now? Maybe latin1, so just take byte for byte instead */
+	    name_buf = (char *) erts_alloc(alloc_type, (size+1)*2);
+	    p = (byte *) name_buf;
+	    while (size--) {
+		*p++ = *bytes++;
+		*p++ = 0;
+	    }
+	    *p++ = 0;
+	    *p++ = 0;
+	} else { /* WIN_WCHAR and valid UTF8 */
+	    name_buf = (char *) erts_alloc(alloc_type, (num_chars+1)*2);
+	    erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars);
+	    name_buf[num_chars*2] = 0;
+	    name_buf[num_chars*2+1] = 0;
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+    } else {
+	return NULL;
+    }
+    return name_buf;
+}
+
+
+Sint erts_native_filename_need(Eterm ioterm, int encoding) 
+{
+    Eterm *objp;
+    Eterm obj;
+    DECLARE_ESTACK(stack);
+    Sint need = 0;
+
+    if (is_atom(ioterm)) {
+	Atom* ap;
+	int i;
+	ap = atom_tab(atom_val(ioterm));
+	switch (encoding) {
+	case ERL_FILENAME_LATIN1:
+	    need = ap->len;
+	    break;
+	case ERL_FILENAME_UTF8_MAC:
+	case ERL_FILENAME_UTF8:
+	    for (i = 0; i < ap->len; i++) {
+		need += (ap->name[i] >= 0x80) ? 2 : 1;
+	    }
+	    break;
+	case ERL_FILENAME_WIN_WCHAR:
+	    need = 2*(ap->len);
+	    break;
+	default:
+	    need = -1;
+	}
+	DESTROY_ESTACK(stack);
+	return need;
+    }
+
+    if (is_nil(ioterm)) {
+	DESTROY_ESTACK(stack);
+	return need;
+    }
+    if (!is_list(ioterm)) {
+	DESTROY_ESTACK(stack);
+	return (Sint) -1;
+    }
+    /* OK a list, needs to be processed in order, handling each flat list-level
+       as they occur, just like io_list_to_binary would */
+    ESTACK_PUSH(stack,ioterm);
+    while (!ESTACK_ISEMPTY(stack)) {
+	ioterm = ESTACK_POP(stack);	
+	if (is_nil(ioterm)) {
+	    /* ignore empty lists */
+	    continue;
+	}
+	if(is_list(ioterm)) {
+L_Again:   /* Restart with sublist, old listend was pushed on stack */
+	    objp = list_val(ioterm);
+	    obj = CAR(objp);
+	    for(;;) { /* loop over one flat list of bytes and binaries
+		         until sublist or list end is encountered */
+		if (is_small(obj)) { /* Always small */
+		    for(;;) {
+			Uint x = unsigned_val(obj);
+			switch (encoding) {
+			case ERL_FILENAME_LATIN1:
+			    if (x > 255) {
+				DESTROY_ESTACK(stack);
+				return ((Sint) -1);
+			    }
+			    need += 1;
+			    break;
+			case ERL_FILENAME_UTF8_MAC:
+			case ERL_FILENAME_UTF8:
+			    if (x < 0x80) {
+				need +=1;
+			    } else if (x < 0x800) {
+				need += 2;
+			    } else if (x < 0x10000) {
+				if ((x >= 0xD800 && x <= 0xDFFF) ||
+				    (x == 0xFFFE) ||
+				    (x == 0xFFFF)) { /* Invalid unicode range */
+				    DESTROY_ESTACK(stack);
+				    return ((Sint) -1);
+				}
+				need += 3;
+			    } else  if (x < 0x110000) {
+				need += 4; 
+			    } else {
+				DESTROY_ESTACK(stack);
+				return ((Sint) -1);
+			    }
+			    break;
+			case ERL_FILENAME_WIN_WCHAR:
+			    if (x <= 0xffff) { 
+				need += 2;
+				break;
+			    } /* else fall throug to error */
+			default:
+			    DESTROY_ESTACK(stack);
+			    return ((Sint) -1);
+			}
+			    
+			/* everything else will give badarg later 
+			   in the process, so we dont check */
+			ioterm = CDR(objp);
+			if (!is_list(ioterm)) {
+			    break;
+			}
+			objp = list_val(ioterm);
+			obj = CAR(objp);
+			if (!is_small(obj))
+			    break;
+		    }
+		} else if (is_nil(obj)) {
+		    ioterm = CDR(objp);
+		    if (!is_list(ioterm)) {
+			break;
+		    }
+		    objp = list_val(ioterm);
+		    obj = CAR(objp);
+		} else if (is_list(obj)) {
+		    /* push rest of list for later processing, start 
+		       again with sublist */
+		    ESTACK_PUSH(stack,CDR(objp));
+		    ioterm = obj;
+		    goto L_Again;
+		} else {
+		    DESTROY_ESTACK(stack);
+		    return ((Sint) -1);
+		} 
+		if (is_nil(ioterm) || !is_list(ioterm)) {
+		    break;
+		}
+	    } /* for(;;) */
+	} /* is_list(ioterm) */
+	
+	if (!is_list(ioterm) && !is_nil(ioterm)) {
+	    /* inproper list end */
+	    DESTROY_ESTACK(stack);
+	    return ((Sint) -1);
+	}
+    } /* while  not estack empty */
+    DESTROY_ESTACK(stack);
+    return need;
+}
+
+void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) 
+{
+    Eterm *objp;
+    Eterm obj;
+    DECLARE_ESTACK(stack);
+
+    if (is_atom(ioterm)) {
+	Atom* ap;
+	int i;
+	ap = atom_tab(atom_val(ioterm));
+	switch (encoding) {
+	case ERL_FILENAME_LATIN1:
+	    for (i = 0; i < ap->len; i++) {
+		*p++ = ap->name[i];
+	    }
+	    break;
+	case ERL_FILENAME_UTF8_MAC:
+	case ERL_FILENAME_UTF8:
+	    for (i = 0; i < ap->len; i++) {
+		if(ap->name[i] < 0x80) {
+		    *p++ = ap->name[i];
+		} else {
+		    *p++ = (((ap->name[i]) >> 6) | ((byte) 0xC0));
+		    *p++ = (((ap->name[i]) & 0x3F) | ((byte) 0x80));
+		}
+	    }
+	    break;
+	case ERL_FILENAME_WIN_WCHAR:
+	    for (i = 0; i < ap->len; i++) {
+		/* Little endian */
+		*p++ = ap->name[i];
+		*p++ = 0;
+	    }
+	    break;
+	default:
+	    ASSERT(0);
+	}
+	DESTROY_ESTACK(stack);
+	return;
+    }
+
+    if (is_nil(ioterm)) {
+	DESTROY_ESTACK(stack);
+	return;
+    }
+    ASSERT(is_list(ioterm));
+    /* OK a list, needs to be processed in order, handling each flat list-level
+       as they occur, just like io_list_to_binary would */
+    ESTACK_PUSH(stack,ioterm);
+    while (!ESTACK_ISEMPTY(stack)) {
+	ioterm = ESTACK_POP(stack);	
+	if (is_nil(ioterm)) {
+	    /* ignore empty lists */
+	    continue;
+	}
+	if(is_list(ioterm)) {
+L_Again:   /* Restart with sublist, old listend was pushed on stack */
+	    objp = list_val(ioterm);
+	    obj = CAR(objp);
+	    for(;;) { /* loop over one flat list of bytes and binaries
+		         until sublist or list end is encountered */
+		if (is_small(obj)) { /* Always small */
+		    for(;;) {
+			Uint x = unsigned_val(obj);
+			switch (encoding) {
+			case ERL_FILENAME_LATIN1:
+			    ASSERT( x < 256);
+			    *p++ = (byte) x;
+			    break;
+			case ERL_FILENAME_UTF8_MAC:
+			case ERL_FILENAME_UTF8:
+			    if (x < 0x80) {
+				*p++ = (byte) x;
+			    }
+			    else if (x < 0x800) {
+				*p++ = (((byte) (x >> 6)) | 
+					((byte) 0xC0));
+				*p++ = (((byte) (x & 0x3F)) | 
+					((byte) 0x80));
+			    } else if (x < 0x10000) {
+				ASSERT(!((x >= 0xD800 && x <= 0xDFFF) ||
+					 (x == 0xFFFE) ||
+					 (x == 0xFFFF)));
+				*p++ = (((byte) (x >> 12)) | 
+					((byte) 0xE0));
+				*p++ = ((((byte) (x >> 6)) & 0x3F)  | 
+					((byte) 0x80));
+				*p++ = (((byte) (x & 0x3F)) | 
+					((byte) 0x80));
+			    } else {
+				ASSERT(x < 0x110000);
+				*p++ = (((byte) (x >> 18)) | 
+					((byte) 0xF0));
+				*p++ = ((((byte) (x >> 12)) & 0x3F)  | 
+					((byte) 0x80));
+				*p++ = ((((byte) (x >> 6)) & 0x3F)  | 
+					((byte) 0x80));
+				*p++ = (((byte) (x & 0x3F)) | 
+					((byte) 0x80));
+			    }
+			    break;
+			case ERL_FILENAME_WIN_WCHAR:
+			    ASSERT(x <= 0xFFFF); 
+			    *p++ = (byte) (x & 0xFFU);
+			    *p++ = (byte) ((x >> 8) & 0xFFU);
+			    break;
+			default:
+			    ASSERT(0);
+			}
+			    
+			/* everything else will give badarg later 
+			   in the process, so we dont check */
+			ioterm = CDR(objp);
+			if (!is_list(ioterm)) {
+			    break;
+			}
+			objp = list_val(ioterm);
+			obj = CAR(objp);
+			if (!is_small(obj))
+			    break;
+		    }
+		} else if (is_nil(obj)) {
+		    ioterm = CDR(objp);
+		    if (!is_list(ioterm)) {
+			break;
+		    }
+		    objp = list_val(ioterm);
+		    obj = CAR(objp);
+		} else if (is_list(obj)) {
+		    /* push rest of list for later processing, start 
+		       again with sublist */
+		    ESTACK_PUSH(stack,CDR(objp));
+		    ioterm = obj;
+		    goto L_Again;
+		} else {
+		    ASSERT(0);
+		} 
+		if (is_nil(ioterm) || !is_list(ioterm)) {
+		    break;
+		}
+	    } /* for(;;) */
+	} /* is_list(ioterm) */
+	
+	ASSERT(is_list(ioterm) || is_nil(ioterm));
+    } /* while  not estack empty */
+    DESTROY_ESTACK(stack);
+    return;
+}
+void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars)
+{
+    Uint unipoint;
+    
+    while (num_chars--) {
+	if (((*bytes) & ((byte) 0x80)) == 0) {
+	    unipoint = (Uint) *bytes;
+	    ++bytes;
+	} else if (((*bytes) & ((byte) 0xE0)) == 0xC0) {
+	    unipoint = 
+		(((Uint) ((*bytes) & ((byte) 0x1F))) << 6) |
+		((Uint) (bytes[1] & ((byte) 0x3F))); 	
+	    bytes += 2;
+	} else if (((*bytes) & ((byte) 0xF0)) == 0xE0) {
+	    unipoint = 
+		(((Uint) ((*bytes) & ((byte) 0xF))) << 12) |
+		(((Uint) (bytes[1] & ((byte) 0x3F))) << 6) |
+		((Uint) (bytes[2] & ((byte) 0x3F)));
+	    bytes +=3;
+	} else if (((*bytes) & ((byte) 0xF8)) == 0xF0) {
+	    unipoint = 
+		(((Uint) ((*bytes) & ((byte) 0x7))) << 18) |
+		(((Uint) (bytes[1] & ((byte) 0x3F))) << 12) |
+		(((Uint) (bytes[2] & ((byte) 0x3F))) << 6) |
+		((Uint) (bytes[3] & ((byte) 0x3F)));
+	    bytes += 4;
+	} else {
+	    erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1");
+	}
+	*target++ = (byte) (unipoint & 0xFF);
+	*target++ = (byte) ((unipoint >> 8) & 0xFF);
+    }
+}
+
+/*
+ * This internal bif converts a filename to whatever format is suitable for the file driver
+ * It also adds zero termination so that prim_file needn't bother with the character encoding
+ * of the file driver 
+ */
+BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
+{
+    int encoding = erts_get_native_filename_encoding();
+    Sint need;
+    Eterm bin_term;
+    byte* bin_p;
+    /* Prim file explicitly does not allow atoms, although we could 
+       very well cope with it. Instead of letting 'file' handle them,
+       it would probably be more efficient to handle them here. Subject to 
+       change in R15. */ 
+    if (is_atom(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (is_binary(BIF_ARG_1)) {
+	byte *temp_alloc = NULL;
+	byte *bytes;
+	byte *err_pos;
+	Uint size,num_chars;
+	/* Uninterpreted encoding except if windows widechar, in case we convert from 
+	   utf8 to win_wchar */
+	size = binary_size(BIF_ARG_1);
+	bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+	if (encoding != ERL_FILENAME_WIN_WCHAR) {
+	    /*Add 0 termination only*/
+	    bin_term = new_binary(BIF_P, NULL, size+1);
+	    bin_p = binary_bytes(bin_term);
+	    memcpy(bin_p,bytes,size);
+	    bin_p[size]=0;
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BIF_RET(bin_term);
+	} 
+	/* In a wchar world, the emulator flags only affect how
+	   binaries are interpreted when sent from the user. */
+	/* Determine real length and create a new binary */
+	if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || 
+	    erts_get_user_requested_filename_encoding() ==  ERL_FILENAME_LATIN1) {
+	    /* What to do now? Maybe latin1, so just take byte for byte instead */
+	    bin_term = new_binary(BIF_P, 0, (size+1)*2);
+	    bin_p = binary_bytes(bin_term);
+	    while (size--) {
+		*bin_p++ = *bytes++;
+		*bin_p++ = 0;
+	    }
+	    *bin_p++ = 0;
+	    *bin_p++ = 0;
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BIF_RET(bin_term);
+	}
+	/* OK, UTF8 ok, number of characters is in num_chars */
+	bin_term = new_binary(BIF_P, 0, (num_chars+1)*2);
+	bin_p = binary_bytes(bin_term);
+	erts_copy_utf8_to_utf16_little(bin_p, bytes, num_chars);
+	/* zero termination */
+	bin_p[num_chars*2] = 0;
+	bin_p[num_chars*2+1] = 0;
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BIF_RET(bin_term);
+    } /* binary */   
+	    
+
+    if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (encoding == ERL_FILENAME_WIN_WCHAR) {
+	need += 2;
+    } else {
+	++need;
+    }
+    
+    bin_term = new_binary(BIF_P, 0, need);
+    bin_p = binary_bytes(bin_term);
+    erts_native_filename_put(BIF_ARG_1,encoding,bin_p); 
+    bin_p[need-1] = 0;
+    if (encoding == ERL_FILENAME_WIN_WCHAR) {
+	bin_p[need-2] = 0;
+    }
+    BIF_RET(bin_term);
+}
+
+BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
+{
+    Eterm real_bin;
+    Uint offset;
+    Uint size,num_chars;
+    Uint bitsize;
+    Uint bitoffs;
+    Eterm *hp;
+    byte *temp_alloc = NULL;
+    byte *bytes;
+    byte *err_pos;
+    Uint num_built; /* characters */
+    Uint num_eaten; /* bytes */
+    Eterm ret;
+    int mac = 0;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    size = binary_size(BIF_ARG_1);
+    ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+    if (bitsize != 0) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (size == 0) {
+	BIF_RET(NIL);
+    }
+    switch (erts_get_native_filename_encoding()) {
+    case ERL_FILENAME_LATIN1:
+	hp = HAlloc(BIF_P, 2 * size);
+	bytes = binary_bytes(real_bin)+offset;
+    
+	BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
+    case ERL_FILENAME_UTF8_MAC:
+	mac = 1;
+    case ERL_FILENAME_UTF8:
+	bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+	if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    goto noconvert;
+	}
+	num_built = 0;
+	num_eaten = 0;
+	if (mac) {
+	    ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size);
+	} else {
+	    ret = do_utf8_to_list(BIF_P, num_chars, bytes, size, num_chars, &num_built, &num_eaten, NIL);
+	} 
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BIF_RET(ret);
+    case ERL_FILENAME_WIN_WCHAR:
+	bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+	if ((size % 2) != 0) { /* Panic fixup to avoid crashing the emulator */
+	    size--;
+	    hp = HAlloc(BIF_P, size+2);
+	    ret = CONS(hp,make_small((Uint) bytes[size]),NIL);
+	    hp += 2;
+	} else {
+	    hp = HAlloc(BIF_P, size);
+	    ret = NIL;
+	}
+	bytes += size-1;
+	while (size > 0) {
+	    Uint x = ((Uint) *bytes--) << 8;
+	    x |= ((Uint) *bytes--);
+	    size -= 2;
+	    ret = CONS(hp,make_small(x),ret);
+	    hp += 2;
+	}	    
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BIF_RET(ret);
+    default:
+	goto noconvert;
+    }
+ noconvert:
+    BIF_RET(BIF_ARG_1);
+}
+
+BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1)
+{
+    Eterm real_bin;
+    Uint offset;
+    Uint size,num_chars;
+    Uint bitsize;
+    Uint bitoffs;
+    Eterm ret;
+    byte *temp_alloc = NULL;
+    byte *bytes;
+    byte *err_pos;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    size = binary_size(BIF_ARG_1);
+    ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+    if (bitsize != 0) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (size == 0) {
+	BIF_RET(NIL);
+    }
+    bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+    if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size);
+    erts_free_aligned_binary_bytes(temp_alloc);
+    BIF_RET(ret);
+}  
+
+BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0)
+{
+    switch (erts_get_native_filename_encoding()) {
+    case ERL_FILENAME_LATIN1:
+	BIF_RET(am_latin1);
+    case ERL_FILENAME_UTF8_MAC:
+    case ERL_FILENAME_UTF8:
+	BIF_RET(am_utf8);
+    case ERL_FILENAME_WIN_WCHAR:
+	if (erts_get_user_requested_filename_encoding() ==  ERL_FILENAME_LATIN1) {
+	    BIF_RET(am_latin1);
+	} else {
+	    BIF_RET(am_utf8);
+	}
+    default:
+	BIF_RET(am_undefined);
+    }
+}
diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h
new file mode 100644
index 0000000000..fb0a111ca2
--- /dev/null
+++ b/erts/emulator/beam/erl_unicode_normalize.h
@@ -0,0 +1,1687 @@
+/*
+* %CopyrightBegin%
+*
+* Copyright Ericsson AB 1999-2010. All Rights Reserved.
+*
+* The contents of this file are subject to the Erlang Public License,
+* Version 1.1, (the "License"); you may not use this file except in
+* compliance with the License. You should have received a copy of the
+* Erlang Public License along with this software. If not, it can be
+* retrieved online at http://www.erlang.org/.
+*
+* Software distributed under the License is distributed on an "AS IS"
+* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+* the License for the specific language governing rights and limitations
+* under the License.
+*
+* %CopyrightEnd%
+*/
+/*
+* This file is automatically generated by dec.erl, do not edit manually
+*/
+#define HASH_SIZE_FACTOR 2
+typedef struct _compose_entry {
+    Uint16 c;
+    Uint16 res;
+    Uint16 num_subs;
+    struct _compose_entry *subs;
+    int *hash;
+} CompEntry;
+
+static int compose_tab_size = 61;
+static int hash_compose_tab_0_15[12] = 
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_0_15 */ 
+static CompEntry compose_tab_0_15[] = {
+{65, 7846, 0, NULL, NULL},
+{69, 7872, 0, NULL, NULL},
+{79, 7890, 0, NULL, NULL},
+{97, 7847, 0, NULL, NULL},
+{101, 7873, 0, NULL, NULL},
+{111, 7891, 0, NULL, NULL}
+}; /* compose_tab_0_15 */ 
+static int hash_compose_tab_0_16[8] = 
+{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_0_16 */ 
+static CompEntry compose_tab_0_16[] = {
+{69, 7700, 0, NULL, NULL},
+{79, 7760, 0, NULL, NULL},
+{101, 7701, 0, NULL, NULL},
+{111, 7761, 0, NULL, NULL}
+}; /* compose_tab_0_16 */ 
+static int hash_compose_tab_0_17[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_0_17 */ 
+static CompEntry compose_tab_0_17[] = {
+{65, 7856, 0, NULL, NULL},
+{97, 7857, 0, NULL, NULL}
+}; /* compose_tab_0_17 */ 
+static int hash_compose_tab_0_18[8] = 
+{-1,2,-1,-1,-1,0,1,3}; /* hash_compose_tab_0_18 */ 
+static CompEntry compose_tab_0_18[] = {
+{85, 475, 0, NULL, NULL},
+{117, 476, 0, NULL, NULL},
+{953, 8146, 0, NULL, NULL},
+{965, 8162, 0, NULL, NULL}
+}; /* compose_tab_0_18 */ 
+static int hash_compose_tab_0_19_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_19_0 */ 
+static CompEntry compose_tab_0_19_0[] = {
+{913, 8074, 0, NULL, NULL},
+{919, 8090, 0, NULL, NULL},
+{937, 8106, 0, NULL, NULL},
+{945, 8066, 0, NULL, NULL},
+{951, 8082, 0, NULL, NULL},
+{969, 8098, 0, NULL, NULL}
+}; /* compose_tab_0_19_0 */ 
+static int hash_compose_tab_0_19[28] = 
+{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_0_19 */ 
+static CompEntry compose_tab_0_19[] = {
+{837, 0, 6, compose_tab_0_19_0, hash_compose_tab_0_19_0},
+{913, 7946, 0, NULL, NULL},
+{917, 7962, 0, NULL, NULL},
+{919, 7978, 0, NULL, NULL},
+{921, 7994, 0, NULL, NULL},
+{927, 8010, 0, NULL, NULL},
+{937, 8042, 0, NULL, NULL},
+{945, 7938, 0, NULL, NULL},
+{949, 7954, 0, NULL, NULL},
+{951, 7970, 0, NULL, NULL},
+{953, 7986, 0, NULL, NULL},
+{959, 8002, 0, NULL, NULL},
+{965, 8018, 0, NULL, NULL},
+{969, 8034, 0, NULL, NULL}
+}; /* compose_tab_0_19 */ 
+static int hash_compose_tab_0_20_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_20_0 */ 
+static CompEntry compose_tab_0_20_0[] = {
+{913, 8075, 0, NULL, NULL},
+{919, 8091, 0, NULL, NULL},
+{937, 8107, 0, NULL, NULL},
+{945, 8067, 0, NULL, NULL},
+{951, 8083, 0, NULL, NULL},
+{969, 8099, 0, NULL, NULL}
+}; /* compose_tab_0_20_0 */ 
+static int hash_compose_tab_0_20[30] = 
+{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5,
+ 12}; /* hash_compose_tab_0_20 */ 
+static CompEntry compose_tab_0_20[] = {
+{837, 0, 6, compose_tab_0_20_0, hash_compose_tab_0_20_0},
+{913, 7947, 0, NULL, NULL},
+{917, 7963, 0, NULL, NULL},
+{919, 7979, 0, NULL, NULL},
+{921, 7995, 0, NULL, NULL},
+{927, 8011, 0, NULL, NULL},
+{933, 8027, 0, NULL, NULL},
+{937, 8043, 0, NULL, NULL},
+{945, 7939, 0, NULL, NULL},
+{949, 7955, 0, NULL, NULL},
+{951, 7971, 0, NULL, NULL},
+{953, 7987, 0, NULL, NULL},
+{959, 8003, 0, NULL, NULL},
+{965, 8019, 0, NULL, NULL},
+{969, 8035, 0, NULL, NULL}
+}; /* compose_tab_0_20 */ 
+static int hash_compose_tab_0_21[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_0_21 */ 
+static CompEntry compose_tab_0_21[] = {
+{79, 7900, 0, NULL, NULL},
+{85, 7914, 0, NULL, NULL},
+{111, 7901, 0, NULL, NULL},
+{117, 7915, 0, NULL, NULL}
+}; /* compose_tab_0_21 */ 
+static int hash_compose_tab_0_22[6] = 
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_0_22 */ 
+static CompEntry compose_tab_0_22[] = {
+{945, 8114, 0, NULL, NULL},
+{951, 8130, 0, NULL, NULL},
+{969, 8178, 0, NULL, NULL}
+}; /* compose_tab_0_22 */ 
+static int hash_compose_tab_0[78] = 
+{38,3,29,-1,-1,-1,-1,4,19,5,20,6,14,30,31,21,32,33,37,7,-1,-1,-1,8,34,-1,-1,9,
+ -1,35,-1,-1,-1,10,36,-1,-1,-1,-1,11,-1,12,-1,13,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,23,-1,22,-1,24,-1,25,-1,26,-1,0,-1,-1,15,1,16,27,17,2,18,28,-1,-1}; /* hash_compose_tab_0 */ 
+static CompEntry compose_tab_0[] = {
+{65, 192, 0, NULL, NULL},
+{69, 200, 0, NULL, NULL},
+{73, 204, 0, NULL, NULL},
+{79, 210, 0, NULL, NULL},
+{85, 217, 0, NULL, NULL},
+{87, 7808, 0, NULL, NULL},
+{89, 7922, 0, NULL, NULL},
+{97, 224, 0, NULL, NULL},
+{101, 232, 0, NULL, NULL},
+{105, 236, 0, NULL, NULL},
+{111, 242, 0, NULL, NULL},
+{117, 249, 0, NULL, NULL},
+{119, 7809, 0, NULL, NULL},
+{121, 7923, 0, NULL, NULL},
+{168, 8173, 0, NULL, NULL},
+{770, 0, 6, compose_tab_0_15, hash_compose_tab_0_15},
+{772, 0, 4, compose_tab_0_16, hash_compose_tab_0_16},
+{774, 0, 2, compose_tab_0_17, hash_compose_tab_0_17},
+{776, 0, 4, compose_tab_0_18, hash_compose_tab_0_18},
+{787, 0, 14, compose_tab_0_19, hash_compose_tab_0_19},
+{788, 0, 15, compose_tab_0_20, hash_compose_tab_0_20},
+{795, 0, 4, compose_tab_0_21, hash_compose_tab_0_21},
+{837, 0, 3, compose_tab_0_22, hash_compose_tab_0_22},
+{913, 8122, 0, NULL, NULL},
+{917, 8136, 0, NULL, NULL},
+{919, 8138, 0, NULL, NULL},
+{921, 8154, 0, NULL, NULL},
+{927, 8184, 0, NULL, NULL},
+{933, 8170, 0, NULL, NULL},
+{937, 8186, 0, NULL, NULL},
+{945, 8048, 0, NULL, NULL},
+{949, 8050, 0, NULL, NULL},
+{951, 8052, 0, NULL, NULL},
+{953, 8054, 0, NULL, NULL},
+{959, 8056, 0, NULL, NULL},
+{965, 8058, 0, NULL, NULL},
+{969, 8060, 0, NULL, NULL},
+{8127, 8141, 0, NULL, NULL},
+{8190, 8157, 0, NULL, NULL}
+}; /* compose_tab_0 */ 
+static int hash_compose_tab_1_39[12] = 
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_1_39 */ 
+static CompEntry compose_tab_1_39[] = {
+{65, 7844, 0, NULL, NULL},
+{69, 7870, 0, NULL, NULL},
+{79, 7888, 0, NULL, NULL},
+{97, 7845, 0, NULL, NULL},
+{101, 7871, 0, NULL, NULL},
+{111, 7889, 0, NULL, NULL}
+}; /* compose_tab_1_39 */ 
+static int hash_compose_tab_1_40[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_40 */ 
+static CompEntry compose_tab_1_40[] = {
+{79, 7756, 0, NULL, NULL},
+{85, 7800, 0, NULL, NULL},
+{111, 7757, 0, NULL, NULL},
+{117, 7801, 0, NULL, NULL}
+}; /* compose_tab_1_40 */ 
+static int hash_compose_tab_1_41[8] = 
+{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_1_41 */ 
+static CompEntry compose_tab_1_41[] = {
+{69, 7702, 0, NULL, NULL},
+{79, 7762, 0, NULL, NULL},
+{101, 7703, 0, NULL, NULL},
+{111, 7763, 0, NULL, NULL}
+}; /* compose_tab_1_41 */ 
+static int hash_compose_tab_1_42[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_1_42 */ 
+static CompEntry compose_tab_1_42[] = {
+{65, 7854, 0, NULL, NULL},
+{97, 7855, 0, NULL, NULL}
+}; /* compose_tab_1_42 */ 
+static int hash_compose_tab_1_43[12] = 
+{-1,0,1,-1,-1,4,5,-1,-1,2,3,-1}; /* hash_compose_tab_1_43 */ 
+static CompEntry compose_tab_1_43[] = {
+{73, 7726, 0, NULL, NULL},
+{85, 471, 0, NULL, NULL},
+{105, 7727, 0, NULL, NULL},
+{117, 472, 0, NULL, NULL},
+{953, 8147, 0, NULL, NULL},
+{965, 8163, 0, NULL, NULL}
+}; /* compose_tab_1_43 */ 
+static int hash_compose_tab_1_44[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_1_44 */ 
+static CompEntry compose_tab_1_44[] = {
+{65, 506, 0, NULL, NULL},
+{97, 507, 0, NULL, NULL}
+}; /* compose_tab_1_44 */ 
+static int hash_compose_tab_1_45_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_45_0 */ 
+static CompEntry compose_tab_1_45_0[] = {
+{913, 8076, 0, NULL, NULL},
+{919, 8092, 0, NULL, NULL},
+{937, 8108, 0, NULL, NULL},
+{945, 8068, 0, NULL, NULL},
+{951, 8084, 0, NULL, NULL},
+{969, 8100, 0, NULL, NULL}
+}; /* compose_tab_1_45_0 */ 
+static int hash_compose_tab_1_45[28] = 
+{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_1_45 */ 
+static CompEntry compose_tab_1_45[] = {
+{837, 0, 6, compose_tab_1_45_0, hash_compose_tab_1_45_0},
+{913, 7948, 0, NULL, NULL},
+{917, 7964, 0, NULL, NULL},
+{919, 7980, 0, NULL, NULL},
+{921, 7996, 0, NULL, NULL},
+{927, 8012, 0, NULL, NULL},
+{937, 8044, 0, NULL, NULL},
+{945, 7940, 0, NULL, NULL},
+{949, 7956, 0, NULL, NULL},
+{951, 7972, 0, NULL, NULL},
+{953, 7988, 0, NULL, NULL},
+{959, 8004, 0, NULL, NULL},
+{965, 8020, 0, NULL, NULL},
+{969, 8036, 0, NULL, NULL}
+}; /* compose_tab_1_45 */ 
+static int hash_compose_tab_1_46_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_46_0 */ 
+static CompEntry compose_tab_1_46_0[] = {
+{913, 8077, 0, NULL, NULL},
+{919, 8093, 0, NULL, NULL},
+{937, 8109, 0, NULL, NULL},
+{945, 8069, 0, NULL, NULL},
+{951, 8085, 0, NULL, NULL},
+{969, 8101, 0, NULL, NULL}
+}; /* compose_tab_1_46_0 */ 
+static int hash_compose_tab_1_46[30] = 
+{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5,
+ 12}; /* hash_compose_tab_1_46 */ 
+static CompEntry compose_tab_1_46[] = {
+{837, 0, 6, compose_tab_1_46_0, hash_compose_tab_1_46_0},
+{913, 7949, 0, NULL, NULL},
+{917, 7965, 0, NULL, NULL},
+{919, 7981, 0, NULL, NULL},
+{921, 7997, 0, NULL, NULL},
+{927, 8013, 0, NULL, NULL},
+{933, 8029, 0, NULL, NULL},
+{937, 8045, 0, NULL, NULL},
+{945, 7941, 0, NULL, NULL},
+{949, 7957, 0, NULL, NULL},
+{951, 7973, 0, NULL, NULL},
+{953, 7989, 0, NULL, NULL},
+{959, 8005, 0, NULL, NULL},
+{965, 8021, 0, NULL, NULL},
+{969, 8037, 0, NULL, NULL}
+}; /* compose_tab_1_46 */ 
+static int hash_compose_tab_1_47[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_47 */ 
+static CompEntry compose_tab_1_47[] = {
+{79, 7898, 0, NULL, NULL},
+{85, 7912, 0, NULL, NULL},
+{111, 7899, 0, NULL, NULL},
+{117, 7913, 0, NULL, NULL}
+}; /* compose_tab_1_47 */ 
+static int hash_compose_tab_1_48[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_1_48 */ 
+static CompEntry compose_tab_1_48[] = {
+{67, 7688, 0, NULL, NULL},
+{99, 7689, 0, NULL, NULL}
+}; /* compose_tab_1_48 */ 
+static int hash_compose_tab_1_49[6] = 
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_1_49 */ 
+static CompEntry compose_tab_1_49[] = {
+{945, 8116, 0, NULL, NULL},
+{951, 8132, 0, NULL, NULL},
+{959, 8180, 0, NULL, NULL}
+}; /* compose_tab_1_49 */ 
+static int hash_compose_tab_1[140] = 
+{-1,-1,-1,-1,-1,-1,-1,68,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,34,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,35,-1,-1,-1,-1,64,-1,0,-1,1,-1,2,39,3,40,4,41,5,6,7,
+ 8,9,10,36,11,12,42,13,43,14,44,15,16,37,45,46,50,47,51,17,52,18,53,19,54,20,
+ 55,21,56,22,23,24,25,26,27,38,28,29,48,30,57,31,58,32,33,59,60,61,62,65,66,
+ 63,67,69,-1,-1,-1,-1,-1,49,-1,-1}; /* hash_compose_tab_1 */ 
+static CompEntry compose_tab_1[] = {
+{65, 193, 0, NULL, NULL},
+{67, 262, 0, NULL, NULL},
+{69, 201, 0, NULL, NULL},
+{71, 500, 0, NULL, NULL},
+{73, 205, 0, NULL, NULL},
+{75, 7728, 0, NULL, NULL},
+{76, 313, 0, NULL, NULL},
+{77, 7742, 0, NULL, NULL},
+{78, 323, 0, NULL, NULL},
+{79, 211, 0, NULL, NULL},
+{80, 7764, 0, NULL, NULL},
+{82, 340, 0, NULL, NULL},
+{83, 346, 0, NULL, NULL},
+{85, 218, 0, NULL, NULL},
+{87, 7810, 0, NULL, NULL},
+{89, 221, 0, NULL, NULL},
+{90, 377, 0, NULL, NULL},
+{97, 225, 0, NULL, NULL},
+{99, 263, 0, NULL, NULL},
+{101, 233, 0, NULL, NULL},
+{103, 501, 0, NULL, NULL},
+{105, 237, 0, NULL, NULL},
+{107, 7729, 0, NULL, NULL},
+{108, 314, 0, NULL, NULL},
+{109, 7743, 0, NULL, NULL},
+{110, 324, 0, NULL, NULL},
+{111, 243, 0, NULL, NULL},
+{112, 7765, 0, NULL, NULL},
+{114, 341, 0, NULL, NULL},
+{115, 347, 0, NULL, NULL},
+{117, 250, 0, NULL, NULL},
+{119, 7811, 0, NULL, NULL},
+{121, 253, 0, NULL, NULL},
+{122, 378, 0, NULL, NULL},
+{168, 8174, 0, NULL, NULL},
+{198, 508, 0, NULL, NULL},
+{216, 510, 0, NULL, NULL},
+{230, 509, 0, NULL, NULL},
+{248, 511, 0, NULL, NULL},
+{770, 0, 6, compose_tab_1_39, hash_compose_tab_1_39},
+{771, 0, 4, compose_tab_1_40, hash_compose_tab_1_40},
+{772, 0, 4, compose_tab_1_41, hash_compose_tab_1_41},
+{774, 0, 2, compose_tab_1_42, hash_compose_tab_1_42},
+{776, 0, 6, compose_tab_1_43, hash_compose_tab_1_43},
+{778, 0, 2, compose_tab_1_44, hash_compose_tab_1_44},
+{787, 0, 14, compose_tab_1_45, hash_compose_tab_1_45},
+{788, 0, 15, compose_tab_1_46, hash_compose_tab_1_46},
+{795, 0, 4, compose_tab_1_47, hash_compose_tab_1_47},
+{807, 0, 2, compose_tab_1_48, hash_compose_tab_1_48},
+{837, 0, 3, compose_tab_1_49, hash_compose_tab_1_49},
+{913, 8123, 0, NULL, NULL},
+{917, 8137, 0, NULL, NULL},
+{919, 8139, 0, NULL, NULL},
+{921, 8155, 0, NULL, NULL},
+{927, 8185, 0, NULL, NULL},
+{933, 8171, 0, NULL, NULL},
+{937, 8187, 0, NULL, NULL},
+{945, 8049, 0, NULL, NULL},
+{949, 8051, 0, NULL, NULL},
+{951, 8053, 0, NULL, NULL},
+{953, 8055, 0, NULL, NULL},
+{959, 8057, 0, NULL, NULL},
+{965, 8059, 0, NULL, NULL},
+{969, 8061, 0, NULL, NULL},
+{1043, 1027, 0, NULL, NULL},
+{1050, 1036, 0, NULL, NULL},
+{1075, 1107, 0, NULL, NULL},
+{1082, 1116, 0, NULL, NULL},
+{8127, 8142, 0, NULL, NULL},
+{8190, 8158, 0, NULL, NULL}
+}; /* compose_tab_1 */ 
+static int hash_compose_tab_2_26[12] = 
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_2_26 */ 
+static CompEntry compose_tab_2_26[] = {
+{65, 7852, 0, NULL, NULL},
+{69, 7878, 0, NULL, NULL},
+{79, 7896, 0, NULL, NULL},
+{97, 7853, 0, NULL, NULL},
+{101, 7879, 0, NULL, NULL},
+{111, 7897, 0, NULL, NULL}
+}; /* compose_tab_2_26 */ 
+static int hash_compose_tab_2[54] = 
+{-1,-1,-1,20,-1,-1,-1,21,-1,22,-1,0,23,1,24,2,25,3,4,5,6,-1,-1,-1,-1,7,-1,-1,
+ -1,8,-1,9,-1,10,-1,11,12,-1,-1,-1,-1,-1,-1,13,-1,14,-1,15,26,16,17,18,19,-1}; /* hash_compose_tab_2 */ 
+static CompEntry compose_tab_2[] = {
+{65, 194, 0, NULL, NULL},
+{67, 264, 0, NULL, NULL},
+{69, 202, 0, NULL, NULL},
+{71, 284, 0, NULL, NULL},
+{72, 292, 0, NULL, NULL},
+{73, 206, 0, NULL, NULL},
+{74, 308, 0, NULL, NULL},
+{79, 212, 0, NULL, NULL},
+{83, 348, 0, NULL, NULL},
+{85, 219, 0, NULL, NULL},
+{87, 372, 0, NULL, NULL},
+{89, 374, 0, NULL, NULL},
+{90, 7824, 0, NULL, NULL},
+{97, 226, 0, NULL, NULL},
+{99, 265, 0, NULL, NULL},
+{101, 234, 0, NULL, NULL},
+{103, 285, 0, NULL, NULL},
+{104, 293, 0, NULL, NULL},
+{105, 238, 0, NULL, NULL},
+{106, 309, 0, NULL, NULL},
+{111, 244, 0, NULL, NULL},
+{115, 349, 0, NULL, NULL},
+{117, 251, 0, NULL, NULL},
+{119, 373, 0, NULL, NULL},
+{121, 375, 0, NULL, NULL},
+{122, 7825, 0, NULL, NULL},
+{803, 0, 6, compose_tab_2_26, hash_compose_tab_2_26}
+}; /* compose_tab_2 */ 
+static int hash_compose_tab_3_16[12] = 
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_3_16 */ 
+static CompEntry compose_tab_3_16[] = {
+{65, 7850, 0, NULL, NULL},
+{69, 7876, 0, NULL, NULL},
+{79, 7894, 0, NULL, NULL},
+{97, 7851, 0, NULL, NULL},
+{101, 7877, 0, NULL, NULL},
+{111, 7895, 0, NULL, NULL}
+}; /* compose_tab_3_16 */ 
+static int hash_compose_tab_3_17[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_3_17 */ 
+static CompEntry compose_tab_3_17[] = {
+{65, 7860, 0, NULL, NULL},
+{97, 7861, 0, NULL, NULL}
+}; /* compose_tab_3_17 */ 
+static int hash_compose_tab_3_18[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_3_18 */ 
+static CompEntry compose_tab_3_18[] = {
+{79, 7904, 0, NULL, NULL},
+{85, 7918, 0, NULL, NULL},
+{111, 7905, 0, NULL, NULL},
+{117, 7919, 0, NULL, NULL}
+}; /* compose_tab_3_18 */ 
+static int hash_compose_tab_3[38] = 
+{-1,-1,3,4,13,14,-1,15,-1,5,6,16,-1,7,17,-1,-1,-1,-1,-1,-1,8,-1,-1,-1,9,-1,0,
+ -1,10,-1,1,-1,-1,11,2,12,18}; /* hash_compose_tab_3 */ 
+static CompEntry compose_tab_3[] = {
+{65, 195, 0, NULL, NULL},
+{69, 7868, 0, NULL, NULL},
+{73, 296, 0, NULL, NULL},
+{78, 209, 0, NULL, NULL},
+{79, 213, 0, NULL, NULL},
+{85, 360, 0, NULL, NULL},
+{86, 7804, 0, NULL, NULL},
+{89, 7928, 0, NULL, NULL},
+{97, 227, 0, NULL, NULL},
+{101, 7869, 0, NULL, NULL},
+{105, 297, 0, NULL, NULL},
+{110, 241, 0, NULL, NULL},
+{111, 245, 0, NULL, NULL},
+{117, 361, 0, NULL, NULL},
+{118, 7805, 0, NULL, NULL},
+{121, 7929, 0, NULL, NULL},
+{770, 0, 6, compose_tab_3_16, hash_compose_tab_3_16},
+{774, 0, 2, compose_tab_3_17, hash_compose_tab_3_17},
+{795, 0, 4, compose_tab_3_18, hash_compose_tab_3_18}
+}; /* compose_tab_3 */ 
+static int hash_compose_tab_4_14[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_4_14 */ 
+static CompEntry compose_tab_4_14[] = {
+{65, 480, 0, NULL, NULL},
+{97, 481, 0, NULL, NULL}
+}; /* compose_tab_4_14 */ 
+static int hash_compose_tab_4_15[8] = 
+{-1,0,2,-1,-1,1,3,-1}; /* hash_compose_tab_4_15 */ 
+static CompEntry compose_tab_4_15[] = {
+{65, 478, 0, NULL, NULL},
+{85, 469, 0, NULL, NULL},
+{97, 479, 0, NULL, NULL},
+{117, 470, 0, NULL, NULL}
+}; /* compose_tab_4_15 */ 
+static int hash_compose_tab_4_16[8] = 
+{-1,-1,1,3,0,2,-1,-1}; /* hash_compose_tab_4_16 */ 
+static CompEntry compose_tab_4_16[] = {
+{76, 7736, 0, NULL, NULL},
+{82, 7772, 0, NULL, NULL},
+{108, 7737, 0, NULL, NULL},
+{114, 7773, 0, NULL, NULL}
+}; /* compose_tab_4_16 */ 
+static int hash_compose_tab_4_17[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_4_17 */ 
+static CompEntry compose_tab_4_17[] = {
+{79, 492, 0, NULL, NULL},
+{111, 493, 0, NULL, NULL}
+}; /* compose_tab_4_17 */ 
+static int hash_compose_tab_4[56] = 
+{-1,22,-1,-1,-1,11,13,-1,-1,0,-1,-1,-1,1,23,2,26,3,18,16,-1,-1,-1,4,17,19,-1,
+ 27,-1,5,12,-1,-1,-1,-1,-1,-1,20,-1,-1,24,6,-1,-1,-1,7,-1,8,14,9,15,21,25,-1,
+ -1,10}; /* hash_compose_tab_4 */ 
+static CompEntry compose_tab_4[] = {
+{65, 256, 0, NULL, NULL},
+{69, 274, 0, NULL, NULL},
+{71, 7712, 0, NULL, NULL},
+{73, 298, 0, NULL, NULL},
+{79, 332, 0, NULL, NULL},
+{85, 362, 0, NULL, NULL},
+{97, 257, 0, NULL, NULL},
+{101, 275, 0, NULL, NULL},
+{103, 7713, 0, NULL, NULL},
+{105, 299, 0, NULL, NULL},
+{111, 333, 0, NULL, NULL},
+{117, 363, 0, NULL, NULL},
+{198, 482, 0, NULL, NULL},
+{230, 483, 0, NULL, NULL},
+{775, 0, 2, compose_tab_4_14, hash_compose_tab_4_14},
+{776, 0, 4, compose_tab_4_15, hash_compose_tab_4_15},
+{803, 0, 4, compose_tab_4_16, hash_compose_tab_4_16},
+{808, 0, 2, compose_tab_4_17, hash_compose_tab_4_17},
+{913, 8121, 0, NULL, NULL},
+{921, 8153, 0, NULL, NULL},
+{933, 8169, 0, NULL, NULL},
+{945, 8113, 0, NULL, NULL},
+{953, 8145, 0, NULL, NULL},
+{965, 8161, 0, NULL, NULL},
+{1048, 1250, 0, NULL, NULL},
+{1059, 1262, 0, NULL, NULL},
+{1080, 1251, 0, NULL, NULL},
+{1091, 1263, 0, NULL, NULL}
+}; /* compose_tab_4 */ 
+static int hash_compose_tab_5_12[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_5_12 */ 
+static CompEntry compose_tab_5_12[] = {
+{65, 7862, 0, NULL, NULL},
+{97, 7863, 0, NULL, NULL}
+}; /* compose_tab_5_12 */ 
+static int hash_compose_tab_5_13[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_5_13 */ 
+static CompEntry compose_tab_5_13[] = {
+{69, 7708, 0, NULL, NULL},
+{101, 7709, 0, NULL, NULL}
+}; /* compose_tab_5_13 */ 
+static int hash_compose_tab_5[60] = 
+{28,-1,-1,-1,-1,0,19,-1,-1,1,-1,2,29,3,14,-1,-1,-1,-1,4,20,15,-1,12,-1,5,21,
+ 13,22,23,-1,-1,-1,16,-1,-1,-1,6,-1,24,-1,7,-1,8,-1,9,17,-1,-1,-1,-1,10,25,18,
+ -1,-1,-1,11,26,27}; /* hash_compose_tab_5 */ 
+static CompEntry compose_tab_5[] = {
+{65, 258, 0, NULL, NULL},
+{69, 276, 0, NULL, NULL},
+{71, 286, 0, NULL, NULL},
+{73, 300, 0, NULL, NULL},
+{79, 334, 0, NULL, NULL},
+{85, 364, 0, NULL, NULL},
+{97, 259, 0, NULL, NULL},
+{101, 277, 0, NULL, NULL},
+{103, 287, 0, NULL, NULL},
+{105, 301, 0, NULL, NULL},
+{111, 335, 0, NULL, NULL},
+{117, 365, 0, NULL, NULL},
+{803, 0, 2, compose_tab_5_12, hash_compose_tab_5_12},
+{807, 0, 2, compose_tab_5_13, hash_compose_tab_5_13},
+{913, 8120, 0, NULL, NULL},
+{921, 8152, 0, NULL, NULL},
+{933, 8168, 0, NULL, NULL},
+{945, 8112, 0, NULL, NULL},
+{953, 8144, 0, NULL, NULL},
+{965, 8160, 0, NULL, NULL},
+{1040, 1232, 0, NULL, NULL},
+{1045, 1238, 0, NULL, NULL},
+{1046, 1217, 0, NULL, NULL},
+{1048, 1049, 0, NULL, NULL},
+{1059, 1038, 0, NULL, NULL},
+{1072, 1233, 0, NULL, NULL},
+{1077, 1239, 0, NULL, NULL},
+{1078, 1218, 0, NULL, NULL},
+{1080, 1081, 0, NULL, NULL},
+{1091, 1118, 0, NULL, NULL}
+}; /* compose_tab_5 */ 
+static int hash_compose_tab_6_36[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_6_36 */ 
+static CompEntry compose_tab_6_36[] = {
+{83, 7780, 0, NULL, NULL},
+{115, 7781, 0, NULL, NULL}
+}; /* compose_tab_6_36 */ 
+static int hash_compose_tab_6_38[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_6_38 */ 
+static CompEntry compose_tab_6_38[] = {
+{83, 7782, 0, NULL, NULL},
+{115, 7783, 0, NULL, NULL}
+}; /* compose_tab_6_38 */ 
+static int hash_compose_tab_6_39[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_6_39 */ 
+static CompEntry compose_tab_6_39[] = {
+{83, 7784, 0, NULL, NULL},
+{115, 7785, 0, NULL, NULL}
+}; /* compose_tab_6_39 */ 
+static int hash_compose_tab_6[80] = 
+{10,-1,11,12,13,39,-1,14,15,16,17,-1,-1,-1,-1,-1,-1,-1,18,19,20,21,22,23,24,
+ -1,-1,-1,-1,25,26,-1,27,-1,28,29,30,-1,-1,31,32,33,34,-1,-1,-1,-1,-1,-1,36,
+ -1,-1,-1,-1,37,-1,-1,-1,-1,-1,38,-1,-1,35,-1,-1,0,1,2,3,4,5,6,7,-1,-1,-1,8,9,
+ -1}; /* hash_compose_tab_6 */ 
+static CompEntry compose_tab_6[] = {
+{66, 7682, 0, NULL, NULL},
+{67, 266, 0, NULL, NULL},
+{68, 7690, 0, NULL, NULL},
+{69, 278, 0, NULL, NULL},
+{70, 7710, 0, NULL, NULL},
+{71, 288, 0, NULL, NULL},
+{72, 7714, 0, NULL, NULL},
+{73, 304, 0, NULL, NULL},
+{77, 7744, 0, NULL, NULL},
+{78, 7748, 0, NULL, NULL},
+{80, 7766, 0, NULL, NULL},
+{82, 7768, 0, NULL, NULL},
+{83, 7776, 0, NULL, NULL},
+{84, 7786, 0, NULL, NULL},
+{87, 7814, 0, NULL, NULL},
+{88, 7818, 0, NULL, NULL},
+{89, 7822, 0, NULL, NULL},
+{90, 379, 0, NULL, NULL},
+{98, 7683, 0, NULL, NULL},
+{99, 267, 0, NULL, NULL},
+{100, 7691, 0, NULL, NULL},
+{101, 279, 0, NULL, NULL},
+{102, 7711, 0, NULL, NULL},
+{103, 289, 0, NULL, NULL},
+{104, 7715, 0, NULL, NULL},
+{109, 7745, 0, NULL, NULL},
+{110, 7749, 0, NULL, NULL},
+{112, 7767, 0, NULL, NULL},
+{114, 7769, 0, NULL, NULL},
+{115, 7777, 0, NULL, NULL},
+{116, 7787, 0, NULL, NULL},
+{119, 7815, 0, NULL, NULL},
+{120, 7819, 0, NULL, NULL},
+{121, 7823, 0, NULL, NULL},
+{122, 380, 0, NULL, NULL},
+{383, 7835, 0, NULL, NULL},
+{769, 0, 2, compose_tab_6_36, hash_compose_tab_6_36},
+{774, 784, 0, NULL, NULL},
+{780, 0, 2, compose_tab_6_38, hash_compose_tab_6_38},
+{803, 0, 2, compose_tab_6_39, hash_compose_tab_6_39}
+}; /* compose_tab_6 */ 
+static int hash_compose_tab_7_23[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_7_23 */ 
+static CompEntry compose_tab_7_23[] = {
+{79, 7758, 0, NULL, NULL},
+{111, 7759, 0, NULL, NULL}
+}; /* compose_tab_7_23 */ 
+static int hash_compose_tab_7_24[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_7_24 */ 
+static CompEntry compose_tab_7_24[] = {
+{85, 7802, 0, NULL, NULL},
+{117, 7803, 0, NULL, NULL}
+}; /* compose_tab_7_24 */ 
+static int hash_compose_tab_7[100] = 
+{48,10,21,-1,11,12,-1,-1,-1,-1,49,13,-1,-1,-1,20,14,15,-1,16,17,18,25,-1,-1,
+ -1,-1,-1,-1,22,30,-1,-1,26,-1,-1,-1,-1,-1,-1,31,-1,-1,-1,-1,32,33,34,35,-1,
+ -1,-1,-1,27,36,-1,-1,-1,-1,37,-1,-1,-1,38,-1,0,28,39,-1,1,-1,23,2,3,24,40,-1,
+ 41,29,4,42,43,44,-1,-1,5,45,6,7,8,-1,46,-1,-1,-1,47,-1,9,-1,19}; /* hash_compose_tab_7 */ 
+static CompEntry compose_tab_7[] = {
+{65, 196, 0, NULL, NULL},
+{69, 203, 0, NULL, NULL},
+{72, 7718, 0, NULL, NULL},
+{73, 207, 0, NULL, NULL},
+{79, 214, 0, NULL, NULL},
+{85, 220, 0, NULL, NULL},
+{87, 7812, 0, NULL, NULL},
+{88, 7820, 0, NULL, NULL},
+{89, 376, 0, NULL, NULL},
+{97, 228, 0, NULL, NULL},
+{101, 235, 0, NULL, NULL},
+{104, 7719, 0, NULL, NULL},
+{105, 239, 0, NULL, NULL},
+{111, 246, 0, NULL, NULL},
+{116, 7831, 0, NULL, NULL},
+{117, 252, 0, NULL, NULL},
+{119, 7813, 0, NULL, NULL},
+{120, 7821, 0, NULL, NULL},
+{121, 255, 0, NULL, NULL},
+{399, 1242, 0, NULL, NULL},
+{415, 1258, 0, NULL, NULL},
+{601, 1243, 0, NULL, NULL},
+{629, 1259, 0, NULL, NULL},
+{771, 0, 2, compose_tab_7_23, hash_compose_tab_7_23},
+{772, 0, 2, compose_tab_7_24, hash_compose_tab_7_24},
+{921, 938, 0, NULL, NULL},
+{933, 939, 0, NULL, NULL},
+{953, 970, 0, NULL, NULL},
+{965, 971, 0, NULL, NULL},
+{978, 980, 0, NULL, NULL},
+{1030, 1031, 0, NULL, NULL},
+{1040, 1234, 0, NULL, NULL},
+{1045, 1025, 0, NULL, NULL},
+{1046, 1244, 0, NULL, NULL},
+{1047, 1246, 0, NULL, NULL},
+{1048, 1252, 0, NULL, NULL},
+{1054, 1254, 0, NULL, NULL},
+{1059, 1264, 0, NULL, NULL},
+{1063, 1268, 0, NULL, NULL},
+{1067, 1272, 0, NULL, NULL},
+{1072, 1235, 0, NULL, NULL},
+{1077, 1105, 0, NULL, NULL},
+{1078, 1245, 0, NULL, NULL},
+{1079, 1247, 0, NULL, NULL},
+{1080, 1253, 0, NULL, NULL},
+{1086, 1255, 0, NULL, NULL},
+{1091, 1265, 0, NULL, NULL},
+{1095, 1269, 0, NULL, NULL},
+{1099, 1273, 0, NULL, NULL},
+{1110, 1111, 0, NULL, NULL}
+}; /* compose_tab_7 */ 
+static int hash_compose_tab_8_12[12] = 
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_8_12 */ 
+static CompEntry compose_tab_8_12[] = {
+{65, 7848, 0, NULL, NULL},
+{69, 7874, 0, NULL, NULL},
+{79, 7892, 0, NULL, NULL},
+{97, 7849, 0, NULL, NULL},
+{101, 7875, 0, NULL, NULL},
+{111, 7893, 0, NULL, NULL}
+}; /* compose_tab_8_12 */ 
+static int hash_compose_tab_8_13[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_8_13 */ 
+static CompEntry compose_tab_8_13[] = {
+{65, 7858, 0, NULL, NULL},
+{97, 7859, 0, NULL, NULL}
+}; /* compose_tab_8_13 */ 
+static int hash_compose_tab_8_14[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_8_14 */ 
+static CompEntry compose_tab_8_14[] = {
+{79, 7902, 0, NULL, NULL},
+{85, 7916, 0, NULL, NULL},
+{111, 7903, 0, NULL, NULL},
+{117, 7917, 0, NULL, NULL}
+}; /* compose_tab_8_14 */ 
+static int hash_compose_tab_8[30] = 
+{-1,11,-1,-1,-1,0,-1,6,-1,1,-1,7,-1,2,-1,8,14,-1,-1,3,12,9,-1,-1,13,4,-1,10,
+ -1,5}; /* hash_compose_tab_8 */ 
+static CompEntry compose_tab_8[] = {
+{65, 7842, 0, NULL, NULL},
+{69, 7866, 0, NULL, NULL},
+{73, 7880, 0, NULL, NULL},
+{79, 7886, 0, NULL, NULL},
+{85, 7910, 0, NULL, NULL},
+{89, 7926, 0, NULL, NULL},
+{97, 7843, 0, NULL, NULL},
+{101, 7867, 0, NULL, NULL},
+{105, 7881, 0, NULL, NULL},
+{111, 7887, 0, NULL, NULL},
+{117, 7911, 0, NULL, NULL},
+{121, 7927, 0, NULL, NULL},
+{770, 0, 6, compose_tab_8_12, hash_compose_tab_8_12},
+{774, 0, 2, compose_tab_8_13, hash_compose_tab_8_13},
+{795, 0, 4, compose_tab_8_14, hash_compose_tab_8_14}
+}; /* compose_tab_8 */ 
+static int hash_compose_tab_9[12] = 
+{-1,1,2,5,-1,0,-1,-1,-1,3,-1,4}; /* hash_compose_tab_9 */ 
+static CompEntry compose_tab_9[] = {
+{65, 197, 0, NULL, NULL},
+{85, 366, 0, NULL, NULL},
+{97, 229, 0, NULL, NULL},
+{117, 367, 0, NULL, NULL},
+{119, 7832, 0, NULL, NULL},
+{121, 7833, 0, NULL, NULL}
+}; /* compose_tab_9 */ 
+static int hash_compose_tab_10[12] = 
+{-1,1,-1,2,4,-1,-1,0,-1,3,-1,5}; /* hash_compose_tab_10 */ 
+static CompEntry compose_tab_10[] = {
+{79, 336, 0, NULL, NULL},
+{85, 368, 0, NULL, NULL},
+{111, 337, 0, NULL, NULL},
+{117, 369, 0, NULL, NULL},
+{1059, 1266, 0, NULL, NULL},
+{1091, 1267, 0, NULL, NULL}
+}; /* compose_tab_10 */ 
+static int hash_compose_tab_11_33[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_11_33 */ 
+static CompEntry compose_tab_11_33[] = {
+{85, 473, 0, NULL, NULL},
+{117, 474, 0, NULL, NULL}
+}; /* compose_tab_11_33 */ 
+static int hash_compose_tab_11[68] = 
+{2,3,-1,4,-1,5,-1,6,7,-1,8,9,-1,-1,10,11,12,13,-1,-1,-1,-1,14,-1,-1,-1,-1,-1,
+ 33,15,-1,16,17,18,31,19,-1,20,21,22,23,-1,24,25,-1,-1,26,27,28,29,32,-1,-1,
+ -1,30,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,-1,1}; /* hash_compose_tab_11 */ 
+static CompEntry compose_tab_11[] = {
+{65, 461, 0, NULL, NULL},
+{67, 268, 0, NULL, NULL},
+{68, 270, 0, NULL, NULL},
+{69, 282, 0, NULL, NULL},
+{71, 486, 0, NULL, NULL},
+{73, 463, 0, NULL, NULL},
+{75, 488, 0, NULL, NULL},
+{76, 317, 0, NULL, NULL},
+{78, 327, 0, NULL, NULL},
+{79, 465, 0, NULL, NULL},
+{82, 344, 0, NULL, NULL},
+{83, 352, 0, NULL, NULL},
+{84, 356, 0, NULL, NULL},
+{85, 467, 0, NULL, NULL},
+{90, 381, 0, NULL, NULL},
+{97, 462, 0, NULL, NULL},
+{99, 269, 0, NULL, NULL},
+{100, 271, 0, NULL, NULL},
+{101, 283, 0, NULL, NULL},
+{103, 487, 0, NULL, NULL},
+{105, 464, 0, NULL, NULL},
+{106, 496, 0, NULL, NULL},
+{107, 489, 0, NULL, NULL},
+{108, 318, 0, NULL, NULL},
+{110, 328, 0, NULL, NULL},
+{111, 466, 0, NULL, NULL},
+{114, 345, 0, NULL, NULL},
+{115, 353, 0, NULL, NULL},
+{116, 357, 0, NULL, NULL},
+{117, 468, 0, NULL, NULL},
+{122, 382, 0, NULL, NULL},
+{439, 494, 0, NULL, NULL},
+{658, 495, 0, NULL, NULL},
+{776, 0, 2, compose_tab_11_33, hash_compose_tab_11_33}
+}; /* compose_tab_11 */ 
+static int hash_compose_tab_12_1[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_12_1 */ 
+static CompEntry compose_tab_12_1[] = {
+{953, 912, 0, NULL, NULL},
+{965, 944, 0, NULL, NULL}
+}; /* compose_tab_12_1 */ 
+static int hash_compose_tab_12[34] = 
+{11,4,12,5,-1,-1,-1,13,-1,6,-1,-1,-1,14,-1,7,-1,15,-1,8,-1,-1,-1,-1,-1,-1,16,
+ 9,1,2,-1,10,0,3}; /* hash_compose_tab_12 */ 
+static CompEntry compose_tab_12[] = {
+{168, 901, 0, NULL, NULL},
+{776, 0, 2, compose_tab_12_1, hash_compose_tab_12_1},
+{913, 902, 0, NULL, NULL},
+{917, 904, 0, NULL, NULL},
+{919, 905, 0, NULL, NULL},
+{921, 906, 0, NULL, NULL},
+{927, 908, 0, NULL, NULL},
+{933, 910, 0, NULL, NULL},
+{937, 911, 0, NULL, NULL},
+{945, 940, 0, NULL, NULL},
+{949, 941, 0, NULL, NULL},
+{951, 942, 0, NULL, NULL},
+{953, 943, 0, NULL, NULL},
+{959, 972, 0, NULL, NULL},
+{965, 973, 0, NULL, NULL},
+{969, 974, 0, NULL, NULL},
+{978, 979, 0, NULL, NULL}
+}; /* compose_tab_12 */ 
+static int hash_compose_tab_13[28] = 
+{-1,5,10,-1,-1,11,-1,-1,-1,0,-1,-1,-1,1,6,-1,-1,2,7,-1,12,8,13,3,-1,-1,4,9}; /* hash_compose_tab_13 */ 
+static CompEntry compose_tab_13[] = {
+{65, 512, 0, NULL, NULL},
+{69, 516, 0, NULL, NULL},
+{73, 520, 0, NULL, NULL},
+{79, 524, 0, NULL, NULL},
+{82, 528, 0, NULL, NULL},
+{85, 532, 0, NULL, NULL},
+{97, 513, 0, NULL, NULL},
+{101, 517, 0, NULL, NULL},
+{105, 521, 0, NULL, NULL},
+{111, 525, 0, NULL, NULL},
+{114, 529, 0, NULL, NULL},
+{117, 533, 0, NULL, NULL},
+{1140, 1142, 0, NULL, NULL},
+{1141, 1143, 0, NULL, NULL}
+}; /* compose_tab_13 */ 
+static int hash_compose_tab_14[24] = 
+{-1,2,6,-1,-1,7,-1,3,-1,8,4,-1,-1,5,-1,9,-1,0,10,-1,-1,1,11,-1}; /* hash_compose_tab_14 */ 
+static CompEntry compose_tab_14[] = {
+{65, 514, 0, NULL, NULL},
+{69, 518, 0, NULL, NULL},
+{73, 522, 0, NULL, NULL},
+{79, 526, 0, NULL, NULL},
+{82, 530, 0, NULL, NULL},
+{85, 534, 0, NULL, NULL},
+{97, 515, 0, NULL, NULL},
+{101, 519, 0, NULL, NULL},
+{105, 523, 0, NULL, NULL},
+{111, 527, 0, NULL, NULL},
+{114, 531, 0, NULL, NULL},
+{117, 535, 0, NULL, NULL}
+}; /* compose_tab_14 */ 
+static int hash_compose_tab_15_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_15_0 */ 
+static CompEntry compose_tab_15_0[] = {
+{913, 8072, 0, NULL, NULL},
+{919, 8088, 0, NULL, NULL},
+{937, 8104, 0, NULL, NULL},
+{945, 8064, 0, NULL, NULL},
+{951, 8080, 0, NULL, NULL},
+{969, 8096, 0, NULL, NULL}
+}; /* compose_tab_15_0 */ 
+static int hash_compose_tab_15[30] = 
+{-1,12,-1,-1,-1,13,-1,6,-1,14,-1,-1,-1,1,-1,7,-1,2,-1,3,8,4,9,10,-1,-1,-1,0,5,
+ 11}; /* hash_compose_tab_15 */ 
+static CompEntry compose_tab_15[] = {
+{837, 0, 6, compose_tab_15_0, hash_compose_tab_15_0},
+{913, 7944, 0, NULL, NULL},
+{917, 7960, 0, NULL, NULL},
+{919, 7976, 0, NULL, NULL},
+{921, 7992, 0, NULL, NULL},
+{927, 8008, 0, NULL, NULL},
+{937, 8040, 0, NULL, NULL},
+{945, 7936, 0, NULL, NULL},
+{949, 7952, 0, NULL, NULL},
+{951, 7968, 0, NULL, NULL},
+{953, 7984, 0, NULL, NULL},
+{959, 8000, 0, NULL, NULL},
+{961, 8164, 0, NULL, NULL},
+{965, 8016, 0, NULL, NULL},
+{969, 8032, 0, NULL, NULL}
+}; /* compose_tab_15 */ 
+static int hash_compose_tab_16_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_16_0 */ 
+static CompEntry compose_tab_16_0[] = {
+{913, 8073, 0, NULL, NULL},
+{919, 8089, 0, NULL, NULL},
+{937, 8105, 0, NULL, NULL},
+{945, 8065, 0, NULL, NULL},
+{951, 8081, 0, NULL, NULL},
+{969, 8097, 0, NULL, NULL}
+}; /* compose_tab_16_0 */ 
+static int hash_compose_tab_16[34] = 
+{11,3,12,4,-1,-1,-1,13,-1,5,14,6,-1,15,-1,7,-1,16,-1,8,-1,0,-1,-1,-1,-1,-1,9,
+ -1,1,-1,10,-1,2}; /* hash_compose_tab_16 */ 
+static CompEntry compose_tab_16[] = {
+{837, 0, 6, compose_tab_16_0, hash_compose_tab_16_0},
+{913, 7945, 0, NULL, NULL},
+{917, 7961, 0, NULL, NULL},
+{919, 7977, 0, NULL, NULL},
+{921, 7993, 0, NULL, NULL},
+{927, 8009, 0, NULL, NULL},
+{929, 8172, 0, NULL, NULL},
+{933, 8025, 0, NULL, NULL},
+{937, 8041, 0, NULL, NULL},
+{945, 7937, 0, NULL, NULL},
+{949, 7953, 0, NULL, NULL},
+{951, 7969, 0, NULL, NULL},
+{953, 7985, 0, NULL, NULL},
+{959, 8001, 0, NULL, NULL},
+{961, 8165, 0, NULL, NULL},
+{965, 8017, 0, NULL, NULL},
+{969, 8033, 0, NULL, NULL}
+}; /* compose_tab_16 */ 
+static int hash_compose_tab_17[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_17 */ 
+static CompEntry compose_tab_17[] = {
+{79, 416, 0, NULL, NULL},
+{85, 431, 0, NULL, NULL},
+{111, 417, 0, NULL, NULL},
+{117, 432, 0, NULL, NULL}
+}; /* compose_tab_17 */ 
+static int hash_compose_tab_18_38[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_18_38 */ 
+static CompEntry compose_tab_18_38[] = {
+{79, 7906, 0, NULL, NULL},
+{85, 7920, 0, NULL, NULL},
+{111, 7907, 0, NULL, NULL},
+{117, 7921, 0, NULL, NULL}
+}; /* compose_tab_18_38 */ 
+static int hash_compose_tab_18[78] = 
+{9,10,-1,-1,11,12,13,14,15,16,-1,17,18,-1,-1,38,-1,-1,-1,19,20,-1,21,22,-1,-1,
+ 23,24,-1,25,26,27,28,29,-1,-1,30,31,32,33,34,35,-1,36,37,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,-1,2,3,-1,-1,4,5,-1,6,7,8}; /* hash_compose_tab_18 */ 
+static CompEntry compose_tab_18[] = {
+{65, 7840, 0, NULL, NULL},
+{66, 7684, 0, NULL, NULL},
+{68, 7692, 0, NULL, NULL},
+{69, 7864, 0, NULL, NULL},
+{72, 7716, 0, NULL, NULL},
+{73, 7882, 0, NULL, NULL},
+{75, 7730, 0, NULL, NULL},
+{76, 7734, 0, NULL, NULL},
+{77, 7746, 0, NULL, NULL},
+{78, 7750, 0, NULL, NULL},
+{79, 7884, 0, NULL, NULL},
+{82, 7770, 0, NULL, NULL},
+{83, 7778, 0, NULL, NULL},
+{84, 7788, 0, NULL, NULL},
+{85, 7908, 0, NULL, NULL},
+{86, 7806, 0, NULL, NULL},
+{87, 7816, 0, NULL, NULL},
+{89, 7924, 0, NULL, NULL},
+{90, 7826, 0, NULL, NULL},
+{97, 7841, 0, NULL, NULL},
+{98, 7685, 0, NULL, NULL},
+{100, 7693, 0, NULL, NULL},
+{101, 7865, 0, NULL, NULL},
+{104, 7717, 0, NULL, NULL},
+{105, 7883, 0, NULL, NULL},
+{107, 7731, 0, NULL, NULL},
+{108, 7735, 0, NULL, NULL},
+{109, 7747, 0, NULL, NULL},
+{110, 7751, 0, NULL, NULL},
+{111, 7885, 0, NULL, NULL},
+{114, 7771, 0, NULL, NULL},
+{115, 7779, 0, NULL, NULL},
+{116, 7789, 0, NULL, NULL},
+{117, 7909, 0, NULL, NULL},
+{118, 7807, 0, NULL, NULL},
+{119, 7817, 0, NULL, NULL},
+{121, 7925, 0, NULL, NULL},
+{122, 7827, 0, NULL, NULL},
+{795, 0, 4, compose_tab_18_38, hash_compose_tab_18_38}
+}; /* compose_tab_18 */ 
+static int hash_compose_tab_19[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_19 */ 
+static CompEntry compose_tab_19[] = {
+{85, 7794, 0, NULL, NULL},
+{117, 7795, 0, NULL, NULL}
+}; /* compose_tab_19 */ 
+static int hash_compose_tab_20[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_20 */ 
+static CompEntry compose_tab_20[] = {
+{65, 7680, 0, NULL, NULL},
+{97, 7681, 0, NULL, NULL}
+}; /* compose_tab_20 */ 
+static int hash_compose_tab_21[40] = 
+{-1,-1,7,8,9,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,10,11,-1,-1,12,13,-1,
+ -1,0,1,14,15,2,3,16,17,4,5,18,6,19}; /* hash_compose_tab_21 */ 
+static CompEntry compose_tab_21[] = {
+{67, 199, 0, NULL, NULL},
+{68, 7696, 0, NULL, NULL},
+{71, 290, 0, NULL, NULL},
+{72, 7720, 0, NULL, NULL},
+{75, 310, 0, NULL, NULL},
+{76, 315, 0, NULL, NULL},
+{78, 325, 0, NULL, NULL},
+{82, 342, 0, NULL, NULL},
+{83, 350, 0, NULL, NULL},
+{84, 354, 0, NULL, NULL},
+{99, 231, 0, NULL, NULL},
+{100, 7697, 0, NULL, NULL},
+{103, 291, 0, NULL, NULL},
+{104, 7721, 0, NULL, NULL},
+{107, 311, 0, NULL, NULL},
+{108, 316, 0, NULL, NULL},
+{110, 326, 0, NULL, NULL},
+{114, 343, 0, NULL, NULL},
+{115, 351, 0, NULL, NULL},
+{116, 355, 0, NULL, NULL}
+}; /* compose_tab_21 */ 
+static int hash_compose_tab_22[20] = 
+{-1,6,-1,-1,-1,0,4,7,-1,1,-1,8,-1,2,-1,-1,-1,5,9,3}; /* hash_compose_tab_22 */ 
+static CompEntry compose_tab_22[] = {
+{65, 260, 0, NULL, NULL},
+{69, 280, 0, NULL, NULL},
+{73, 302, 0, NULL, NULL},
+{79, 490, 0, NULL, NULL},
+{85, 370, 0, NULL, NULL},
+{97, 261, 0, NULL, NULL},
+{101, 281, 0, NULL, NULL},
+{105, 303, 0, NULL, NULL},
+{111, 491, 0, NULL, NULL},
+{117, 371, 0, NULL, NULL}
+}; /* compose_tab_22 */ 
+static int hash_compose_tab_23[24] = 
+{-1,-1,-1,-1,2,6,3,7,-1,-1,-1,-1,4,5,8,9,-1,-1,-1,-1,0,1,10,11}; /* hash_compose_tab_23 */ 
+static CompEntry compose_tab_23[] = {
+{68, 7698, 0, NULL, NULL},
+{69, 7704, 0, NULL, NULL},
+{76, 7740, 0, NULL, NULL},
+{78, 7754, 0, NULL, NULL},
+{84, 7792, 0, NULL, NULL},
+{85, 7798, 0, NULL, NULL},
+{100, 7699, 0, NULL, NULL},
+{101, 7705, 0, NULL, NULL},
+{108, 7741, 0, NULL, NULL},
+{110, 7755, 0, NULL, NULL},
+{116, 7793, 0, NULL, NULL},
+{117, 7799, 0, NULL, NULL}
+}; /* compose_tab_23 */ 
+static int hash_compose_tab_24[4] = 
+{0,1,-1,-1}; /* hash_compose_tab_24 */ 
+static CompEntry compose_tab_24[] = {
+{72, 7722, 0, NULL, NULL},
+{104, 7723, 0, NULL, NULL}
+}; /* compose_tab_24 */ 
+static int hash_compose_tab_25[12] = 
+{-1,1,2,-1,-1,3,-1,-1,-1,0,4,5}; /* hash_compose_tab_25 */ 
+static CompEntry compose_tab_25[] = {
+{69, 7706, 0, NULL, NULL},
+{73, 7724, 0, NULL, NULL},
+{85, 7796, 0, NULL, NULL},
+{101, 7707, 0, NULL, NULL},
+{105, 7725, 0, NULL, NULL},
+{117, 7797, 0, NULL, NULL}
+}; /* compose_tab_25 */ 
+static int hash_compose_tab_26[34] = 
+{1,-1,10,-1,-1,11,12,2,3,13,4,-1,14,-1,5,15,6,-1,-1,-1,16,-1,7,-1,-1,-1,-1,-1,
+ -1,-1,8,-1,0,9}; /* hash_compose_tab_26 */ 
+static CompEntry compose_tab_26[] = {
+{66, 7686, 0, NULL, NULL},
+{68, 7694, 0, NULL, NULL},
+{75, 7732, 0, NULL, NULL},
+{76, 7738, 0, NULL, NULL},
+{78, 7752, 0, NULL, NULL},
+{82, 7774, 0, NULL, NULL},
+{84, 7790, 0, NULL, NULL},
+{90, 7828, 0, NULL, NULL},
+{98, 7687, 0, NULL, NULL},
+{100, 7695, 0, NULL, NULL},
+{104, 7830, 0, NULL, NULL},
+{107, 7733, 0, NULL, NULL},
+{108, 7739, 0, NULL, NULL},
+{110, 7753, 0, NULL, NULL},
+{114, 7775, 0, NULL, NULL},
+{116, 7791, 0, NULL, NULL},
+{122, 7829, 0, NULL, NULL}
+}; /* compose_tab_26 */ 
+static int hash_compose_tab_27_1[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_27_1 */ 
+static CompEntry compose_tab_27_1[] = {
+{953, 8151, 0, NULL, NULL},
+{965, 8167, 0, NULL, NULL}
+}; /* compose_tab_27_1 */ 
+static int hash_compose_tab_27_2_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_2_0 */ 
+static CompEntry compose_tab_27_2_0[] = {
+{913, 8078, 0, NULL, NULL},
+{919, 8094, 0, NULL, NULL},
+{937, 8110, 0, NULL, NULL},
+{945, 8070, 0, NULL, NULL},
+{951, 8086, 0, NULL, NULL},
+{969, 8102, 0, NULL, NULL}
+}; /* compose_tab_27_2_0 */ 
+static int hash_compose_tab_27_2[20] = 
+{-1,3,-1,-1,-1,5,8,-1,-1,9,-1,6,-1,1,7,-1,-1,0,4,2}; /* hash_compose_tab_27_2 */ 
+static CompEntry compose_tab_27_2[] = {
+{837, 0, 6, compose_tab_27_2_0, hash_compose_tab_27_2_0},
+{913, 7950, 0, NULL, NULL},
+{919, 7982, 0, NULL, NULL},
+{921, 7998, 0, NULL, NULL},
+{937, 8046, 0, NULL, NULL},
+{945, 7942, 0, NULL, NULL},
+{951, 7974, 0, NULL, NULL},
+{953, 7990, 0, NULL, NULL},
+{965, 8022, 0, NULL, NULL},
+{969, 8038, 0, NULL, NULL}
+}; /* compose_tab_27_2 */ 
+static int hash_compose_tab_27_3_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_3_0 */ 
+static CompEntry compose_tab_27_3_0[] = {
+{913, 8079, 0, NULL, NULL},
+{919, 8095, 0, NULL, NULL},
+{937, 8111, 0, NULL, NULL},
+{945, 8071, 0, NULL, NULL},
+{951, 8087, 0, NULL, NULL},
+{969, 8103, 0, NULL, NULL}
+}; /* compose_tab_27_3_0 */ 
+static int hash_compose_tab_27_3[22] = 
+{-1,0,10,-1,-1,7,-1,8,-1,4,-1,1,-1,5,-1,-1,-1,2,-1,3,9,6}; /* hash_compose_tab_27_3 */ 
+static CompEntry compose_tab_27_3[] = {
+{837, 0, 6, compose_tab_27_3_0, hash_compose_tab_27_3_0},
+{913, 7951, 0, NULL, NULL},
+{919, 7983, 0, NULL, NULL},
+{921, 7999, 0, NULL, NULL},
+{933, 8031, 0, NULL, NULL},
+{937, 8047, 0, NULL, NULL},
+{945, 7943, 0, NULL, NULL},
+{951, 7975, 0, NULL, NULL},
+{953, 7991, 0, NULL, NULL},
+{965, 8023, 0, NULL, NULL},
+{969, 8039, 0, NULL, NULL}
+}; /* compose_tab_27_3 */ 
+static int hash_compose_tab_27_4[6] = 
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_27_4 */ 
+static CompEntry compose_tab_27_4[] = {
+{945, 8119, 0, NULL, NULL},
+{951, 8135, 0, NULL, NULL},
+{969, 8183, 0, NULL, NULL}
+}; /* compose_tab_27_4 */ 
+static int hash_compose_tab_27[24] = 
+{0,-1,-1,-1,-1,8,11,-1,1,5,9,-1,-1,-1,-1,6,10,7,-1,2,3,4,-1,-1}; /* hash_compose_tab_27 */ 
+static CompEntry compose_tab_27[] = {
+{168, 8129, 0, NULL, NULL},
+{776, 0, 2, compose_tab_27_1, hash_compose_tab_27_1},
+{787, 0, 10, compose_tab_27_2, hash_compose_tab_27_2},
+{788, 0, 11, compose_tab_27_3, hash_compose_tab_27_3},
+{837, 0, 3, compose_tab_27_4, hash_compose_tab_27_4},
+{945, 8118, 0, NULL, NULL},
+{951, 8134, 0, NULL, NULL},
+{953, 8150, 0, NULL, NULL},
+{965, 8166, 0, NULL, NULL},
+{969, 8182, 0, NULL, NULL},
+{8127, 8143, 0, NULL, NULL},
+{8190, 8159, 0, NULL, NULL}
+}; /* compose_tab_27 */ 
+static int hash_compose_tab_28[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_28 */ 
+static CompEntry compose_tab_28[] = {
+{913, 8124, 0, NULL, NULL},
+{919, 8140, 0, NULL, NULL},
+{937, 8188, 0, NULL, NULL},
+{945, 8115, 0, NULL, NULL},
+{951, 8131, 0, NULL, NULL},
+{969, 8179, 0, NULL, NULL}
+}; /* compose_tab_28 */ 
+static int hash_compose_tab_29[4] = 
+{0,-1,1,-1}; /* hash_compose_tab_29 */ 
+static CompEntry compose_tab_29[] = {
+{1488, 64302, 0, NULL, NULL},
+{1522, 64287, 0, NULL, NULL}
+}; /* compose_tab_29 */ 
+static int hash_compose_tab_30[2] = 
+{0,-1}; /* hash_compose_tab_30 */ 
+static CompEntry compose_tab_30[] = {
+{1488, 64303, 0, NULL, NULL}
+}; /* compose_tab_30 */ 
+static int hash_compose_tab_31[2] = 
+{-1,0}; /* hash_compose_tab_31 */ 
+static CompEntry compose_tab_31[] = {
+{1493, 64331, 0, NULL, NULL}
+}; /* compose_tab_31 */ 
+static int hash_compose_tab_32[44] = 
+{7,8,9,10,11,-1,12,-1,13,14,-1,15,16,-1,17,18,19,20,21,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,5,6,-1}; /* hash_compose_tab_32 */ 
+static CompEntry compose_tab_32[] = {
+{1488, 64304, 0, NULL, NULL},
+{1489, 64305, 0, NULL, NULL},
+{1490, 64306, 0, NULL, NULL},
+{1491, 64307, 0, NULL, NULL},
+{1492, 64308, 0, NULL, NULL},
+{1493, 64309, 0, NULL, NULL},
+{1494, 64310, 0, NULL, NULL},
+{1496, 64312, 0, NULL, NULL},
+{1497, 64313, 0, NULL, NULL},
+{1498, 64314, 0, NULL, NULL},
+{1499, 64315, 0, NULL, NULL},
+{1500, 64316, 0, NULL, NULL},
+{1502, 64318, 0, NULL, NULL},
+{1504, 64320, 0, NULL, NULL},
+{1505, 64321, 0, NULL, NULL},
+{1507, 64323, 0, NULL, NULL},
+{1508, 64324, 0, NULL, NULL},
+{1510, 64326, 0, NULL, NULL},
+{1511, 64327, 0, NULL, NULL},
+{1512, 64328, 0, NULL, NULL},
+{1513, 64329, 0, NULL, NULL},
+{1514, 64330, 0, NULL, NULL}
+}; /* compose_tab_32 */ 
+static int hash_compose_tab_33[6] = 
+{-1,0,2,-1,-1,1}; /* hash_compose_tab_33 */ 
+static CompEntry compose_tab_33[] = {
+{1489, 64332, 0, NULL, NULL},
+{1499, 64333, 0, NULL, NULL},
+{1508, 64334, 0, NULL, NULL}
+}; /* compose_tab_33 */ 
+static int hash_compose_tab_34_0[2] = 
+{-1,0}; /* hash_compose_tab_34_0 */ 
+static CompEntry compose_tab_34_0[] = {
+{1513, 64300, 0, NULL, NULL}
+}; /* compose_tab_34_0 */ 
+static int hash_compose_tab_34[4] = 
+{0,1,-1,-1}; /* hash_compose_tab_34 */ 
+static CompEntry compose_tab_34[] = {
+{1468, 0, 1, compose_tab_34_0, hash_compose_tab_34_0},
+{1513, 64298, 0, NULL, NULL}
+}; /* compose_tab_34 */ 
+static int hash_compose_tab_35_0[2] = 
+{-1,0}; /* hash_compose_tab_35_0 */ 
+static CompEntry compose_tab_35_0[] = {
+{1513, 64301, 0, NULL, NULL}
+}; /* compose_tab_35_0 */ 
+static int hash_compose_tab_35[4] = 
+{0,1,-1,-1}; /* hash_compose_tab_35 */ 
+static CompEntry compose_tab_35[] = {
+{1468, 0, 1, compose_tab_35_0, hash_compose_tab_35_0},
+{1513, 64299, 0, NULL, NULL}
+}; /* compose_tab_35 */ 
+static int hash_compose_tab_36[22] = 
+{3,10,-1,-1,-1,4,5,-1,-1,-1,-1,-1,6,-1,-1,0,1,2,7,8,9,-1}; /* hash_compose_tab_36 */ 
+static CompEntry compose_tab_36[] = {
+{2325, 2392, 0, NULL, NULL},
+{2326, 2393, 0, NULL, NULL},
+{2327, 2394, 0, NULL, NULL},
+{2332, 2395, 0, NULL, NULL},
+{2337, 2396, 0, NULL, NULL},
+{2338, 2397, 0, NULL, NULL},
+{2344, 2345, 0, NULL, NULL},
+{2347, 2398, 0, NULL, NULL},
+{2351, 2399, 0, NULL, NULL},
+{2352, 2353, 0, NULL, NULL},
+{2355, 2356, 0, NULL, NULL}
+}; /* compose_tab_36 */ 
+static int hash_compose_tab_37[8] = 
+{-1,0,1,-1,2,-1,-1,3}; /* hash_compose_tab_37 */ 
+static CompEntry compose_tab_37[] = {
+{2465, 2524, 0, NULL, NULL},
+{2466, 2525, 0, NULL, NULL},
+{2476, 2480, 0, NULL, NULL},
+{2479, 2527, 0, NULL, NULL}
+}; /* compose_tab_37 */ 
+static int hash_compose_tab_38[2] = 
+{-1,0}; /* hash_compose_tab_38 */ 
+static CompEntry compose_tab_38[] = {
+{2503, 2507, 0, NULL, NULL}
+}; /* compose_tab_38 */ 
+static int hash_compose_tab_39[2] = 
+{-1,0}; /* hash_compose_tab_39 */ 
+static CompEntry compose_tab_39[] = {
+{2503, 2508, 0, NULL, NULL}
+}; /* compose_tab_39 */ 
+static int hash_compose_tab_40[10] = 
+{-1,-1,0,1,3,4,-1,-1,2,-1}; /* hash_compose_tab_40 */ 
+static CompEntry compose_tab_40[] = {
+{2582, 2649, 0, NULL, NULL},
+{2583, 2650, 0, NULL, NULL},
+{2588, 2651, 0, NULL, NULL},
+{2593, 2652, 0, NULL, NULL},
+{2603, 2654, 0, NULL, NULL}
+}; /* compose_tab_40 */ 
+static int hash_compose_tab_41[6] = 
+{1,2,-1,-1,-1,0}; /* hash_compose_tab_41 */ 
+static CompEntry compose_tab_41[] = {
+{2849, 2908, 0, NULL, NULL},
+{2850, 2909, 0, NULL, NULL},
+{2863, 2911, 0, NULL, NULL}
+}; /* compose_tab_41 */ 
+static int hash_compose_tab_42[2] = 
+{-1,0}; /* hash_compose_tab_42 */ 
+static CompEntry compose_tab_42[] = {
+{2887, 2891, 0, NULL, NULL}
+}; /* compose_tab_42 */ 
+static int hash_compose_tab_43[2] = 
+{-1,0}; /* hash_compose_tab_43 */ 
+static CompEntry compose_tab_43[] = {
+{2887, 2888, 0, NULL, NULL}
+}; /* compose_tab_43 */ 
+static int hash_compose_tab_44[2] = 
+{-1,0}; /* hash_compose_tab_44 */ 
+static CompEntry compose_tab_44[] = {
+{2887, 2892, 0, NULL, NULL}
+}; /* compose_tab_44 */ 
+static int hash_compose_tab_45[4] = 
+{-1,-1,0,1}; /* hash_compose_tab_45 */ 
+static CompEntry compose_tab_45[] = {
+{3014, 3018, 0, NULL, NULL},
+{3015, 3019, 0, NULL, NULL}
+}; /* compose_tab_45 */ 
+static int hash_compose_tab_46[4] = 
+{-1,-1,0,1}; /* hash_compose_tab_46 */ 
+static CompEntry compose_tab_46[] = {
+{2962, 2964, 0, NULL, NULL},
+{3014, 3020, 0, NULL, NULL}
+}; /* compose_tab_46 */ 
+static int hash_compose_tab_47[2] = 
+{0,-1}; /* hash_compose_tab_47 */ 
+static CompEntry compose_tab_47[] = {
+{3142, 3144, 0, NULL, NULL}
+}; /* compose_tab_47 */ 
+static int hash_compose_tab_48[2] = 
+{0,-1}; /* hash_compose_tab_48 */ 
+static CompEntry compose_tab_48[] = {
+{3270, 3274, 0, NULL, NULL}
+}; /* compose_tab_48 */ 
+static int hash_compose_tab_49_1[2] = 
+{0,-1}; /* hash_compose_tab_49_1 */ 
+static CompEntry compose_tab_49_1[] = {
+{3270, 3275, 0, NULL, NULL}
+}; /* compose_tab_49_1 */ 
+static int hash_compose_tab_49[6] = 
+{2,-1,1,-1,-1,0}; /* hash_compose_tab_49 */ 
+static CompEntry compose_tab_49[] = {
+{3263, 3264, 0, NULL, NULL},
+{3266, 0, 1, compose_tab_49_1, hash_compose_tab_49_1},
+{3270, 3271, 0, NULL, NULL}
+}; /* compose_tab_49 */ 
+static int hash_compose_tab_50[2] = 
+{0,-1}; /* hash_compose_tab_50 */ 
+static CompEntry compose_tab_50[] = {
+{3270, 3272, 0, NULL, NULL}
+}; /* compose_tab_50 */ 
+static int hash_compose_tab_51[4] = 
+{-1,-1,0,1}; /* hash_compose_tab_51 */ 
+static CompEntry compose_tab_51[] = {
+{3398, 3402, 0, NULL, NULL},
+{3399, 3403, 0, NULL, NULL}
+}; /* compose_tab_51 */ 
+static int hash_compose_tab_52[2] = 
+{0,-1}; /* hash_compose_tab_52 */ 
+static CompEntry compose_tab_52[] = {
+{3398, 3404, 0, NULL, NULL}
+}; /* compose_tab_52 */ 
+static int hash_compose_tab_53[2] = 
+{-1,0}; /* hash_compose_tab_53 */ 
+static CompEntry compose_tab_53[] = {
+{3661, 3635, 0, NULL, NULL}
+}; /* compose_tab_53 */ 
+static int hash_compose_tab_54[2] = 
+{-1,0}; /* hash_compose_tab_54 */ 
+static CompEntry compose_tab_54[] = {
+{3789, 3763, 0, NULL, NULL}
+}; /* compose_tab_54 */ 
+static int hash_compose_tab_55_2[4] = 
+{-1,-1,0,1}; /* hash_compose_tab_55_2 */ 
+static CompEntry compose_tab_55_2[] = {
+{4018, 3959, 0, NULL, NULL},
+{4019, 3961, 0, NULL, NULL}
+}; /* compose_tab_55_2 */ 
+static int hash_compose_tab_55[6] = 
+{0,-1,1,2,-1,-1}; /* hash_compose_tab_55 */ 
+static CompEntry compose_tab_55[] = {
+{3954, 3955, 0, NULL, NULL},
+{3956, 3957, 0, NULL, NULL},
+{3968, 0, 2, compose_tab_55_2, hash_compose_tab_55_2}
+}; /* compose_tab_55 */ 
+static int hash_compose_tab_56[4] = 
+{-1,-1,0,1}; /* hash_compose_tab_56 */ 
+static CompEntry compose_tab_56[] = {
+{4018, 3958, 0, NULL, NULL},
+{4019, 3960, 0, NULL, NULL}
+}; /* compose_tab_56 */ 
+static int hash_compose_tab_57[4] = 
+{0,1,-1,-1}; /* hash_compose_tab_57 */ 
+static CompEntry compose_tab_57[] = {
+{3904, 3945, 0, NULL, NULL},
+{3984, 4025, 0, NULL, NULL}
+}; /* compose_tab_57 */ 
+static int hash_compose_tab_58[20] = 
+{-1,2,7,-1,-1,-1,0,3,5,8,-1,4,9,-1,-1,-1,1,6,-1,-1}; /* hash_compose_tab_58 */ 
+static CompEntry compose_tab_58[] = {
+{3906, 3907, 0, NULL, NULL},
+{3916, 3917, 0, NULL, NULL},
+{3921, 3922, 0, NULL, NULL},
+{3926, 3927, 0, NULL, NULL},
+{3931, 3932, 0, NULL, NULL},
+{3986, 3987, 0, NULL, NULL},
+{3996, 3997, 0, NULL, NULL},
+{4001, 4002, 0, NULL, NULL},
+{4006, 4007, 0, NULL, NULL},
+{4011, 4012, 0, NULL, NULL}
+}; /* compose_tab_58 */ 
+static int hash_compose_tab_59[96] = 
+{33,12,34,-1,13,35,14,36,15,37,-1,-1,-1,-1,-1,16,38,-1,17,39,-1,18,40,-1,19,
+ 41,-1,20,42,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,43,44,45,
+ 46,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,21,47,-1,-1,-1,-1,-1,-1,-1,0,22,-1,-1,-1,1,
+ 23,2,24,3,25,4,26,5,27,6,28,7,29,8,30,9,31,10,32,11}; /* hash_compose_tab_59 */ 
+static CompEntry compose_tab_59[] = {
+{12358, 12436, 0, NULL, NULL},
+{12363, 12364, 0, NULL, NULL},
+{12365, 12366, 0, NULL, NULL},
+{12367, 12368, 0, NULL, NULL},
+{12369, 12370, 0, NULL, NULL},
+{12371, 12372, 0, NULL, NULL},
+{12373, 12374, 0, NULL, NULL},
+{12375, 12376, 0, NULL, NULL},
+{12377, 12378, 0, NULL, NULL},
+{12379, 12380, 0, NULL, NULL},
+{12381, 12382, 0, NULL, NULL},
+{12383, 12384, 0, NULL, NULL},
+{12385, 12386, 0, NULL, NULL},
+{12388, 12389, 0, NULL, NULL},
+{12390, 12391, 0, NULL, NULL},
+{12392, 12393, 0, NULL, NULL},
+{12399, 12400, 0, NULL, NULL},
+{12402, 12403, 0, NULL, NULL},
+{12405, 12406, 0, NULL, NULL},
+{12408, 12409, 0, NULL, NULL},
+{12411, 12412, 0, NULL, NULL},
+{12445, 12446, 0, NULL, NULL},
+{12454, 12532, 0, NULL, NULL},
+{12459, 12460, 0, NULL, NULL},
+{12461, 12462, 0, NULL, NULL},
+{12463, 12464, 0, NULL, NULL},
+{12465, 12466, 0, NULL, NULL},
+{12467, 12468, 0, NULL, NULL},
+{12469, 12470, 0, NULL, NULL},
+{12471, 12472, 0, NULL, NULL},
+{12473, 12474, 0, NULL, NULL},
+{12475, 12476, 0, NULL, NULL},
+{12477, 12478, 0, NULL, NULL},
+{12479, 12480, 0, NULL, NULL},
+{12481, 12482, 0, NULL, NULL},
+{12484, 12485, 0, NULL, NULL},
+{12486, 12487, 0, NULL, NULL},
+{12488, 12489, 0, NULL, NULL},
+{12495, 12496, 0, NULL, NULL},
+{12498, 12499, 0, NULL, NULL},
+{12501, 12502, 0, NULL, NULL},
+{12504, 12505, 0, NULL, NULL},
+{12507, 12508, 0, NULL, NULL},
+{12527, 12535, 0, NULL, NULL},
+{12528, 12536, 0, NULL, NULL},
+{12529, 12537, 0, NULL, NULL},
+{12530, 12538, 0, NULL, NULL},
+{12541, 12542, 0, NULL, NULL}
+}; /* compose_tab_59 */ 
+static int hash_compose_tab_60[20] = 
+{-1,7,1,-1,8,2,-1,9,3,-1,-1,4,-1,-1,-1,5,-1,-1,6,0}; /* hash_compose_tab_60 */ 
+static CompEntry compose_tab_60[] = {
+{12399, 12401, 0, NULL, NULL},
+{12402, 12404, 0, NULL, NULL},
+{12405, 12407, 0, NULL, NULL},
+{12408, 12410, 0, NULL, NULL},
+{12411, 12413, 0, NULL, NULL},
+{12495, 12497, 0, NULL, NULL},
+{12498, 12500, 0, NULL, NULL},
+{12501, 12503, 0, NULL, NULL},
+{12504, 12506, 0, NULL, NULL},
+{12507, 12509, 0, NULL, NULL}
+}; /* compose_tab_60 */ 
+static int hash_compose_tab[122] = 
+{30,31,52,60,32,-1,-1,33,-1,34,35,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,-1,5,6,7,8,9,10,11,12,36,13,37,14,
+ 38,15,16,55,40,-1,-1,-1,-1,17,56,-1,-1,-1,-1,-1,41,18,19,20,42,21,22,-1,45,
+ 39,-1,23,24,-1,25,26,-1,-1,-1,-1,-1,-1,-1,-1,48,-1,43,44,51,53,-1,-1,27,46,
+ 54,28,-1,-1,47,-1,-1,-1,-1,49,50,-1,-1,57,-1,58,59,29}; /* hash_compose_tab */ 
+static CompEntry compose_tab[] = {
+{768, 0, 39, compose_tab_0, hash_compose_tab_0},
+{769, 0, 70, compose_tab_1, hash_compose_tab_1},
+{770, 0, 27, compose_tab_2, hash_compose_tab_2},
+{771, 0, 19, compose_tab_3, hash_compose_tab_3},
+{772, 0, 28, compose_tab_4, hash_compose_tab_4},
+{774, 0, 30, compose_tab_5, hash_compose_tab_5},
+{775, 0, 40, compose_tab_6, hash_compose_tab_6},
+{776, 0, 50, compose_tab_7, hash_compose_tab_7},
+{777, 0, 15, compose_tab_8, hash_compose_tab_8},
+{778, 0, 6, compose_tab_9, hash_compose_tab_9},
+{779, 0, 6, compose_tab_10, hash_compose_tab_10},
+{780, 0, 34, compose_tab_11, hash_compose_tab_11},
+{781, 0, 17, compose_tab_12, hash_compose_tab_12},
+{783, 0, 14, compose_tab_13, hash_compose_tab_13},
+{785, 0, 12, compose_tab_14, hash_compose_tab_14},
+{787, 0, 15, compose_tab_15, hash_compose_tab_15},
+{788, 0, 17, compose_tab_16, hash_compose_tab_16},
+{795, 0, 4, compose_tab_17, hash_compose_tab_17},
+{803, 0, 39, compose_tab_18, hash_compose_tab_18},
+{804, 0, 2, compose_tab_19, hash_compose_tab_19},
+{805, 0, 2, compose_tab_20, hash_compose_tab_20},
+{807, 0, 20, compose_tab_21, hash_compose_tab_21},
+{808, 0, 10, compose_tab_22, hash_compose_tab_22},
+{813, 0, 12, compose_tab_23, hash_compose_tab_23},
+{814, 0, 2, compose_tab_24, hash_compose_tab_24},
+{816, 0, 6, compose_tab_25, hash_compose_tab_25},
+{817, 0, 17, compose_tab_26, hash_compose_tab_26},
+{834, 0, 12, compose_tab_27, hash_compose_tab_27},
+{837, 0, 6, compose_tab_28, hash_compose_tab_28},
+{1463, 0, 2, compose_tab_29, hash_compose_tab_29},
+{1464, 0, 1, compose_tab_30, hash_compose_tab_30},
+{1465, 0, 1, compose_tab_31, hash_compose_tab_31},
+{1468, 0, 22, compose_tab_32, hash_compose_tab_32},
+{1471, 0, 3, compose_tab_33, hash_compose_tab_33},
+{1473, 0, 2, compose_tab_34, hash_compose_tab_34},
+{1474, 0, 2, compose_tab_35, hash_compose_tab_35},
+{2364, 0, 11, compose_tab_36, hash_compose_tab_36},
+{2492, 0, 4, compose_tab_37, hash_compose_tab_37},
+{2494, 0, 1, compose_tab_38, hash_compose_tab_38},
+{2519, 0, 1, compose_tab_39, hash_compose_tab_39},
+{2620, 0, 5, compose_tab_40, hash_compose_tab_40},
+{2876, 0, 3, compose_tab_41, hash_compose_tab_41},
+{2878, 0, 1, compose_tab_42, hash_compose_tab_42},
+{2902, 0, 1, compose_tab_43, hash_compose_tab_43},
+{2903, 0, 1, compose_tab_44, hash_compose_tab_44},
+{3006, 0, 2, compose_tab_45, hash_compose_tab_45},
+{3031, 0, 2, compose_tab_46, hash_compose_tab_46},
+{3158, 0, 1, compose_tab_47, hash_compose_tab_47},
+{3266, 0, 1, compose_tab_48, hash_compose_tab_48},
+{3285, 0, 3, compose_tab_49, hash_compose_tab_49},
+{3286, 0, 1, compose_tab_50, hash_compose_tab_50},
+{3390, 0, 2, compose_tab_51, hash_compose_tab_51},
+{3415, 0, 1, compose_tab_52, hash_compose_tab_52},
+{3634, 0, 1, compose_tab_53, hash_compose_tab_53},
+{3762, 0, 1, compose_tab_54, hash_compose_tab_54},
+{3953, 0, 3, compose_tab_55, hash_compose_tab_55},
+{3968, 0, 2, compose_tab_56, hash_compose_tab_56},
+{4021, 0, 2, compose_tab_57, hash_compose_tab_57},
+{4023, 0, 10, compose_tab_58, hash_compose_tab_58},
+{12441, 0, 48, compose_tab_59, hash_compose_tab_59},
+{12442, 0, 10, compose_tab_60, hash_compose_tab_60}
+}; /* compose_tab */ 
+#define COMP_CANDIDATE_MAP_OFFSET 24
+static Uint32 comp_candidate_map[] = {
+    0x081ABFDFU,
+    0x000361B8U,
+    0x00000024U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x93800000U,
+    0x00000006U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x10000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x50000000U,
+    0x00800000U,
+    0x00000000U,
+    0x00000000U,
+    0x10000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x50000000U,
+    0x00C00000U,
+    0x00000000U,
+    0x00000000U,
+    0x40000000U,
+    0x00800000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00400000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00600004U,
+    0x00000000U,
+    0x00000000U,
+    0x40000000U,
+    0x00800000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00040000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00040000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00020000U,
+    0x00000001U,
+    0x00A00000U
+};
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 50b3e5b61c..e7fd144ec3 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -47,13 +47,23 @@
 #define SEQ_TRACE 1
 
 #define CONTEXT_REDS 2000	/* Swap process out after this number */
-#define MAX_ARG 256	        /* Max number of arguments allowed */
+#define MAX_ARG 255	        /* Max number of arguments allowed */
 #define MAX_REG 1024            /* Max number of x(N) registers used */
 
+/* Scheduler stores data for temporary heaps if
+   !HEAP_ON_C_STACK. Macros (*TmpHeap*) in global.h selects if we put temporary
+   heap data on the C stack or if we use the buffers in the scheduler data. */
+#define TMP_HEAP_SIZE 128            /* Number of Eterm in the schedulers
+				        small heap for transient heap data */
+#define CMP_TMP_HEAP_SIZE       2    /* cmp wants its own tmp-heap... */
+#define ERL_ARITH_TMP_HEAP_SIZE 4    /* as does erl_arith... */
+#define BEAM_EMU_TMP_HEAP_SIZE  2    /* and beam_emu... */
+
 /*
  * The new arithmetic operations need some extra X registers in the register array.
+ * so does the gc_bif's (i_gc_bif3 need 3 extra).
  */
-#define ERTS_X_REGS_ALLOCATED (MAX_REG+2)
+#define ERTS_X_REGS_ALLOCATED (MAX_REG+3)
 
 #define INPUT_REDUCTIONS (2 * CONTEXT_REDS)
 
@@ -74,6 +84,7 @@
 
 #define ErtsHAllocLockCheck(P) \
   ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks((P))) \
+      	             || ((P)->id == ERTS_INVALID_PID) \
 		     || ((P)->scheduler_data \
 			 && (P) == (P)->scheduler_data->match_pseudo_process) \
 		     || erts_is_system_blocked(0))
@@ -109,14 +120,15 @@
  * Allocate heap memory, first on the ordinary heap;
  * failing that, in a heap fragment.
  */
-#define HAlloc(p, sz)			                              \
+#define HAllocX(p, sz, xtra)		                              \
     (ASSERT_EXPR((sz) >= 0),					      \
      ErtsHAllocLockCheck(p),					      \
      (IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \
-      ? erts_heap_alloc((p),(sz))                                     \
+      ? erts_heap_alloc((p),(sz),(xtra))                              \
       : (INIT_HEAP_MEM(p,sz),		                              \
          HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz))))
 
+#define HAlloc(P, SZ) HAllocX(P,SZ,0)
 
 #define HRelease(p, endp, ptr)					\
   if ((ptr) == (endp)) {					\
@@ -130,8 +142,12 @@
 #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p))
 
 #if defined(DEBUG) || defined(CHECK_FOR_HOLES)
+#if HALFWORD_HEAP
+# define ERTS_HOLE_MARKER (0xaf5e78ccU)
+#else
 # define ERTS_HOLE_MARKER (((0xaf5e78ccUL << 24) << 8) | 0xaf5e78ccUL)
 #endif
+#endif
 
 /*
  * Allocate heap memory on the ordinary heap, NEVER in a heap
@@ -184,6 +200,7 @@ extern int BIN_VH_MIN_SIZE;	/* minimum virtual (bin) heap */
 extern int erts_atom_table_size;/* Atom table size */
 
 #define ORIG_CREATION 0
+#define INTERNAL_CREATION 255
 
 /* macros for extracting bytes from uint16's */
 
diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h
index 4930def4ed..ddc2c1396d 100644
--- a/erts/emulator/beam/error.h
+++ b/erts/emulator/beam/error.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -187,10 +187,10 @@ extern Eterm exception_tag[NUMBER_EXC_TAGS];
 struct StackTrace {
     Eterm header;	/* bignum header - must be first in struct */
     Eterm freason; /* original exception reason is saved in the struct */
-    Eterm* pc;
-    Eterm* current;
+    BeamInstr* pc;
+    BeamInstr* current;
     int depth;	/* number of saved pointers in trace[] */
-    Eterm *trace[1];  /* varying size - must be last in struct */
+    BeamInstr *trace[1];  /* varying size - must be last in struct */
 };
 
 #endif /* __ERROR_H__ */
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 271b40cf0f..5bc402fe22 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -43,11 +43,9 @@ static erts_smp_rwmtx_t export_table_lock; /* Locks the secondary export table.
 #define export_read_unlock()	erts_smp_rwmtx_runlock(&export_table_lock)
 #define export_write_lock()	erts_smp_rwmtx_rwlock(&export_table_lock)
 #define export_write_unlock()	erts_smp_rwmtx_rwunlock(&export_table_lock)
-#define export_init_lock()	erts_smp_rwmtx_init(&export_table_lock, \
-						    "export_tab")
 
-extern Eterm* em_call_error_handler;
-extern Uint* em_call_traced_function;
+extern BeamInstr* em_call_error_handler;
+extern BeamInstr* em_call_traced_function;
 
 void
 export_info(int to, void *to_arg)
@@ -93,7 +91,7 @@ export_alloc(Export* tmpl)
     obj->code[2] = tmpl->code[2];
     obj->slot.index = -1;
     obj->address = obj->code+3;
-    obj->code[3] = (Eterm) em_call_error_handler;
+    obj->code[3] = (BeamInstr) em_call_error_handler;
     obj->code[4] = 0;
     obj->match_prog_set = NULL;
     return obj;
@@ -111,8 +109,12 @@ void
 init_export_table(void)
 {
     HashFunctions f;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+    erts_smp_rwmtx_init_opt(&export_table_lock, &rwmtx_opt, "export_tab");
 
-    export_init_lock();
     f.hash = (H_FUN) export_hash;
     f.cmp  = (HCMP_FUN) export_cmp;
     f.alloc = (HALLOC_FUN) export_alloc;
@@ -140,7 +142,7 @@ init_export_table(void)
 Export*
 erts_find_export_entry(Eterm m, Eterm f, unsigned int a)
 {
-    HashValue hval = EXPORT_HASH(m, f, a);
+    HashValue hval = EXPORT_HASH((BeamInstr) m, (BeamInstr) f, (BeamInstr) a);
     int ix;
     HashBucket* b;
 
@@ -185,7 +187,7 @@ erts_find_function(Eterm m, Eterm f, unsigned int a)
 
     ep = hash_get(&export_table.htable, (void*) &e);
     if (ep != NULL && ep->address == ep->code+3 &&
-	ep->code[3] != (Uint) em_call_traced_function) {
+	ep->code[3] != (BeamInstr) em_call_traced_function) {
 	ep = NULL;
     }
     return ep;
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index cd6af6dd85..c604fdf7c3 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -37,7 +37,7 @@ typedef struct export
     void* address;		/* Pointer to code for function. */
     struct binary* match_prog_set; /* Match program for tracing. */
 
-    Eterm fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */
+    BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */
     /*
      * code[0]: Tagged atom for module.
      * code[1]: Tagged atom for function.
@@ -52,7 +52,7 @@ typedef struct export
      *		on_load function that has not been run yet.
      *		Otherwise: 0.
      */
-    Eterm code[5];
+    BeamInstr code[5];
 } Export;
 
 
@@ -74,6 +74,6 @@ Export *export_get(Export*);
 #include "beam_load.h" /* For em_* extern declarations */ 
 #define ExportIsBuiltIn(EntryPtr) 			\
 (((EntryPtr)->address == (EntryPtr)->code + 3) && 	\
- ((EntryPtr)->code[3] == (Uint) em_apply_bif))
+ ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif))
 
 #endif
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 099eddd195..1a102f7187 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -49,10 +49,8 @@
 #define in_area(ptr,start,nbytes) ((Uint)((char*)(ptr) - (char*)(start)) < (nbytes))
 
 #define MAX_STRING_LEN 0xffff
-#define dec_set_creation(nodename,creat)				\
-  (((nodename) == erts_this_node->sysname && (creat) == ORIG_CREATION)	\
-   ? erts_this_node->creation						\
-   : (creat))
+
+#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION || (Cre) == INTERNAL_CREATION)
 
 #undef ERTS_DEBUG_USE_DIST_SEP
 #ifdef DEBUG
@@ -65,11 +63,9 @@
 #  endif
 #endif
 
-/*
- * For backward compatibility reasons, only encode integers that
- * fit in 28 bits (signed) using INTEGER_EXT.
+/* Does Sint fit in Sint32?
  */
-#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2)
+#define IS_SSMALL32(x) (((Uint) (((x) >> (32-1)) + 1)) < 2)
 
 /*
  *   Valid creations for nodes are 1, 2, or 3. 0 can also be sent
@@ -85,14 +81,14 @@
  *
  */
 
-static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
+static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap);
 static Uint is_external_string(Eterm obj, int* p_is_string);
 static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
 static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
 static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
 static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
 static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
-static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins);
+static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins, int internal_tags);
 
 
 static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned);
@@ -463,6 +459,12 @@ Uint erts_encode_ext_size(Eterm term)
 	+ 1 /* VERSION_MAGIC */;
 }
 
+Uint erts_encode_ext_size_ets(Eterm term)
+{
+    return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS);
+}
+
+
 void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp)
 {
     byte *ep = *ext;
@@ -470,7 +472,7 @@ void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap
     if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
 #endif
 	*ep++ = VERSION_MAGIC;
-    ep = enc_term(acmp, term, ep, flags);
+    ep = enc_term(acmp, term, ep, flags, NULL);
     if (!ep)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "%s:%d:erts_encode_dist_ext(): Internal data structure error\n",
@@ -482,7 +484,7 @@ void erts_encode_ext(Eterm term, byte **ext)
 {
     byte *ep = *ext;
     *ep++ = VERSION_MAGIC;
-    ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS);
+    ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL);
     if (!ep)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "%s:%d:erts_encode_ext(): Internal data structure error\n",
@@ -490,6 +492,12 @@ void erts_encode_ext(Eterm term, byte **ext)
     *ext = ep;
 }
 
+byte* erts_encode_ext_ets(Eterm term, byte *ep, struct erl_off_heap_header** off_heap)
+{
+    return enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS,
+		    off_heap);
+}
+
 ErtsDistExternal *
 erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize)
 {
@@ -504,7 +512,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize)
     ASSERT(edep->ext_endp >= edep->extp);
     ext_sz = edep->ext_endp - edep->extp;
 
-    align_sz = ERTS_WORD_ALIGN_PAD_SZ(dist_ext_sz + ext_sz);
+    align_sz = ERTS_EXTRA_DATA_ALIGN_SZ(dist_ext_sz + ext_sz);
 
     new_edep = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA,
 			  dist_ext_sz + ext_sz + align_sz + xsize);
@@ -815,7 +823,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins)
 	    goto fail;
 	ep = edep->extp+1;
     }
-    res = decoded_size(ep, edep->ext_endp, no_refc_bins);
+    res = decoded_size(ep, edep->ext_endp, no_refc_bins, 0);
     if (res >= 0)
 	return res;
  fail:
@@ -827,9 +835,17 @@ Sint erts_decode_ext_size(byte *ext, Uint size, int no_refc_bins)
 {
     if (size == 0 || *ext != VERSION_MAGIC)
 	return -1;
-    return decoded_size(ext+1, ext+size, no_refc_bins);
+    return decoded_size(ext+1, ext+size, no_refc_bins, 0);
+}
+
+Sint erts_decode_ext_size_ets(byte *ext, Uint size)
+{
+    Sint sz = decoded_size(ext, ext+size, 0, 1);
+    ASSERT(sz >= 0);
+    return sz;
 }
 
+
 /*
 ** hpp is set to either a &p->htop or
 ** a pointer to a memory pointer (form message buffers)
@@ -889,7 +905,13 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
     return obj;
 }
 
-
+Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext)
+{
+    Eterm obj;
+    ext = dec_term(NULL, hpp, ext, off_heap, &obj);
+    ASSERT(ext);
+    return obj;
+}
 
 /**********************************************************************/
 
@@ -966,6 +988,7 @@ term_to_binary_1(Process* p, Eterm Term)
     return erts_term_to_binary(p, Term, 0, TERM_TO_BINARY_DFLAGS);
 }
 
+
 Eterm
 term_to_binary_2(Process* p, Eterm Term, Eterm Flags)
 {
@@ -1077,7 +1100,7 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
 	    goto error;
 	size = (Sint) dest_len;
     }
-    res = decoded_size(state->extp, state->extp + size, 0);
+    res = decoded_size(state->extp, state->extp + size, 0, 0);
     if (res < 0)
 	goto error;
     return res;
@@ -1185,7 +1208,8 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
         opt = CAR(list_val(opts));
         if (opt == am_safe) {
 	    fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE;
-        } else {
+        }
+	else {
             goto error;
         }
         opts = CDR(list_val(opts));
@@ -1240,7 +1264,7 @@ external_size_1(Process* p, Eterm Term)
 Eterm
 erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags)
 {
-    int size;
+    Uint size;
     Eterm bin;
     size_t real_size;
     byte* endp;
@@ -1257,7 +1281,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags)
 	    bytes = erts_alloc(ERTS_ALC_T_TMP, size);
 	}
 
-	if ((endp = enc_term(NULL, Term, bytes, flags))
+	if ((endp = enc_term(NULL, Term, bytes, flags, NULL))
 	    == NULL) {
 	    erl_exit(1, "%s, line %d: bad term: %x\n",
 		     __FILE__, __LINE__, Term);
@@ -1302,7 +1326,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags)
 	bin = new_binary(p, (byte *)NULL, size);
 	bytes = binary_bytes(bin);
 	bytes[0] = VERSION_MAGIC;
-	if ((endp = enc_term(NULL, Term, bytes+1, flags))	
+	if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL))
 	    == NULL) {
 	    erl_exit(1, "%s, line %d: bad term: %x\n",
 		     __FILE__, __LINE__, Term);
@@ -1332,6 +1356,21 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
 
     ASSERT(is_atom(atom));
 
+    if (dflags & DFLAGS_INTERNAL_TAGS) {
+	Uint aval = atom_val(atom);
+	ASSERT(aval < (1<<24));
+	if (aval >= (1 << 16)) {
+	    *ep++ = ATOM_INTERNAL_REF3;
+	    put_int24(aval, ep);
+	    ep += 3;
+	}
+	else {
+	    *ep++ = ATOM_INTERNAL_REF2;
+	    put_int16(aval, ep);
+	    ep += 2;
+	}
+	return ep;
+    }
     /*
      * term_to_binary/1,2 and the initial distribution message
      * don't use the cache.
@@ -1381,7 +1420,8 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
     ep += 4;
     put_int32(os, ep);
     ep += 4;
-    *ep++ = pid_creation(pid);
+    *ep++ = (is_internal_pid(pid) && (dflags & DFLAGS_INTERNAL_TAGS)) ?
+	INTERNAL_CREATION : pid_creation(pid);
     return ep;
 }
 
@@ -1420,6 +1460,23 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp)
         }
 	ep += len;
 	break;
+    case ATOM_INTERNAL_REF2:
+	n = get_int16(ep);
+	ep += 2;
+	if (n >= atom_table_size()) {
+	    goto error;
+	}
+	*objp = make_atom(n);
+	break;
+    case ATOM_INTERNAL_REF3:
+	n = get_int24(ep);
+	ep += 3;
+	if (n >= atom_table_size()) {
+	    goto error;
+	}
+	*objp = make_atom(n);
+	break;
+
     default:
     error:
 	*objp = NIL;	/* Don't leave a hole in the heap */
@@ -1428,6 +1485,19 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp)
     return ep;
 }
 
+static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation)
+{
+    switch (creation) {
+    case INTERNAL_CREATION:
+	return erts_this_node;
+    case ORIG_CREATION:
+	if (sysname == erts_this_node->sysname) {
+	    creation = erts_this_node->creation;
+	}
+    }
+    return erts_find_or_insert_node(sysname,creation);
+}
+
 static byte*
 dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp)
 {
@@ -1451,18 +1521,20 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
     ep += 4;
     if (ser > ERTS_MAX_PID_SERIAL)
 	return NULL;
-    if ((cre = get_int8(ep)) >= MAX_CREATION)
-	return NULL;
+    cre = get_int8(ep);
     ep += 1;
 
+    if (!is_valid_creation(cre)) {
+	return NULL;
+    }
+    data = make_pid_data(ser, num);
+
     /*
      * We are careful to create the node entry only after all
      * validity tests are done.
      */
-    cre = dec_set_creation(sysname,cre);
-    node = erts_find_or_insert_node(sysname,cre);
+    node = dec_get_node(sysname, cre);
 
-    data = make_pid_data(ser, num);
     if(node == erts_this_node) {
 	*objp = make_internal_pid(data);
     } else {
@@ -1470,11 +1542,11 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
 	*hpp += EXTERNAL_THING_HEAD_SIZE + 1;
 
 	etp->header = make_external_pid_header(1);
-	etp->next = off_heap->externals;
+	etp->next = off_heap->first;
 	etp->node = node;
 	etp->data.ui[0] = data;
 
-	off_heap->externals = etp;
+	off_heap->first = (struct erl_off_heap_header*) etp;
 	*objp = make_external_pid(etp);
     }
     return ep;
@@ -1487,22 +1559,31 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
 #define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3)
 
 static byte*
-enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
+enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
+	 struct erl_off_heap_header** off_heap)
 {
-    DECLARE_ESTACK(s);
+    DECLARE_WSTACK(s);
     Uint n;
     Uint i;
     Uint j;
     Uint* ptr;
     Eterm val;
     FloatDef f;
+#if HALFWORD_HEAP
+    UWord wobj;
+#endif
+
 
     goto L_jump_start;
 
  outer_loop:
-    while (!ESTACK_ISEMPTY(s)) {
-	obj = ESTACK_POP(s);
-	switch (val = ESTACK_POP(s)) {
+    while (!WSTACK_ISEMPTY(s)) {
+#if HALFWORD_HEAP
+	obj = (Eterm) (wobj = WSTACK_POP(s));
+#else
+	obj = WSTACK_POP(s);
+#endif
+	switch (val = WSTACK_POP(s)) {
 	case ENC_TERM:
 	    break;
 	case ENC_ONE_CONS:
@@ -1513,29 +1594,40 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 
 		obj = CAR(cons);
 		tl = CDR(cons);
-		ESTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM);
-		ESTACK_PUSH(s, tl);
+		WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM);
+		WSTACK_PUSH(s, tl);
 	    }
 	    break;
 	case ENC_PATCH_FUN_SIZE:
 	    {
+#if HALFWORD_HEAP
+		byte* size_p = (byte *) wobj;
+#else
 		byte* size_p = (byte *) obj;
-
+#endif
 		put_int32(ep - size_p, size_p);
 	    }
 	    goto outer_loop;
 	case ENC_LAST_ARRAY_ELEMENT:
 	    {
+#if HALFWORD_HEAP
+		Eterm* ptr = (Eterm *) wobj;
+#else
 		Eterm* ptr = (Eterm *) obj;
+#endif
 		obj = *ptr;
 	    }
 	    break;
 	default:		/* ENC_LAST_ARRAY_ELEMENT+1 and upwards */
 	    {
+#if HALFWORD_HEAP
+		Eterm* ptr = (Eterm *) wobj;
+#else
 		Eterm* ptr = (Eterm *) obj;
+#endif
 		obj = *ptr++;
-		ESTACK_PUSH(s, val-1);
-		ESTACK_PUSH(s, (Eterm) ptr);
+		WSTACK_PUSH(s, val-1);
+		WSTACK_PUSH(s, (UWord) ptr);
 	    }
 	    break;
 	}
@@ -1552,19 +1644,23 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 
 	case SMALL_DEF:
 	    {
+		/* From R14B we no longer restrict INTEGER_EXT to 28 bits,
+		 * as done earlier for backward compatibility reasons. */
 		Sint val = signed_val(obj);
 
 		if ((Uint)val < 256) {
 		    *ep++ = SMALL_INTEGER_EXT;
 		    put_int8(val, ep);
 		    ep++;
-		} else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) {
+		} else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) {
 		    *ep++ = INTEGER_EXT;
 		    put_int32(val, ep);
 		    ep += 4;
 		} else {
-		    Eterm tmp_big[2];
-		    Eterm big = small_to_big(val, tmp_big);
+		    DeclareTmpHeapNoproc(tmp_big,2);
+		    Eterm big;
+		    UseTmpHeapNoproc(2);
+		    big = small_to_big(val, tmp_big);
 		    *ep++ = SMALL_BIG_EXT;
 		    n = big_bytes(big);
 		    ASSERT(n < 256);
@@ -1572,23 +1668,38 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		    ep += 1;
 		    *ep++ = big_sign(big);
 		    ep = big_to_bytes(big, ep);
+		    UnUseTmpHeapNoproc(2);
 		}
 	    }
 	    break;
 
 	case BIG_DEF:
-	    if ((n = big_bytes(obj)) < 256) {
-		*ep++ = SMALL_BIG_EXT;
-		put_int8(n, ep);
-		ep += 1;
-	    }
-	    else {
-		*ep++ = LARGE_BIG_EXT;
-		put_int32(n, ep);
-		ep += 4;
+	    {
+		int sign = big_sign(obj);
+		n = big_bytes(obj);
+		if (sizeof(Sint)==4 && n<=4) {
+		    Uint dig = big_digit(obj,0);		   
+		    Sint val = sign ? -dig : dig;
+		    if ((val<0) == sign) {
+			*ep++ = INTEGER_EXT;
+			put_int32(val, ep);
+			ep += 4;
+			break;
+		    }
+		}
+		if (n < 256) {
+		    *ep++ = SMALL_BIG_EXT;
+		    put_int8(n, ep);
+		    ep += 1;
+		}
+		else {
+		    *ep++ = LARGE_BIG_EXT;
+		    put_int32(n, ep);
+		    ep += 4;
+		}
+		*ep++ = sign;
+		ep = big_to_bytes(obj, ep);
 	    }
-	    *ep++ = big_sign(obj);
-	    ep = big_to_bytes(obj, ep);
 	    break;
 
 	case PID_DEF:
@@ -1601,12 +1712,14 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 	    Uint32 *ref_num;
 
 	    ASSERT(dflags & DFLAG_EXTENDED_REFERENCES);
+
 	    *ep++ = NEW_REFERENCE_EXT;
 	    i = ref_no_of_numbers(obj);
 	    put_int16(i, ep);
 	    ep += 2;
 	    ep = enc_atom(acmp,ref_node_name(obj),ep,dflags);
-	    *ep++ = ref_creation(obj);
+	    *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_ref(obj)) ?
+		INTERNAL_CREATION : ref_creation(obj);
 	    ref_num = ref_numbers(obj);
 	    for (j = 0; j < i; j++) {
 		put_int32(ref_num[j], ep);
@@ -1622,7 +1735,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 	    j = port_number(obj);
 	    put_int32(j, ep);
 	    ep += 4;
-	    *ep++ = port_creation(obj);
+	    *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_port(obj)) ?
+		INTERNAL_CREATION : port_creation(obj);
 	    break;
 
 	case LIST_DEF:
@@ -1662,8 +1776,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		ep += 4;
 	    }
 	    if (i > 0) {
-		ESTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1);
-		ESTACK_PUSH(s, (Eterm) ptr);
+		WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1);
+		WSTACK_PUSH(s, (UWord) ptr);
 	    }
 	    break;
 
@@ -1702,6 +1816,41 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		byte* bytes;
 
 		ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize);
+		if (dflags & DFLAGS_INTERNAL_TAGS) {
+		    ProcBin* pb = (ProcBin*) binary_val(obj);
+		    Uint bytesize = pb->size;
+		    if (pb->thing_word == HEADER_SUB_BIN) {
+			ErlSubBin* sub = (ErlSubBin*)pb;
+			pb = (ProcBin*) binary_val(sub->orig);
+			ASSERT(bytesize == sub->size);
+			bytesize += (bitoffs + bitsize + 7) / 8;
+		    }
+		    if (pb->thing_word == HEADER_PROC_BIN
+			&& heap_bin_size(bytesize) > PROC_BIN_SIZE) {
+			ProcBin tmp;
+			if (bitoffs || bitsize) {
+			    *ep++ = BIT_BINARY_INTERNAL_REF;
+			    *ep++ = bitoffs;
+			    *ep++ = bitsize;
+			}
+			else {
+			    *ep++ = BINARY_INTERNAL_REF;
+			}
+			if (pb->flags) {
+			    erts_emasculate_writable_binary(pb);
+			}
+			erts_refc_inc(&pb->val->refc, 2);
+
+			sys_memcpy(&tmp, pb, sizeof(ProcBin));
+			tmp.next = *off_heap;
+			tmp.bytes = bytes;
+			tmp.size = bytesize;
+			sys_memcpy(ep, &tmp, sizeof(ProcBin));
+			*off_heap = (struct erl_off_heap_header*) ep;
+			ep += sizeof(ProcBin);
+			break;
+		    }
+		}
 		if (bitsize == 0) {
 		    /* Plain old byte-sized binary. */
 		    *ep++ = BINARY_EXT;
@@ -1737,16 +1886,16 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		    *ep++ = SMALL_INTEGER_EXT;
 		    *ep++ = bitsize;
 		}
-		break;
 	    }
+	    break;
 	case EXPORT_DEF:
 	    {
-		Export* exp = (Export *) (export_val(obj))[1];
+		Export* exp = *((Export **) (export_val(obj) + 1));
 		if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) {
 		    *ep++ = EXPORT_EXT;
 		    ep = enc_atom(acmp, exp->code[0], ep, dflags);
 		    ep = enc_atom(acmp, exp->code[1], ep, dflags);
-		    ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags);
+		    ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags, off_heap);
 		} else {
 		    /* Tag, arity */
 		    *ep++ = SMALL_TUPLE_EXT;
@@ -1770,8 +1919,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		    int ei;
 
 		    *ep++ = NEW_FUN_EXT;
-		    ESTACK_PUSH(s, ENC_PATCH_FUN_SIZE);
-		    ESTACK_PUSH(s, (Eterm) ep); /* Position for patching in size */
+		    WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE);
+		    WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */
 		    ep += 4;
 		    *ep = funp->arity;
 		    ep += 1;
@@ -1782,14 +1931,14 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		    put_int32(funp->num_free, ep);
 		    ep += 4;
 		    ep = enc_atom(acmp, funp->fe->module, ep, dflags);
-		    ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags);
-		    ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags);
+		    ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap);
+		    ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap);
 		    ep = enc_pid(acmp, funp->creator, ep, dflags);
 
 		fun_env:
 		    for (ei = funp->num_free-1; ei > 0; ei--) {
-			ESTACK_PUSH(s, ENC_TERM);
-			ESTACK_PUSH(s, funp->env[ei]);
+			WSTACK_PUSH(s, ENC_TERM);
+			WSTACK_PUSH(s, (UWord) funp->env[ei]);
 		    }
 		    if (funp->num_free != 0) {
 			obj = funp->env[0];
@@ -1832,11 +1981,12 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 	    break;
 	}
     }
-    DESTROY_ESTACK(s);
+    DESTROY_WSTACK(s);
     return ep;
 }
 
-static Uint
+static
+Uint
 is_external_string(Eterm list, int* p_is_string)
 {
     Uint len = 0;
@@ -1874,69 +2024,30 @@ is_external_string(Eterm list, int* p_is_string)
     return len;
 }
 
-/* Assumes that the ones to undo are preluding the lists. */ 
+/* Assumes that the ones to undo are preluding the list. */ 
 static void
 undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end)
 {
     const Uint area_sz = (end - start) * sizeof(Eterm);
-    struct proc_bin* mso;
-    struct proc_bin** mso_nextp = NULL;
-#ifndef HYBRID /* FIND ME! */
-    struct erl_fun_thing* funs;
-    struct erl_fun_thing** funs_nextp = NULL;
-#endif
-    struct external_thing_* ext;
-    struct external_thing_** ext_nextp = NULL;
-
-    for (mso = off_heap->mso; ; mso=mso->next) {
-	if (!in_area(mso, start, area_sz)) {
-	    if (mso_nextp != NULL) {
-		*mso_nextp = NULL;
-		erts_cleanup_mso(off_heap->mso);
-		off_heap->mso = mso;
+    struct erl_off_heap_header* hdr;
+    struct erl_off_heap_header** hdr_nextp = NULL;
+
+    for (hdr = off_heap->first; ; hdr=hdr->next) {
+	if (!in_area(hdr, start, area_sz)) {
+	    if (hdr_nextp != NULL) {
+		*hdr_nextp = NULL;
+		erts_cleanup_offheap(off_heap);
+		off_heap->first = hdr;
 	    }
 	    break;
 	}
-	mso_nextp = &mso->next;
+	hdr_nextp = &hdr->next;
     }    
 
-#ifndef HYBRID /* FIND ME! */
-    for (funs = off_heap->funs; ; funs=funs->next) {
-	if (!in_area(funs, start, area_sz)) {
-	    if (funs_nextp != NULL) {
-		*funs_nextp = NULL;
-		erts_cleanup_funs(off_heap->funs);
-		off_heap->funs = funs;
-	    }
-	    break;
-	}
-	funs_nextp = &funs->next;
-    }    
-#endif
-    for (ext = off_heap->externals; ; ext=ext->next) {
-	if (!in_area(ext, start, area_sz)) {
-	    if (ext_nextp != NULL) {
-		*ext_nextp = NULL;
-		erts_cleanup_externals(off_heap->externals);
-		off_heap->externals = ext;
-	    }
-	    break;
-	}
-	ext_nextp = &ext->next;
-    }
-
-    /* Assert that the ones to undo were indeed preluding the lists. */ 
+    /* Assert that the ones to undo were indeed preluding the list. */ 
 #ifdef DEBUG
-    for (mso = off_heap->mso; mso != NULL; mso=mso->next) {
-	ASSERT(!in_area(mso, start, area_sz));
-    }    
-# ifndef HYBRID /* FIND ME! */
-    for (funs = off_heap->funs; funs != NULL; funs=funs->next) {
-	ASSERT(!in_area(funs, start, area_sz));
-    }    
-# endif
-    for (ext = off_heap->externals; ext != NULL; ext=ext->next) {
-	ASSERT(!in_area(ext, start, area_sz));
+    for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) {
+	ASSERT(!in_area(hdr, start, area_sz));
     }    
 #endif /* DEBUG */
 }
@@ -1952,11 +2063,11 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et
     register Eterm* hp = *hpp;	/* Please don't take the address of hp */
     Eterm* next = objp;
 
-    *next = (Eterm) NULL;
+    *next = (Eterm) (UWord) NULL;
 
     while (next != NULL) {
 	objp = next;
-	next = (Eterm *) (*objp);
+	next = (Eterm *) EXPAND_POINTER(*objp);
 
 	switch (*ep++) {
 	case INTEGER_EXT:
@@ -1964,7 +2075,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et
 		Sint sn = get_int32(ep);
 
 		ep += 4;
-#if defined(ARCH_64)
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		*objp = make_small(sn);
 #else
 		if (MY_IS_SSMALL(sn)) {
@@ -2061,7 +2172,7 @@ dec_term_atom_common:
 	    hp += n;
 	    objp = hp - 1;
 	    while (n-- > 0) {
-		objp[0] = (Eterm) next;
+		objp[0] = (Eterm) COMPRESS_POINTER(next);
 		next = objp;
 		objp--;
 	    }
@@ -2079,12 +2190,12 @@ dec_term_atom_common:
 	    *objp = make_list(hp);
 	    hp += 2*n;
 	    objp = hp - 2;
-	    objp[0] = (Eterm) (objp+1);
-	    objp[1] = (Eterm) next;
+	    objp[0] = (Eterm) COMPRESS_POINTER((objp+1));
+	    objp[1] = (Eterm) COMPRESS_POINTER(next);
 	    next = objp;
 	    objp -= 2;
 	    while (--n > 0) {
-		objp[0] = (Eterm) next;
+		objp[0] = (Eterm) COMPRESS_POINTER(next);
 		objp[1] = make_list(objp + 2);
 		next = objp;
 		objp -= 2;
@@ -2165,13 +2276,13 @@ dec_term_atom_common:
 		    goto error;
 		}
 		ep += 4;
-		if ((cre = get_int8(ep)) >= MAX_CREATION) {
+		cre = get_int8(ep);
+		ep++;
+		if (!is_valid_creation(cre)) {
 		    goto error;
 		}
-		ep++;
-		cre = dec_set_creation(sysname,cre);
-		node = erts_find_or_insert_node(sysname, cre);
 
+		node = dec_get_node(sysname, cre);
 		if(node == erts_this_node) {
 		    *objp = make_internal_port(num);
 		}
@@ -2180,11 +2291,11 @@ dec_term_atom_common:
 		    hp += EXTERNAL_THING_HEAD_SIZE + 1;
 		    
 		    etp->header = make_external_port_header(1);
-		    etp->next = off_heap->externals;
+		    etp->next = off_heap->first;
 		    etp->node = node;
 		    etp->data.ui[0] = num;
 
-		    off_heap->externals = etp;
+		    off_heap->first = (struct erl_off_heap_header*)etp;
 		    *objp = make_external_port(etp);
 		}
 
@@ -2208,9 +2319,11 @@ dec_term_atom_common:
 		    goto error;
 		ep += 4;
 
-		if ((cre = get_int8(ep)) >= MAX_CREATION)
-		    goto error;
+		cre = get_int8(ep);
 		ep += 1;
+		if (!is_valid_creation(cre)) {
+		    goto error;
+		}
 		goto ref_ext_common;
 
 	    case NEW_REFERENCE_EXT:
@@ -2223,10 +2336,11 @@ dec_term_atom_common:
 		if ((ep = dec_atom(edep, ep, &sysname)) == NULL)
 		    goto error;
 
-		if ((cre = get_int8(ep)) >= MAX_CREATION)
-		    goto error;
+		cre = get_int8(ep);
 		ep += 1;
-
+		if (!is_valid_creation(cre)) {
+		    goto error;
+		}
 		r0 = get_int32(ep);
 		ep += 4;
 		if (r0 >= MAX_REFERENCE)
@@ -2234,36 +2348,42 @@ dec_term_atom_common:
 
 	    ref_ext_common:
 
-		cre = dec_set_creation(sysname, cre);
-		node = erts_find_or_insert_node(sysname, cre);
+		node = dec_get_node(sysname, cre);
 		if(node == erts_this_node) {
 		    RefThing *rtp = (RefThing *) hp;
-		    hp += REF_THING_HEAD_SIZE;
-#ifdef ARCH_64
+		    ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE);
+
+#if defined(ARCH_64) && !HALFWORD_HEAP
+		    hp += REF_THING_HEAD_SIZE + ref_words/2 + 1;
 		    rtp->header = make_ref_thing_header(ref_words/2 + 1);
 #else
+		    hp += REF_THING_HEAD_SIZE + ref_words;
 		    rtp->header = make_ref_thing_header(ref_words);
 #endif
 		    *objp = make_internal_ref(rtp);
 		}
 		else {
 		    ExternalThing *etp = (ExternalThing *) hp;
-		    hp += EXTERNAL_THING_HEAD_SIZE;
-		    
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
+		    hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1;
+#else
+		    hp += EXTERNAL_THING_HEAD_SIZE + ref_words;
+#endif
+
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		    etp->header = make_external_ref_header(ref_words/2 + 1);
 #else
 		    etp->header = make_external_ref_header(ref_words);
 #endif
-		    etp->next = off_heap->externals;
+		    etp->next = off_heap->first;
 		    etp->node = node;
 
-		    off_heap->externals = etp;
+		    off_heap->first = (struct erl_off_heap_header*)etp;
 		    *objp = make_external_ref(etp);
+		    ref_num = &(etp->data.ui32[0]);
 		}
 
-		ref_num = (Uint32 *) hp;
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		*(ref_num++) = ref_words /* 32-bit arity */;
 #endif
 		ref_num[0] = r0;
@@ -2271,12 +2391,9 @@ dec_term_atom_common:
 		    ref_num[i] = get_int32(ep);
 		    ep += 4;
 		}
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		if ((1 + ref_words) % 2)
 		    ref_num[ref_words] = 0;
-		hp += ref_words/2 + 1;
-#else
-		hp += ref_words;
 #endif
 		break;
 	    }
@@ -2304,8 +2421,8 @@ dec_term_atom_common:
 		    hp += PROC_BIN_SIZE;
 		    pb->thing_word = HEADER_PROC_BIN;
 		    pb->size = n;
-		    pb->next = off_heap->mso;
-		    off_heap->mso = pb;
+		    pb->next = off_heap->first;
+		    off_heap->first = (struct erl_off_heap_header*)pb;
 		    pb->val = dbin;
 		    pb->bytes = (byte*) dbin->orig_bytes;
 		    pb->flags = 0;
@@ -2341,8 +2458,8 @@ dec_term_atom_common:
 		    pb = (ProcBin *) hp;
 		    pb->thing_word = HEADER_PROC_BIN;
 		    pb->size = n;
-		    pb->next = off_heap->mso;
-		    off_heap->mso = pb;
+		    pb->next = off_heap->first;
+		    off_heap->first = (struct erl_off_heap_header*)pb;
 		    pb->val = dbin;
 		    pb->bytes = (byte*) dbin->orig_bytes;
 		    pb->flags = 0;
@@ -2398,7 +2515,12 @@ dec_term_atom_common:
                 }
 		*objp = make_export(hp);
 		*hp++ = HEADER_EXPORT;
+#if HALFWORD_HEAP
+		*((UWord *) (UWord) hp) =  (UWord) erts_export_get_or_make_stub(mod, name, arity);
+		hp += 2;
+#else
 		*hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity);
+#endif
 		break;
 	    }
 	    break;
@@ -2457,8 +2579,8 @@ dec_term_atom_common:
 		 * It is safe to link the fun into the fun list only when
 		 * no more validity tests can fail.
 		 */
-		funp->next = off_heap->funs;
-		off_heap->funs = funp;
+		funp->next = off_heap->first;
+		off_heap->first = (struct erl_off_heap_header*)funp;
 #endif
 
 		funp->fe = erts_put_fun_entry2(module, old_uniq, old_index,
@@ -2474,11 +2596,11 @@ dec_term_atom_common:
 
 		/* Environment */
 		for (i = num_free-1; i >= 0; i--) {
-		    funp->env[i] = (Eterm) next;
+		    funp->env[i] = (Eterm) COMPRESS_POINTER(next);
 		    next = funp->env + i;
 		}
 		/* Creator */
-		funp->creator = (Eterm) next;
+		funp->creator = (Eterm) COMPRESS_POINTER(next);
 		next = &(funp->creator);
 		break;
 	    }
@@ -2535,8 +2657,8 @@ dec_term_atom_common:
 		 * It is safe to link the fun into the fun list only when
 		 * no more validity tests can fail.
 		 */
-		funp->next = off_heap->funs;
-		off_heap->funs = funp;
+		funp->next = off_heap->first;
+		off_heap->first = (struct erl_off_heap_header*)funp;
 #endif
 		old_uniq = unsigned_val(temp);
 
@@ -2549,11 +2671,71 @@ dec_term_atom_common:
 
 		/* Environment */
 		for (i = num_free-1; i >= 0; i--) {
-		    funp->env[i] = (Eterm) next;
+		    funp->env[i] = (Eterm) COMPRESS_POINTER(next);
 		    next = funp->env + i;
 		}
 		break;
 	    }
+	case ATOM_INTERNAL_REF2:
+	    n = get_int16(ep);
+	    ep += 2;
+	    if (n >= atom_table_size()) {
+		goto error;
+	    }
+	    *objp = make_atom(n);
+	    break;
+	case ATOM_INTERNAL_REF3:
+	    n = get_int24(ep);
+	    ep += 3;
+	    if (n >= atom_table_size()) {
+		goto error;
+	    }
+	    *objp = make_atom(n);
+	    break;
+
+	case BINARY_INTERNAL_REF:
+	    {
+		ProcBin* pb = (ProcBin*) hp;
+		sys_memcpy(pb, ep, sizeof(ProcBin));
+		ep += sizeof(ProcBin);
+
+		erts_refc_inc(&pb->val->refc, 1);
+		hp += PROC_BIN_SIZE;
+		pb->next = off_heap->first;
+		off_heap->first = (struct erl_off_heap_header*)pb;
+		pb->flags = 0;
+		*objp = make_binary(pb);
+		break;
+	    }
+	case BIT_BINARY_INTERNAL_REF:
+	    {
+		Sint bitoffs = *ep++;
+		Sint bitsize = *ep++;
+		ProcBin* pb = (ProcBin*) hp;
+		ErlSubBin* sub;
+		sys_memcpy(pb, ep, sizeof(ProcBin));
+		ep += sizeof(ProcBin);
+
+		erts_refc_inc(&pb->val->refc, 1);
+		hp += PROC_BIN_SIZE;
+		pb->next = off_heap->first;
+		off_heap->first = (struct erl_off_heap_header*)pb;
+		pb->flags = 0;
+
+		sub = (ErlSubBin*)hp;
+		sub->thing_word = HEADER_SUB_BIN;
+		sub->size = pb->size - (bitoffs + bitsize + 7)/8;
+		sub->offs = 0;
+		sub->bitoffs = bitoffs;
+		sub->bitsize = bitsize;
+		sub->is_writable = 0;
+		sub->orig = make_binary(pb);
+
+		hp += ERL_SUB_BIN_SIZE;
+		*objp = make_binary(sub);
+		break;
+	    }
+
 	default:
 	error:
 	    /* UNDO:
@@ -2580,26 +2762,35 @@ dec_term_atom_common:
 static Uint
 encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 {
-    DECLARE_ESTACK(s);
+    DECLARE_WSTACK(s);
     Uint m, i, arity;
     Uint result = 0;
+#if HALFWORD_HEAP
+    UWord wobj = 0;
+#endif
 
     goto L_jump_start;
 
  outer_loop:
-    while (!ESTACK_ISEMPTY(s)) {
-	obj = ESTACK_POP(s);
-
+    while (!WSTACK_ISEMPTY(s)) {
+#if HALFWORD_HEAP
+	obj = (Eterm) (wobj = WSTACK_POP(s));
+#else
+	obj = WSTACK_POP(s);
+#endif
     handle_popped_obj:
-	if (is_CP(obj)) {
+	if (is_CP(obj)) { /* Does not look for CP, looks for "no tag" */
+#if HALFWORD_HEAP
+	    Eterm* ptr = (Eterm *) wobj;
+#else
 	    Eterm* ptr = (Eterm *) obj;
-
+#endif
 	    /*
 	     * Pointer into a tuple.
 	     */
 	    obj = *ptr--;
 	    if (!is_header(obj)) {
-		ESTACK_PUSH(s, (Eterm)ptr);
+		WSTACK_PUSH(s, (UWord)ptr);
 	    } else {
 		/* Reached tuple header */
 		ASSERT(header_is_arityval(obj));
@@ -2611,7 +2802,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 
 	    tl = CDR(cons);
 	    obj = CAR(cons);
-	    ESTACK_PUSH(s, tl);
+	    WSTACK_PUSH(s, tl);
 	} else if (is_nil(obj)) {
 	    result++;
 	    goto outer_loop;
@@ -2627,37 +2818,51 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 	case NIL_DEF:
 	    result++;
 	    break;
-	case ATOM_DEF: {
-	    int alen = atom_tab(atom_val(obj))->len;
-	    if ((MAX_ATOM_LENGTH <= 255 || alen <= 255)
-		&& (dflags & DFLAG_SMALL_ATOM_TAGS)) {
-		/* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */
-		result += 1 + 1 + alen;
+	case ATOM_DEF:
+	    if (dflags & DFLAGS_INTERNAL_TAGS) {
+		if (atom_val(obj) >= (1<<16)) {
+		    result += 1 + 3;
+		}
+		else {
+		    result += 1 + 2;
+		}
 	    }
 	    else {
-		/* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */
-		result += 1 + 2 + alen;
+		int alen = atom_tab(atom_val(obj))->len;
+		if ((MAX_ATOM_LENGTH <= 255 || alen <= 255)
+		    && (dflags & DFLAG_SMALL_ATOM_TAGS)) {
+		    /* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */
+			result += 1 + 1 + alen;
+		}
+		else {
+		    /* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */
+			result += 1 + 2 + alen;
+		}
+		insert_acache_map(acmp, obj);
 	    }
-	    insert_acache_map(acmp, obj);
 	    break;
-	}
 	case SMALL_DEF:
 	    {
 		Sint val = signed_val(obj);
 
 		if ((Uint)val < 256)
 		    result += 1 + 1;		/* SMALL_INTEGER_EXT */
-		else if (sizeof(Sint) == 4 || IS_SSMALL28(val))
+		else if (sizeof(Sint) == 4 || IS_SSMALL32(val))
 		    result += 1 + 4;		/* INTEGER_EXT */
 		else {
-		    Eterm tmp_big[2];
+		    DeclareTmpHeapNoproc(tmp_big,2);
+		    UseTmpHeapNoproc(2);
 		    i = big_bytes(small_to_big(val, tmp_big));
 		    result += 1 + 1 + 1 + i;	/* SMALL_BIG_EXT */
+		    UnUseTmpHeapNoproc(2);
 		}
 	    }
 	    break;
 	case BIG_DEF:
-	    if ((i = big_bytes(obj)) < 256)
+	    i = big_bytes(obj);
+	    if (sizeof(Sint)==4 && i <= 4 && (big_digit(obj,0)-big_sign(obj)) < (1<<31))
+		result += 1 + 4;          /* INTEGER_EXT */
+	    else if (i < 256)
 		result += 1 + 1 + 1 + i;  /* tag,size,sign,digits */
 	    else
 		result += 1 + 4 + 1 + i;  /* tag,size,sign,digits */
@@ -2698,7 +2903,11 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 		    result += 1 + 4;
 		}
 		ptr += arity;
+#if HALFWORD_HEAP
+		obj = (Eterm) (wobj = (UWord) ptr);
+#else
 		obj = (Eterm) ptr;
+#endif
 		goto handle_popped_obj;
 	    }
 	    break;
@@ -2710,8 +2919,25 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 	    }
 	    break;
 	case BINARY_DEF:
+	    if (dflags & DFLAGS_INTERNAL_TAGS) {
+		ProcBin* pb = (ProcBin*) binary_val(obj);
+		Uint sub_extra = 0;
+		Uint tot_bytes = pb->size;
+		if (pb->thing_word == HEADER_SUB_BIN) {
+		    ErlSubBin* sub = (ErlSubBin*) pb;
+		    pb = (ProcBin*) binary_val(sub->orig);
+		    sub_extra = 2;  /* bitoffs and bitsize */
+		    tot_bytes += (sub->bitoffs + sub->bitsize+ 7) / 8;
+		}
+		if (pb->thing_word == HEADER_PROC_BIN
+		    && heap_bin_size(tot_bytes) > PROC_BIN_SIZE) {
+
+		    result += 1 + sub_extra + sizeof(ProcBin);
+		    break;
+		}
+	    }
 	    result += 1 + 4 + binary_size(obj) +
-		5;			/* For unaligned binary */
+		    5;			/* For unaligned binary */
 	    break;
 	case FUN_DEF:
 	    {
@@ -2740,14 +2966,14 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 
 		    if (is_not_list(obj)) {
 			/* Push any non-list terms on the stack */
-			ESTACK_PUSH(s, obj);
+			WSTACK_PUSH(s, obj);
 		    } else {
 			/* Lists must be handled specially. */
 			if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) {
 			    result += m + 2 + 1;
 			} else {
 			    result += 5;
-			    ESTACK_PUSH(s, obj);
+			    WSTACK_PUSH(s, obj);
 			}
 		    }
 		}
@@ -2760,8 +2986,12 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 
 	case EXPORT_DEF:
 	    {
-		Export* ep = (Export *) (export_val(obj))[1];
+		Export* ep = *((Export **) (export_val(obj) + 1));
+#if HALFWORD_HEAP
+		result += 2;
+#else
 		result += 1;
+#endif
 		result += encode_size_struct2(acmp, ep->code[0], dflags);
 		result += encode_size_struct2(acmp, ep->code[1], dflags);
 		result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags);
@@ -2774,12 +3004,12 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 	}
     }
 
-    DESTROY_ESTACK(s);
+    DESTROY_WSTACK(s);
     return result;
 }
 
 static Sint
-decoded_size(byte *ep, byte* endp, int no_refc_bins)
+decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags)
 {
     int heap_size = 0;
     int terms;
@@ -2886,7 +3116,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins)
 		ep += 2;
 		atom_extra_skip = 1 + 4*id_words;
 		/* In case it is an external ref */
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1;
 #else
 		heap_size += EXTERNAL_THING_HEAD_SIZE + id_words;
@@ -2961,7 +3191,11 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins)
 	    break;
 	case EXPORT_EXT:
 	    terms += 3;
+#if HALFWORD_HEAP
+	    heap_size += 3;
+#else
 	    heap_size += 2;
+#endif
 	    break;
 	case NEW_FUN_EXT:
 	    {
@@ -2985,6 +3219,29 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins)
 		heap_size += ERL_FUN_SIZE + num_free;
 		break;
 	    }
+	case ATOM_INTERNAL_REF2:
+	    SKIP(2+atom_extra_skip);
+	    atom_extra_skip = 0;
+	    break;
+	case ATOM_INTERNAL_REF3:
+	    SKIP(3+atom_extra_skip);
+	    atom_extra_skip = 0;
+	    break;
+
+	case BINARY_INTERNAL_REF:
+	    if (!internal_tags) {
+		return -1;
+	    }
+	    SKIP(sizeof(ProcBin));
+	    heap_size += PROC_BIN_SIZE;
+	    break;
+	case BIT_BINARY_INTERNAL_REF:
+	    if (!internal_tags) {
+		return -1;
+	    }
+	    SKIP(2+sizeof(ProcBin));
+	    heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE;
+	    break;
 	default:
 	    return -1;
 	}
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index eada6d4f27..d8287b96a4 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -54,6 +54,10 @@
 
 #define DIST_HEADER       'D'
 #define ATOM_CACHE_REF    'R'
+#define ATOM_INTERNAL_REF2 'I'
+#define ATOM_INTERNAL_REF3 'K'
+#define BINARY_INTERNAL_REF 'J'
+#define BIT_BINARY_INTERNAL_REF 'L'
 #define COMPRESSED        'P'
 
 #if 0
@@ -156,7 +160,9 @@ Uint erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap *);
 void erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *);
 
 Uint erts_encode_ext_size(Eterm);
+Uint erts_encode_ext_size_ets(Eterm);
 void erts_encode_ext(Eterm, byte **);
+byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap);
 
 #ifdef ERTS_WANT_EXTERNAL_TAGS
 ERTS_GLB_INLINE void erts_peek_dist_header(ErtsDistHeaderPeek *, byte *, Uint);
@@ -172,7 +178,9 @@ Sint erts_decode_dist_ext_size(ErtsDistExternal *, int);
 Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *);
 
 Sint erts_decode_ext_size(byte*, Uint, int);
+Sint erts_decode_ext_size_ets(byte*, Uint);
 Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**);
+Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*);
 
 Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags);
 
@@ -211,8 +219,8 @@ ERTS_GLB_INLINE void *
 erts_dist_ext_trailer(ErtsDistExternal *edep)
 {
     void *res = (void *) (edep->ext_endp
-			  + ERTS_WORD_ALIGN_PAD_SZ(edep->ext_endp));
-    ASSERT((((Uint) res) % sizeof(Uint)) == 0);
+			  + ERTS_EXTRA_DATA_ALIGN_SZ(edep->ext_endp));
+    ASSERT((((UWord) res) % sizeof(Uint)) == 0);
     return res;
 }
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index cab249a53f..499bdd77ba 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -76,12 +76,6 @@ typedef struct line_buf {  /* Buffer used in line oriented I/O */
 			      The rest is the overflow buffer. */
 } LineBuf;
 
-/* Temporary object header, auto-deallocated when NIF returns. */
-struct enif_tmp_obj_t {
-    struct enif_tmp_obj_t* next;
-    void (*dtor)(struct enif_tmp_obj_t*);
-    /*char data[];*/
-};
 struct enif_environment_t /* ErlNifEnv */
 {
     struct erl_module_nif* mod_nif;
@@ -189,7 +183,7 @@ struct port {
 				    process to get (line oriented I/O)*/
     Uint32 status;		 /* Status and type flags */
     int control_flags;		 /* Flags for port_control()  */
-    Uint32 snapshot;             /* Next snapshot that port should be part of */
+    erts_aint32_t snapshot;      /* Next snapshot that port should be part of */
     struct reg_proc *reg;
     ErlDrvPDL port_data_lock;
 
@@ -403,7 +397,7 @@ extern Eterm erts_ddll_monitor_driver(Process *p,
 
 /* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */
 #define ERTS_INTERNAL_BINARY_FIELDS				\
-    Uint flags;							\
+    UWord flags;							\
     erts_refc_t refc;						\
     ERTS_BINARY_STRUCT_ALIGNMENT
 
@@ -470,7 +464,10 @@ typedef union {
 typedef struct proc_bin {
     Eterm thing_word;		/* Subtag REFC_BINARY_SUBTAG. */
     Uint size;			/* Binary size in bytes. */
-    struct proc_bin *next;	/* Pointer to next ProcBin. */
+#if HALFWORD_HEAP
+    void* dummy_ptr_padding__;
+#endif
+    struct erl_off_heap_header *next;
     Binary *val;		/* Pointer to Binary structure. */
     byte *bytes;		/* Pointer to the actual data bytes. */
     Uint flags;			/* Flag word. */
@@ -500,8 +497,8 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp)
 
     pb->thing_word = HEADER_PROC_BIN;
     pb->size = 0;
-    pb->next = ohp->mso;
-    ohp->mso = pb;
+    pb->next = ohp->first;
+    ohp->first = (struct erl_off_heap_header*) pb;
     pb->val = mbp;
     pb->bytes = (byte *) mbp->orig_bytes;
     pb->flags = 0;
@@ -518,13 +515,22 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp)
    && (thing_subtag(*binary_val((T))) == REFC_BINARY_SUBTAG) \
    && (((ProcBin *) binary_val((T)))->val->flags & BIN_FLAG_MAGIC))
 
+
+union erl_off_heap_ptr {
+    struct erl_off_heap_header* hdr;
+    ProcBin *pb;
+    struct erl_fun_thing* fun;
+    struct external_thing_* ext;
+    Eterm* ep;
+    void* voidp;
+};
+
 /* arrays that get malloced at startup */
 extern Port* erts_port;
-extern erts_smp_atomic_t erts_ports_alive;
 
 extern Uint erts_max_ports;
 extern Uint erts_port_tab_index_mask;
-extern erts_smp_atomic_t erts_ports_snapshot;
+extern erts_smp_atomic32_t erts_ports_snapshot;
 extern erts_smp_atomic_t erts_dead_ports_ptr;
 
 ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt);
@@ -534,12 +540,12 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt);
 ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck));
-    if (prt->snapshot != erts_smp_atomic_read(&erts_ports_snapshot)) {
+    if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) {
 	/* Dead ports are added from the end of the snapshot buffer */
 	Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr,
-							    -(long)sizeof(Eterm));
+							    -(erts_aint_t)sizeof(Eterm));
 	ASSERT(tombstone+1 != NULL);
-	ASSERT(prt->snapshot == (Uint32) erts_smp_atomic_read(&erts_ports_snapshot) - 1);
+	ASSERT(prt->snapshot == erts_smp_atomic32_read(&erts_ports_snapshot) - 1);
 	*tombstone = prt->id;
     }
     /*else no ongoing snapshot or port was already included or created after snapshot */
@@ -556,7 +562,7 @@ extern Uint display_items;	/* no of items to display in traces etc */
 extern Uint display_loads;	/* print info about loaded modules */
 
 extern int erts_backtrace_depth;
-extern erts_smp_atomic_t erts_max_gen_gcs;
+extern erts_smp_atomic32_t erts_max_gen_gcs;
 
 extern int erts_disable_tolerant_timeofday;
 
@@ -723,6 +729,60 @@ do {									\
 #define ESTACK_POP(s) (*(--ESTK_CONCAT(s,_sp)))
 
 
+void erl_grow_wstack(UWord** start, UWord** sp, UWord** end);
+#define WSTK_CONCAT(a,b) a##b
+#define WSTK_SUBSCRIPT(s,i) *((UWord *)((byte *)WSTK_CONCAT(s,_start) + (i)))
+#define DEF_WSTACK_SIZE (16)
+
+#define DECLARE_WSTACK(s)						\
+    UWord WSTK_CONCAT(s,_default_stack)[DEF_WSTACK_SIZE];		\
+    UWord* WSTK_CONCAT(s,_start) = WSTK_CONCAT(s,_default_stack);	\
+    UWord* WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start);			\
+    UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE
+
+#define DESTROY_WSTACK(s)						\
+do {									\
+    if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) {	\
+	erts_free(ERTS_ALC_T_ESTACK, WSTK_CONCAT(s,_start));		\
+    }									\
+} while(0)
+
+#define WSTACK_PUSH(s, x)						\
+do {									\
+    if (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_end)) {			\
+	erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp),	\
+	               &WSTK_CONCAT(s,_end));				\
+    }									\
+    *WSTK_CONCAT(s,_sp)++ = (x);					\
+} while(0)
+
+#define WSTACK_PUSH2(s, x, y)						\
+do {									\
+    if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 2) {			\
+	erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp),	\
+		&WSTK_CONCAT(s,_end));	\
+    }									\
+    *WSTK_CONCAT(s,_sp)++ = (x);					\
+    *WSTK_CONCAT(s,_sp)++ = (y);					\
+} while(0)
+
+#define WSTACK_PUSH3(s, x, y, z)					\
+do {									\
+    if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 3) {			\
+	erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp),	\
+		&WSTK_CONCAT(s,_end));					\
+    }									\
+    *WSTK_CONCAT(s,_sp)++ = (x);					\
+    *WSTK_CONCAT(s,_sp)++ = (y);					\
+    *WSTK_CONCAT(s,_sp)++ = (z);					\
+} while(0)
+
+#define WSTACK_COUNT(s) (WSTK_CONCAT(s,_sp) - WSTK_CONCAT(s,_start))
+
+#define WSTACK_ISEMPTY(s) (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_start))
+#define WSTACK_POP(s) (*(--WSTK_CONCAT(s,_sp)))
+
+
 /* port status flags */
 
 #define ERTS_PORT_SFLG_CONNECTED	((Uint32) (1 <<  0))
@@ -773,9 +833,8 @@ do {									\
 void erts_emasculate_writable_binary(ProcBin* pb);
 Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap);
 Eterm erts_new_mso_binary(Process*, byte*, int);
-Eterm new_binary(Process*, byte*, int);
+Eterm new_binary(Process*, byte*, Uint);
 Eterm erts_realloc_binary(Eterm bin, size_t size);
-void erts_cleanup_mso(ProcBin* pb);
 
 /* erl_bif_info.c */
 
@@ -802,7 +861,7 @@ void erts_system_profile_clear(Process *c_p);
 int erts_load_module(Process *c_p, ErtsProcLocks c_p_locks,
 		     Eterm group_leader, Eterm* mod, byte* code, int size);
 void init_load(void);
-Eterm* find_function_from_pc(Eterm* pc);
+BeamInstr* find_function_from_pc(BeamInstr* pc);
 Eterm erts_module_info_0(Process* p, Eterm module);
 Eterm erts_module_info_1(Process* p, Eterm module, Eterm what);
 Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info);
@@ -830,9 +889,33 @@ void erl_error(char*, va_list);
 /* copy.c */
 void init_copy(void);
 Eterm copy_object(Eterm, Process*);
+
+#if HALFWORD_HEAP
+Uint size_object_rel(Eterm, Eterm*);
+#  define size_object(A) size_object_rel(A,NULL)
+
+Eterm copy_struct_rel(Eterm, Uint, Eterm**, ErlOffHeap*, Eterm* src_base, Eterm* dst_base);
+#  define copy_struct(OBJ,SZ,HPP,OH) copy_struct_rel(OBJ,SZ,HPP,OH, NULL,NULL)
+
+Eterm copy_shallow_rel(Eterm*, Uint, Eterm**, ErlOffHeap*, Eterm* src_base);
+#  define copy_shallow(A,B,C,D) copy_shallow_rel(A,B,C,D,NULL)
+
+#else /* !HALFWORD_HEAP */
+
 Uint size_object(Eterm);
+#  define size_object_rel(A,B) size_object(A)
+
 Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
+#  define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH)
+
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
+#  define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D)
+
+#endif
+
+
+void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
+		      Eterm* refs, unsigned nrefs);
 
 #ifdef HYBRID
 #define RRMA_DEFAULT_SIZE 256
@@ -965,7 +1048,7 @@ void print_pass_through(int, byte*, int);
 
 /* beam_emu.c */
 int catchlevel(Process*);
-void init_emulator(_VOID_);
+void init_emulator(void);
 void process_main(void);
 Eterm build_stacktrace(Process* c_p, Eterm exc);
 Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value);
@@ -1030,6 +1113,7 @@ Eterm erts_heap_sizes(Process* p);
 void erts_offset_off_heap(ErlOffHeap *, Sint, Eterm*, Eterm*);
 void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*);
 void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
+void erts_free_heap_frags(Process* p);
 
 #ifdef HYBRID
 int erts_global_garbage_collect(Process*, int, Eterm*, int);
@@ -1143,13 +1227,12 @@ ERTS_GLB_INLINE void
 erts_smp_port_unlock(Port *prt)
 {
 #ifdef ERTS_SMP
-    long refc;
+    erts_aint_t refc;
+    erts_smp_mtx_unlock(prt->lock);
     refc = erts_smp_atomic_dectest(&prt->refc);
     ASSERT(refc >= 0);
     if (refc == 0)
 	erts_port_cleanup(prt);
-    else
-	erts_smp_mtx_unlock(prt->lock);
 #endif
 }
 
@@ -1171,7 +1254,7 @@ erts_smp_port_unlock(Port *prt)
   ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)
 
 #define ERTS_PORT_SCHED_ID(P, ID) \
-  ((Uint) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (ID)))
+  ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID)))
 
 #ifdef ERTS_SMP
 Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks);
@@ -1363,84 +1446,6 @@ void erl_drv_thr_init(void);
 
 /* time.c */
 
-ERTS_GLB_INLINE long do_time_read_and_reset(void);
-#ifdef ERTS_TIMER_THREAD
-ERTS_GLB_INLINE int next_time(void);
-ERTS_GLB_INLINE void bump_timer(long);
-#else
-int next_time(void);
-void bump_timer(long);
-extern erts_smp_atomic_t do_time;	/* set at clock interrupt */
-ERTS_GLB_INLINE void do_time_add(long);
-#endif
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-#ifdef ERTS_TIMER_THREAD
-ERTS_GLB_INLINE long do_time_read_and_reset(void) { return 0; }
-ERTS_GLB_INLINE int next_time(void) { return -1; }
-ERTS_GLB_INLINE void bump_timer(long ignore) { }
-#else
-ERTS_GLB_INLINE long do_time_read_and_reset(void)
-{
-    return erts_smp_atomic_xchg(&do_time, 0L);
-}
-ERTS_GLB_INLINE void do_time_add(long elapsed)
-{
-    erts_smp_atomic_add(&do_time, elapsed);
-}
-#endif
-
-#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
-
-void init_time(void);
-void erl_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint);
-void erl_cancel_timer(ErlTimer*);
-Uint time_left(ErlTimer *);
-
-Uint erts_timer_wheel_memory_size(void);
-
-#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME))
-#  ifndef HAVE_ERTS_NOW_CPU
-#    define HAVE_ERTS_NOW_CPU
-#    ifdef HAVE_GETHRVTIME
-#      define erts_start_now_cpu() sys_start_hrvtime()
-#      define erts_stop_now_cpu()  sys_stop_hrvtime()
-#    endif
-#  endif
-void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
-#endif
-
-void erts_get_timeval(SysTimeval *tv);
-long erts_get_time(void);
-
-extern SysTimeval erts_first_emu_time;
-
-void erts_get_emu_time(SysTimeval *);
-
-ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE int
-erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p)
-{
-    if (t1p->tv_sec == t2p->tv_sec) {
-	if (t1p->tv_usec < t2p->tv_usec)
-	    return -1;
-	else if (t1p->tv_usec > t2p->tv_usec)
-	    return 1;
-	return 0;
-    }
-    return t1p->tv_sec < t2p->tv_sec ? -1 : 1;
-}
-
-#endif
-
-#ifdef DEBUG
-void p_slpq(_VOID_);
-#endif
-
 /* utils.c */
 
 /*
@@ -1449,7 +1454,6 @@ void p_slpq(_VOID_);
 void erts_silence_warn_unused_result(long unused);
 
 void erts_cleanup_offheap(ErlOffHeap *offheap);
-void erts_cleanup_externals(ExternalThing *);
 
 Uint erts_fit_in_bits(Uint);
 int list_length(Eterm);
@@ -1463,6 +1467,7 @@ Uint32 make_hash(Eterm);
 
 Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str);
 Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui);
+Eterm erts_bld_uword(Uint **hpp, Uint *szp, UWord uw);
 Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64);
 Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64);
 Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr);
@@ -1481,7 +1486,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length,
 			      Eterm atoms[], Uint uints1[], Uint uints2[]);
 
 Eterm store_external_or_ref_in_proc_(Process *, Eterm);
-Eterm store_external_or_ref_(Uint **, ExternalThing **, Eterm);
+Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm);
 
 #define NC_HEAP_SIZE(NC) \
  (ASSERT_EXPR(is_node_container((NC))), \
@@ -1499,23 +1504,39 @@ void erts_init_utils_mem(void);
 erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint);
 void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *);
 
+#if HALFWORD_HEAP
+int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base);
+#  define eq(A,B) eq_rel(A,NULL,B,NULL)
+#else
 int eq(Eterm, Eterm);
+#  define eq_rel(A,A_BASE,B,B_BASE) eq(A,B)
+#endif
+
 #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
 
+#if HALFWORD_HEAP
+Sint cmp_rel(Eterm, Eterm*, Eterm, Eterm*);
+#define CMP(A,B) cmp_rel(A,NULL,B,NULL)
+#else
 Sint cmp(Eterm, Eterm);
-#define cmp_lt(a,b)	(cmp((a),(b)) < 0)
-#define cmp_le(a,b)	(cmp((a),(b)) <= 0)
-#define cmp_eq(a,b)	(cmp((a),(b)) == 0)
-#define cmp_ne(a,b)	(cmp((a),(b)) != 0)
-#define cmp_ge(a,b)	(cmp((a),(b)) >= 0)
-#define cmp_gt(a,b)	(cmp((a),(b)) > 0)
+#define cmp_rel(A,A_BASE,B,B_BASE) cmp(A,B)
+#define CMP(A,B) cmp(A,B)
+#endif
+#define cmp_lt(a,b)	(CMP((a),(b)) < 0)
+#define cmp_le(a,b)	(CMP((a),(b)) <= 0)
+#define cmp_eq(a,b)	(CMP((a),(b)) == 0)
+#define cmp_ne(a,b)	(CMP((a),(b)) != 0)
+#define cmp_ge(a,b)	(CMP((a),(b)) >= 0)
+#define cmp_gt(a,b)	(CMP((a),(b)) > 0)
 
 #define CMP_LT(a,b)	((a) != (b) && cmp_lt((a),(b)))
 #define CMP_GE(a,b)	((a) == (b) || cmp_ge((a),(b)))
 #define CMP_EQ(a,b)	((a) == (b) || cmp_eq((a),(b)))
 #define CMP_NE(a,b)	((a) != (b) && cmp_ne((a),(b)))
 
+/* duplicates from big.h */
 int term_to_Uint(Eterm term, Uint *up);
+int term_to_UWord(Eterm, UWord*);
 
 #ifdef HAVE_ERTS_NOW_CPU
 extern int erts_cpu_timestamp;
@@ -1525,9 +1546,26 @@ void erts_init_bif_chksum(void);
 /* erl_bif_re.c */
 void erts_init_bif_re(void);
 Sint erts_re_set_loop_limit(Sint limit);
+/* erl_bif_binary.c */
+void erts_init_bif_binary(void);
+Sint erts_binary_set_loop_limit(Sint limit);
+
 /* erl_unicode.c */
 void erts_init_unicode(void);
 Sint erts_unicode_set_loop_limit(Sint limit);
+
+void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ;
+Sint erts_native_filename_need(Eterm ioterm, int encoding);
+void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars);
+int erts_analyze_utf8(byte *source, Uint size, 
+			byte **err_pos, Uint *num_chars, int *left);
+char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty);
+
+#define ERTS_UTF8_OK 0
+#define ERTS_UTF8_INCOMPLETE 1
+#define ERTS_UTF8_ERROR 2
+#define ERTS_UTF8_ANALYZE_MORE 3
+
 /* erl_trace.c */
 void erts_init_trace(void);
 void erts_trace_check_exiting(Eterm exiting);
@@ -1554,12 +1592,12 @@ void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *);
 void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
 void trace_send(Process*, Eterm, Eterm);
 void trace_receive(Process*, Eterm);
-Uint32 erts_call_trace(Process *p, Eterm mfa[], Binary *match_spec, Eterm* args,
+Uint32 erts_call_trace(Process *p, BeamInstr mfa[], Binary *match_spec, Eterm* args,
 		       int local, Eterm *tracer_pid);
-void erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid);
-void erts_trace_exception(Process* p, Eterm mfa[], Eterm class, Eterm value, 
+void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid);
+void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value,
 			  Eterm *tracer);
-void erts_trace_return_to(Process *p, Uint *pc);
+void erts_trace_return_to(Process *p, BeamInstr *pc);
 void trace_sched(Process*, Eterm);
 void trace_proc(Process*, Process*, Eterm, Eterm);
 void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args);
@@ -1589,7 +1627,7 @@ Uint erts_trace_flag2bit(Eterm flag);
 int erts_trace_flags(Eterm List, 
 		 Uint *pMask, Eterm *pTracer, int *pCpuTimestamp);
 Eterm erts_bif_trace(int bif_index, Process* p, 
-		     Eterm arg1, Eterm arg2, Eterm arg3, Uint *I);
+		     Eterm arg1, Eterm arg2, Eterm arg3, BeamInstr *I);
 
 #ifdef ERTS_SMP
 void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp);
@@ -1606,7 +1644,7 @@ void bin_write(int, void*, byte*, int);
 int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */
 
 struct Sint_buf {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     char s[22];
 #else
     char s[12];
@@ -1614,12 +1652,16 @@ struct Sint_buf {
 };	
 char* Sint_to_buf(Sint, struct Sint_buf*);
 
+#define ERTS_IOLIST_OK 0
+#define ERTS_IOLIST_OVERFLOW 1
+#define ERTS_IOLIST_TYPE 2
+
 Eterm buf_to_intlist(Eterm**, char*, int, Eterm); /* most callers pass plain char*'s */
 int io_list_to_buf(Eterm, char*, int);
 int io_list_to_buf2(Eterm, char*, int);
-int io_list_len(Eterm);
+int erts_iolist_size(Eterm, Uint *);
 int is_string(Eterm);
-void erl_at_exit(FUNCTION(void,(*),(void*)), void*);
+void erl_at_exit(void (*) (void*), void*);
 Eterm collect_memory(Process *);
 void dump_memory_to_fd(int);
 int dump_memory_data(const char *);
@@ -1654,17 +1696,14 @@ Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live);
+Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live);
+Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live);
 
 Uint erts_current_reductions(Process* current, Process *p);
 
 int erts_print_system_version(int to, void *arg, Process *c_p);
 
-/*
- * Interface to erl_init
- */
-void erl_init(void);
-void erts_first_process(Eterm modname, void* code, unsigned size, int argc, char** argv);
-
+int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);
 #define seq_trace_output(token, msg, type, receiver, process) \
 seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL)
 #define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \
@@ -1683,8 +1722,10 @@ struct trace_pattern_flags {
     unsigned int local      : 1; /* Local call trace breakpoint */
     unsigned int meta       : 1; /* Metadata trace breakpoint */
     unsigned int call_count : 1; /* Fast call count breakpoint */
+    unsigned int call_time  : 1; /* Fast call time breakpoint */
 };
 extern const struct trace_pattern_flags erts_trace_pattern_flags_off;
+extern int erts_call_time_breakpoint_tracing;
 int erts_set_trace_pattern(Eterm* mfa, int specified, 
 			   Binary* match_prog_set, Binary *meta_match_prog_set,
 			   int on, struct trace_pattern_flags,
@@ -1720,26 +1761,35 @@ do {								\
 extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr);
 Eterm erts_match_set_lint(Process *p, Eterm matchexpr); 
 extern void erts_match_set_release_result(Process* p);
+
+enum erts_pam_run_flags {
+    ERTS_PAM_TMP_RESULT=0,
+    ERTS_PAM_COPY_RESULT=1,
+    ERTS_PAM_CONTIGUOUS_TUPLE=2
+};
 extern Eterm erts_match_set_run(Process *p, Binary *mpsp, 
 				Eterm *args, int num_args,
+				enum erts_pam_run_flags in_flags,
 				Uint32 *return_flags);
 extern Eterm erts_match_set_get_source(Binary *mpsp);
 extern void erts_match_prog_foreach_offheap(Binary *b,
 					    void (*)(ErlOffHeap *, void *),
 					    void *);
 
-#define MATCH_SET_RETURN_TRACE 0x1 /* return trace requested */
-#define MATCH_SET_RETURN_TO_TRACE 0x2 /* Misleading name, it is not actually
-					 set by the match program, but by the
-					 breakpoint functions */
-#define MATCH_SET_EXCEPTION_TRACE 0x4 /* exception trace requested */
+#define MATCH_SET_RETURN_TRACE    (0x1) /* return trace requested */
+#define MATCH_SET_RETURN_TO_TRACE (0x2) /* Misleading name, it is not actually
+					   set by the match program, but by the
+					   breakpoint functions */
+#define MATCH_SET_EXCEPTION_TRACE (0x4) /* exception trace requested */
 #define MATCH_SET_RX_TRACE (MATCH_SET_RETURN_TRACE|MATCH_SET_EXCEPTION_TRACE)
 /*
  * Flag values when tracing bif
+ * Future note: flag field is 8 bits
  */
-#define BIF_TRACE_AS_LOCAL  0x1
-#define BIF_TRACE_AS_GLOBAL 0x2
-#define BIF_TRACE_AS_META   0x4
+#define BIF_TRACE_AS_LOCAL      (0x1)
+#define BIF_TRACE_AS_GLOBAL     (0x2)
+#define BIF_TRACE_AS_META       (0x4)
+#define BIF_TRACE_AS_CALL_TIME  (0x8)
 
 extern erts_driver_t vanilla_driver;
 extern erts_driver_t spawn_driver;
@@ -1781,7 +1831,7 @@ erts_alloc_message_heap(Uint size,
 #endif
 
     if (size > (Uint) INT_MAX)
-	erl_exit(ERTS_ABORT_EXIT, "HUGE size (%bpu)\n", size);
+	erl_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size);
 
     if (
 #if defined(ERTS_SMP)
@@ -1839,4 +1889,67 @@ erts_alloc_message_heap(Uint size,
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#endif
+#if !HEAP_ON_C_STACK
+#  if defined(DEBUG)
+#    define DeclareTmpHeap(VariableName,Size,Process) \
+       Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process)
+#    define DeclareTypedTmpHeap(Type,VariableName,Process)		\
+      Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process)
+#    define DeclareTmpHeapNoproc(VariableName,Size) \
+       Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL)
+#    define UseTmpHeap(Size,Proc) \
+       do { \
+         erts_debug_use_tmp_heap((Size),(Proc)); \
+       } while (0)
+#    define UnUseTmpHeap(Size,Proc) \
+       do { \
+         erts_debug_unuse_tmp_heap((Size),(Proc)); \
+       } while (0)
+#    define UseTmpHeapNoproc(Size) \
+       do { \
+         erts_debug_use_tmp_heap(Size,NULL); \
+       } while (0)
+#    define UnUseTmpHeapNoproc(Size) \
+       do { \
+         erts_debug_unuse_tmp_heap(Size,NULL); \
+       } while (0)
+#  else
+#    define DeclareTmpHeap(VariableName,Size,Process) \
+       Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
+#    define DeclareTypedTmpHeap(Type,VariableName,Process)		\
+      Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
+#    define DeclareTmpHeapNoproc(VariableName,Size) \
+       Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used)
+#    define UseTmpHeap(Size,Proc) \
+       do { \
+         ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used += (Size); \
+       } while (0)
+#    define UnUseTmpHeap(Size,Proc) \
+       do { \
+         ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used -= (Size); \
+       } while (0)
+#    define UseTmpHeapNoproc(Size) \
+       do { \
+         erts_get_scheduler_data()->num_tmp_heap_used += (Size); \
+       } while (0)
+#    define UnUseTmpHeapNoproc(Size) \
+       do { \
+         erts_get_scheduler_data()->num_tmp_heap_used -= (Size); \
+       } while (0)
+
+
+#  endif
+
+#else
+#  define DeclareTmpHeap(VariableName,Size,Process) \
+     Eterm VariableName[Size]
+#  define DeclareTypedTmpHeap(Type,VariableName,Process)	\
+     Type VariableName[1]
+#  define DeclareTmpHeapNoproc(VariableName,Size) \
+     Eterm VariableName[Size]
+#  define UseTmpHeap(Size,Proc) /* Nothing */
+#  define UnUseTmpHeap(Size,Proc) /* Nothing */
+#  define UseTmpHeapNoproc(Size) /* Nothing */
+#  define UnUseTmpHeapNoproc(Size) /* Nothing */
+#endif /* HEAP_ON_C_STACK */
+#endif /* !__GLOBAL_H__ */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 3309b77086..df5f8b22a3 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -56,7 +56,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key;  /* Save last DDLL error o
 							  per thread basis (for BC interfaces) */
 
 Port*      erts_port; /* The port table */
-erts_smp_atomic_t erts_ports_alive;
 erts_smp_atomic_t erts_bytes_out;	/* No bytes sent out of the system */
 erts_smp_atomic_t erts_bytes_in;	/* No bytes gotten into the system */
 
@@ -72,6 +71,18 @@ erts_driver_t fd_driver;
 static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *);
 static void terminate_port(Port *p);
 static void pdl_init(void);
+#ifdef ERTS_SMP
+static void driver_monitor_lock_pdl(Port *p);
+static void driver_monitor_unlock_pdl(Port *p);
+#define DRV_MONITOR_LOCK_PDL(Port) driver_monitor_lock_pdl(Port)
+#define DRV_MONITOR_UNLOCK_PDL(Port) driver_monitor_unlock_pdl(Port)
+#else
+#define DRV_MONITOR_LOCK_PDL(Port) /* nothing */
+#define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */
+#endif
+
+#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT)
+#define SMALL_WRITE_VEC  16
 
 static ERTS_INLINE ErlIOQueue*
 drvport2ioq(ErlDrvPort drvport)
@@ -181,7 +192,7 @@ typedef struct line_buf_context {
 static erts_smp_spinlock_t get_free_port_lck;
 static Uint last_port_num;
 static Uint port_num_mask;
-erts_smp_atomic_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */
+erts_smp_atomic32_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */
 
 
 static ERTS_INLINE void
@@ -271,10 +282,36 @@ erts_test_next_port(int set, Uint next)
     return res;
 }
 
+
+static void port_cleanup(Port *prt);
+
+#ifdef ERTS_SMP
+
+static void
+sched_port_cleanup(void *vprt)
+{
+    Port *prt = (Port *) vprt;
+    erts_smp_mtx_lock(prt->lock);
+    port_cleanup(prt);
+}
+
+#endif
+
 void
 erts_port_cleanup(Port *prt)
 {
 #ifdef ERTS_SMP
+    if (erts_smp_mtx_trylock(prt->lock) == EBUSY)
+	erts_schedule_misc_op(sched_port_cleanup, (void *) prt);
+    else
+#endif
+	port_cleanup(prt);
+}
+
+void
+port_cleanup(Port *prt)
+{
+#ifdef ERTS_SMP
     Uint32 port_specific;
     erts_smp_mtx_t *mtx;
 #endif
@@ -386,14 +423,13 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver,
     new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1);
     sys_strcpy(new_name, name);
     erts_smp_runq_lock(runq);
-    erts_smp_atomic_inc(&erts_ports_alive);
     erts_smp_port_state_lock(prt);    
     prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus;
-    prt->snapshot = (Uint32) erts_smp_atomic_read(&erts_ports_snapshot);    
+    prt->snapshot = erts_smp_atomic32_read(&erts_ports_snapshot);    
     old_name = prt->name;
     prt->name = new_name;
 #ifdef ERTS_SMP
-    erts_smp_atomic_set(&prt->run_queue, (long) runq);
+    erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) runq);
 #endif
     ASSERT(!prt->drv_ptr);
     prt->drv_ptr = driver;
@@ -635,7 +671,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 #ifdef ERTS_SMP
 	erts_cancel_smp_ptimer(port->ptimer);
 #else
-	erl_cancel_timer(&(port->tm));
+	erts_cancel_timer(&(port->tm));
 #endif
 	stopq(port);
 	kill_port(port);
@@ -919,13 +955,14 @@ do {									\
     int _bitoffs;							\
     int _bitsize;							\
     ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize);		\
-    ASSERT(_bitsize == 0);						\
+    if (_bitsize != 0) goto L_type_error;				\
     if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG &&	\
 	_bitoffs == 0) {						\
 	b_size += _size;						\
+        if (b_size < _size) goto L_overflow_error;			\
 	in_clist = 0;							\
 	v_size++;							\
-        if (_size >= bin_limit) {					\
+        if (_size >= ERL_SMALL_IO_BIN_LIMIT) {				\
             p_in_clist = 0;						\
             p_v_size++;							\
         } else {							\
@@ -937,6 +974,7 @@ do {									\
         }								\
     } else {								\
 	c_size += _size;						\
+        if (c_size < _size) goto L_overflow_error;			\
 	if (!in_clist) {						\
 	    in_clist = 1;						\
 	    v_size++;							\
@@ -951,29 +989,30 @@ do {									\
 
 
 /* 
-** Size of a io list in bytes
-** return -1 if error
-** returns:            - Total size of io list
-**           vsize     - SysIOVec size needed for a writev
-**           csize     - Number of bytes not in binary (in the common binary)
-**           pvsize    - SysIOVec size needed if packing small binaries
-**           pcsize    - Number of bytes in the common binary if packing
-*/
+ * Returns 0 if successful and a non-zero value otherwise.
+ *
+ * Return values through pointers:
+ *    *vsize      - SysIOVec size needed for a writev
+ *    *csize      - Number of bytes not in binary (in the common binary)
+ *    *pvsize     - SysIOVec size needed if packing small binaries
+ *    *pcsize     - Number of bytes in the common binary if packing
+ *    *total_size - Total size of iolist in bytes
+ */
 
 static int 
-io_list_vec_len(Eterm obj, int* vsize, int* csize, 
-		int bin_limit, /* small binaries limit */
-		int * pvsize, int * pcsize)
+io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize,
+		Uint* pvsize, Uint* pcsize, Uint* total_size)
 {
     DECLARE_ESTACK(s);
     Eterm* objp;
-    int v_size = 0;
-    int c_size = 0;
-    int b_size = 0;
-    int in_clist = 0;
-    int p_v_size = 0;
-    int p_c_size = 0;
-    int p_in_clist = 0;
+    Uint v_size = 0;
+    Uint c_size = 0;
+    Uint b_size = 0;
+    Uint in_clist = 0;
+    Uint p_v_size = 0;
+    Uint p_c_size = 0;
+    Uint p_in_clist = 0;
+    Uint total;
 
     goto L_jump_start;  /* avoid a push */
 
@@ -987,6 +1026,9 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize,
 
 	    if (is_byte(obj)) {
 		c_size++;
+		if (c_size == 0) {
+		    goto L_overflow_error;
+		}
 		if (!in_clist) {
 		    in_clist = 1;
 		    v_size++;
@@ -1026,32 +1068,31 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize,
 	}
     }
 
+    total = c_size + b_size;
+    if (total < c_size) {
+	goto L_overflow_error;
+    }
+    *total_size = total;
+
     DESTROY_ESTACK(s);
-    if (vsize != NULL)
-	*vsize = v_size;
-    if (csize != NULL)
-	*csize = c_size;
-    if (pvsize != NULL)
-	*pvsize = p_v_size;
-    if (pcsize != NULL)
-	*pcsize = p_c_size;
-    return c_size + b_size;
+    *vsize = v_size;
+    *csize = c_size;
+    *pvsize = p_v_size;
+    *pcsize = p_c_size;
+    return 0;
 
  L_type_error:
+ L_overflow_error:
     DESTROY_ESTACK(s);
-    return -1;
+    return 1;
 }
 
-#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT)
-#define SMALL_WRITE_VEC  16
-
-
 /* write data to a port */
 int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 {
     char *buf;
     erts_driver_t *drv = p->drv_ptr;
-    int size;
+    Uint size;
     int fpe_was_unmasked;
     
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
@@ -1059,10 +1100,10 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 
     p->caller = caller_id;
     if (drv->outputv != NULL) {
-	int vsize;
-	int csize;
-	int pvsize;
-	int pcsize;
+	Uint vsize;
+	Uint csize;
+	Uint pvsize;
+	Uint pcsize;
 	int blimit;
 	SysIOVec iv[SMALL_WRITE_VEC];
 	ErlDrvBinary* bv[SMALL_WRITE_VEC];
@@ -1071,9 +1112,8 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 	ErlDrvBinary* cbin;
 	ErlIOVec ev;
 
-	if ((size = io_list_vec_len(list, &vsize, &csize, 
-				    ERL_SMALL_IO_BIN_LIMIT,
-				    &pvsize, &pcsize)) < 0) {
+	if (io_list_vec_len(list, &vsize, &csize,
+			    &pvsize, &pcsize, &size)) {
 	    goto bad_value;
 	}
 	/* To pack or not to pack (small binaries) ...? */
@@ -1148,7 +1188,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 	else {
 	    ASSERT(r == -1); /* Overflow */
 	    erts_free(ERTS_ALC_T_TMP, buf);
-	    if ((size = io_list_len(list)) < 0) {
+	    if (erts_iolist_size(list, &size)) {
 		goto bad_value;
 	    }
 
@@ -1191,7 +1231,6 @@ void init_io(void)
 {
     int i;
     ErlDrvEntry** dp;
-    ErlDrvEntry* drv;
     char maxports[21]; /* enough for any 64-bit integer */
     size_t maxportssize = sizeof(maxports);
     Uint ports_bits = ERTS_PORTS_BITS;
@@ -1240,7 +1279,6 @@ void init_io(void)
 
     erts_smp_atomic_init(&erts_bytes_out, 0);
     erts_smp_atomic_init(&erts_bytes_in, 0);
-    erts_smp_atomic_init(&erts_ports_alive, 0);
 
     for (i = 0; i < erts_max_ports; i++) {
 	erts_port_task_init_sched(&erts_port[i].sched);
@@ -1262,7 +1300,7 @@ void init_io(void)
 	erts_port[i].port_data_lock = NULL;
     }
 
-    erts_smp_atomic_init(&erts_ports_snapshot, (long) 0);
+    erts_smp_atomic32_init(&erts_ports_snapshot, (erts_aint32_t) 0);
     last_port_num = 0;
     erts_smp_spinlock_init(&get_free_port_lck, "get_free_port");
 
@@ -1274,10 +1312,8 @@ void init_io(void)
     init_driver(&fd_driver, &fd_driver_entry, NULL);
     init_driver(&vanilla_driver, &vanilla_driver_entry, NULL);
     init_driver(&spawn_driver, &spawn_driver_entry, NULL);
-    for (dp = driver_tab; *dp != NULL; dp++) {
-	drv = *dp;
+    for (dp = driver_tab; *dp != NULL; dp++)
 	erts_add_driver_entry(*dp, NULL, 1);
-    }
 
     erts_smp_tsd_set(driver_list_lock_status_key, NULL);
     erts_smp_mtx_unlock(&erts_driver_list_lock);
@@ -1540,14 +1576,14 @@ static void deliver_read_message(Port* prt, Eterm to,
 	pb = (ProcBin *) hp;
 	pb->thing_word = HEADER_PROC_BIN;
 	pb->size = len;
-	pb->next = ohp->mso;
-	ohp->mso = pb;
+	pb->next = ohp->first;
+	ohp->first = (struct erl_off_heap_header*)pb;
 	pb->val = bptr;
 	pb->bytes = (byte*) bptr->orig_bytes;
 	pb->flags = 0;
 	hp += PROC_BIN_SIZE;
 
-	ohp->overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
 	listp = make_binary(pb);
     }
 
@@ -1690,14 +1726,14 @@ deliver_vec_message(Port* prt,			/* Port */
 	    }
 	    pb->thing_word = HEADER_PROC_BIN;
 	    pb->size = iov->iov_len;
-	    pb->next = ohp->mso;
-	    ohp->mso = pb;
+	    pb->next = ohp->first;
+	    ohp->first = (struct erl_off_heap_header*)pb;
 	    pb->val = ErlDrvBinary2Binary(b);
 	    pb->bytes = base;
 	    pb->flags = 0;
 	    hp += PROC_BIN_SIZE;
 	    
-	    ohp->overhead += iov->iov_len / sizeof(Eterm);
+	    OH_OVERHEAD(ohp, iov->iov_len / sizeof(Eterm));
 
 	    if (listp == NIL) {  /* compatible with deliver_bin_message */
 		listp = make_binary(pb);
@@ -1804,7 +1840,7 @@ terminate_port(Port *prt)
 #ifdef ERTS_SMP
     erts_cancel_smp_ptimer(prt->ptimer);
 #else
-    erl_cancel_timer(&prt->tm);
+    erts_cancel_timer(&prt->tm);
 #endif
 
     drv = prt->drv_ptr;
@@ -1998,12 +2034,13 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason)
        p->nlinks = NULL;
        erts_sweep_links(lnk, &sweep_one_link, &sc);
    }
+   DRV_MONITOR_LOCK_PDL(p);
    {
        ErtsMonitor *moni = p->monitors;
        p->monitors = NULL;
        erts_sweep_monitors(moni, &sweep_one_monitor, NULL);
    } 
-
+   DRV_MONITOR_UNLOCK_PDL(p);
 
    if ((p->status & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) {
        erts_do_net_exits(p->dist_entry, rreason);
@@ -2114,7 +2151,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
     byte* to_port = NULL;	/* Buffer to write to port. */
 				/* Initialization is for shutting up
 				   warning about use before set. */
-    int to_len = 0;		/* Length of buffer. */
+    Uint to_len = 0;		/* Length of buffer. */
     int must_free = 0;		/* True if the buffer should be freed. */
     char port_result[ERL_ONHEAP_BIN_LIMIT];  /* Default buffer for result from port. */
     char* port_resp;		/* Pointer to result buffer. */
@@ -2159,7 +2196,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
 	} else {
 	    ASSERT(r == -1);	/* Overflow */
 	    erts_free(ERTS_ALC_T_TMP, (void *) to_port);
-	    if ((to_len = io_list_len(iolist)) < 0) { /* Type error */
+	    if (erts_iolist_size(iolist, &to_len)) { /* Type error */
 		return THE_NON_VALUE;
 	    }
 	    must_free = 1;
@@ -2223,12 +2260,12 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
 		ProcBin* pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
 		pb->thing_word = HEADER_PROC_BIN;
 		pb->size = dbin->orig_size;
-		pb->next = MSO(p).mso;
-		MSO(p).mso = pb;
+		pb->next = MSO(p).first;
+		MSO(p).first = (struct erl_off_heap_header*)pb;
 		pb->val = ErlDrvBinary2Binary(dbin);
 		pb->bytes = (byte*) dbin->orig_bytes;
 		pb->flags = 0;
-		MSO(p).overhead += dbin->orig_size / sizeof(Eterm);
+		OH_OVERHEAD(&(MSO(p)), dbin->orig_size / sizeof(Eterm));
 		return make_binary(pb);
 	    }
 	    port_resp = dbin->orig_bytes;
@@ -2384,7 +2421,7 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len)
 
     if (len > (Uint) INT_MAX)
 	erl_exit(ERTS_ABORT_EXIT,
-		 "Absurdly large data buffer (%bpu bytes) passed to"
+		 "Absurdly large data buffer (%beu bytes) passed to"
 		 "output callback of %s driver.\n",
 		 len,
 		 p->drv_ptr->name ? p->drv_ptr->name : "unknown");
@@ -2766,17 +2803,25 @@ driver_deliver_term(ErlDrvPort port,
 	    break;
 	case ERL_DRV_INT:  /* signed int argument */
 	    ERTS_DDT_CHK_ENOUGH_ARGS(1);
+#if HALFWORD_HEAP
+	    erts_bld_sint64(NULL, &need, (Sint64)ptr[0]);
+#else
 	    /* check for bignum */
 	    if (!IS_SSMALL((Sint)ptr[0]))
 		need += BIG_UINT_HEAP_SIZE;  /* use small_to_big */
+#endif
 	    ptr++;
 	    depth++;
 	    break;
 	case ERL_DRV_UINT:  /* unsigned int argument */
 	    ERTS_DDT_CHK_ENOUGH_ARGS(1);
+#if HALFWORD_HEAP
+	    erts_bld_uint64(NULL, &need, (Uint64)ptr[0]);
+#else
 	    /* check for bignum */
 	    if (!IS_USMALL(0, (Uint)ptr[0]))
 		need += BIG_UINT_HEAP_SIZE;  /* use small_to_big */
+#endif
 	    ptr++;
 	    depth++;
 	    break;
@@ -2943,22 +2988,30 @@ driver_deliver_term(ErlDrvPort port,
 	    break;
 
 	case ERL_DRV_INT:  /* signed int argument */
+#if HALFWORD_HEAP
+	    mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]);
+#else
 	    if (IS_SSMALL((Sint)ptr[0]))
 		mess = make_small((Sint)ptr[0]);
 	    else {
 		mess = small_to_big((Sint)ptr[0], hp);
 		hp += BIG_UINT_HEAP_SIZE;
 	    }
+#endif
 	    ptr++;
 	    break;
 
 	case ERL_DRV_UINT:  /* unsigned int argument */
+#if HALFWORD_HEAP
+	    mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]);
+#else
 	    if (IS_USMALL(0, (Uint)ptr[0]))
 		mess = make_small((Uint)ptr[0]);
 	    else {
 		mess = uint_to_big((Uint)ptr[0], hp);
 		hp += BIG_UINT_HEAP_SIZE;
 	    }
+#endif
 	    ptr++;
 	    break;
 
@@ -2997,14 +3050,14 @@ driver_deliver_term(ErlDrvPort port,
 		driver_binary_inc_refc(b);  /* caller will free binary */
 		pb->thing_word = HEADER_PROC_BIN;
 		pb->size = size;
-		pb->next = ohp->mso;
-		ohp->mso = pb;
+		pb->next = ohp->first;
+		ohp->first = (struct erl_off_heap_header*)pb;
 		pb->val = ErlDrvBinary2Binary(b);
 		pb->bytes = ((byte*) b->orig_bytes) + offset;
 		pb->flags = 0;
 		mess =  make_binary(pb);
 		hp += PROC_BIN_SIZE;
-		ohp->overhead += pb->size / sizeof(Eterm);
+		OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
 	    }
 	    ptr += 3;
 	    break;
@@ -3036,12 +3089,12 @@ driver_deliver_term(ErlDrvPort port,
 		hp += PROC_BIN_SIZE;
 		pbp->thing_word = HEADER_PROC_BIN;
 		pbp->size = size;
-		pbp->next = ohp->mso;
-		ohp->mso = pbp;
+		pbp->next = ohp->first;
+		ohp->first = (struct erl_off_heap_header*)pbp;
 		pbp->val = bp;
 		pbp->bytes = (byte*) bp->orig_bytes;
 		pbp->flags = 0;
-		ohp->overhead += (pbp->size / sizeof(Eterm));
+		OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm));
 		mess = make_binary(pbp);
 	    }
 	    ptr += 2;
@@ -3200,7 +3253,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, int hlen,
 	return 0;
 
     prt->bytes_in += (hlen + len);
-    erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + len));
+    erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len));
     if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) {
 	return erts_net_message(prt,
 				prt->dist_entry,
@@ -3235,7 +3288,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, int hlen, char* buf, int len)
 	return 0;
     
     prt->bytes_in += (hlen + len);
-    erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + len));
+    erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len));
     if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) {
 	if (len == 0)
 	    return erts_net_message(prt,
@@ -3312,7 +3365,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, int hlen, ErlIOVec* vec, int skip)
 
     /* XXX handle distribution !!! */
     prt->bytes_in += (hlen + size);
-    erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + size));
+    erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + size));
     deliver_vec_message(prt, prt->connected, hbuf, hlen, binv, iov, n, size);
     return 0;
 }
@@ -3356,25 +3409,25 @@ int len;
  * reference count on driver binaries...
  */
 
-long
+ErlDrvSInt
 driver_binary_get_refc(ErlDrvBinary *dbp)
 {
     Binary* bp = ErlDrvBinary2Binary(dbp);
-    return erts_refc_read(&bp->refc, 1);
+    return (ErlDrvSInt) erts_refc_read(&bp->refc, 1);
 }
 
-long
+ErlDrvSInt
 driver_binary_inc_refc(ErlDrvBinary *dbp)
 {
     Binary* bp = ErlDrvBinary2Binary(dbp);
-    return erts_refc_inctest(&bp->refc, 2);
+    return (ErlDrvSInt) erts_refc_inctest(&bp->refc, 2);
 }
 
-long
+ErlDrvSInt
 driver_binary_dec_refc(ErlDrvBinary *dbp)
 {
     Binary* bp = ErlDrvBinary2Binary(dbp);
-    return erts_refc_dectest(&bp->refc, 1);
+    return (ErlDrvSInt) erts_refc_dectest(&bp->refc, 1);
 }
 
 
@@ -3489,12 +3542,12 @@ pdl_init_refc(ErlDrvPDL pdl)
     erts_atomic_init(&pdl->refc, 1);
 }
 
-static ERTS_INLINE long
+static ERTS_INLINE ErlDrvSInt
 pdl_read_refc(ErlDrvPDL pdl)
 {
-    long refc = erts_atomic_read(&pdl->refc);
+    erts_aint_t refc = erts_atomic_read(&pdl->refc);
     ERTS_LC_ASSERT(refc >= 0);
-    return refc;
+    return (ErlDrvSInt) refc;
 }
 
 static ERTS_INLINE void
@@ -3504,12 +3557,12 @@ pdl_inc_refc(ErlDrvPDL pdl)
     ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 1);
 }
 
-static ERTS_INLINE long
+static ERTS_INLINE ErlDrvSInt
 pdl_inctest_refc(ErlDrvPDL pdl)
 {
-    long refc = erts_atomic_inctest(&pdl->refc);
+    erts_aint_t refc = erts_atomic_inctest(&pdl->refc);
     ERTS_LC_ASSERT(refc > 1);
-    return refc;
+    return (ErlDrvSInt) refc;
 }
 
 #if 0 /* unused */
@@ -3521,12 +3574,12 @@ pdl_dec_refc(ErlDrvPDL pdl)
 }
 #endif
 
-static ERTS_INLINE long
+static ERTS_INLINE ErlDrvSInt
 pdl_dectest_refc(ErlDrvPDL pdl)
 {
-    long refc = erts_atomic_dectest(&pdl->refc);
+    erts_aint_t refc = erts_atomic_dectest(&pdl->refc);
     ERTS_LC_ASSERT(refc >= 0);
-    return refc;
+    return (ErlDrvSInt) refc;
 }
 
 static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl)
@@ -3536,6 +3589,32 @@ static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl)
     erts_free(ERTS_ALC_T_PORT_DATA_LOCK, pdl);
 }
 
+#ifdef ERTS_SMP
+
+static void driver_monitor_lock_pdl(Port *p) {
+    if (p->port_data_lock) {
+	driver_pdl_lock(p->port_data_lock);
+    }
+    /* Now we either have the port lock or the port_data_lock */
+    ERTS_LC_ASSERT(!p->port_data_lock
+		   || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx)));
+    ERTS_SMP_LC_ASSERT(p->port_data_lock
+		       || erts_lc_is_port_locked(p));
+}
+
+static void driver_monitor_unlock_pdl(Port *p) {
+    /* We should either have the port lock or the port_data_lock */
+    ERTS_LC_ASSERT(!p->port_data_lock
+		   || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx)));
+    ERTS_SMP_LC_ASSERT(p->port_data_lock
+		       || erts_lc_is_port_locked(p));
+    if (p->port_data_lock) {
+	driver_pdl_unlock(p->port_data_lock);
+    }
+}
+
+#endif
+
 /*
  * exported driver_pdl_* functions ...
  */
@@ -3571,7 +3650,7 @@ driver_pdl_lock(ErlDrvPDL pdl)
 void
 driver_pdl_unlock(ErlDrvPDL pdl)
 {
-    long refc;
+    ErlDrvSInt refc;
 #ifdef HARDDEBUG
     erts_fprintf(stderr, "driver_pdl_unlock(0x%08X)\r\n",(unsigned) pdl);
 #endif
@@ -3581,28 +3660,30 @@ driver_pdl_unlock(ErlDrvPDL pdl)
 	pdl_destroy(pdl);
 }
 
-long
+ErlDrvSInt
 driver_pdl_get_refc(ErlDrvPDL pdl)
 {
     return pdl_read_refc(pdl);
 }
 
-long
+ErlDrvSInt
 driver_pdl_inc_refc(ErlDrvPDL pdl)
 {
-    long refc = pdl_inctest_refc(pdl);
+    ErlDrvSInt refc = pdl_inctest_refc(pdl);
 #ifdef HARDDEBUG
-    erts_fprintf(stderr, "driver_pdl_inc_refc(0x%08X) -> %ld\r\n",(unsigned) pdl, refc);
+    erts_fprintf(stderr, "driver_pdl_inc_refc(%p) -> %bed\r\n",
+		 pdl, refc);
 #endif
     return refc;
 }
 
-long
+ErlDrvSInt
 driver_pdl_dec_refc(ErlDrvPDL pdl)
 {
-    long refc = pdl_dectest_refc(pdl);
+    ErlDrvSInt refc = pdl_dectest_refc(pdl);
 #ifdef HARDDEBUG
-    erts_fprintf(stderr, "driver_pdl_dec_refc(0x%08X) -> %ld\r\n",(unsigned) pdl, refc);
+    erts_fprintf(stderr, "driver_pdl_dec_refc(%p) -> %bpd\r\n",
+		 pdl, refc);
 #endif
     if (!refc)
 	pdl_destroy(pdl);
@@ -3988,13 +4069,13 @@ drv_cancel_timer(Port *prt)
 #ifdef ERTS_SMP
     erts_cancel_smp_ptimer(prt->ptimer);
 #else
-    erl_cancel_timer(&prt->tm);
+    erts_cancel_timer(&prt->tm);
 #endif
     if (erts_port_task_is_scheduled(&prt->timeout_task))
 	erts_port_task_abort(prt->id, &prt->timeout_task);
 }
 
-int driver_set_timer(ErlDrvPort ix, Uint t)
+int driver_set_timer(ErlDrvPort ix, UWord t)
 {
     Port* prt = erts_drvport2port(ix);
 
@@ -4012,7 +4093,7 @@ int driver_set_timer(ErlDrvPort ix, Uint t)
 			   (ErlTimeoutProc) schedule_port_timeout,
 			   t);
 #else
-    erl_set_timer(&prt->tm,
+    erts_set_timer(&prt->tm,
 		  (ErlTimeoutProc) schedule_port_timeout,
 		  NULL,
 		  prt,
@@ -4043,9 +4124,9 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t)
 	return -1;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 #ifdef ERTS_SMP
-    *t = prt->ptimer ? time_left(&prt->ptimer->timer.tm) : 0;
+    *t = prt->ptimer ? erts_time_left(&prt->ptimer->timer.tm) : 0;
 #else
-    *t = time_left(&prt->tm);
+    *t = erts_time_left(&prt->tm);
 #endif
     return 0;
 }
@@ -4053,12 +4134,16 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t)
 int 
 driver_get_now(ErlDrvNowData *now_data)
 {
+    Uint mega,secs,micro;
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
     if (now_data == NULL) {
 	return -1;
     }
-    get_now(&(now_data->megasecs),&(now_data->secs),&(now_data->microsecs));
+    get_now(&mega,&secs,&micro);
+    now_data->megasecs = (unsigned long) mega;
+    now_data->secs = (unsigned long) secs;
+    now_data->microsecs = (unsigned long) micro;
     return 0;
 }
 
@@ -4072,14 +4157,15 @@ static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon)
     memcpy(mon,refp,sizeof(RefThing));
 }
 
-int driver_monitor_process(ErlDrvPort port, 
-			   ErlDrvTermData process, 
-			   ErlDrvMonitor *monitor)
+
+static int do_driver_monitor_process(Port *prt,
+				     Eterm *buf,
+				     ErlDrvTermData process,
+				     ErlDrvMonitor *monitor)
 {
-    Port *prt = erts_drvport2port(port);
     Process *rp;
     Eterm ref;
-    Eterm buf[REF_THING_SIZE];
+
     if (prt->drv_ptr->process_exit == NULL) {
 	return -1;
     }
@@ -4089,22 +4175,76 @@ int driver_monitor_process(ErlDrvPort port,
     if (!rp) {
 	return 1;
     }
+
     ref = erts_make_ref_in_buffer(buf);
     erts_add_monitor(&(prt->monitors), MON_ORIGIN, ref, rp->id, NIL);
     erts_add_monitor(&(rp->monitors), MON_TARGET, ref, prt->id, NIL);
-    
+
     erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
     ref_to_driver_monitor(ref,monitor);
     return 0;
 }
 
-int driver_demonitor_process(ErlDrvPort port,
-			     const ErlDrvMonitor *monitor)
+/*
+ * This can be called from a non scheduler thread iff a port_data_lock exists
+ */
+int driver_monitor_process(ErlDrvPort port,
+			   ErlDrvTermData process,
+			   ErlDrvMonitor *monitor)
+{
+    Port *prt;
+    int ret;
+    Uint32 status;
+    ErtsSchedulerData *sched = erts_get_scheduler_data();
+    int ix = (int) port;
+    if (ix < 0 || erts_max_ports <= ix) {
+	return -1;
+    }
+    prt = &erts_port[ix];
+
+    DRV_MONITOR_LOCK_PDL(prt);
+
+    if (sched) {
+	status = erts_port[ix].status;
+    } else {
+	erts_smp_port_state_lock(prt);
+	status = erts_port[ix].status;
+	erts_smp_port_state_unlock(prt);
+    }
+
+    if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
+	DRV_MONITOR_UNLOCK_PDL(prt);
+	return -1;
+    }
+
+    /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock
+       (if we're a driver thread) */
+    ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
+
+#if !HEAP_ON_C_STACK
+    if (!sched) {
+	/* Need a separate allocation for the ref :( */
+	Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM,
+				sizeof(Eterm)*REF_THING_SIZE);
+	ret = do_driver_monitor_process(prt,buf,process,monitor);
+	erts_free(ERTS_ALC_T_TEMP_TERM,buf);
+    } else
+#endif
+    {
+	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
+	UseTmpHeapNoproc(REF_THING_SIZE);
+	ret = do_driver_monitor_process(prt,buf,process,monitor);
+	UnUseTmpHeapNoproc(REF_THING_SIZE);
+    }
+    DRV_MONITOR_UNLOCK_PDL(prt);
+    return ret;
+}
+
+static int do_driver_demonitor_process(Port *prt, Eterm *buf,
+				       const ErlDrvMonitor *monitor)
 {
-    Port *prt = erts_drvport2port(port);
     Process *rp;
     Eterm ref;
-    Eterm buf[REF_THING_SIZE];
     ErtsMonitor *mon;
     Eterm to;
 
@@ -4137,12 +4277,60 @@ int driver_demonitor_process(ErlDrvPort port,
     return 0;
 }
 
-ErlDrvTermData driver_get_monitored_process(ErlDrvPort port,
+int driver_demonitor_process(ErlDrvPort port,
+			     const ErlDrvMonitor *monitor)
+{
+    Port *prt;
+    int ret;
+    Uint32 status;
+    ErtsSchedulerData *sched = erts_get_scheduler_data();
+    int ix = (int) port;
+    if (ix < 0 || erts_max_ports <= ix) {
+	return -1;
+    }
+    prt = &erts_port[ix];
+
+    DRV_MONITOR_LOCK_PDL(prt);
+
+    if (sched) {
+	status = erts_port[ix].status;
+    } else {
+	erts_smp_port_state_lock(prt);
+	status = erts_port[ix].status;
+	erts_smp_port_state_unlock(prt);
+    }
+
+    if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
+	DRV_MONITOR_UNLOCK_PDL(prt);
+	return -1;
+    }
+
+    /* Now we should have either the port lock (if we have a scheduler) or the port data lock
+       (if we're a driver thread) */
+    ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
+#if !HEAP_ON_C_STACK
+    if (!sched) {
+	/* Need a separate allocation for the ref :( */
+	Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM,
+				sizeof(Eterm)*REF_THING_SIZE);
+	ret = do_driver_demonitor_process(prt,buf,monitor);
+	erts_free(ERTS_ALC_T_TEMP_TERM,buf);
+    } else
+#endif
+    {
+	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
+	UseTmpHeapNoproc(REF_THING_SIZE);
+	ret = do_driver_demonitor_process(prt,buf,monitor);
+	UnUseTmpHeapNoproc(REF_THING_SIZE);
+    }
+    DRV_MONITOR_UNLOCK_PDL(prt);
+    return ret;
+}
+
+static ErlDrvTermData do_driver_get_monitored_process(Port *prt, Eterm *buf,
 					    const ErlDrvMonitor *monitor)
 {
-    Port *prt = erts_drvport2port(port);
     Eterm ref;
-    Eterm buf[REF_THING_SIZE];
     ErtsMonitor *mon;
     Eterm to;
 
@@ -4158,6 +4346,59 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort port,
     return (ErlDrvTermData) to;
 }
 
+
+ErlDrvTermData driver_get_monitored_process(ErlDrvPort port,
+					    const ErlDrvMonitor *monitor)
+{
+    Port *prt;
+    ErlDrvTermData ret;
+    Uint32 status;
+    ErtsSchedulerData *sched = erts_get_scheduler_data();
+    int ix = (int) port;
+    if (ix < 0 || erts_max_ports <= ix) {
+	return driver_term_nil;
+    }
+    prt = &erts_port[ix];
+
+    DRV_MONITOR_LOCK_PDL(prt);
+
+    if (sched) {
+	status = erts_port[ix].status;
+    } else {
+	erts_smp_port_state_lock(prt);
+	status = erts_port[ix].status;
+	erts_smp_port_state_unlock(prt);
+    }
+
+    if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
+	DRV_MONITOR_UNLOCK_PDL(prt);
+	return driver_term_nil;
+    }
+
+    /* Now we should have either the port lock (if we have a scheduler) or the port data lock
+       (if we're a driver thread) */
+    ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
+
+#if !HEAP_ON_C_STACK
+    if (!sched) {
+	/* Need a separate allocation for the ref :( */
+	Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM,
+				sizeof(Eterm)*REF_THING_SIZE);
+	ret = do_driver_get_monitored_process(prt,buf,monitor);
+	erts_free(ERTS_ALC_T_TEMP_TERM,buf);
+    } else
+#endif
+    {
+	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
+	UseTmpHeapNoproc(REF_THING_SIZE);
+	ret = do_driver_get_monitored_process(prt,buf,monitor);
+	UnUseTmpHeapNoproc(REF_THING_SIZE);
+    }
+    DRV_MONITOR_UNLOCK_PDL(prt);
+    return ret;
+}
+
+
 int driver_compare_monitors(const ErlDrvMonitor *monitor1,
 			    const ErlDrvMonitor *monitor2)
 {
@@ -4173,18 +4414,22 @@ void erts_fire_port_monitor(Port *prt, Eterm ref)
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
     ASSERT(prt->drv_ptr != NULL);    
-
+    DRV_MONITOR_LOCK_PDL(prt);
     if (erts_lookup_monitor(prt->monitors,ref) == NULL) {
+	DRV_MONITOR_UNLOCK_PDL(prt);
 	return;
     }
     callback = prt->drv_ptr->process_exit;
     ASSERT(callback != NULL);
     ref_to_driver_monitor(ref,&drv_monitor);
+    DRV_MONITOR_UNLOCK_PDL(prt);
     fpe_was_unmasked = erts_block_fpe();
     (*callback)((ErlDrvData) (prt->drv_data), &drv_monitor);
     erts_unblock_fpe(fpe_was_unmasked);
+    DRV_MONITOR_LOCK_PDL(prt);
     /* remove monitor *after* callback */
     rmon = erts_remove_monitor(&(prt->monitors),ref);
+    DRV_MONITOR_UNLOCK_PDL(prt);
     if (rmon) {
 	erts_destroy_monitor(rmon);
     }
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 87d13b3607..694e4ab72f 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -29,8 +29,8 @@ typedef struct erl_module {
     IndexSlot slot;		/* Must be located at top of struct! */
     int module;			/* Atom index for module (not tagged). */
 
-    Eterm* code;
-    Eterm* old_code;
+    BeamInstr* code;
+    BeamInstr* old_code;
     int code_length;		/* Length of loaded code in bytes. */
     int old_code_length;	/* Length of old loaded code in bytes */
     unsigned catches, old_catches;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index ce1df74f03..8a5763b4bb 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1,19 +1,19 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 
@@ -60,12 +60,17 @@ func_info M=a a==am_module_info A=u==0 | label L | move n r => too_old_compiler
 func_info M=a a==am_module_info A=u==1 | label L | move n r => too_old_compiler
 
 # The undocumented and unsupported guard BIF is_constant/1 was removed
-# in R13. The is_constant/2 operation is marked as obosolete in genop.tab,
+# in R13. The is_constant/2 operation is marked as obsolete in genop.tab,
 # so the loader will automatically generate a too_old_compiler message
 # it is used, but we need to handle the is_constant/1 BIF specially here.
 
 bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler
 
+# Since the constant pool was introduced in R12B, empty tuples ({})
+# are literals. Therefore we no longer need to allow put_tuple/2
+# with a tuple size of zero.
+
+put_tuple u==0 d => too_old_compiler
 
 #
 # All the other instructions.
@@ -79,6 +84,8 @@ i_trace_breakpoint
 i_mtrace_breakpoint
 i_debug_breakpoint
 i_count_breakpoint
+i_time_breakpoint
+i_return_time_trace
 i_return_to_trace
 i_yield
 i_global_cons
@@ -94,16 +101,16 @@ return
 %macro: test_heap TestHeap -pack
 
 allocate t t
-allocate_heap I I I
+allocate_heap t I t
 deallocate I
 init y
 allocate_zero t t
-allocate_heap_zero I I I
+allocate_heap_zero t I t
 
 trim N Remaining => i_trim N
 i_trim I
 
-test_heap I I
+test_heap I t
 
 allocate_heap S u==0 R => allocate S R
 allocate_heap_zero S u==0 R => allocate_zero S R
@@ -115,15 +122,9 @@ init Y1 | init Y2 => init2 Y1 Y2
 %macro: init2 Init2 -pack
 %macro: init3 Init3 -pack
 
-#
-# Warning: The put_string instruction is specially treated in the loader.
-# Don't change the instruction format unless you change the loader too.
-#
-put_string I I d
-
 # Selecting values
 
-select_val S=q Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest)
+select_val S=aiq Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest)
 
 select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
   gen_jump_tab(S, Fail, Size, Rest)
@@ -131,34 +132,59 @@ select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
 is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
   gen_jump_tab(S, Fail, Size, Rest)
 
+is_integer TypeFail=f S | select_val S=s Fail=f Size=u Rest=* | \
+	   mixed_types(Size, Rest) => \
+  gen_split_values(S, TypeFail, Fail, Size, Rest)
+
 select_val S=s Fail=f Size=u Rest=* | mixed_types(Size, Rest) => \
-  gen_split_values(S, Fail, Size, Rest)
+  gen_split_values(S, Fail, Fail, Size, Rest)
 
-is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | \
+is_integer Fail=f S | select_val S=d Fail=f Size=u Rest=* | \
   fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest)
 
-is_atom Fail=f S | select_val S=s Fail=f Size=u Rest=* | \
+is_atom Fail=f S | select_val S=d Fail=f Size=u Rest=* | \
   fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest)
 
-select_val S=s Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \
-  gen_select_val(S, Fail, Size, Rest)
+select_val S=s Fail=f Size=u Rest=* | floats_or_bignums(Size, Rest) => \
+  gen_select_literals(S, Fail, Size, Rest)
 
-select_val S=s Fail=f Size=u Rest=* | all_values_are_big(Size, Rest) => \
-  gen_select_big(S, Fail, Size, Rest)
+select_val S=d Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \
+  gen_select_val(S, Fail, Size, Rest)
 
-is_tuple Fail=f S | select_tuple_arity S=s Fail=f Size=u Rest=* => \
+is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \
   gen_select_tuple_arity(S, Fail, Size, Rest)
 
-select_tuple_arity S=s Fail=f Size=u Rest=* => \
+select_tuple_arity S=d Fail=f Size=u Rest=* => \
   gen_select_tuple_arity(S, Fail, Size, Rest)
 
-i_select_val s f I
-i_select_tuple_arity s f I
-i_select_big s f
-i_select_float s f I
+i_select_val r f I
+i_select_val x f I
+i_select_val y f I
+
+i_select_val2 r f c f c f
+i_select_val2 x f c f c f
+i_select_val2 y f c f c f
+
+i_select_tuple_arity2 r f A f A f
+i_select_tuple_arity2 x f A f A f
+i_select_tuple_arity2 y f A f A f
+
+i_select_tuple_arity r f I
+i_select_tuple_arity x f I
+i_select_tuple_arity y f I
 
-i_jump_on_val_zero s f I
-i_jump_on_val s f I I
+i_jump_on_val_zero r f I
+i_jump_on_val_zero x f I
+i_jump_on_val_zero y f I
+
+i_jump_on_val r f I I
+i_jump_on_val x f I I
+i_jump_on_val y f I I
+
+jump Target | label Lbl | same_label(Target, Lbl) => label Lbl
+
+is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \
+  is_eq_exact Fail S1 S2 | label L2
 
 %macro: get_list GetList -pack
 get_list x x x
@@ -233,11 +259,17 @@ is_number Fail Literal=q => move Literal x | is_number Fail x
 
 jump f
 
-case_end Literal=q => move Literal x | case_end x
-badmatch Literal=q => move Literal x | badmatch x
+case_end Literal=cq => move Literal x | case_end x
+badmatch Literal=cq => move Literal x | badmatch x
+
+case_end r
+case_end x
+case_end y
+
+badmatch r
+badmatch x
+badmatch y
 
-case_end s
-badmatch s
 if_end
 raise s s
 
@@ -247,12 +279,33 @@ system_limit j
 
 move R R =>
 
+move C=cxy r | jump Lbl => move_jump Lbl C
+
+%macro: move_jump MoveJump -nonext
+move_jump f n
+move_jump f c
+move_jump f x
+move_jump f y
+
 move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2
 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2
+move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4
+
+move C=aiq X=x==1 => move_x1 C
+move C=aiq X=x==2 => move_x2 C
+
+move_x1 c
+move_x2 c
 
 %macro: move2 Move2 -pack
 move2 x y x y
 move2 y x y x
+move2 x x x x
+
+# The compiler almost never generates a "move Literal y(Y)" instruction,
+# so let's cheat if we encounter one.
+move S=n D=y => init D
+move S=c D=y => move S x | move x D
 
 %macro:move Move -pack -gen_dest
 move x x
@@ -264,15 +317,10 @@ move r x
 move r y
 move c r
 move c x
-move c y
 move n x
 move n r
 move y y
 
-%cold
-move s d
-%hot
-
 # Receive operations.
 
 loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src
@@ -305,58 +353,78 @@ i_wait_error_locked
 send
 
 #
-# Comparisions.
+# Optimized comparisons with one immediate/literal operand.
+#
+
+is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C
+is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C
+
+is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C
+is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C
+
+%macro: i_is_eq_exact_immed EqualImmed -fail_action
+i_is_eq_exact_immed f r c
+i_is_eq_exact_immed f x c
+i_is_eq_exact_immed f y c
+
+i_is_eq_exact_literal r f c
+i_is_eq_exact_literal x f c
+i_is_eq_exact_literal y f c
+
+%macro: i_is_ne_exact_immed NotEqualImmed -fail_action
+i_is_ne_exact_immed f r c
+i_is_ne_exact_immed f x c
+i_is_ne_exact_immed f y c
+
+i_is_ne_exact_literal r f c
+i_is_ne_exact_literal x f c
+i_is_ne_exact_literal y f c
+
+#
+# All other comparisons.
 #
 
-is_eq_exact Lbl=f R=rxy C=ian => i_is_eq_immed Lbl R C
-is_eq Lbl=f R=rxy C=an => i_is_eq_immed Lbl R C
+is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl
+is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl
 
 is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl
 is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl
 is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl
 is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl
 
-is_eq_exact Lbl=f S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl
-is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl
-
+i_is_eq_exact f
+i_is_ne_exact f
 i_is_lt f
 i_is_ge f
 i_is_eq f
 i_is_ne f
-i_is_eq_exact f
-i_is_ne_exact f
-
-%macro: i_is_eq_immed EqualImmed -fail_action
-i_is_eq_immed f r c
-i_is_eq_immed f x c
-i_is_eq_immed f y c
 
 #
 # Putting things.
 #
 
-put_tuple u==0 Dst => i_put_tuple_only u Dst
-put_tuple Arity Dst | put V => i_put_tuple Arity V Dst
+put_tuple Arity Dst => i_put_tuple Dst u
 
-i_put_tuple_only A d
+i_put_tuple Dst Arity Puts=* | put S1 | put S2 | \
+  put S3 | put S4 | put S5 => \
+	    tuple_append_put5(Arity, Dst, Puts, S1, S2, S3, S4, S5)
 
-%macro: i_put_tuple PutTuple -pack
-i_put_tuple A x x
-i_put_tuple A y x
-i_put_tuple A r x
-i_put_tuple A n x
-i_put_tuple A c x
-i_put_tuple A x y
-i_put_tuple A x r
-i_put_tuple A y r
-i_put_tuple A n r
-i_put_tuple A c r
+i_put_tuple Dst Arity Puts=* | put S => \
+	    tuple_append_put(Arity, Dst, Puts, S)
 
-%cold
-i_put_tuple A r y
-i_put_tuple A y y
-i_put_tuple A c y
-%hot
+i_put_tuple/2
+
+%macro:i_put_tuple PutTuple -pack -goto:do_put_tuple
+i_put_tuple r I
+i_put_tuple x I
+i_put_tuple y I
+
+#
+# The instruction "put_list Const [] Dst" will not be generated by
+# the current BEAM compiler. But until R15A, play it safe by handling
+# that instruction with the following transformation.
+#
+put_list Const=c n Dst => move Const x | put_list x n Dst
 
 %macro:put_list PutList -pack -gen_dest
 
@@ -364,10 +432,8 @@ put_list x n x
 put_list y n x
 put_list x x x
 put_list y x x
-put_list c n x
 put_list x x r
 put_list y r r
-put_list c n r
 
 put_list y y x
 put_list x y x
@@ -378,6 +444,13 @@ put_list y y r
 put_list y r x
 put_list r n x
 
+put_list x r x
+put_list x y r
+put_list y x r
+put_list y x x
+
+put_list x r r
+
 # put_list SrcReg Constant Dst
 put_list r c r
 put_list r c x
@@ -405,17 +478,9 @@ put_list c y x
 put_list c y y
 
 %cold
-put_list x r r
 put_list s s d
 %hot
 
-%macro: put Put
-put x
-put r
-put y
-put c
-put n
-
 %macro: i_fetch FetchArgs -pack
 i_fetch c c
 i_fetch c r
@@ -466,19 +531,20 @@ move_return n r
 
 move S r | deallocate D | return => move_deallocate_return S r D
 
-%macro: move_deallocate_return MoveDeallocateReturn -nonext
-move_deallocate_return x r P
-move_deallocate_return y r P
-move_deallocate_return c r P
-move_deallocate_return n r P
+%macro: move_deallocate_return MoveDeallocateReturn -pack -nonext
+move_deallocate_return x r Q
+move_deallocate_return y r Q
+move_deallocate_return c r Q
+move_deallocate_return n r Q
 
 deallocate D | return => deallocate_return D
 
 %macro: deallocate_return DeallocateReturn -nonext
-deallocate_return P
+deallocate_return Q
 
 test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y
 
+%macro: test_heap_1_put_list TestHeapPutList -pack
 test_heap_1_put_list I y
 
 # Test tuple & arity (head)
@@ -578,14 +644,14 @@ is_list f y
 
 is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs
 
-%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action
-is_nonempty_list_allocate f x I I
-is_nonempty_list_allocate f r I I
+%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack
+is_nonempty_list_allocate f x I t
+is_nonempty_list_allocate f r I t
 
 is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2
 
-%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action
-is_non_empty_list_test_heap f r I I
+%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack
+is_non_empty_list_test_heap f r I t
 
 %macro: is_nonempty_list IsNonemptyList -fail_action
 is_nonempty_list f x
@@ -914,8 +980,13 @@ node x
 node y
 %hot
 
-i_fast_element j I s d
-i_element j s s d
+i_fast_element r j I d
+i_fast_element x j I d
+i_fast_element y j I d
+
+i_element r j s d
+i_element x j s d
+i_element y j s d
 
 bif1 f b s d
 bif1_body b s d
@@ -942,11 +1013,11 @@ move S r | call_last Ar P=f D => move_call_last S r P D
 
 i_move_call_last f P c r
 
-%macro:move_call_last MoveCallLast -arg_f -nonext
+%macro:move_call_last MoveCallLast -arg_f -nonext -pack
 
 move_call_last/4
-move_call_last x r f P
-move_call_last y r f P
+move_call_last x r f Q
+move_call_last y r f Q
 
 move S=c r | call_only Ar P=f => i_move_call_only P S r
 move S=x r | call_only Ar P=f => move_call_only S r P
@@ -995,7 +1066,7 @@ is_function f y
 is_function f r
 is_function Fail=f c => jump Fail
 
-func_info M=a F=a A=u | label L => gen_func_info(M, F, A, L)
+func_info M F A => i_func_info u M F A
 
 # ================================================================
 # New bit syntax matching (R11B).
@@ -1184,12 +1255,6 @@ i_bs_init_bits_fail_heap I j I d
 i_bs_init_bits I I d
 i_bs_init_bits_heap I I I d
 
-bs_bits_to_bytes Fail Src Dst => i_bs_bits_to_bytes Src Fail Dst
-
-i_bs_bits_to_bytes r j d
-i_bs_bits_to_bytes x j d
-i_bs_bits_to_bytes y j d
-
 bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D
 bs_add Fail S1 S2 Unit D => i_fetch S1 S2 | i_bs_add Fail Unit D
 
@@ -1311,10 +1376,12 @@ fmul p FR1 FR2 FR3 => i_fmul FR1 FR2 FR3
 fdiv p FR1 FR2 FR3 => i_fdiv FR1 FR2 FR3
 fnegate p FR1 FR2 => i_fnegate FR1 FR2
 
-fconv Int=iq Dst=l => move Int x | fconv x Dst
+fconv Arg=iqan Dst=l => move Arg x | fconv x Dst
 
 fmove q l
 fmove d l
+fmove l d
+
 fconv d l
 
 i_fadd l l l
@@ -1330,12 +1397,6 @@ fcheckerror p => i_fcheckerror
 i_fcheckerror
 fclearerror
 
-fmove FR=l Dst=d | new_float_allocation() => fmove_new FR Dst
- 
-# The new instruction for moving a float out of a floating point register.
-# (No allocation.)
-fmove_new l d
-
 #
 # New apply instructions in R10B.
 #
@@ -1344,7 +1405,21 @@ apply I
 apply_last I P
 
 #
-# New GCing arithmetic instructions.
+# Optimize addition and subtraction of small literals using
+# the i_increment/4 instruction (in bodies, not in guards).
+#
+
+gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \
+	gen_increment(Reg, Int, Live, Dst)
+gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \
+	gen_increment(Reg, Int, Live, Dst)
+
+gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \
+	negation_is_small(Int) => \
+	gen_increment_from_minus(Reg, Int, Live, Dst)
+
+#
+# GCing arithmetic instructions.
 #
 
 gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst
@@ -1367,6 +1442,10 @@ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
 gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst
 gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst
 
+i_increment r I I d
+i_increment x I I d
+i_increment y I I d
+
 i_plus j I d
 i_minus j I d
 i_times j I d
@@ -1397,34 +1476,60 @@ bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler
 # Guard BIFs.
 #
 gc_bif1 Fail I Bif=u$bif:erlang:length/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+	gen_guard_bif1(Fail, I, Bif, Src, Dst)
 
 gc_bif1 Fail I Bif=u$bif:erlang:size/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+	gen_guard_bif1(Fail, I, Bif, Src, Dst)
 
 gc_bif1 Fail I Bif=u$bif:erlang:bit_size/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+	gen_guard_bif1(Fail, I, Bif, Src, Dst)
 
 gc_bif1 Fail I Bif=u$bif:erlang:byte_size/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+	gen_guard_bif1(Fail, I, Bif, Src, Dst)
 
 gc_bif1 Fail I Bif=u$bif:erlang:abs/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+	gen_guard_bif1(Fail, I, Bif, Src, Dst)
 
 gc_bif1 Fail I Bif=u$bif:erlang:float/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+	gen_guard_bif1(Fail, I, Bif, Src, Dst)
 
 gc_bif1 Fail I Bif=u$bif:erlang:round/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+	gen_guard_bif1(Fail, I, Bif, Src, Dst)
 
 gc_bif1 Fail I Bif=u$bif:erlang:trunc/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+	gen_guard_bif1(Fail, I, Bif, Src, Dst)
+
+gc_bif2 Fail I Bif=u$bif:erlang:binary_part/2 S1 S2 Dst=d => \
+	gen_guard_bif2(Fail, I, Bif, S1, S2, Dst)
+
+gc_bif3 Fail I Bif=u$bif:erlang:binary_part/3 S1 S2 S3 Dst=d => \
+	gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst)
 
 i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D
 
 i_gc_bif1 j I s I d
 
+ii_gc_bif2/6
+
+ii_gc_bif2 Fail Bif S1 S2 Live D => i_fetch S1 S2 | i_gc_bif2 Fail Bif Live D
+
+i_gc_bif2 j I I d
+
+ii_gc_bif3/7
+
+ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D
+
+i_gc_bif3 j I s I d
 #
 # R13B03
 #
 on_load
+
+#
+# R14A.
+#
+recv_mark f
+
+recv_set Fail | label Lbl | loop_rec Lf Reg => \
+   i_recv_set | label Lbl | loop_rec Lf Reg
+i_recv_set
diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c
index 8c8029d450..a66d60aa22 100644
--- a/erts/emulator/beam/packet_parser.c
+++ b/erts/emulator/beam/packet_parser.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -47,11 +47,6 @@
                              (((unsigned char*) (s))[1] << 8) | \
                              (((unsigned char*) (s))[0]))
 
-#define put_int24(s, x) ((((unsigned char*)(s))[0] = ((x) >> 16) & 0xff), \
-                         (((unsigned char*)(s))[1] = ((x) >> 8) & 0xff), \
-                         (((unsigned char*)(s))[2] = (x) & 0xff))
-
-
 #if !defined(__WIN32__) && !defined(HAVE_STRNCASECMP)
 #define STRNCASECMP my_strncasecmp
 
@@ -679,7 +674,7 @@ int packet_parse_http(const char* buf, int len, int* statep,
             while (n && SP(ptr)) {
                 ptr++; n--;
             }
-            if (ptr==p0) return -1;
+            if (ptr==p0 && n>0) return -1;
             
             /* NOTE: the syntax allows empty reason phrases */
             (*statep) = !0;
@@ -833,7 +828,7 @@ int packet_parse_ssl(const char* buf, int len,
         char prefix[4];
         /* <<1:8,Length:24,Data/binary>> */
         prefix[0] = 1;
-        put_int24(&prefix[1],len-3);
+        put_int24(len-3,&prefix[1]);
         return pcb->ssl_tls(arg, 22, major, minor, buf+3, len-3, prefix, sizeof(prefix));
     } 
     else {
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 964c10a380..26d64887d0 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -39,8 +39,6 @@ static Hash process_reg;
 
 static erts_smp_rwmtx_t regtab_rwmtx;
 
-#define reg_lock_init()			erts_smp_rwmtx_init(&regtab_rwmtx, \
-							    "reg_tab")
 #define reg_try_read_lock()		erts_smp_rwmtx_tryrlock(&regtab_rwmtx)
 #define reg_try_write_lock()		erts_smp_rwmtx_tryrwlock(&regtab_rwmtx)
 #define reg_read_lock()			erts_smp_rwmtx_rlock(&regtab_rwmtx)
@@ -147,8 +145,11 @@ static void reg_free(RegProc *obj)
 void init_register_table(void)
 {
     HashFunctions f;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
-    reg_lock_init();
+    erts_smp_rwmtx_init_opt(&regtab_rwmtx, &rwmtx_opt, "reg_tab");
 
     f.hash = (H_FUN) reg_hash;
     f.cmp  = (HCMP_FUN) reg_cmp;
@@ -476,8 +477,9 @@ int erts_unregister_name(Process *c_p,
      *           on c_prt.
      */
 
-    if (!c_p)
+    if (!c_p) {
 	c_p_locks = 0;
+    }
     current_c_p_locks = c_p_locks;
 
  restart:
@@ -489,9 +491,15 @@ int erts_unregister_name(Process *c_p,
     if (is_non_value(name)) {
 	/* Unregister current process name */
 	ASSERT(c_p);
-	if (c_p->reg)
+#ifdef ERTS_SMP
+	if (current_c_p_locks != c_p_locks) {
+	    erts_smp_proc_lock(c_p, c_p_locks);
+	    current_c_p_locks = c_p_locks;
+	}
+#endif
+	if (c_p->reg) {
 	    r.name = c_p->reg->name;
-	else {
+	} else {
 	    /* Name got unregistered while main lock was released */
 	    res = 0;
 	    goto done;
@@ -533,24 +541,25 @@ int erts_unregister_name(Process *c_p,
 	    }
 
 	} else if (rp->p) {
-	    Process* p = rp->p;
+
 #ifdef ERTS_SMP
 	    erts_proc_safelock(c_p,
 			       current_c_p_locks,
 			       c_p_locks,
 			       rp->p,
-			       0,
+			       (c_p == rp->p) ?  current_c_p_locks : 0,
 			       ERTS_PROC_LOCK_MAIN);
 	    current_c_p_locks = c_p_locks;
 #endif
-	    p->reg = NULL;
+	    rp->p->reg = NULL;
+	    if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) {
+		trace_proc(c_p, rp->p, am_unregister, r.name);
+	    }
 #ifdef ERTS_SMP
-	    if (rp->p != c_p)
+	    if (rp->p != c_p) {
 		erts_smp_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN);
-#endif
-	    if (IS_TRACED_FL(p, F_TRACE_PROCS)) {
-		trace_proc(c_p, p, am_unregister, r.name);
 	    }
+#endif
 	}
 	hash_erase(&process_reg, (void*) &r);
 	res = 1;
@@ -560,14 +569,17 @@ int erts_unregister_name(Process *c_p,
 
     reg_write_unlock();
     if (c_prt != port) {
-	if (port)
+	if (port) {
 	    erts_smp_port_unlock(port);
-	if (c_prt)
+	}
+	if (c_prt) {
 	    erts_smp_port_lock(c_prt);
+	}
     }
 #ifdef ERTS_SMP
-    if (c_p && !current_c_p_locks)
+    if (c_p && !current_c_p_locks) {
 	erts_smp_proc_lock(c_p, c_p_locks);
+    }
 #endif
     return res;
 }
diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index 21d6ce9304..4c54e19cdb 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -99,7 +99,7 @@ static void rehash(SafeHash* h, int grow_limit)
 	erts_free(h->type, (void *) old_tab);
     }
     /*else already done */
-    erts_smp_atomic_set(&h->is_rehashing, 0);
+    erts_smp_atomic_set_relb(&h->is_rehashing, 0);
 }
 
 
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 4b949523fa..e64c43de6e 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1,37 +1,30 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
 #ifndef __SYS_H__
 #define __SYS_H__
 
+
 #if defined(VALGRIND) && !defined(NO_FPE_SIGNALS)
 #  define NO_FPE_SIGNALS
 #endif
 
-/* Never use elib-malloc when purify-memory-tracing */
-#if defined(PURIFY)
-#undef ENABLE_ELIB_MALLOC
-#undef ELIB_HEAP_SBRK
-#undef ELIB_ALLOC_IS_CLIB
-#endif
-
-
 /* xxxP __VXWORKS__ */
 #ifdef VXWORKS
 #include <vxWorks.h>
@@ -46,19 +39,10 @@
 #define ENABLE_CHILD_WAITER_THREAD 1
 #endif
 
-/* The ERTS_TIMER_TREAD #define must be visible to the
-   erl_${OS}_sys.h #include files: it controls whether
-   certain optional facilities should be defined or not. */
-#if defined(ERTS_SMP) && 0
-#define ERTS_TIMER_THREAD
-#endif
-
 #if defined (__WIN32__)
 #  include "erl_win_sys.h"
 #elif defined (VXWORKS) 
 #  include "erl_vxworks_sys.h"
-#elif defined (_OSE_) 
-#  include "erl_ose_sys.h"
 #else 
 #  include "erl_unix_sys.h"
 #ifndef UNIX
@@ -172,23 +156,6 @@ void erl_assert_error(char* expr, char* file, int line);
 
 #include <stdarg.h>
 
-#if defined(__STDC__) || defined(_MSC_VER)
-#  define EXTERN_FUNCTION(t, f, x)  extern t f x
-#  define FUNCTION(t, f, x) t f x
-#  define _DOTS_ ...
-#  define _VOID_      void
-#elif defined(__cplusplus)
-#  define EXTERN_FUNCTION(f, x) extern "C" { f x }
-#  define FUNCTION(t, f, x) t f x
-#  define _DOTS_ ...
-#  define _VOID_    void
-#else
-#  define EXTERN_FUNCTION(t, f, x) extern t f (/*x*/)
-#  define FUNCTION(t, f, x) t f (/*x*/)
-#  define _DOTS_
-#  define _VOID_
-#endif
-
 /* This isn't sys-dependent, but putting it here benefits sys.c and drivers
    - allow use of 'const' regardless of compiler */
 
@@ -198,7 +165,7 @@ void erl_assert_error(char* expr, char* file, int line);
 
 #ifdef VXWORKS
 /* Replace VxWorks' printf with a real one that does fprintf(stdout, ...) */
-EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...));
+int real_printf(const char *fmt, ...);
 #  define printf real_printf
 #endif
 
@@ -230,9 +197,14 @@ EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...));
 ** Data types:
 **
 ** Eterm: A tagged erlang term (possibly 64 bits)
+** BeamInstr: A beam code instruction unit, possibly larger than Eterm, not smaller.
 ** UInt:  An unsigned integer exactly as large as an Eterm.
 ** SInt:  A signed integer exactly as large as an eterm and therefor large
 **        enough to hold the return value of the signed_val() macro.
+** UWord: An unsigned integer at least as large as a void * and also as large
+**          or larger than an Eterm
+** SWord: A signed integer at least as large as a void * and also as large
+**          or larger than an Eterm
 ** Uint32: An unsigned integer of 32 bits exactly
 ** Sint32: A signed integer of 32 bits exactly
 ** Uint16: An unsigned integer of 16 bits exactly
@@ -253,11 +225,43 @@ EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...));
 #else
 #error Neither 32 nor 64 bit architecture
 #endif
+#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR)
+#    define HALFWORD_HEAP 1
+#    define HALFWORD_ASSERT 0
+#    define ASSERT_HALFWORD(COND) ASSERT(COND)
+#else
+#    define HALFWORD_HEAP 0
+#    define HALFWORD_ASSERT 0
+#    define ASSERT_HALFWORD(COND)
+#endif
 
 #if SIZEOF_VOID_P != SIZEOF_SIZE_T
 #error sizeof(void*) != sizeof(size_t)
 #endif
 
+#if HALFWORD_HEAP
+
+#if SIZEOF_INT == 4
+typedef unsigned int Eterm;
+typedef unsigned int Uint;
+typedef int          Sint;
+#define ERTS_SIZEOF_ETERM SIZEOF_INT
+#else
+#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
+#endif
+
+#if SIZEOF_VOID_P == SIZEOF_LONG
+typedef unsigned long UWord;
+typedef long          SWord;
+#elif SIZEOF_VOID_P == SIZEOF_INT
+typedef unsigned int UWord;
+typedef int          SWord;
+#else
+#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
+#endif
+
+#else /* !HALFWORD_HEAP */
+
 #if SIZEOF_VOID_P == SIZEOF_LONG
 typedef unsigned long Eterm;
 typedef unsigned long Uint;
@@ -272,6 +276,13 @@ typedef int          Sint;
 #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
 #endif
 
+typedef Uint UWord;
+typedef Sint SWord;
+
+#endif /* HALFWORD_HEAP */
+
+typedef UWord BeamInstr;
+
 #ifndef HAVE_INT64
 #if SIZEOF_LONG == 8
 #define HAVE_INT64 1
@@ -316,23 +327,20 @@ typedef unsigned char byte;
 #error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found 
 #endif
 
-#if defined(ARCH_64)
-#  define ERTS_WORD_ALIGN_PAD_SZ(X) \
+#  define ERTS_EXTRA_DATA_ALIGN_SZ(X) \
     (((size_t) 8) - (((size_t) (X)) & ((size_t) 7)))
-#elif defined(ARCH_32)
-#  define ERTS_WORD_ALIGN_PAD_SZ(X) \
-    (((size_t) 4) - (((size_t) (X)) & ((size_t) 3)))
-#else
-#error "Not supported..."
-#endif
 
 #include "erl_lock_check.h"
+
+/* needed by erl_smp.h */
+int erts_send_warning_to_logger_str_nogl(char *);
+
 #include "erl_smp.h"
 
 #ifdef ERTS_WANT_BREAK_HANDLING
 #  ifdef ERTS_SMP
-extern erts_smp_atomic_t erts_break_requested;
-#    define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic_read(&erts_break_requested))
+extern erts_smp_atomic32_t erts_break_requested;
+#    define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic32_read(&erts_break_requested))
 #  else
 extern volatile int erts_break_requested;
 #    define ERTS_BREAK_REQUESTED erts_break_requested
@@ -345,8 +353,8 @@ void erts_do_break_handling(void);
 #    define ERTS_GOT_SIGUSR1 0
 #  else
 #    ifdef ERTS_SMP
-extern erts_smp_atomic_t erts_got_sigusr1;
-#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic_read(&erts_got_sigusr1))
+extern erts_smp_atomic32_t erts_got_sigusr1;
+#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read(&erts_got_sigusr1))
 #    else
 extern volatile int erts_got_sigusr1;
 #      define ERTS_GOT_SIGUSR1 erts_got_sigusr1
@@ -413,13 +421,6 @@ extern volatile int erts_writing_erl_crash_dump;
    in non-blocking mode - and ioctl FIONBIO on AIX *doesn't* work for
    pipes or ttys (O_NONBLOCK does)!!! For now, we'll use FIONBIO for AIX. */
 
-# ifdef _OSE_
-static const int zero_value = 0, one_value = 1;
-#      define SET_BLOCKING(fd)	        ioctl((fd), FIONBIO, (char*)&zero_value)
-#      define SET_NONBLOCKING(fd)	ioctl((fd), FIONBIO, (char*)&one_value)
-#      define ERRNO_BLOCK EWOULDBLOCK
-# else
-
 #  ifdef __WIN32__
 
 static unsigned long zero_value = 0, one_value = 1;
@@ -460,11 +461,8 @@ static const int zero_value = 0, one_value = 1;
 #      endif /* !NB_FIONBIO */
 #    endif /* _WXWORKS_ */
 #  endif /* !__WIN32__ */
-# endif /* _OSE_ */
 #endif /* WANT_NONBLOCKING */
 
-extern erts_cpu_info_t *erts_cpuinfo; /* erl_init.c */
-
 __decl_noreturn void __noreturn erl_exit(int n, char*, ...);
 
 /* Some special erl_exit() codes: */
@@ -523,7 +521,8 @@ int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *);
 int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *);
 int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *);
 int erts_send_info_to_logger_str_nogl(char *);
-int erts_send_warning_to_logger_str_nogl(char *);
+/* needed by erl_smp.h (declared above)
+   int erts_send_warning_to_logger_str_nogl(char *); */
 int erts_send_error_to_logger_str_nogl(char *);
 
 typedef struct preload {
@@ -538,11 +537,9 @@ typedef struct preload {
  * None of the drivers use all of the fields.
  */
 
-/* OSE: Want process_type and priority in here as well! Needs updates in erl_bif_ports.c! */
-
 typedef struct _SysDriverOpts {
-    int ifd;			/* Input file descriptor (fd driver). */
-    int ofd;			/* Outputfile descriptor (fd driver). */
+    Uint ifd;			/* Input file descriptor (fd driver). */
+    Uint ofd;			/* Outputfile descriptor (fd driver). */
     int packet_bytes;		/* Number of bytes in packet header. */
     int read_write;		/* Read and write bits. */
     int use_stdio;		/* Use standard I/O: TRUE or FALSE. */
@@ -556,12 +553,6 @@ typedef struct _SysDriverOpts {
     char *wd;			/* Working directory. */
     unsigned spawn_type;        /* Bitfield of ERTS_SPAWN_DRIVER | 
 				   ERTS_SPAWN_EXTERNAL | both*/ 
-
-#ifdef _OSE_
-    enum PROCESS_TYPE process_type;
-    OSPRIORITY priority;
-#endif /* _OSE_ */
-
 } SysDriverOpts;
 
 extern char *erts_default_arg0;
@@ -569,11 +560,7 @@ extern char *erts_default_arg0;
 extern char os_type[];
 
 extern int sys_init_time(void);
-#if defined(ERTS_TIMER_THREAD)
-#define erts_deliver_time()
-#else
 extern void erts_deliver_time(void);
-#endif
 extern void erts_time_remaining(SysTimeval *);
 extern int erts_init_time_sup(void);
 extern void erts_sys_init_float(void);
@@ -621,7 +608,7 @@ extern char *erts_sys_ddll_error(int code);
 
 
 /*
- * System interfaces for startup/sae code (functions found in respective sys.c)
+ * System interfaces for startup.
  */
 
 
@@ -638,18 +625,14 @@ extern void erts_sys_pre_init(void);
 extern void erl_sys_init(void);
 extern void erl_sys_args(int *argc, char **argv);
 extern void erl_sys_schedule(int);
-#ifdef _OSE_
-extern void erl_sys_init_final(void);
-#else
-void sys_tty_reset(void);
-#endif
+void sys_tty_reset(int);
 
-EXTERN_FUNCTION(int, sys_max_files, (_VOID_));
+int sys_max_files(void);
 void sys_init_io(void);
 Preload* sys_preloaded(void);
-EXTERN_FUNCTION(unsigned char*, sys_preload_begin, (Preload*));
-EXTERN_FUNCTION(void, sys_preload_end, (Preload*));
-EXTERN_FUNCTION(int, sys_get_key, (int));
+unsigned char* sys_preload_begin(Preload*);
+void sys_preload_end(Preload*);
+int sys_get_key(int);
 void elapsed_time_both(unsigned long *ms_user, unsigned long *ms_sys, 
 		       unsigned long *ms_user_diff, unsigned long *ms_sys_diff);
 void wall_clock_elapsed_time_both(unsigned long *ms_total, 
@@ -666,7 +649,7 @@ int local_to_univ(Sint *year, Sint *month, Sint *day,
 		  Sint *hour, Sint *minute, Sint *second, int isdst);
 void get_now(Uint*, Uint*, Uint*);
 void get_sys_now(Uint*, Uint*, Uint*);
-EXTERN_FUNCTION(void, set_break_quit, (void (*)(void), void (*)(void)));
+void set_break_quit(void (*)(void), void (*)(void));
 
 void os_flavor(char*, unsigned);
 void os_version(int*, int*, int*);
@@ -706,7 +689,7 @@ int erts_write_env(char *key, char *value);
 #define ERTS_DEFAULT_MMAP_THRESHOLD  (128 * 1024)
 #define ERTS_DEFAULT_MMAP_MAX        64
 
-EXTERN_FUNCTION(int, sys_alloc_opt, (int, int));
+int sys_alloc_opt(int, int);
 
 typedef struct {
   Sint trim_threshold;
@@ -715,7 +698,7 @@ typedef struct {
   Sint mmap_max;
 } SysAllocStat;
 
-EXTERN_FUNCTION(void, sys_alloc_stat, (SysAllocStat *));
+void sys_alloc_stat(SysAllocStat *);
 
 /* Block the whole system... */
 
@@ -739,11 +722,11 @@ typedef enum {
 } erts_activity_error_t;
 
 typedef struct {
-    erts_smp_atomic_t do_block;
+    erts_smp_atomic32_t do_block;
     struct {
-	erts_smp_atomic_t wait;
-	erts_smp_atomic_t gc;
-	erts_smp_atomic_t io;
+	erts_smp_atomic32_t wait;
+	erts_smp_atomic32_t gc;
+	erts_smp_atomic32_t io;
     } in_activity;
 } erts_system_block_state_t;
 
@@ -894,7 +877,7 @@ ERTS_GLB_INLINE int
 erts_smp_pending_system_block(void)
 {
 #ifdef ERTS_SMP
-    return erts_smp_atomic_read(&erts_system_block_state.do_block);
+    return (int) erts_smp_atomic32_read(&erts_system_block_state.do_block);
 #else
     return 0;
 #endif
@@ -930,7 +913,7 @@ erts_smp_set_activity(erts_activity_t old_activity,
     case ERTS_ACTIVITY_UNDEFINED:
 	break;
     case ERTS_ACTIVITY_WAIT:
-	erts_smp_atomic_dec(&erts_system_block_state.in_activity.wait);
+	erts_smp_atomic32_dec(&erts_system_block_state.in_activity.wait);
 	if (locked) {
 	    /* You are not allowed to leave activity waiting
 	     * without supplying the possibility to block
@@ -941,10 +924,10 @@ erts_smp_set_activity(erts_activity_t old_activity,
 	}
 	break;
     case ERTS_ACTIVITY_GC:
-	erts_smp_atomic_dec(&erts_system_block_state.in_activity.gc);
+	erts_smp_atomic32_dec(&erts_system_block_state.in_activity.gc);
 	break;
     case ERTS_ACTIVITY_IO:
-	erts_smp_atomic_dec(&erts_system_block_state.in_activity.io);
+	erts_smp_atomic32_dec(&erts_system_block_state.in_activity.io);
 	break;
     default:
 	erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY,
@@ -960,13 +943,13 @@ erts_smp_set_activity(erts_activity_t old_activity,
     case ERTS_ACTIVITY_UNDEFINED:
 	break;
     case ERTS_ACTIVITY_WAIT:
-	erts_smp_atomic_inc(&erts_system_block_state.in_activity.wait);
+	erts_smp_atomic32_inc(&erts_system_block_state.in_activity.wait);
 	break;
     case ERTS_ACTIVITY_GC:
-	erts_smp_atomic_inc(&erts_system_block_state.in_activity.gc);
+	erts_smp_atomic32_inc(&erts_system_block_state.in_activity.gc);
 	break;
     case ERTS_ACTIVITY_IO:
-	erts_smp_atomic_inc(&erts_system_block_state.in_activity.io);
+	erts_smp_atomic32_inc(&erts_system_block_state.in_activity.io);
 	break;
     default:
 	erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY,
@@ -1001,27 +984,31 @@ erts_smp_set_activity(erts_activity_t old_activity,
 
 typedef erts_smp_atomic_t erts_refc_t;
 
-ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, long val);
-ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, long min_val);
-ERTS_GLB_INLINE long erts_refc_inctest(erts_refc_t *refcp, long min_val);
-ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, long min_val);
-ERTS_GLB_INLINE long erts_refc_dectest(erts_refc_t *refcp, long min_val);
-ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, long diff, long min_val);
-ERTS_GLB_INLINE long erts_refc_read(erts_refc_t *refcp, long min_val);
+ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val);
+ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp,
+					      erts_aint_t min_val);
+ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp,
+					      erts_aint_t min_val);
+ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff,
+				   erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp,
+					   erts_aint_t min_val);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
-erts_refc_init(erts_refc_t *refcp, long val)
+erts_refc_init(erts_refc_t *refcp, erts_aint_t val)
 {
     erts_smp_atomic_init((erts_smp_atomic_t *) refcp, val);
 }
 
 ERTS_GLB_INLINE void
-erts_refc_inc(erts_refc_t *refcp, long min_val)
+erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
 {
 #ifdef ERTS_REFC_DEBUG
-    long val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp);
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
@@ -1031,10 +1018,10 @@ erts_refc_inc(erts_refc_t *refcp, long min_val)
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_refc_inctest(erts_refc_t *refcp, long min_val)
+ERTS_GLB_INLINE erts_aint_t
+erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
 {
-    long val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -1045,10 +1032,10 @@ erts_refc_inctest(erts_refc_t *refcp, long min_val)
 }
 
 ERTS_GLB_INLINE void
-erts_refc_dec(erts_refc_t *refcp, long min_val)
+erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
 {
 #ifdef ERTS_REFC_DEBUG
-    long val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp);
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
@@ -1058,10 +1045,10 @@ erts_refc_dec(erts_refc_t *refcp, long min_val)
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_refc_dectest(erts_refc_t *refcp, long min_val)
+ERTS_GLB_INLINE erts_aint_t
+erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
 {
-    long val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -1072,10 +1059,10 @@ erts_refc_dectest(erts_refc_t *refcp, long min_val)
 }
 
 ERTS_GLB_INLINE void
-erts_refc_add(erts_refc_t *refcp, long diff, long min_val)
+erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
 {
 #ifdef ERTS_REFC_DEBUG
-    long val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff);
+    erts_aint_t val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff);
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
@@ -1085,10 +1072,10 @@ erts_refc_add(erts_refc_t *refcp, long diff, long min_val)
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_refc_read(erts_refc_t *refcp, long min_val)
+ERTS_GLB_INLINE erts_aint_t
+erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
 {
-    long val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -1104,13 +1091,10 @@ erts_refc_read(erts_refc_t *refcp, long min_val)
 extern int erts_use_kernel_poll;
 #endif
 
-void elib_ensure_initialized(void);
-
-
-#if (defined(VXWORKS) || defined(_OSE_))
+#if defined(VXWORKS)
 /* NOTE! sys_calloc2 does not exist on other 
    platforms than VxWorks and OSE */
-EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint));
+void* sys_calloc2(Uint, Uint);
 #endif /* VXWORKS || OSE */
 
 
@@ -1150,14 +1134,14 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint));
 
 /* Standard set of integer macros  .. */
 
-#define get_int64(s) ((((unsigned char*) (s))[0] << 56) | \
-                      (((unsigned char*) (s))[1] << 48) | \
-                      (((unsigned char*) (s))[2] << 40) | \
-                      (((unsigned char*) (s))[3] << 32) | \
-                      (((unsigned char*) (s))[4] << 24) | \
-                      (((unsigned char*) (s))[5] << 16) | \
-                      (((unsigned char*) (s))[6] << 8)  | \
-                      (((unsigned char*) (s))[7]))
+#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \
+                      (((Uint64)((unsigned char*) (s))[1]) << 48) | \
+                      (((Uint64)((unsigned char*) (s))[2]) << 40) | \
+                      (((Uint64)((unsigned char*) (s))[3]) << 32) | \
+                      (((Uint64)((unsigned char*) (s))[4]) << 24) | \
+                      (((Uint64)((unsigned char*) (s))[5]) << 16) | \
+                      (((Uint64)((unsigned char*) (s))[6]) << 8)  | \
+                      (((Uint64)((unsigned char*) (s))[7])))
 
 #define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\
                             ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\
@@ -1180,6 +1164,15 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint));
                             ((char*)(s))[3] = (char)(i)         & 0xff;} \
                         while (0)
 
+#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \
+                      (((unsigned char*) (s))[1] << 8)  | \
+                      (((unsigned char*) (s))[2]))
+
+#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff;  \
+                            ((char*)(s))[1] = (char)((i) >> 8)  & 0xff;  \
+                            ((char*)(s))[2] = (char)(i)         & 0xff;} \
+                        while (0)
+
 #define get_int16(s) ((((unsigned char*)  (s))[0] << 8) | \
                       (((unsigned char*)  (s))[1]))
 
@@ -1193,6 +1186,7 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint));
 
 #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)
 
+
 /*
  * Use DEBUGF as you would use printf, but use double parentheses:
  *
@@ -1202,8 +1196,8 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint));
  */
 
 #ifdef DEBUG
-EXTERN_FUNCTION(void, erl_debug, (char* format, ...));
-EXTERN_FUNCTION(void, erl_bin_write, (unsigned char *, int, int));
+void erl_debug(char* format, ...);
+void erl_bin_write(unsigned char *, int, int);
 
 #  define DEBUGF(x) erl_debug x
 #else
@@ -1257,6 +1251,22 @@ char* win32_errorstr(int);
 
 #endif
 
+/************************************************************************
+ * Find out the native filename encoding of the process (look at locale of 
+ * Unix processes and just do UTF16 on windows 
+ ************************************************************************/
+#define ERL_FILENAME_UNKNOWN 0
+#define ERL_FILENAME_LATIN1 1
+#define ERL_FILENAME_UTF8 2
+#define ERL_FILENAME_UTF8_MAC 3
+#define ERL_FILENAME_WIN_WCHAR 4
+
+int erts_get_native_filename_encoding(void);
+/* The set function is only to be used by erl_init! */
+void erts_set_user_requested_filename_encoding(int encoding); 
+int erts_get_user_requested_filename_encoding(void);
+
+void erts_init_sys_common_misc(void);
 
 #endif
 
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index a07d6a5327..a00faff912 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -83,24 +83,8 @@
 #define ASSERT_NO_LOCKED_LOCKS
 #endif
 
+static erts_smp_mtx_t tiw_lock;
 
-#if defined(ERTS_TIMER_THREAD) || 1
-/* I don't yet know why, but using a mutex instead of a spinlock
-   or spin-based rwlock avoids excessive delays at startup. */
-static erts_smp_rwmtx_t tiw_lock;
-#define tiw_read_lock()		erts_smp_rwmtx_rlock(&tiw_lock)
-#define tiw_read_unlock()	erts_smp_rwmtx_runlock(&tiw_lock)
-#define tiw_write_lock()	erts_smp_rwmtx_rwlock(&tiw_lock)
-#define tiw_write_unlock()	erts_smp_rwmtx_rwunlock(&tiw_lock)
-#define tiw_init_lock()		erts_smp_rwmtx_init(&tiw_lock, "timer_wheel")
-#else
-static erts_smp_rwlock_t tiw_lock;
-#define tiw_read_lock()		erts_smp_read_lock(&tiw_lock)
-#define tiw_read_unlock()	erts_smp_read_unlock(&tiw_lock)
-#define tiw_write_lock()	erts_smp_write_lock(&tiw_lock)
-#define tiw_write_unlock()	erts_smp_write_unlock(&tiw_lock)
-#define tiw_init_lock()		erts_smp_rwlock_init(&tiw_lock, "timer_wheel")
-#endif
 
 /* BEGIN tiw_lock protected variables 
 **
@@ -115,80 +99,37 @@ static erts_smp_rwlock_t tiw_lock;
 static ErlTimer** tiw;		/* the timing wheel, allocated in init_time() */
 static Uint tiw_pos;		/* current position in wheel */
 static Uint tiw_nto;		/* number of timeouts in wheel */
+static Uint tiw_min;
+static ErlTimer *tiw_min_ptr;
 
 /* END tiw_lock protected variables */
 
 /* Actual interval time chosen by sys_init_time() */
 static int itime; /* Constant after init */
 
-#if defined(ERTS_TIMER_THREAD)
-static SysTimeval time_start;	/* start of current time interval */
-static long ticks_end;		/* time_start+ticks_end == time_wakeup */
-static long ticks_latest;	/* delta from time_start at latest time update*/
-
-static ERTS_INLINE long time_gettimeofday(SysTimeval *now)
-{
-    long elapsed;
-
-    erts_get_timeval(now);
-    now->tv_usec = 1000 * (now->tv_usec / 1000); /* ms resolution */
-    elapsed = (1000 * (now->tv_sec - time_start.tv_sec) +
-	       (now->tv_usec - time_start.tv_usec) / 1000);
-    // elapsed /= CLOCK_RESOLUTION;
-    return elapsed;
-}
-
-static long do_time_update(void)
-{
-    SysTimeval now;
-    long elapsed;
-
-    elapsed = time_gettimeofday(&now);
-    ticks_latest = elapsed;
-    return elapsed;
-}
-
-static ERTS_INLINE long do_time_read(void)
-{
-    return ticks_latest;
-}
-
-static long do_time_reset(void)
-{
-    SysTimeval now;
-    long elapsed;
-
-    elapsed = time_gettimeofday(&now);
-    time_start = now;
-    ticks_end = LONG_MAX;
-    ticks_latest = 0;
-    return elapsed;
-}
-
-static ERTS_INLINE void do_time_init(void)
-{
-    (void)do_time_reset();
-}
-
-#else
 erts_smp_atomic_t do_time;	/* set at clock interrupt */
-static ERTS_INLINE long do_time_read(void) { return erts_smp_atomic_read(&do_time); }
-static ERTS_INLINE long do_time_update(void) { return do_time_read(); }
+static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read(&do_time); }
+static ERTS_INLINE erts_aint_t do_time_update(void) { return do_time_read(); }
 static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init(&do_time, 0L); }
-#endif
 
 /* get the time (in units of itime) to the next timeout,
    or -1 if there are no timeouts                     */
 
-static int next_time_internal(void) /* PRE: tiw_lock taken by caller */
+static erts_aint_t next_time_internal(void) /* PRE: tiw_lock taken by caller */
 {
     int i, tm, nto;
     unsigned int min;
     ErlTimer* p;
-    long dt;
+    erts_aint_t dt;
   
     if (tiw_nto == 0)
 	return -1;	/* no timeouts in wheel */
+
+    if (tiw_min_ptr) {
+	min = tiw_min;
+	dt  = do_time_read();
+	return ((min >= dt) ? (min - dt) : 0);
+    }
   
     /* start going through wheel to find next timeout */
     tm = nto = 0;
@@ -201,11 +142,17 @@ static int next_time_internal(void) /* PRE: tiw_lock taken by caller */
 	    if (p->count == 0) {
 		/* found next timeout */
 		dt = do_time_read();
+		/* p->count is zero */
+		tiw_min_ptr = p;
+		tiw_min     = tm;
 		return ((tm >= dt) ? (tm - dt) : 0);
 	    } else {
 		/* keep shortest time in 'min' */
-		if (tm + p->count*TIW_SIZE < min)
+		if (tm + p->count*TIW_SIZE < min) {
 		    min = tm + p->count*TIW_SIZE;
+		    tiw_min_ptr = p;
+		    tiw_min     = min;
+		}
 	    }
 	    p = p->next;
 	}
@@ -218,30 +165,53 @@ static int next_time_internal(void) /* PRE: tiw_lock taken by caller */
     return ((min >= dt) ? (min - dt) : 0);
 }
 
-#if !defined(ERTS_TIMER_THREAD)
+static void remove_timer(ErlTimer *p) {
+    /* first */
+    if (!p->prev) {
+	tiw[p->slot] = p->next;
+	if(p->next)
+	    p->next->prev = NULL;
+    } else {
+	p->prev->next = p->next;
+    }
+
+    /* last */
+    if (!p->next) {
+	if (p->prev)
+	    p->prev->next = NULL;
+    } else {
+	p->next->prev = p->prev;
+    }
+
+    p->next = NULL;
+    p->prev = NULL;
+    /* Make sure cancel callback isn't called */
+    p->active = 0;
+    tiw_nto--;
+}
+
 /* Private export to erl_time_sup.c */
-int next_time(void)
+erts_aint_t erts_next_time(void)
 {
-    int ret;
+    erts_aint_t ret;
 
-    tiw_write_lock();
+    erts_smp_mtx_lock(&tiw_lock);
     (void)do_time_update();
     ret = next_time_internal();
-    tiw_write_unlock();
+    erts_smp_mtx_unlock(&tiw_lock);
     return ret;
 }
-#endif
 
-static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-locked */
+static ERTS_INLINE void bump_timer_internal(erts_aint_t dt) /* PRE: tiw_lock is write-locked */
 {
     Uint keep_pos;
     Uint count;
     ErlTimer *p, **prev, *timeout_head, **timeout_tail;
-    Uint dtime = (unsigned long)dt;  
+    Uint dtime = (Uint) dt;  
 
     /* no need to bump the position if there aren't any timeouts */
     if (tiw_nto == 0) {
-	tiw_write_unlock();
+	erts_smp_mtx_unlock(&tiw_lock);
 	return;
     }
 
@@ -258,12 +228,16 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l
 	if (tiw_pos == keep_pos) count--;
 	prev = &tiw[tiw_pos];
 	while ((p = *prev) != NULL) {
+	    ASSERT( p != p->next);
 	    if (p->count < count) {     /* we have a timeout */
-		*prev = p->next;	/* Remove from list */
-		tiw_nto--;
-		p->next = NULL;
-		p->active = 0;		/* Make sure cancel callback
-					   isn't called */
+		/* remove min time */
+		if (tiw_min_ptr == p) {
+		    tiw_min_ptr = NULL;
+		    tiw_min = 0;
+		}
+
+		/* Remove from list */
+		remove_timer(p);
 		*timeout_tail = p;	/* Insert in timeout queue */
 		timeout_tail = &p->next;
 	    }
@@ -277,8 +251,10 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l
 	dtime--;
     }
     tiw_pos = keep_pos;
+    if (tiw_min_ptr)
+	tiw_min -= dt;
     
-    tiw_write_unlock();
+    erts_smp_mtx_unlock(&tiw_lock);
     
     /* Call timedout timers callbacks */
     while (timeout_head) {
@@ -291,24 +267,17 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l
 	 * callback is called.
 	 */
 	p->next = NULL;
+	p->prev = NULL;
 	p->slot = 0;
 	(*p->timeout)(p->arg);
     }
 }
 
-#if defined(ERTS_TIMER_THREAD)
-static void timer_thread_bump_timer(void)
+void erts_bump_timer(erts_aint_t dt) /* dt is value from do_time */
 {
-    tiw_write_lock();
-    bump_timer_internal(do_time_reset());
-}
-#else
-void bump_timer(long dt) /* dt is value from do_time */
-{
-    tiw_write_lock();
+    erts_smp_mtx_lock(&tiw_lock);
     bump_timer_internal(dt);
 }
-#endif
 
 Uint
 erts_timer_wheel_memory_size(void)
@@ -316,82 +285,10 @@ erts_timer_wheel_memory_size(void)
     return (Uint) TIW_SIZE * sizeof(ErlTimer*);
 }
 
-#if defined(ERTS_TIMER_THREAD)
-static struct erts_iwait *timer_thread_iwait;
-
-static int timer_thread_setup_delay(SysTimeval *rem_time)
-{
-    long elapsed;
-    int ticks;
-
-    tiw_write_lock();
-    elapsed = do_time_update();
-    ticks = next_time_internal();
-    if (ticks == -1)	/* timer queue empty */
-	ticks = 100*1000*1000;
-    if (elapsed > ticks)
-	elapsed = ticks;
-    ticks -= elapsed;
-    //ticks *= CLOCK_RESOLUTION;
-    rem_time->tv_sec = ticks / 1000;
-    rem_time->tv_usec = 1000 * (ticks % 1000);
-    ticks_end = ticks;
-    tiw_write_unlock();
-    return ticks;
-}
-
-static void *timer_thread_start(void *ignore)
-{
-    SysTimeval delay;
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_set_thread_name("timer");
-#endif
-    erts_register_blockable_thread();
-
-    for(;;) {
-	if (timer_thread_setup_delay(&delay)) {
-	    erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
-	    ASSERT_NO_LOCKED_LOCKS;
-	    erts_iwait_wait(timer_thread_iwait, &delay);
-	    ASSERT_NO_LOCKED_LOCKS;
-	    erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
-	}
-	else
-	    erts_smp_chk_system_block(NULL, NULL, NULL);
-	timer_thread_bump_timer();
-	ASSERT_NO_LOCKED_LOCKS;
-    }
-    /*NOTREACHED*/
-    return NULL;
-}
-
-static ERTS_INLINE void timer_thread_post_insert(Uint ticks)
-{
-    if ((Sint)ticks < ticks_end)
-	erts_iwait_interrupt(timer_thread_iwait);
-}
-
-static void timer_thread_init(void)
-{
-    erts_thr_opts_t opts = ERTS_THR_OPTS_DEFAULT_INITER;
-    erts_tid_t tid;
-
-    opts->detached = 1;
-
-    timer_thread_iwait = erts_iwait_init();
-    erts_thr_create(&tid, timer_thread_start, NULL, &opts);
-}
-
-#else
-static ERTS_INLINE void timer_thread_post_insert(Uint ticks) { }
-static ERTS_INLINE void timer_thread_init(void) { }
-#endif
-
 /* this routine links the time cells into a free list at the start
    and sets the time queue as empty */
 void
-init_time(void)
+erts_init_time(void)
 {
     int i;
 
@@ -399,7 +296,7 @@ init_time(void)
        if timer thread is enabled */
     itime = erts_init_time_sup();
 
-    tiw_init_lock();
+    erts_smp_mtx_init(&tiw_lock, "timer_wheel");
 
     tiw = (ErlTimer**) erts_alloc(ERTS_ALC_T_TIMER_WHEEL,
 				  TIW_SIZE * sizeof(ErlTimer*));
@@ -407,10 +304,13 @@ init_time(void)
 	tiw[i] = NULL;
     do_time_init();
     tiw_pos = tiw_nto = 0;
-
-    timer_thread_init();
+    tiw_min_ptr = NULL;
+    tiw_min = 0;
 }
 
+
+
+
 /*
 ** Insert a process into the time queue, with a timeout 't'
 */
@@ -440,20 +340,35 @@ insert_timer(ErlTimer* p, Uint t)
   
     /* insert at head of list at slot */
     p->next = tiw[tm];
+    p->prev = NULL;
+    if (p->next != NULL)
+	p->next->prev = p;
     tiw[tm] = p;
-    tiw_nto++;
 
-    timer_thread_post_insert(ticks);
+
+    /* insert min time */
+    if ((tiw_nto == 0) || ((tiw_min_ptr != NULL) && (ticks < tiw_min))) {
+	tiw_min = ticks;
+	tiw_min_ptr = p;
+    }
+    if ((tiw_min_ptr == p) && (ticks > tiw_min)) {
+	/* some other timer might be 'min' now */
+	tiw_min = 0;
+	tiw_min_ptr = NULL;
+    }
+
+    tiw_nto++;
 }
 
 void
-erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
+erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
 	      void* arg, Uint t)
 {
+
     erts_deliver_time();
-    tiw_write_lock();
+    erts_smp_mtx_lock(&tiw_lock);
     if (p->active) { /* XXX assert ? */
-	tiw_write_unlock();
+	erts_smp_mtx_unlock(&tiw_lock);
 	return;
     }
     p->timeout = timeout;
@@ -461,45 +376,37 @@ erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
     p->arg = arg;
     p->active = 1;
     insert_timer(p, t);
-    tiw_write_unlock();
-#if defined(ERTS_SMP) && !defined(ERTS_TIMER_THREAD)
+    erts_smp_mtx_unlock(&tiw_lock);
+#if defined(ERTS_SMP)
     if (t <= (Uint) LONG_MAX)
 	erts_sys_schedule_interrupt_timed(1, (long) t);
 #endif
 }
 
 void
-erl_cancel_timer(ErlTimer* p)
+erts_cancel_timer(ErlTimer* p)
 {
-    ErlTimer *tp;
-    ErlTimer **prev;
-
-    tiw_write_lock();
+    erts_smp_mtx_lock(&tiw_lock);
     if (!p->active) { /* allow repeated cancel (drivers) */
-	tiw_write_unlock();
+	erts_smp_mtx_unlock(&tiw_lock);
 	return;
     }
-    /* find p in linked list at slot p->slot and remove it */
-    prev = &tiw[p->slot];
-    while ((tp = *prev) != NULL) {
-	if (tp == p) {
-	    *prev = p->next;	/* Remove from list */
-	    tiw_nto--;
-	    p->next = NULL;
-	    p->slot = p->count = 0;
-	    p->active = 0;
-	    if (p->cancel != NULL) {
-		tiw_write_unlock();
-		(*p->cancel)(p->arg);
-	    } else {
-		tiw_write_unlock();
-	    }
-	    return;
-	} else {
-	    prev = &tp->next;
-	}
+
+    /* is it the 'min' timer, remove min */
+    if (p == tiw_min_ptr) {
+	tiw_min_ptr = NULL;
+	tiw_min     = 0;
     }
-    tiw_write_unlock();
+
+    remove_timer(p);
+    p->slot = p->count = 0;
+
+    if (p->cancel != NULL) {
+	erts_smp_mtx_unlock(&tiw_lock);
+	(*p->cancel)(p->arg);
+	return;
+    }
+    erts_smp_mtx_unlock(&tiw_lock);
 }
 
 /*
@@ -509,15 +416,15 @@ erl_cancel_timer(ErlTimer* p)
   immediately if it hadn't been cancelled).
 */
 Uint
-time_left(ErlTimer *p)
+erts_time_left(ErlTimer *p)
 {
     Uint left;
-    long dt;
+    erts_aint_t dt;
 
-    tiw_read_lock();
+    erts_smp_mtx_lock(&tiw_lock);
 
     if (!p->active) {
-	tiw_read_unlock();
+	erts_smp_mtx_unlock(&tiw_lock);
 	return 0;
     }
 
@@ -531,19 +438,18 @@ time_left(ErlTimer *p)
     else
 	left -= dt;
 
-    tiw_read_unlock();
+    erts_smp_mtx_unlock(&tiw_lock);
 
-    return left * itime;
+    return (Uint) left * itime;
 }
 
 #ifdef DEBUG
-
-void p_slpq()
+void erts_p_slpq()
 {
     int i;
     ErlTimer* p;
   
-    tiw_read_lock();
+    erts_smp_mtx_lock(&tiw_lock);
 
     /* print the whole wheel, starting at the current position */
     erts_printf("\ntiw_pos = %d tiw_nto %d\n", tiw_pos, tiw_nto);
@@ -565,7 +471,6 @@ void p_slpq()
 	}
     }
 
-    tiw_read_unlock();
+    erts_smp_mtx_unlock(&tiw_lock);
 }
-
 #endif /* DEBUG */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 31efddc0f2..a17de717bc 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -32,6 +32,7 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "packet_parser.h"
+#include "erl_gc.h"
 #define ERTS_WANT_DB_INTERNAL__
 #include "erl_db.h"
 #include "erl_threads.h"
@@ -47,24 +48,17 @@
 #undef M_MMAP_THRESHOLD
 #undef M_MMAP_MAX
 
-#if !defined(ELIB_ALLOC_IS_CLIB) && defined(__GLIBC__) && defined(HAVE_MALLOC_H)
+#if defined(__GLIBC__) && defined(HAVE_MALLOC_H)
 #include <malloc.h>
 #endif
 
-#if defined(ELIB_ALLOC_IS_CLIB) || !defined(HAVE_MALLOPT)
+#if !defined(HAVE_MALLOPT)
 #undef  HAVE_MALLOPT
 #define HAVE_MALLOPT 0
 #endif
 
 /* profile_scheduler mini message queue */
 
-#ifdef ERTS_TIMER_THREAD
-/* A timer thread is not welcomed with this lock violation work around.
- * - Bj�rn-Egil
- */
-#error Timer thread may not be enabled due to lock violation.
-#endif
-
 typedef struct {
     Uint scheduler_id;
     Uint no_schedulers;
@@ -97,7 +91,7 @@ dispatch_profile_msg_q(profile_sched_msg_q *psmq)
 
 
 Eterm*
-erts_heap_alloc(Process* p, Uint need)
+erts_heap_alloc(Process* p, Uint need, Uint xtra)
 {
     ErlHeapFragment* bp;
     Eterm* htop;
@@ -123,9 +117,9 @@ erts_heap_alloc(Process* p, Uint need)
     p->space_verified_from = NULL;
 #endif /* FORCE_HEAP_FRAGS */
 
-    n = need;
+    n = need + xtra;
     bp = MBUF(p);
-    if (bp != NULL && need <= (bp->size - bp->used_size)) {
+    if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) {
 	Eterm* ret = bp->mem + bp->used_size;
 	bp->used_size += need;
 	return ret;
@@ -158,16 +152,11 @@ erts_heap_alloc(Process* p, Uint need)
 
     bp->next = MBUF(p);
     MBUF(p) = bp;
-    bp->size = n;
-    bp->used_size = n;
+    bp->alloc_size = n;
+    bp->used_size = need;
     MBUF_SIZE(p) += n;
-    bp->off_heap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    bp->off_heap.funs = NULL;
-#endif
-    bp->off_heap.externals = NULL;
+    bp->off_heap.first = NULL;
     bp->off_heap.overhead = 0;
-
     return bp->mem;
 }
 
@@ -204,6 +193,25 @@ erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end)
     *end = *start + new_size;
     *sp = *start + sp_offs;
 }
+/*
+ * Helper function for the ESTACK macros defined in global.h.
+ */
+void
+erl_grow_wstack(UWord** start, UWord** sp, UWord** end)
+{
+    Uint old_size = (*end - *start);
+    Uint new_size = old_size * 2;
+    Uint sp_offs = *sp - *start;
+    if (new_size > 2 * DEF_ESTACK_SIZE) {
+	*start = erts_realloc(ERTS_ALC_T_ESTACK, (void *) *start, new_size*sizeof(UWord));
+    } else {
+	UWord* new_ptr = erts_alloc(ERTS_ALC_T_ESTACK, new_size*sizeof(UWord));
+	sys_memcpy(new_ptr, *start, old_size*sizeof(UWord));
+	*start = new_ptr;
+    }
+    *end = *start + new_size;
+    *sp = *start + sp_offs;
+}
 
 /* CTYPE macros */
 
@@ -354,6 +362,31 @@ erts_bld_uint(Uint **hpp, Uint *szp, Uint ui)
     return res;
 }
 
+/*
+ * Erts_bld_uword is more or less similar to erts_bld_uint, but a pointer
+ * can safely be passed.
+ */
+
+Eterm
+erts_bld_uword(Uint **hpp, Uint *szp, UWord uw)
+{
+    Eterm res = THE_NON_VALUE;
+    if (IS_USMALL(0, uw)) {
+	if (hpp)
+	    res = make_small((Uint) uw);
+    }
+    else {
+	if (szp)
+	    *szp += BIG_UWORD_HEAP_SIZE(uw);
+	if (hpp) {
+	    res = uword_to_big(uw, *hpp);
+	    *hpp += BIG_UWORD_HEAP_SIZE(uw);
+	}
+    }
+    return res;
+}
+
+
 Eterm
 erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64)
 {
@@ -364,7 +397,7 @@ erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64)
     }
     else {
 	if (szp)
-	    *szp = ERTS_UINT64_HEAP_SIZE(ui64);
+	    *szp += ERTS_UINT64_HEAP_SIZE(ui64);
 	if (hpp)
 	    res = erts_uint64_to_big(ui64, hpp);
     }
@@ -381,7 +414,7 @@ erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64)
     }
     else {
 	if (szp)
-	    *szp = ERTS_SINT64_HEAP_SIZE(si64);
+	    *szp += ERTS_SINT64_HEAP_SIZE(si64);
 	if (hpp)
 	    res = erts_sint64_to_big(si64, hpp);
     }
@@ -465,7 +498,7 @@ erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len)
     if (hpp) {
 	res = NIL;
 	while (--i >= 0) {
-	    res = CONS(*hpp, make_small(str[i]), res);
+	    res = CONS(*hpp, make_small((byte) str[i]), res);
 	    *hpp += 2;
 	}
     }
@@ -711,7 +744,7 @@ hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash)
 
 Uint32 make_hash(Eterm term_arg)
 {
-    DECLARE_ESTACK(stack);
+    DECLARE_WSTACK(stack);
     Eterm term = term_arg;
     Eterm hash = 0;
     unsigned op;
@@ -770,7 +803,7 @@ tail_recur:
 	    Uint y2 = y1 < 0 ? -(Uint)y1 : y1;
 
 	    UINT32_HASH_STEP(y2, FUNNY_NUMBER2);
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	    if (y2 >> 32)
 		UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2);
 #endif
@@ -787,7 +820,7 @@ tail_recur:
 	}
     case EXPORT_DEF:
 	{
-	    Export* ep = (Export *) (export_val(term))[1];
+	    Export* ep = *((Export **) (export_val(term) + 1));
 
 	    hash = hash * FUNNY_NUMBER11 + ep->code[2];
 	    hash = hash*FUNNY_NUMBER1 + 
@@ -809,7 +842,7 @@ tail_recur:
 	    hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq;
 	    if (num_free > 0) {
 		if (num_free > 1) {
-		    ESTACK_PUSH3(stack, (Eterm) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
+		    WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
 		}
 		term = funp->env[0];
 		goto tail_recur;
@@ -837,9 +870,9 @@ tail_recur:
 	}
 
     case MAKE_HASH_CDR_PRE_OP:
-	term = ESTACK_POP(stack);
+	term = (Eterm) WSTACK_POP(stack);
 	if (is_not_list(term)) {
-	    ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP);
+	    WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP);
 	    goto tail_recur;
 	}
 	/* fall through */
@@ -854,13 +887,13 @@ tail_recur:
 		hash = hash*FUNNY_NUMBER2 + unsigned_val(*list);
 		
 		if (is_not_list(CDR(list))) {
-		    ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP);
+		    WSTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP);
 		    term = CDR(list);
 		    goto tail_recur;
 		}		
 		list = list_val(CDR(list));
 	    }
-	    ESTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP);
+	    WSTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP);
 	    term = CAR(list);
 	    goto tail_recur;
 	}
@@ -888,7 +921,7 @@ tail_recur:
 	    }
 	    d = BIG_DIGIT(ptr, k);
 	    k = sizeof(ErtsDigit);
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	    if (!(d >> 32))
 		k /= 2;
 #endif
@@ -904,21 +937,21 @@ tail_recur:
 	    Eterm* ptr = tuple_val(term);
 	    Uint arity = arityval(*ptr);
 
-	    ESTACK_PUSH3(stack, arity, (Eterm)(ptr+1), arity);
+	    WSTACK_PUSH3(stack, (UWord) arity, (UWord)(ptr+1), (UWord) arity);
 	    op = MAKE_HASH_TUPLE_OP;	    
 	}/*fall through*/
     case MAKE_HASH_TUPLE_OP:
     case MAKE_HASH_FUN_OP:
 	{
-	    Uint i = ESTACK_POP(stack);
-	    Eterm* ptr = (Eterm*) ESTACK_POP(stack);
+	    Uint i = (Uint) WSTACK_POP(stack);
+	    Eterm* ptr = (Eterm*) WSTACK_POP(stack);
 	    if (i != 0) {
 		term = *ptr;
-		ESTACK_PUSH3(stack, (Eterm)(ptr+1), i-1, op);
+		WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op);
 		goto tail_recur;
 	    }
 	    if (op == MAKE_HASH_TUPLE_OP) {
-		Uint32 arity = ESTACK_POP(stack);
+		Uint32 arity = (Uint32) WSTACK_POP(stack);
 		hash = hash*FUNNY_NUMBER9 + arity;
 	    }
 	    break;
@@ -928,10 +961,10 @@ tail_recur:
 	erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op);
 	return 0;
       }
-      if (ESTACK_ISEMPTY(stack)) break;
-      op = ESTACK_POP(stack);
+      if (WSTACK_ISEMPTY(stack)) break;
+      op = WSTACK_POP(stack);
     }
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return hash;
 
 #undef UINT32_HASH_STEP
@@ -1002,7 +1035,7 @@ Uint32
 make_hash2(Eterm term)
 {
     Uint32 hash;
-    Eterm tmp_big[2];
+    DeclareTmpHeapNoproc(tmp_big,2);
 
 /* (HCONST * {2, ..., 14}) mod 2^32 */
 #define HCONST_2 0x3c6ef372UL
@@ -1041,7 +1074,6 @@ make_hash2(Eterm term)
 	} while(0)
 
 #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2)
-
     /* Optimization. Simple cases before declaration of estack. */
     if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
 	switch (term & _TAG_IMMED1_MASK) {
@@ -1070,6 +1102,7 @@ make_hash2(Eterm term)
     Eterm tmp;
     DECLARE_ESTACK(s);
 
+    UseTmpHeapNoproc(2);
     hash = 0;
     for (;;) {
 	switch (primary_tag(term)) {
@@ -1123,7 +1156,7 @@ make_hash2(Eterm term)
 	    break;
 	    case EXPORT_SUBTAG:
 	    {
-		Export* ep = (Export *) (export_val(term))[1];
+		Export* ep = *((Export **) (export_val(term) + 1));
 
 		UINT32_HASH_2
 		    (ep->code[2], 
@@ -1314,6 +1347,7 @@ make_hash2(Eterm term)
 	hash2_common:
 	    if (ESTACK_ISEMPTY(s)) {
 		DESTROY_ESTACK(s);
+		UnUseTmpHeapNoproc(2);
 		return hash;
 	    }
 	    term = ESTACK_POP(s);
@@ -1332,7 +1366,7 @@ make_hash2(Eterm term)
 Uint32 make_broken_hash(Eterm term)
 {
     Uint32 hash = 0;
-    DECLARE_ESTACK(stack);
+    DECLARE_WSTACK(stack);
     unsigned op;
 tail_recur:
     op = tag_val_def(term); 
@@ -1346,7 +1380,7 @@ tail_recur:
 	    (atom_tab(atom_val(term))->slot.bucket.hvalue);
 	break;
     case SMALL_DEF:
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     {
 	Sint y1 = signed_val(term);
 	Uint y2 = y1 < 0 ? -(Uint)y1 : y1;
@@ -1399,7 +1433,7 @@ tail_recur:
 
     case EXPORT_DEF:
 	{
-	    Export* ep = (Export *) (export_val(term))[1];
+	    Export* ep = *((Export **) (export_val(term) + 1));
 
 	    hash = hash * FUNNY_NUMBER11 + ep->code[2];
 	    hash = hash*FUNNY_NUMBER1 + 
@@ -1421,7 +1455,7 @@ tail_recur:
 	    hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq;
 	    if (num_free > 0) {
 		if (num_free > 1) {
-		    ESTACK_PUSH3(stack, (Eterm) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
+		    WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
 		}
 		term = funp->env[0];
 		goto tail_recur;
@@ -1456,16 +1490,17 @@ tail_recur:
 	break;
 
     case MAKE_HASH_CDR_PRE_OP:
-	term = ESTACK_POP(stack);
+	term = (Eterm) WSTACK_POP(stack);
 	if (is_not_list(term)) {
-	    ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP);
+	    WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP);
 	    goto tail_recur;
 	}
 	/*fall through*/
     case LIST_DEF:
 	{
 	    Eterm* list = list_val(term);
-	    ESTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP);
+	    WSTACK_PUSH2(stack, (UWord) CDR(list),
+			 (UWord) MAKE_HASH_CDR_PRE_OP);
 	    term = CAR(list);
 	    goto tail_recur;
 	}
@@ -1538,21 +1573,21 @@ tail_recur:
 	    Eterm* ptr = tuple_val(term);
 	    Uint arity = arityval(*ptr);
 
-	    ESTACK_PUSH3(stack, arity, (Eterm)(ptr+1), arity);
+	    WSTACK_PUSH3(stack, (UWord) arity, (UWord) (ptr+1), (UWord) arity);
 	    op = MAKE_HASH_TUPLE_OP;
 	}/*fall through*/ 
     case MAKE_HASH_TUPLE_OP:
     case MAKE_HASH_FUN_OP:
 	{
-	    Uint i = ESTACK_POP(stack);
-	    Eterm* ptr = (Eterm*) ESTACK_POP(stack);
+	    Uint i = (Uint) WSTACK_POP(stack);
+	    Eterm* ptr = (Eterm*) WSTACK_POP(stack);
 	    if (i != 0) {
 		term = *ptr;
-		ESTACK_PUSH3(stack, (Eterm)(ptr+1), i-1, op);
+		WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op);
 		goto tail_recur;
 	    }
 	    if (op == MAKE_HASH_TUPLE_OP) {
-		Uint32 arity = ESTACK_POP(stack);
+		Uint32 arity = (UWord) WSTACK_POP(stack);
 		hash = hash*FUNNY_NUMBER9 + arity;
 	    }
 	    break;
@@ -1562,11 +1597,11 @@ tail_recur:
 	erl_exit(1, "Invalid tag in make_broken_hash\n");
 	return 0;
       }
-      if (ESTACK_ISEMPTY(stack)) break;
-      op = ESTACK_POP(stack);
+      if (WSTACK_ISEMPTY(stack)) break;
+      op = (Uint) WSTACK_POP(stack);
     }
 
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return hash;
     
 #undef MAKE_HASH_TUPLE_OP
@@ -1859,42 +1894,44 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp)
     erts_free(ERTS_ALC_T_TMP_DSBUF, (void *) dsbufp);
 }
 
-
 /* eq and cmp are written as separate functions a eq is a little faster */
 
 /*
  * Test for equality of two terms.
  * Returns 0 if not equal, or a non-zero value otherwise.
  */
-
+#if HALFWORD_HEAP
+int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
+#else
 int eq(Eterm a, Eterm b)
+#endif
 {
-    DECLARE_ESTACK(stack);
+    DECLARE_WSTACK(stack);
     Sint sz;
     Eterm* aa;
-    Eterm* bb;	
+    Eterm* bb;
 
 tailrecur:
-    if (a == b) goto pop_next; 
+    if (is_same(a, a_base, b, b_base)) goto pop_next;
 tailrecur_ne:
 
     switch (primary_tag(a)) {
     case TAG_PRIMARY_LIST:
 	if (is_list(b)) {
-	    Eterm* aval = list_val(a);
-	    Eterm* bval = list_val(b);
+	    Eterm* aval = list_val_rel(a, a_base);
+	    Eterm* bval = list_val_rel(b, b_base);
 	    while (1) {
 		Eterm atmp = CAR(aval);
 		Eterm btmp = CAR(bval);
-		if (atmp != btmp) {
-		    ESTACK_PUSH2(stack,CDR(bval),CDR(aval));
+		if (!is_same(atmp,a_base,btmp,b_base)) {
+		    WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval));
 		    a = atmp;
 		    b = btmp;
 		    goto tailrecur_ne;
 		}
 		atmp = CDR(aval);
 		btmp = CDR(bval);
-		if (atmp == btmp) {
+		if (is_same(atmp,a_base,btmp,b_base)) {
 		    goto pop_next;
 		}
 		if (is_not_list(atmp) || is_not_list(btmp)) {
@@ -1902,22 +1939,22 @@ tailrecur_ne:
 		    b = btmp;
 		    goto tailrecur_ne;
 		}
-		aval = list_val(atmp);
-		bval = list_val(btmp);
+		aval = list_val_rel(atmp, a_base);
+		bval = list_val_rel(btmp, b_base);
 	    }
 	}
 	break; /* not equal */
 
     case TAG_PRIMARY_BOXED:
 	{	
-	    Eterm hdr = *boxed_val(a);
+	    Eterm hdr = *boxed_val_rel(a,a_base);
 	    switch (hdr & _TAG_HEADER_MASK) {
 	    case ARITYVAL_SUBTAG:
 		{
-		    aa = tuple_val(a);
-		    if (!is_boxed(b) || *boxed_val(b) != *aa)
+		    aa = tuple_val_rel(a, a_base);
+		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
 			goto not_equal;
-		    bb = tuple_val(b);
+		    bb = tuple_val_rel(b,b_base);
 		    if ((sz = arityval(*aa)) == 0) goto pop_next;
 		    ++aa;
 		    ++bb;
@@ -1936,16 +1973,16 @@ tailrecur_ne:
 		    Uint a_bitoffs;
 		    Uint b_bitoffs;
 		    
-		    if (is_not_binary(b)) {
+		    if (!is_binary_rel(b,b_base)) {
 			goto not_equal;
 		    }
-		    a_size = binary_size(a);
-		    b_size = binary_size(b); 
+		    a_size = binary_size_rel(a,a_base);
+		    b_size = binary_size_rel(b,b_base);
 		    if (a_size != b_size) {
 			goto not_equal;
 		    }
-		    ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
-		    ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
+		    ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base);
+		    ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base);
 		    if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
 			if (sys_memcmp(a_ptr, b_ptr, a_size) == 0) goto pop_next;
 		    } else if (a_bitsize == b_bitsize) {
@@ -1956,9 +1993,9 @@ tailrecur_ne:
 		}
 	    case EXPORT_SUBTAG:
 		{
-		    if (is_export(b)) {
-			Export* a_exp = (Export *) (export_val(a))[1];
-			Export* b_exp = (Export *) (export_val(b))[1];
+		    if (is_export_rel(b,b_base)) {
+			Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1));
+			Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1));
 			if (a_exp == b_exp) goto pop_next;
 		    }
 		    break; /* not equal */
@@ -1968,10 +2005,10 @@ tailrecur_ne:
 		    ErlFunThing* f1;
 		    ErlFunThing* f2;
   
-		    if (is_not_fun(b))
+		    if (!is_fun_rel(b,b_base))
 			goto not_equal;
-		    f1 = (ErlFunThing *) fun_val(a);
-		    f2 = (ErlFunThing *) fun_val(b);
+		    f1 = (ErlFunThing *) fun_val_rel(a,a_base);
+		    f2 = (ErlFunThing *) fun_val_rel(b,b_base);
 		    if (f1->fe->module != f2->fe->module ||
 			f1->fe->old_index != f2->fe->old_index ||
 			f1->fe->old_uniq != f2->fe->old_uniq ||
@@ -1989,15 +2026,15 @@ tailrecur_ne:
 		ExternalThing *ap;
 		ExternalThing *bp;
 
-		if(is_not_external(b))
+		if(!is_external_rel(b,b_base))
 		    goto not_equal;
 
-		ap = external_thing_ptr(a);
-		bp = external_thing_ptr(b);
+		ap = external_thing_ptr_rel(a,a_base);
+		bp = external_thing_ptr_rel(b,b_base);
 
 		if(ap->header == bp->header && ap->node == bp->node) {
-		    ASSERT(1 == external_data_words(a));
-		    ASSERT(1 == external_data_words(b));
+		    ASSERT(1 == external_data_words_rel(a,a_base));
+		    ASSERT(1 == external_data_words_rel(b,b_base));
 		    
 		    if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next;
 		}
@@ -2015,27 +2052,36 @@ tailrecur_ne:
 		Uint alen;
 		Uint blen;
 		Uint i;
+		ExternalThing* athing;
+		ExternalThing* bthing;
 
-		if(is_not_external_ref(b))
+		if(!is_external_ref_rel(b,b_base))
 		    goto not_equal;
 
-		if(external_node(a) != external_node(b))
+		athing = external_thing_ptr_rel(a,a_base);
+		bthing = external_thing_ptr_rel(b,b_base);
+
+		if(athing->node != bthing->node)
 		    goto not_equal;
 
-		anum = external_ref_numbers(a);
-		bnum = external_ref_numbers(b);
-		alen = external_ref_no_of_numbers(a);
-		blen = external_ref_no_of_numbers(b);
+		anum = external_thing_ref_numbers(athing);
+		bnum = external_thing_ref_numbers(bthing);
+		alen = external_thing_ref_no_of_numbers(athing);
+		blen = external_thing_ref_no_of_numbers(bthing);
 
 		goto ref_common;
 	    case REF_SUBTAG:
-  
-		    if (is_not_internal_ref(b))
+		    if (!is_internal_ref_rel(b,b_base))
 			goto not_equal;
-		    alen = internal_ref_no_of_numbers(a);
-		    blen = internal_ref_no_of_numbers(b);
-		    anum = internal_ref_numbers(a);
-		    bnum = internal_ref_numbers(b);
+
+		    {
+			RefThing* athing = ref_thing_ptr_rel(a,a_base);
+			RefThing* bthing = ref_thing_ptr_rel(b,b_base);
+			alen = internal_thing_ref_no_of_numbers(athing);
+			blen = internal_thing_ref_no_of_numbers(bthing);
+			anum = internal_thing_ref_numbers(athing);
+			bnum = internal_thing_ref_numbers(bthing);
+		    }
 
 	    ref_common:
 		    ASSERT(alen > 0 && blen > 0);
@@ -2080,10 +2126,10 @@ tailrecur_ne:
 		{
 		    int i;
   
-		    if (is_not_big(b))
+		    if (!is_big_rel(b,b_base))
 			goto not_equal;
-		    aa = big_val(a); /* get pointer to thing */
-		    bb = big_val(b);
+		    aa = big_val_rel(a,a_base);
+		    bb = big_val_rel(b,b_base);
 		    if (*aa != *bb)
 			goto not_equal;
 		    i = BIG_ARITY(aa);
@@ -2098,9 +2144,9 @@ tailrecur_ne:
 		    FloatDef af;
 		    FloatDef bf;
   
-		    if (is_float(b)) {
-			GET_DOUBLE(a, af);
-			GET_DOUBLE(b, bf);
+		    if (is_float_rel(b,b_base)) {
+			GET_DOUBLE_REL(a, af, a_base);
+			GET_DOUBLE_REL(b, bf, b_base);
 			if (af.fd == bf.fd) goto pop_next;
 		    }
 		    break; /* not equal */
@@ -2119,7 +2165,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */
 	Eterm* bp = bb;
 	Sint i = sz;
 	for (;;) {
-	    if (*ap != *bp) break;
+	    if (!is_same(*ap,a_base,*bp,b_base)) break;
 	    if (--i == 0) goto pop_next;
 	    ++ap;
 	    ++bp;
@@ -2130,32 +2176,32 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */
 	    goto not_equal;
 	}
 	if (i > 1) { /* push the rest */
-	    ESTACK_PUSH3(stack, i-1, (Eterm)(bp+1),
-			 ((Eterm)(ap+1)) | TAG_PRIMARY_HEADER);
+	    WSTACK_PUSH3(stack, i-1, (UWord)(bp+1),
+			 ((UWord)(ap+1)) | TAG_PRIMARY_HEADER);
 	    /* We (ab)use TAG_PRIMARY_HEADER to recognize a term_array */
 	}
 	goto tailrecur_ne;
     }
    
 pop_next:
-    if (!ESTACK_ISEMPTY(stack)) {
-	Eterm something  = ESTACK_POP(stack);	
-	if (primary_tag(something) == TAG_PRIMARY_HEADER) { /* a term_array */
+    if (!WSTACK_ISEMPTY(stack)) {
+	UWord something  = WSTACK_POP(stack);
+	if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */
 	    aa = (Eterm*) something;
-	    bb = (Eterm*) ESTACK_POP(stack);
-	    sz = ESTACK_POP(stack);
+	    bb = (Eterm*) WSTACK_POP(stack);
+	    sz = WSTACK_POP(stack);
 	    goto term_array;
 	}
 	a = something;
-	b = ESTACK_POP(stack);
+	b = WSTACK_POP(stack);
 	goto tailrecur;
     }
 
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return 1;
 
 not_equal:
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return 0;
 }
 
@@ -2208,9 +2254,13 @@ static int cmp_atoms(Eterm a, Eterm b)
 		    bb->name+3, bb->len-3);
 }
 
+#if HALFWORD_HEAP
+Sint cmp_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
+#else
 Sint cmp(Eterm a, Eterm b)
+#endif
 {
-    DECLARE_ESTACK(stack);
+    DECLARE_WSTACK(stack);
     Eterm* aa;
     Eterm* bb;
     int i;
@@ -2242,7 +2292,7 @@ Sint cmp(Eterm a, Eterm b)
 
 
 tailrecur:
-    if (a == b) {		/* Equal values or pointers. */
+    if (is_same(a,a_base,b,b_base)) {	/* Equal values or pointers. */
 	goto pop_next;
     }
 tailrecur_ne:
@@ -2268,9 +2318,9 @@ tailrecur_ne:
 	    if (is_internal_port(b)) {
 		bnode = erts_this_node;
 		bdata = internal_port_data(b);
-	    } else if (is_external_port(b)) {
-		bnode = external_port_node(b);
-		bdata = external_port_data(b);
+	    } else if (is_external_port_rel(b,b_base)) {
+		bnode = external_port_node_rel(b,b_base);
+		bdata = external_port_data_rel(b,b_base);
 	    } else {
 		a_tag = PORT_DEF;
 		goto mixed_types;
@@ -2286,9 +2336,9 @@ tailrecur_ne:
 	    if (is_internal_pid(b)) {
 		bnode = erts_this_node;
 		bdata = internal_pid_data(b);
-	    } else if (is_external_pid(b)) {
-		bnode = external_pid_node(b);
-		bdata = external_pid_data(b);
+	    } else if (is_external_pid_rel(b,b_base)) {
+		bnode = external_pid_node_rel(b,b_base);
+		bdata = external_pid_data_rel(b,b_base);
 	    } else {
 		a_tag = PID_DEF;
 		goto mixed_types;
@@ -2321,20 +2371,20 @@ tailrecur_ne:
 	    a_tag = LIST_DEF;
 	    goto mixed_types;
 	}
-	aa = list_val(a);
-	bb = list_val(b);
+	aa = list_val_rel(a,a_base);
+	bb = list_val_rel(b,b_base);
 	while (1) {
 	    Eterm atmp = CAR(aa);
 	    Eterm btmp = CAR(bb);
-	    if (atmp != btmp) {
-		ESTACK_PUSH2(stack,CDR(bb),CDR(aa));
+	    if (!is_same(atmp,a_base,btmp,b_base)) {
+		WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa));
 		a = atmp;
 		b = btmp;
 		goto tailrecur_ne;
 	    }
 	    atmp = CDR(aa);
 	    btmp = CDR(bb);
-	    if (atmp == btmp) {
+	    if (is_same(atmp,a_base,btmp,b_base)) {
 		goto pop_next;
 	    }
 	    if (is_not_list(atmp) || is_not_list(btmp)) {
@@ -2342,20 +2392,20 @@ tailrecur_ne:
 		b = btmp;
 		goto tailrecur_ne;
 	    }
-	    aa = list_val(atmp);
-	    bb = list_val(btmp);
+	    aa = list_val_rel(atmp,a_base);
+	    bb = list_val_rel(btmp,b_base);
 	}
     case TAG_PRIMARY_BOXED:
 	{
-	    Eterm ahdr = *boxed_val(a);
+	    Eterm ahdr = *boxed_val_rel(a,a_base);
 	    switch ((ahdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) {
 	    case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE):
-		if (is_not_tuple(b)) {
+		if (!is_tuple_rel(b,b_base)) {
 		    a_tag = TUPLE_DEF;
 		    goto mixed_types;
 		}
-		aa = tuple_val(a);
-		bb = tuple_val(b);
+		aa = tuple_val_rel(a,a_base);
+		bb = tuple_val_rel(b,b_base);
 		/* compare the arities */
 		i = arityval(ahdr);	/* get the arity*/
 		if (i != arityval(*bb)) {
@@ -2369,31 +2419,31 @@ tailrecur_ne:
 		goto term_array;
 
 	    case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
-		if (is_not_float(b)) {
+		if (!is_float_rel(b,b_base)) {
 		    a_tag = FLOAT_DEF;
 		    goto mixed_types;
 		} else {
 		    FloatDef af;
 		    FloatDef bf; 
 
-		    GET_DOUBLE(a, af);
-		    GET_DOUBLE(b, bf);
+		    GET_DOUBLE_REL(a, af, a_base);
+		    GET_DOUBLE_REL(b, bf, b_base);
 		    ON_CMP_GOTO(float_comp(af.fd, bf.fd));
 		}
 	    case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):
 	    case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):
-		if (is_not_big(b)) {
+		if (!is_big_rel(b,b_base)) {
 		    a_tag = BIG_DEF;
 		    goto mixed_types;
 		}
-		ON_CMP_GOTO(big_comp(a, b));
+		ON_CMP_GOTO(big_comp(rterm2wterm(a,a_base), rterm2wterm(b,b_base)));
 	    case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE):
-		if (is_not_export(b)) {
+		if (!is_export_rel(b,b_base)) {
 		    a_tag = EXPORT_DEF;
 		    goto mixed_types;
 		} else {
-		    Export* a_exp = (Export *) (export_val(a))[1];
-		    Export* b_exp = (Export *) (export_val(b))[1];
+		    Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1));
+		    Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1));
 
 		    if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
 			RETURN_NEQ(j);
@@ -2405,12 +2455,12 @@ tailrecur_ne:
 		}
 		break;
 	    case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE):
-		if (is_not_fun(b)) {
+		if (!is_fun_rel(b,b_base)) {
 		    a_tag = FUN_DEF;
 		    goto mixed_types;
 		} else {
-		    ErlFunThing* f1 = (ErlFunThing *) fun_val(a);
-		    ErlFunThing* f2 = (ErlFunThing *) fun_val(b);
+		    ErlFunThing* f1 = (ErlFunThing *) fun_val_rel(a,a_base);
+		    ErlFunThing* f2 = (ErlFunThing *) fun_val_rel(b,b_base);
 		    Sint diff;
 
 		    diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name,
@@ -2442,51 +2492,57 @@ tailrecur_ne:
 		if (is_internal_pid(b)) {
 		    bnode = erts_this_node;
 		    bdata = internal_pid_data(b);
-		} else if (is_external_pid(b)) {
-		    bnode = external_pid_node(b);
-		    bdata = external_pid_data(b);
+		} else if (is_external_pid_rel(b,b_base)) {
+		    bnode = external_pid_node_rel(b,b_base);
+		    bdata = external_pid_data_rel(b,b_base);
 		} else {
 		    a_tag = EXTERNAL_PID_DEF;
 		    goto mixed_types;
 		}
-		anode = external_pid_node(a);
-		adata = external_pid_data(a);
+		anode = external_pid_node_rel(a,a_base);
+		adata = external_pid_data_rel(a,a_base);
 		goto pid_common;
 	    case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE):
 		if (is_internal_port(b)) {
 		    bnode = erts_this_node;
 		    bdata = internal_port_data(b);
-		} else if (is_external_port(b)) {
-		    bnode = external_port_node(b);
-		    bdata = external_port_data(b);
+		} else if (is_external_port_rel(b,b_base)) {
+		    bnode = external_port_node_rel(b,b_base);
+		    bdata = external_port_data_rel(b,b_base);
 		} else {
 		    a_tag = EXTERNAL_PORT_DEF;
 		    goto mixed_types;
 		}
-		anode = external_port_node(a);
-		adata = external_port_data(a);
+		anode = external_port_node_rel(a,a_base);
+		adata = external_port_data_rel(a,a_base);
 		goto port_common;
 	    case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
 		/*
 		 * Note! When comparing refs we need to compare ref numbers
 		 * (32-bit words), *not* ref data words.
 		 */
+
 		
-		if (is_internal_ref(b)) {
+		if (is_internal_ref_rel(b,b_base)) {
+		    RefThing* bthing = ref_thing_ptr_rel(b,b_base);
 		    bnode = erts_this_node;
-		    bnum = internal_ref_numbers(b);
-		    blen = internal_ref_no_of_numbers(b);
-		} else if(is_external_ref(b)) {
-		    bnode = external_ref_node(b);
-		    bnum = external_ref_numbers(b);
-		    blen = external_ref_no_of_numbers(b);
+		    bnum = internal_thing_ref_numbers(bthing);
+		    blen = internal_thing_ref_no_of_numbers(bthing);
+		} else if(is_external_ref_rel(b,b_base)) {
+		    ExternalThing* bthing = external_thing_ptr_rel(b,b_base);
+		    bnode = bthing->node;
+		    bnum = external_thing_ref_numbers(bthing);
+		    blen = external_thing_ref_no_of_numbers(bthing);
 		} else {
 		    a_tag = REF_DEF;
 		    goto mixed_types;
 		}
-		anode = erts_this_node;
-		anum = internal_ref_numbers(a);
-		alen = internal_ref_no_of_numbers(a);
+		{
+		    RefThing* athing = ref_thing_ptr_rel(a,a_base);
+		    anode = erts_this_node;
+		    anum = internal_thing_ref_numbers(athing);
+		    alen = internal_thing_ref_no_of_numbers(athing);
+		}
 		
 	    ref_common:
 		CMP_NODES(anode, bnode);
@@ -2515,31 +2571,36 @@ tailrecur_ne:
 			RETURN_NEQ((Sint32) (anum[i] - bnum[i]));
 		goto pop_next;
 	    case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):
-		if (is_internal_ref(b)) {
+		if (is_internal_ref_rel(b,b_base)) {
+		    RefThing* bthing = ref_thing_ptr_rel(b,b_base);
 		    bnode = erts_this_node;
-		    bnum = internal_ref_numbers(b);
-		    blen = internal_ref_no_of_numbers(b);
-		} else if (is_external_ref(b)) {
-		    bnode = external_ref_node(b);
-		    bnum = external_ref_numbers(b);
-		    blen = external_ref_no_of_numbers(b);
+		    bnum = internal_thing_ref_numbers(bthing);
+		    blen = internal_thing_ref_no_of_numbers(bthing);
+		} else if (is_external_ref_rel(b,b_base)) {
+		    ExternalThing* bthing = external_thing_ptr_rel(b,b_base);
+		    bnode = bthing->node;
+		    bnum = external_thing_ref_numbers(bthing);
+		    blen = external_thing_ref_no_of_numbers(bthing);
 		} else {
 		    a_tag = EXTERNAL_REF_DEF;
 		    goto mixed_types;
 		}
-		anode = external_ref_node(a);
-		anum = external_ref_numbers(a);
-		alen = external_ref_no_of_numbers(a);
+		{
+		    ExternalThing* athing = external_thing_ptr_rel(a,a_base);
+		    anode = athing->node;
+		    anum = external_thing_ref_numbers(athing);
+		    alen = external_thing_ref_no_of_numbers(athing);
+		}
 		goto ref_common;
 	    default:
 		/* Must be a binary */
-		ASSERT(is_binary(a));
-		if (is_not_binary(b)) {
+		ASSERT(is_binary_rel(a,a_base));
+		if (!is_binary_rel(b,b_base)) {
 		    a_tag = BINARY_DEF;
 		    goto mixed_types;
 		} else {
-		    Uint a_size = binary_size(a);
-		    Uint b_size = binary_size(b);
+		    Uint a_size = binary_size_rel(a,a_base);
+		    Uint b_size = binary_size_rel(b,b_base);
 		    Uint a_bitsize;
 		    Uint b_bitsize;
 		    Uint a_bitoffs;
@@ -2548,8 +2609,8 @@ tailrecur_ne:
 		    int cmp;
 		    byte* a_ptr;
 		    byte* b_ptr;
-		    ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
-		    ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
+		    ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base);
+		    ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base);
 		    if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
 			min_size = (a_size < b_size) ? a_size : b_size;
 			if ((cmp = sys_memcmp(a_ptr, b_ptr, min_size)) != 0) {
@@ -2576,45 +2637,56 @@ tailrecur_ne:
      */
 
  mixed_types:
-    b_tag = tag_val_def(b);
 
     {
 	FloatDef f1, f2;
 	Eterm big;
-	Eterm big_buf[2];
+#if HEAP_ON_C_STACK
+	Eterm big_buf[2]; /* If HEAP_ON_C_STACK */
+#else
+	Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap;
+#endif
+#if HALFWORD_HEAP
+	Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base);
+	Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base);
+#else
+	Eterm aw = a;
+	Eterm bw = b;
+#endif
+	b_tag = tag_val_def(bw);
 
 	switch(_NUMBER_CODE(a_tag, b_tag)) {
 	case SMALL_BIG:
 	    big = small_to_big(signed_val(a), big_buf);
-	    j = big_comp(big, b);
+	    j = big_comp(big, bw);
 	    break;
 	case SMALL_FLOAT:
 	    f1.fd = signed_val(a);
-	    GET_DOUBLE(b, f2);
+	    GET_DOUBLE(bw, f2);
 	    j = float_comp(f1.fd, f2.fd);
 	    break;
 	case BIG_SMALL:
 	    big = small_to_big(signed_val(b), big_buf);
-	    j = big_comp(a, big);
+	    j = big_comp(aw, big);
 	    break;
 	case BIG_FLOAT:
-	    if (big_to_double(a, &f1.fd) < 0) {
+	    if (big_to_double(aw, &f1.fd) < 0) {
 		j = big_sign(a) ? -1 : 1;
 	    } else {
-		GET_DOUBLE(b, f2);
+		GET_DOUBLE(bw, f2);
 		j = float_comp(f1.fd, f2.fd);
 	    }
 	    break;
 	case FLOAT_SMALL:
-	    GET_DOUBLE(a, f1);
+	    GET_DOUBLE(aw, f1);
 	    f2.fd = signed_val(b);
 	    j = float_comp(f1.fd, f2.fd);
 	    break;
 	case FLOAT_BIG:
-	    if (big_to_double(b, &f2.fd) < 0) {
+	    if (big_to_double(bw, &f2.fd) < 0) {
 		j = big_sign(b) ? 1 : -1;
 	    } else {
-		GET_DOUBLE(a, f1);
+		GET_DOUBLE(aw, f1);
 		j = float_comp(f1.fd, f2.fd);
 	    }
 	    break;
@@ -2633,7 +2705,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
     while (--i) {
 	a = *aa++;
 	b = *bb++;
-	if (a != b) {
+	if (!is_same(a,a_base, b,b_base)) {
 	    if (is_atom(a) && is_atom(b)) {
 		if ((j = cmp_atoms(a, b)) != 0) {
 		    goto not_equal;
@@ -2644,7 +2716,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
 		}
 	    } else {
 		/* (ab)Use TAG_PRIMARY_HEADER to recognize a term_array */
-		ESTACK_PUSH3(stack, i, (Eterm)bb, (Eterm)aa | TAG_PRIMARY_HEADER);
+		WSTACK_PUSH3(stack, i, (UWord)bb, (UWord)aa | TAG_PRIMARY_HEADER);
 		goto tailrecur_ne;
 	    }
 	}
@@ -2654,20 +2726,20 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
     goto tailrecur;    
    
 pop_next:
-    if (!ESTACK_ISEMPTY(stack)) {
-	Eterm something = ESTACK_POP(stack);
-	if (primary_tag(something) == TAG_PRIMARY_HEADER) { /* a term_array */
+    if (!WSTACK_ISEMPTY(stack)) {
+	UWord something = WSTACK_POP(stack);
+	if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */
 	    aa = (Eterm*) something;
-	    bb = (Eterm*) ESTACK_POP(stack);
-	    i = ESTACK_POP(stack);
+	    bb = (Eterm*) WSTACK_POP(stack);
+	    i = WSTACK_POP(stack);
 	    goto term_array;
 	}
-	a = something;
-	b = ESTACK_POP(stack);
+	a = (Eterm) something;
+	b = (Eterm) WSTACK_POP(stack);
 	goto tailrecur;
     }
 
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return 0;
 
 not_equal:
@@ -2678,21 +2750,8 @@ not_equal:
 }
 
 
-void
-erts_cleanup_externals(ExternalThing *etp)
-{
-    ExternalThing *tetp;
-
-    tetp = etp;
-
-    while(tetp) {
-	erts_deref_node_entry(tetp->node);
-	tetp = tetp->next;
-    }
-}
-
 Eterm
-store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns)
+store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns)
 {
     Uint i;
     Uint size;
@@ -2711,8 +2770,8 @@ store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns)
 
 	erts_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2);
 
-	((ExternalThing *) to_hp)->next = *etpp;
-	*etpp = (ExternalThing *) to_hp;
+	((struct erl_off_heap_header*) to_hp)->next = oh->first;
+	oh->first = (struct erl_off_heap_header*) to_hp;
 
 	return make_external(to_hp);
     }
@@ -2741,7 +2800,7 @@ store_external_or_ref_in_proc_(Process *proc, Eterm ns)
     sz = NC_HEAP_SIZE(ns);
     ASSERT(sz > 0);
     hp = HAlloc(proc, sz);
-    return store_external_or_ref_(&hp, &MSO(proc).externals, ns);
+    return store_external_or_ref_(&hp, &MSO(proc), ns);
 }
 
 void bin_write(int to, void *to_arg, byte* buf, int sz)
@@ -2962,13 +3021,25 @@ int io_list_to_buf(Eterm obj, char* buf, int len)
     return -1;
 }
 
-int io_list_len(Eterm obj)
+/*
+ * Return 0 if successful, and non-zero if unsuccessful.
+ */
+int erts_iolist_size(Eterm obj, Uint* sizep)
 {
     Eterm* objp;
-    Sint len = 0;
+    Uint size = 0;
     DECLARE_ESTACK(s);
     goto L_again;
 
+#define SAFE_ADD(Var, Val)			\
+    do {					\
+        Uint valvar = (Val);			\
+	Var += valvar;				\
+	if (Var < valvar) {			\
+	    goto L_overflow_error;		\
+	}					\
+    } while (0)
+
     while (!ESTACK_ISEMPTY(s)) {
 	obj = ESTACK_POP(s);
     L_again:
@@ -2978,9 +3049,12 @@ int io_list_len(Eterm obj)
 	    /* Head */
 	    obj = CAR(objp);
 	    if (is_byte(obj)) {
-		len++;
+		size++;
+		if (size == 0) {
+		    goto L_overflow_error;
+		}
 	    } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
-		len += binary_size(obj);
+		SAFE_ADD(size, binary_size(obj));
 	    } else if (is_list(obj)) {
 		ESTACK_PUSH(s, CDR(objp));
 		goto L_iter_list; /* on head */
@@ -2992,23 +3066,29 @@ int io_list_len(Eterm obj)
 	    if (is_list(obj))
 		goto L_iter_list; /* on tail */
 	    else if (is_binary(obj) && binary_bitsize(obj) == 0) {
-		len += binary_size(obj);
+		SAFE_ADD(size, binary_size(obj));
 	    } else if (is_not_nil(obj)) {
 		goto L_type_error;
 	    }
 	} else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */
-	    len += binary_size(obj);
+	    SAFE_ADD(size, binary_size(obj));
 	} else if (is_not_nil(obj)) {
 	    goto L_type_error;
 	}
     }
+#undef SAFE_ADD
 
     DESTROY_ESTACK(s);
-    return len;
+    *sizep = size;
+    return ERTS_IOLIST_OK;
+
+ L_overflow_error:
+    DESTROY_ESTACK(s);
+    return ERTS_IOLIST_OVERFLOW;
 
  L_type_error:
     DESTROY_ESTACK(s);
-    return -1;
+    return ERTS_IOLIST_TYPE;
 }
 
 /* return 0 if item is not a non-empty flat list of bytes */
@@ -3150,7 +3230,7 @@ erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
 
     *timer_ref = res;
 
-    erl_set_timer(&res->timer.tm,
+    erts_set_timer(&res->timer.tm,
 		  (ErlTimeoutProc) ptimer_timeout,
 		  (ErlCancelProc) ptimer_cancelled,
 		  (void*) res,
@@ -3164,7 +3244,7 @@ erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer)
 	ASSERT(*ptimer->timer.timer_ref == ptimer);
 	*ptimer->timer.timer_ref = NULL;
 	ptimer->timer.flags |= ERTS_PTMR_FLG_CANCELLED;
-	erl_cancel_timer(&ptimer->timer.tm);
+	erts_cancel_timer(&ptimer->timer.tm);
     }
 }
 
@@ -3604,19 +3684,19 @@ erts_set_activity_error(erts_activity_error_t error, char *file, int line)
 }
 
 
-static ERTS_INLINE int
+static ERTS_INLINE erts_aint32_t
 threads_not_under_control(void)
 {
-    int res = system_block_state.threads_to_block;
+    erts_aint32_t res = system_block_state.threads_to_block;
 
     /* Waiting is always an allowed activity... */
-    res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.wait);
+    res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.wait);
 
     if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_GC)
-	res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.gc);
+	res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.gc);
 
     if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_IO)
-	res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.io);
+	res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.io);
 
     if (res < 0) {
 	ASSERT(0);
@@ -3676,7 +3756,7 @@ erts_block_system(Uint32 allowed_activities)
     }
     else {
 
-	erts_smp_atomic_inc(&erts_system_block_state.do_block);
+	erts_smp_atomic32_inc(&erts_system_block_state.do_block);
 
 	/* Someone else might be waiting for us to block... */
 	if (do_block) {
@@ -3728,11 +3808,11 @@ erts_emergency_block_system(long timeout, Uint32 allowed_activities)
 
     another_blocker = erts_smp_pending_system_block();
     system_block_state.emergency = 1;
-    erts_smp_atomic_inc(&erts_system_block_state.do_block);
+    erts_smp_atomic32_inc(&erts_system_block_state.do_block);
 
     if (another_blocker) {
 	if (is_blocker()) {
-	    erts_smp_atomic_dec(&erts_system_block_state.do_block);
+	    erts_smp_atomic32_dec(&erts_system_block_state.do_block);
 	    res = 0;
 	    goto done;
 	}
@@ -3789,7 +3869,7 @@ erts_release_system(void)
     if (system_block_state.recursive_block)
 	system_block_state.recursive_block--;
     else {
-	do_block = erts_smp_atomic_dectest(&erts_system_block_state.do_block);
+	do_block = erts_smp_atomic32_dectest(&erts_system_block_state.do_block);
 	system_block_state.have_blocker = 0;
 	if (is_blockable_thread())
 	    system_block_state.threads_to_block++;
@@ -3924,10 +4004,10 @@ erts_system_block_init(void)
 
     /* Global state... */
 
-    erts_smp_atomic_init(&erts_system_block_state.do_block, 0L);
-    erts_smp_atomic_init(&erts_system_block_state.in_activity.wait, 0L);
-    erts_smp_atomic_init(&erts_system_block_state.in_activity.gc, 0L);
-    erts_smp_atomic_init(&erts_system_block_state.in_activity.io, 0L);
+    erts_smp_atomic32_init(&erts_system_block_state.do_block, 0);
+    erts_smp_atomic32_init(&erts_system_block_state.in_activity.wait, 0);
+    erts_smp_atomic32_init(&erts_system_block_state.in_activity.gc, 0);
+    erts_smp_atomic32_init(&erts_system_block_state.in_activity.io, 0);
 
     /* Make sure blockable threads unregister when exiting... */
     erts_smp_install_exit_handler(erts_unregister_blockable_thread);
diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index 95510a16b2..f0ff3f54c5 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -53,6 +53,8 @@
 #define FILE_IPREAD             27
 #define FILE_ALTNAME            28
 #define FILE_READ_LINE          29
+#define FILE_FDATASYNC          30
+#define FILE_FADVISE            31
 
 /* Return codes */
 
@@ -65,6 +67,8 @@
 #define FILE_RESP_LDATA      6
 #define FILE_RESP_N2DATA     7
 #define FILE_RESP_EOF        8
+#define FILE_RESP_FNAME      9
+#define FILE_RESP_ALL_DATA  10
 
 /* Options */
 
@@ -102,16 +106,16 @@
 #include <ctype.h>
 #include <sys/types.h>
 
-extern void erl_exit(int n, char *fmt, _DOTS_);
+void erl_exit(int n, char *fmt, ...);
 
 static ErlDrvSysInfo sys_info;
 
 
-/*#define TRACE 1*/
+/* #define TRACE 1 */
 #ifdef TRACE
-#    define TRACE_C(c) (putchar(c))
-#    define TRACE_S(s) (fputs((s), stdout))
-#    define TRACE_F(args) (printf args)
+#    define TRACE_C(c) do { putchar(c); fflush(stdout); } while (0)
+#    define TRACE_S(s) do { fputs((s), stdout); fflush(stdout); } while (0)
+#    define TRACE_F(args) do { printf args ;fflush(stdout); } while (0)
 #else
 #    define TRACE_C(c) ((void)(0))
 #    define TRACE_S(s) ((void)(0))
@@ -135,24 +139,54 @@ static ErlDrvSysInfo sys_info;
 #define MUTEX_UNLOCK(m)
 #endif
 
-
-
 #if 0
 /* Experimental, for forcing all file operations to use the same thread. */
-static unsigned file_fixed_key = 1;
-#define KEY(desc) (&file_fixed_key)
+   static unsigned file_fixed_key = 1;
+#  define KEY(desc) (&file_fixed_key)
 #else
-#define KEY(desc) (&(desc)->key)
+#  define KEY(desc) (&(desc)->key)
 #endif
 
 
+#ifdef FILENAMES_16BIT
+#  define FILENAME_BYTELEN(Str) filename_len_16bit(Str)
+#  define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) 
+#  define FILENAME_CHARSIZE 2
+
+   static int filename_len_16bit(char *str) 
+   {
+       char *p = str;
+       while(*p != '\0' || p[1] != '\0') {
+	   p += 2;
+       }
+       return (p - str);
+   }
+
+   static void filename_cpy_16bit(char *to, char *from) 
+   {
+       while(*from != '\0' || from[1] != '\0') {
+	   *to++ = *from++;
+	   *to++ = *from++;
+       }
+       *to++ = *from++;
+       *to++ = *from++;
+   }
 
-#if     MAXPATHLEN >= BUFSIZ
-#define    RESBUFSIZE  MAXPATHLEN+1
 #else
-#define    RESBUFSIZE  BUFSIZ
+#  define FILENAME_BYTELEN(Str) strlen(Str)
+#  define FILENAME_COPY(To,From) strcpy(To,From) 
+#  define FILENAME_CHARSIZE 1
 #endif
 
+#if     (MAXPATHLEN+1)*FILENAME_CHARSIZE+1 > BUFSIZ
+#  define    RESBUFSIZE  ((MAXPATHLEN+1)*FILENAME_CHARSIZE+1)
+#else
+#  define    RESBUFSIZE  BUFSIZ
+#endif
+
+
+
+
 #define GET_TIME(i, b) \
     (i).year  = get_int32((b) + 0 * 4); \
     (i).month = get_int32((b) + 1 * 4); \
@@ -196,9 +230,9 @@ enum e_timer {timer_idle, timer_again, timer_write};
 struct t_data;
 
 typedef struct {
-    Sint            fd;
+    SWord           fd;
     ErlDrvPort      port;
-    unsigned        key;      /* Async queue key */
+    unsigned int    key;      /* Async queue key */
     unsigned        flags;    /* Original flags from FILE_OPEN. */
     void          (*invoke)(void *);
     struct t_data  *d;
@@ -284,9 +318,9 @@ struct t_preadv {
 };
 
 #define READDIR_BUFSIZE (8*1024)
-#if READDIR_BUFSIZE < (2*MAXPATHLEN)
-#undef READDIR_BUFSIZE
-#define READDIR_BUFSIZE (2*MAXPATHLEN)
+#if READDIR_BUFSIZE < (FILENAME_CHARSIZE*2*(MAXPATHLEN+1))
+#  undef READDIR_BUFSIZE
+#  define READDIR_BUFSIZE (FILENAME_CHARSIZE*2*(MAXPATHLEN+1))
 #endif
 
 struct t_readdir_buf {
@@ -306,7 +340,7 @@ struct t_data
     int            result_ok;
     Efile_error    errInfo;
     int            flags;
-    Sint           fd;
+    SWord          fd;
     /**/
     Efile_info        info;
     EFILE_DIR_HANDLE  dir_handle; /* Handle to open directory. */
@@ -351,17 +385,22 @@ struct t_data
 	    ErlDrvBinary *binp;
 	    int           size;
 	    int           offset;
-	    char          name[1];
 	} read_file;
 	struct {
 	    struct t_readdir_buf *first_buf;
 	    struct t_readdir_buf *last_buf;
 	} read_dir;
+	struct {
+	    Sint64 offset;
+	    Sint64 length;
+	    int advise;
+	} fadvise;
     } c;
     char b[1];
 };
 
 
+
 #define EF_ALLOC(S)		driver_alloc((S))
 #define EF_REALLOC(P, S)	driver_realloc((P), (S))
 #define EF_SAFE_ALLOC(S)	ef_safe_alloc((S))
@@ -371,7 +410,7 @@ struct t_data
 static void *ef_safe_alloc(Uint s)
 {
     void *p = EF_ALLOC(s);
-    if (!p) erl_exit(1, "efile drv: Can't allocate %d bytes of memory\n", s);
+    if (!p) erl_exit(1, "efile drv: Can't allocate %lu bytes of memory\n", (unsigned long)s);
     return p;
 }
 
@@ -605,7 +644,7 @@ file_start(ErlDrvPort port, char* command)
     }
     desc->fd = FILE_FD_INVALID;
     desc->port = port;
-    desc->key = (unsigned) (Uint) port;
+    desc->key = (unsigned int) (UWord) port;
     desc->flags = 0;
     desc->invoke = NULL;
     desc->d = NULL;
@@ -630,7 +669,7 @@ static void free_data(void *data)
     EF_FREE(data);
 }
 
-static void do_close(int flags, Sint fd) {
+static void do_close(int flags, SWord fd) {
     if (flags & EFILE_COMPRESSED) {
 	erts_gzclose((gzFile)(fd));
     } else {
@@ -709,7 +748,7 @@ static void reply_Uint_posix_error(file_descriptor *desc, Uint num,
     TRACE_C('N');
 
     response[0] = FILE_RESP_NUMERR;
-#if SIZEOF_VOID_P == 4
+#if SIZEOF_VOID_P == 4 || HALFWORD_HEAP
     put_int32(0, response+1);
 #else
     put_int32(num>>32, response+1);
@@ -767,7 +806,7 @@ static int reply_Uint(file_descriptor *desc, Uint result) {
     TRACE_C('R');
 
     tmp[0] = FILE_RESP_NUMBER;
-#if SIZEOF_VOID_P == 4
+#if SIZEOF_VOID_P == 4 || HALFWORD_HEAP
     put_int32(0, tmp+1);
 #else
     put_int32(result>>32, tmp+1);
@@ -883,6 +922,15 @@ static void invoke_chdir(void *data)
     invoke_name(data, efile_chdir);
 }
 
+static void invoke_fdatasync(void *data)
+{
+    struct t_data *d = (struct t_data *) data;
+    int fd = (int) d->fd;
+
+    d->again = 0;
+    d->result_ok = efile_fdatasync(&d->errInfo, fd);
+}
+
 static void invoke_fsync(void *data)
 {
     struct t_data *d = (struct t_data *) data;
@@ -1068,7 +1116,7 @@ static void invoke_read_file(void *data)
 	Sint64 size;
 	
 	if (! (d->result_ok = 
-	       efile_openfile(&d->errInfo, d->c.read_file.name, 
+	       efile_openfile(&d->errInfo, d->b, 
 			      EFILE_MODE_READ, &fd, &size))) {
 	    goto done;
 	}
@@ -1272,7 +1320,7 @@ static void invoke_writev(void *data) {
 	 p < size && iovcnt < iovlen;
 	 p += iov0[iovcnt++].iov_len)
 	;
-    iov = EF_ALLOC(sizeof(SysIOVec)*iovcnt);
+    iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovcnt);
     memcpy(iov,iov0,iovcnt*sizeof(SysIOVec));
     MUTEX_UNLOCK(d->c.writev.q_mtx);
     /* Let go of lock until we deque from original vector */
@@ -1352,7 +1400,7 @@ static void invoke_readlink(void *data)
     d->result_ok = efile_readlink(&d->errInfo, d->b, resbuf+1,
 				  RESBUFSIZE-1);
     if (d->result_ok != 0)
-	strcpy((char *) d->b + 1, resbuf+1);
+	FILENAME_COPY((char *) d->b + 1, resbuf+1);
 }
 
 static void invoke_altname(void *data)
@@ -1364,7 +1412,7 @@ static void invoke_altname(void *data)
     d->result_ok = efile_altname(&d->errInfo, d->b, resbuf+1,
 				  RESBUFSIZE-1);
     if (d->result_ok != 0)
-	strcpy((char *) d->b + 1, resbuf+1);
+	FILENAME_COPY((char *) d->b + 1, resbuf+1);
 }
 
 static void invoke_pwritev(void *data) {
@@ -1389,7 +1437,7 @@ static void invoke_pwritev(void *data) {
     /* Lock the queue just for a while, we don't want it locked during write */
     MUTEX_LOCK(c->q_mtx);
     iov0 = driver_peekq(c->port, &iovlen);
-    iov = EF_ALLOC(sizeof(SysIOVec)*iovlen);
+    iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovlen);
     memcpy(iov,iov0,sizeof(SysIOVec)*iovlen);
     MUTEX_UNLOCK(c->q_mtx);
 
@@ -1483,7 +1531,7 @@ static void invoke_link(void *data)
     char *new_name;
 
     d->again = 0;
-    new_name = name+strlen(name)+1;
+    new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
     d->result_ok = efile_link(&d->errInfo, name, new_name);
 }
 
@@ -1494,7 +1542,7 @@ static void invoke_symlink(void *data)
     char *new_name;
 
     d->again = 0;
-    new_name = name+strlen(name)+1;
+    new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
     d->result_ok = efile_symlink(&d->errInfo, name, new_name);
 }
 
@@ -1505,7 +1553,7 @@ static void invoke_rename(void *data)
     char *new_name;
 
     d->again = 0;
-    new_name = name+strlen(name)+1;
+    new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
     d->result_ok = efile_rename(&d->errInfo, name, new_name);
 }
 
@@ -1553,13 +1601,15 @@ static void invoke_readdir(void *data)
     int s;
     char *p = NULL;
     int buf_sz = 0;
+    size_t tmp_bs;
 
     d->again = 0;
     d->errInfo.posix_errno = 0;
 
     while (1) {
 	char *str;
-	if (buf_sz < (4 /* sz */ + 1 /* cmd */ + MAXPATHLEN + 1 /* '\0' */)) {
+	if (buf_sz < (4 /* sz */ + 1 /* cmd */ + 
+		      FILENAME_CHARSIZE*(MAXPATHLEN + 1))) {
 	    struct t_readdir_buf *b;
 	    if (p) {
 		put_int32(0, p); /* EOB */
@@ -1575,18 +1625,18 @@ static void invoke_readdir(void *data)
 	    buf_sz = READDIR_BUFSIZE - 4/* EOB */;
 	}
 
-	p[4] = FILE_RESP_OK;
+	p[4] = FILE_RESP_FNAME;
 	buf_sz -= 4 + 1;
 	str = p + 4 + 1;
 	ASSERT(buf_sz >= MAXPATHLEN + 1);
-	s = efile_readdir(&d->errInfo, d->b, &d->dir_handle, str, buf_sz);
+	tmp_bs = buf_sz;
+	s = efile_readdir(&d->errInfo, d->b, &d->dir_handle, str, &tmp_bs);
 
 	if (s) {
-	    int str_sz = strlen(str);
-	    int sz = str_sz + 1;
-	    put_int32(sz, p);
-	    p += 4 + sz;
-	    buf_sz -= str_sz;
+	    put_int32(tmp_bs + 1 /* 1 byte for opcode */, p);
+	    p += 4 + tmp_bs + 1;
+	    ASSERT(p == (str + tmp_bs));
+	    buf_sz -= tmp_bs;
 	}
 	else {
 	    put_int32(1, p);
@@ -1620,7 +1670,7 @@ static void invoke_open(void *data)
 	    status = efile_may_openfile(&d->errInfo, d->b);
 	    if (status || (d->errInfo.posix_errno != EISDIR)) {
 		mode = (d->flags & EFILE_MODE_READ) ? "rb" : "wb";
-		d->fd = (Sint) erts_gzopen(d->b, mode);
+		d->fd = (SWord) erts_gzopen(d->b, mode);
 		if ((gzFile)d->fd) {
 		    status = 1;
 		} else {
@@ -1637,6 +1687,18 @@ static void invoke_open(void *data)
     d->result_ok = status;
 }
 
+static void invoke_fadvise(void *data)
+{
+    struct t_data *d = (struct t_data *) data;
+    int fd = (int) d->fd;
+    off_t offset = (off_t) d->c.fadvise.offset;
+    off_t length = (off_t) d->c.fadvise.length;
+    int advise = (int) d->c.fadvise.advise;
+
+    d->again = 0;
+    d->result_ok = efile_fadvise(&d->errInfo, fd, offset, length, advise);
+}
+
 static void free_readdir(void *data)
 {
     struct t_data *d = (struct t_data *) data;
@@ -1883,7 +1945,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	if (!d->result_ok)
 	    reply_error(desc, &d->errInfo);
 	else {
-	    header[0] = FILE_RESP_OK;
+	    header[0] = FILE_RESP_ALL_DATA;
 	    TRACE_C('R');
 	    driver_output_binary(desc->port, header, 1,
 				 d->c.read_file.binp,
@@ -1919,12 +1981,14 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
       case FILE_RMDIR:
       case FILE_CHDIR:
       case FILE_DELETE:
+      case FILE_FDATASYNC:
       case FILE_FSYNC:
       case FILE_TRUNCATE:
       case FILE_LINK:
       case FILE_SYMLINK:
       case FILE_RENAME:
       case FILE_WRITE_INFO:
+      case FILE_FADVISE:
 	reply(desc, d->result_ok, &d->errInfo);
 	free_data(data);
 	break;
@@ -1938,10 +2002,10 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	    if (!d->result_ok)
 		reply_error(desc, &d->errInfo);
 	    else {
-		resbuf[0] = FILE_RESP_OK;
-		length = 1+strlen((char*) resbuf+1);
+		resbuf[0] = FILE_RESP_FNAME;
+		length = 1+FILENAME_BYTELEN((char*) resbuf+1);
 		TRACE_C('R');
-		driver_output2(desc->port, resbuf, length, NULL, 0);
+		driver_output2(desc->port, resbuf, 1, resbuf+1, length-1);
 	    }
 	    free_data(data);
 	    break;
@@ -2001,13 +2065,18 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 		int sz = get_int32(p);
 		while (sz) { /* 0 == EOB */
 		    p += 4;
-		    driver_output2(desc->port, p, sz, NULL, 0);
+		    if (sz - 1 > 0) {
+			driver_output2(desc->port, p, 1, p+1, sz-1);
+		    } else {
+			driver_output2(desc->port, p, 1, NULL, 0);
+		    }
 		    p += sz;
 		    sz = get_int32(p);
 		}
 		b1 = b1->next;
 		EF_FREE(b2);
 	    }
+	    
 	    d->c.read_dir.first_buf = NULL;
 	    d->c.read_dir.last_buf = NULL;
 	}
@@ -2083,9 +2152,9 @@ file_output(ErlDrvData e, char* buf, int count)
 
     case FILE_MKDIR:
     {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
 	
-	strcpy(d->b, name);
+	FILENAME_COPY(d->b, name);
 	d->command = command;
 	d->invoke = invoke_mkdir;
 	d->free = free_data;
@@ -2094,9 +2163,9 @@ file_output(ErlDrvData e, char* buf, int count)
     }
     case FILE_RMDIR:
     {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
 	
-	strcpy(d->b, name);
+	FILENAME_COPY(d->b, name);
 	d->command = command;
 	d->invoke = invoke_rmdir;
 	d->free = free_data;
@@ -2105,9 +2174,9 @@ file_output(ErlDrvData e, char* buf, int count)
     }
     case FILE_DELETE:
     {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
 	
-	strcpy(d->b, name);
+	FILENAME_COPY(d->b, name);
 	d->command = command;
 	d->invoke = invoke_delete_file;
 	d->free = free_data;
@@ -2117,14 +2186,14 @@ file_output(ErlDrvData e, char* buf, int count)
     case FILE_RENAME:
 	{
 	    char* new_name;
-
-	    new_name = name+strlen(name)+1;
+	    int namelen = FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
+	    new_name = name+namelen;
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + strlen(name) + 1
-			      + strlen(new_name) + 1);
+			      + namelen
+			      + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE);
 	
-	    strcpy(d->b, name);
-	    strcpy(d->b + strlen(name) + 1, new_name);
+	    FILENAME_COPY(d->b, name);
+	    FILENAME_COPY(d->b + namelen, new_name);
 	    d->flags = desc->flags;
 	    d->fd = fd;
 	    d->command = command;
@@ -2135,9 +2204,9 @@ file_output(ErlDrvData e, char* buf, int count)
 	}
     case FILE_CHDIR:
     {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
 	
-	strcpy(d->b, name);
+	FILENAME_COPY(d->b, name);
 	d->command = command;
 	d->invoke = invoke_chdir;
 	d->free = free_data;
@@ -2160,9 +2229,10 @@ file_output(ErlDrvData e, char* buf, int count)
 #ifdef USE_THREADS
 	if (sys_info.async_threads > 0)
 	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + 
+			      FILENAME_CHARSIZE);
 	
-	    strcpy(d->b, name);
+	    FILENAME_COPY(d->b, name);
 	    d->dir_handle = NULL;
 	    d->command = command;
 	    d->invoke = invoke_readdir;
@@ -2175,17 +2245,19 @@ file_output(ErlDrvData e, char* buf, int count)
 	else   
 #endif
 	{
+	    size_t resbufsize;
 	    char resbuf[RESBUFSIZE+1];
 	    EFILE_DIR_HANDLE dir_handle; /* Handle to open directory. */
 
 	    errInfo.posix_errno = 0;
 	    dir_handle = NULL;
-	    resbuf[0] = FILE_RESP_OK;
+	    resbuf[0] = FILE_RESP_FNAME;
+	    resbufsize = RESBUFSIZE;
 
 	    while (efile_readdir(&errInfo, name, &dir_handle,
-				 resbuf+1, RESBUFSIZE)) {
-		int length = 1 + strlen(resbuf+1);
-		driver_output2(desc->port, resbuf, length, NULL, 0);
+				 resbuf+1, &resbufsize)) {
+		driver_output2(desc->port, resbuf, 1, resbuf+1, resbufsize);
+		resbufsize = RESBUFSIZE;
 	    }
 	    if (errInfo.posix_errno != 0) {
 		reply_error(desc, &errInfo);
@@ -2197,11 +2269,12 @@ file_output(ErlDrvData e, char* buf, int count)
 	}
     case FILE_OPEN:
 	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(buf+4) + 1);
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(buf+4) + 
+			      FILENAME_CHARSIZE);
 	
 	    d->flags = get_int32((uchar*)buf);
 	    name = buf+4;
-	    strcpy(d->b, name);
+	    FILENAME_COPY(d->b, name);
 	    d->command = command;
 	    d->invoke = invoke_open;
 	    d->free = free_data;
@@ -2209,33 +2282,46 @@ file_output(ErlDrvData e, char* buf, int count)
 	    goto done;
 	}
 
+    case FILE_FDATASYNC:
+	{
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data));
+	    
+	    d->fd = fd;
+	    d->command = command;
+	    d->invoke = invoke_fdatasync;
+	    d->free = free_data;
+	    d->level = 2;
+	    goto done;
+	}
+
     case FILE_FSYNC:
-    {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data));
-	
-	d->fd = fd;
-	d->command = command;
-	d->invoke = invoke_fsync;
-	d->free = free_data;
-	d->level = 2;
-	goto done;
-    }
+	{
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data));
+	    
+	    d->fd = fd;
+	    d->command = command;
+	    d->invoke = invoke_fsync;
+	    d->free = free_data;
+	    d->level = 2;
+	    goto done;
+	}
 
 
     case FILE_FSTAT: 
     case FILE_LSTAT:
-    {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	{
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + 
+			      FILENAME_CHARSIZE);
+	    
+	    FILENAME_COPY(d->b, name);
+	    d->fd = fd;
+	    d->command = command;
+	    d->invoke = invoke_flstat;
+	    d->free = free_data;
+	    d->level = 2;
+	    goto done;
+	}
 	
-	strcpy(d->b, name);
-	d->fd = fd;
-	d->command = command;
-	d->invoke = invoke_flstat;
-	d->free = free_data;
-	d->level = 2;
-	goto done;
-    }
-
     case FILE_TRUNCATE:
         {
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data));
@@ -2252,7 +2338,7 @@ file_output(ErlDrvData e, char* buf, int count)
     case FILE_WRITE_INFO:
 	{
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + strlen(buf+21*4) + 1);
+			      + FILENAME_BYTELEN(buf+21*4) + FILENAME_CHARSIZE);
 	    
 	    d->info.mode = get_int32(buf + 0 * 4);
 	    d->info.uid = get_int32(buf + 1 * 4);
@@ -2260,7 +2346,7 @@ file_output(ErlDrvData e, char* buf, int count)
 	    GET_TIME(d->info.accessTime, buf + 3 * 4);
 	    GET_TIME(d->info.modifyTime, buf + 9 * 4);
 	    GET_TIME(d->info.cTime, buf + 15 * 4);
-	    strcpy(d->b, buf+21*4);
+	    FILENAME_COPY(d->b, buf+21*4);
 	    d->command = command;
 	    d->invoke = invoke_write_info;
 	    d->free = free_data;
@@ -2272,7 +2358,7 @@ file_output(ErlDrvData e, char* buf, int count)
 	{
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1);
 	
-	    strcpy(d->b, name);
+	    FILENAME_COPY(d->b, name);
 	    d->command = command;
 	    d->invoke = invoke_readlink;
 	    d->free = free_data;
@@ -2281,28 +2367,29 @@ file_output(ErlDrvData e, char* buf, int count)
 	}
 
     case FILE_ALTNAME:
-    {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1);
-	strcpy(d->b, name);
-	d->command = command;
-	d->invoke = invoke_altname;
-	d->free = free_data;
-	d->level = 2;
-	goto done;
-    }
+	{
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1);
+	    FILENAME_COPY(d->b, name);
+	    d->command = command;
+	    d->invoke = invoke_altname;
+	    d->free = free_data;
+	    d->level = 2;
+	    goto done;
+	}
 
 
     case FILE_LINK:
 	{
 	    char* new_name;
+	    int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
 
-	    new_name = name+strlen(name)+1;
+	    new_name = name+namelen;
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + strlen(name) + 1
-			      + strlen(new_name) + 1);
+			      + namelen
+			      + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE);
 	
-	    strcpy(d->b, name);
-	    strcpy(d->b + strlen(name) + 1, new_name);
+	    FILENAME_COPY(d->b, name);
+	    FILENAME_COPY(d->b + namelen, new_name);
 	    d->flags = desc->flags;
 	    d->fd = fd;
 	    d->command = command;
@@ -2315,14 +2402,15 @@ file_output(ErlDrvData e, char* buf, int count)
     case FILE_SYMLINK:
 	{
 	    char* new_name;
+	    int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
 
-	    new_name = name+strlen(name)+1;
+	    new_name = name+namelen;
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + strlen(name) + 1
-			      + strlen(new_name) + 1);
+			      + namelen
+			      + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE);
 	
-	    strcpy(d->b, name);
-	    strcpy(d->b + strlen(name) + 1, new_name);
+	    FILENAME_COPY(d->b, name);
+	    FILENAME_COPY(d->b + namelen, new_name);
 	    d->flags = desc->flags;
 	    d->fd = fd;
 	    d->command = command;
@@ -2332,6 +2420,21 @@ file_output(ErlDrvData e, char* buf, int count)
 	    goto done;
 	}
 
+    case FILE_FADVISE:
+    {
+        d = EF_SAFE_ALLOC(sizeof(struct t_data));
+
+        d->fd = fd;
+        d->command = command;
+        d->invoke = invoke_fadvise;
+        d->free = free_data;
+        d->level = 2;
+        d->c.fadvise.offset = get_int64((uchar*) buf);
+        d->c.fadvise.length = get_int64(((uchar*) buf) + sizeof(Sint64));
+        d->c.fadvise.advise = get_int32(((uchar*) buf) + 2 * sizeof(Sint64));
+        goto done;
+    }
+
     }
 
     /*
@@ -2947,6 +3050,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 
     case FILE_READ_FILE: {
 	struct t_data *d;
+	char *filename;
 	if (ev->size < 1+1) {
 	    /* Buffer contains empty name */
 	    reply_posix_error(desc, ENOENT);
@@ -2957,7 +3061,8 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	    reply_posix_error(desc, EINVAL);
 	    goto done;
 	}
-	d = EF_ALLOC(sizeof(struct t_data) + ev->size);
+	filename = EV_CHAR_P(ev, p, q);
+	d = EF_ALLOC(sizeof(struct t_data) -1 + FILENAME_BYTELEN(filename) + FILENAME_CHARSIZE);
 	if (! d) {
 	    reply_posix_error(desc, ENOMEM);
 	    goto done;
@@ -2965,8 +3070,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	d->command = command;
 	d->reply = !0;
 	/* Copy name */
-	memcpy(d->c.read_file.name, EV_CHAR_P(ev, p, q), ev->size-1);
-	d->c.read_file.name[ev->size-1] = '\0';
+	FILENAME_COPY(d->b, filename);
 	d->c.read_file.binp = NULL;
 	d->invoke = invoke_read_file;
 	d->free = free_read_file;
diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h
index 9aa941e550..3097ded3f1 100644
--- a/erts/emulator/drivers/common/erl_efile.h
+++ b/erts/emulator/drivers/common/erl_efile.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1997-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -32,7 +32,8 @@
 #define EFILE_MODE_READ_WRITE 	3
 #define EFILE_MODE_APPEND	4
 #define EFILE_COMPRESSED 	8
-#define EFILE_NO_TRUNCATE      16 /* Special for reopening on VxWorks */
+#define EFILE_MODE_EXCL        16
+#define EFILE_NO_TRUNCATE      32 /* Special for reopening on VxWorks */
 
 /*
  * Seek modes for efile_seek().
@@ -58,6 +59,14 @@
 #define FA_WRITE 	1
 #define FA_READ		2
 
+/* Some OS'es (i.e. Windows) has filenames in wide charaqcters. That requires special handling */
+/* Note that we do *not* honor alignment in the communication to the OS specific driver, */
+/* which is not a problem on x86, but might be on other platforms. The OS specific efile */
+/* implementation is expected to align if needed */
+#ifdef __WIN32__
+#define FILENAMES_16BIT 1
+#endif
+
 /*
  * An handle to an open directory.  To be cast to the correct type
  * in the system-dependent directory functions.
@@ -122,10 +131,11 @@ int efile_getdcwd(Efile_error* errInfo, int drive,
 		  char* buffer, size_t size);
 int efile_readdir(Efile_error* errInfo, char* name, 
 		  EFILE_DIR_HANDLE* dir_handle,
-		  char* buffer, size_t size);
+		  char* buffer, size_t *size);
 int efile_openfile(Efile_error* errInfo, char* name, int flags,
 		   int* pfd, Sint64* pSize);
 void efile_closefile(int fd);
+int efile_fdatasync(Efile_error* errInfo, int fd);
 int efile_fsync(Efile_error* errInfo, int fd);
 int efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
 		   char *name, int info_for_link);
@@ -150,3 +160,5 @@ int efile_altname(Efile_error* errInfo, char *name,
 int efile_link(Efile_error* errInfo, char* old, char* new);
 int efile_symlink(Efile_error* errInfo, char* old, char* new);
 int efile_may_openfile(Efile_error* errInfo, char *name);
+int efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length,
+		  int advise);
diff --git a/erts/emulator/drivers/common/gzio.c b/erts/emulator/drivers/common/gzio.c
index 801bc61d4d..741cb6ae20 100644
--- a/erts/emulator/drivers/common/gzio.c
+++ b/erts/emulator/drivers/common/gzio.c
@@ -28,6 +28,7 @@
 
 #ifdef __WIN32__
 #define HAVE_CONFLICTING_FREAD_DECLARATION
+#define FILENAMES_16BIT 1
 #endif
 
 #ifdef STDC
@@ -102,6 +103,40 @@ local uLong  getLong      OF((gz_stream *s));
 # define ERTS_GZREAD(File, Buf, Count) fread((Buf), 1, (Count), (File))
 #endif
 
+/*
+ * Ripped from efile_drv.c
+ */
+
+#ifdef FILENAMES_16BIT
+#  define FILENAME_BYTELEN(Str) filename_len_16bit(Str)
+#  define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) 
+#  define FILENAME_CHARSIZE 2
+
+   static int filename_len_16bit(const char *str) 
+   {
+       const char *p = str;
+       while(*p != '\0' || p[1] != '\0') {
+	   p += 2;
+       }
+       return (p - str);
+   }
+
+   static void filename_cpy_16bit(char *to, const char *from) 
+   {
+       while(*from != '\0' || from[1] != '\0') {
+	   *to++ = *from++;
+	   *to++ = *from++;
+       }
+       *to++ = *from++;
+       *to++ = *from++;
+   }
+
+#else
+#  define FILENAME_BYTELEN(Str) strlen(Str)
+#  define FILENAME_COPY(To,From) strcpy(To,From) 
+#  define FILENAME_CHARSIZE 1
+#endif
+
 /* ===========================================================================
      Opens a gzip (.gz) file for reading or writing. The mode parameter
    is as in fopen ("rb" or "wb"). The file is given either by file descriptor
@@ -144,11 +179,11 @@ local gzFile gz_open (path, mode)
     s->position = 0;
     s->destroy = destroy;
 
-    s->path = (char*)ALLOC(strlen(path)+1);
+    s->path = (char*)ALLOC(FILENAME_BYTELEN(path)+FILENAME_CHARSIZE);
     if (s->path == NULL) {
         return s->destroy(s), (gzFile)Z_NULL;
     }
-    strcpy(s->path, path); /* do this early for debugging */
+    FILENAME_COPY(s->path, path); /* do this early for debugging */
 
     s->mode = '\0';
     do {
@@ -194,7 +229,22 @@ local gzFile gz_open (path, mode)
     s->stream.avail_out = Z_BUFSIZE;
 
     errno = 0;
-#ifdef UNIX
+#if defined(FILENAMES_16BIT)
+    {
+	char wfmode[160];
+	int i=0,j;
+	for(j=0;fmode[j] != '\0';++j) {
+	    wfmode[i++]=fmode[j];
+	    wfmode[i++]='\0';
+	}
+	wfmode[i++] = '\0';
+	wfmode[i++] = '\0';
+	s->file = F_OPEN(path, wfmode);
+	if (s->file == NULL) {
+	    return s->destroy(s), (gzFile)Z_NULL;
+	}
+    }
+#elif defined(UNIX)
     if (s->mode == 'r') {
 	s->file = open(path, O_RDONLY);
     } else {
@@ -582,6 +632,7 @@ erts_gzseek(gzFile file, int offset, int whence)
     while (s->position < pos) {
 	char buf[512];
 	int n;
+	int save_pos = s->position;
 
 	n = pos - s->position;
 	if (n > sizeof(buf))
@@ -593,6 +644,7 @@ erts_gzseek(gzFile file, int offset, int whence)
 	    memset(buf, '\0', n);
 	    erts_gzwrite(file, buf, n);
 	}
+	if (save_pos == s->position) break;
     }
 
     return s->position;
diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index c6e23ee647..40c4a0df08 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -41,15 +41,22 @@
 #define STRINGIFY_1(b) IDENTITY(#b)
 #define STRINGIFY(a) STRINGIFY_1(a)
 
-#ifndef _OSE_
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 #ifdef HAVE_SYS_UIO_H
 #include <sys/uio.h>
 #endif
-#endif
 
+#ifdef HAVE_NET_IF_DL_H
+#include <net/if_dl.h>
+#endif
+#ifdef HAVE_IFADDRS_H
+#include <ifaddrs.h>
+#endif
+#ifdef HAVE_NETPACKET_PACKET_H
+#include <netpacket/packet.h>
+#endif
 
 /* All platforms fail on malloc errors. */
 #define FATAL_MALLOC
@@ -57,6 +64,21 @@
 
 #include "erl_driver.h"
 
+/* The IS_SOCKET_ERROR macro below is used for portability reasons. While
+   POSIX specifies that errors from socket-related system calls should be
+   indicated with a -1 return value, some users have experienced non-Windows
+   OS kernels that return negative values other than -1. While one can argue
+   that such kernels are technically broken, comparing against values less
+   than 0 covers their out-of-spec return values without imposing incorrect
+   semantics on systems that manage to correctly return -1 for errors, thus
+   increasing Erlang's portability.
+*/
+#ifdef __WIN32__
+#define IS_SOCKET_ERROR(val) ((val) == SOCKET_ERROR)
+#else
+#define IS_SOCKET_ERROR(val) ((val) < 0)
+#endif
+
 #ifdef __WIN32__
 #define  STRNCASECMP strncasecmp
 
@@ -66,8 +88,21 @@
 #include <winsock2.h>
 #endif
 #include <windows.h>
+#include <Ws2tcpip.h>   /* NEED VC 6.0 or higher */
+
+/* Visual studio 2008+: NTDDI_VERSION needs to be set for iphlpapi.h
+   to define the right structures. It needs to be set to WINXP (or LONGHORN)
+   for IPV6 to work and it's set lower by default, so we need to change it. */
+#ifdef HAVE_SDKDDKVER_H
+#  include <sdkddkver.h>
+#  ifdef NTDDI_VERSION 
+#    undef NTDDI_VERSION
+#  endif
+#  define NTDDI_VERSION NTDDI_WINXP
+#endif
+
+#include <iphlpapi.h>
 
-#include <Ws2tcpip.h>   /* NEED VC 6.0 !!! */
 
 #undef WANT_NONBLOCKING
 #include "sys.h"
@@ -186,18 +221,8 @@ static unsigned long one_value = 1;
 #include <netdb.h>
 #endif
 
-#ifndef _OSE_
 #include <sys/socket.h>
 #include <netinet/in.h>
-#else
-/* datatypes and macros from Solaris socket.h */
-struct  linger {
-        int     l_onoff;                /* option on/off */
-        int     l_linger;               /* linger time */
-};
-#define SO_OOBINLINE    0x0100          /* leave received OOB data in line */
-#define SO_LINGER       0x0080          /* linger on close if data present */
-#endif
 
 #ifdef VXWORKS
 #include <rpc/rpctypes.h>
@@ -206,12 +231,10 @@ struct  linger {
 #include <rpc/types.h>
 #endif
 
-#ifndef _OSE_
 #include <netinet/tcp.h>
 #include <arpa/inet.h>
-#endif
 
-#if (!defined(VXWORKS) && !defined(_OSE_))
+#if (!defined(VXWORKS))
 #include <sys/param.h>
 #ifdef HAVE_ARPA_NAMESER_H
 #include <arpa/nameser.h>
@@ -226,33 +249,11 @@ struct  linger {
 #include <sys/ioctl.h>
 #endif
 
-#ifndef _OSE_
 #include <net/if.h>
-#else
-#define IFF_MULTICAST 0x00000800
-#endif
-
-#ifdef _OSE_
-#include "inet.h"
-#include "ineterr.h"
-#include "ose_inet_drv.h"
-#include "nameser.h" 
-#include "resolv.h"
-#define SET_ASYNC(s) setsockopt((s), SOL_SOCKET, SO_OSEEVENT, (&(s)), sizeof(int))
-
-extern void select_release(void);
-
-#endif /* _OSE_ */
-
-/* Solaris headers, only to be used with SFK */
-#ifdef _OSE_SFK_
-#include <ctype.h>
-#include <string.h>
-#endif
 
 /* SCTP support -- currently for UNIX platforms only: */
 #undef HAVE_SCTP
-#if (!defined(VXWORKS) && !defined(_OSE_) && !defined(__WIN32__) && defined(HAVE_SCTP_H))
+#if (!defined(VXWORKS) && !defined(__WIN32__) && defined(HAVE_SCTP_H))
 
 #include <netinet/sctp.h>
 
@@ -315,7 +316,7 @@ static int (*p_sctp_bindx)(int sd, struct sockaddr *addrs,
 #define DEBUGF(X) printf X
 #endif
 
-#if !defined(__WIN32__) && !defined(HAVE_STRNCASECMP)
+#if !defined(HAVE_STRNCASECMP)
 #define STRNCASECMP my_strncasecmp
 
 static int my_strncasecmp(const char *s1, const char *s2, size_t n)
@@ -335,6 +336,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define INVALID_SOCKET -1
 #define INVALID_EVENT  -1
 #define SOCKET_ERROR   -1
+
 #define SOCKET int
 #define HANDLE long int
 #define FD_READ    ERL_DRV_READ
@@ -362,20 +364,6 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define sock_htons(x)               htons((x))
 #define sock_htonl(x)               htonl((x))
 
-#ifdef _OSE_
-#define sock_accept(s, addr, len)   ose_inet_accept((s), (addr), (len))
-#define sock_send(s,buf,len,flag)   ose_inet_send((s),(buf),(len),(flag))
-#define sock_sendto(s,buf,blen,flag,addr,alen) \
-                ose_inet_sendto((s),(buf),(blen),(flag),(addr),(alen))
-#define sock_sendv(s, vec, size, np, flag) \
-		(*(np) = ose_inet_sendv((s), (SysIOVec*)(vec), (size)))
-#define sock_open(af, type, proto)  ose_inet_socket((af), (type), (proto))
-#define sock_close(s)               ose_inet_close((s))
-#define sock_hostname(buf, len)     ose_gethostname((buf), (len))
-#define sock_getservbyname(name,proto) ose_getservbyname((name), (proto))
-#define sock_getservbyport(port,proto) ose_getservbyport((port), (proto))
-
-#else
 #define sock_accept(s, addr, len)   accept((s), (addr), (len))
 #define sock_send(s,buf,len,flag)   send((s),(buf),(len),(flag))
 #define sock_sendto(s,buf,blen,flag,addr,alen) \
@@ -391,7 +379,6 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define sock_hostname(buf, len)     gethostname((buf), (len))
 #define sock_getservbyname(name,proto) getservbyname((name), (proto))
 #define sock_getservbyport(port,proto) getservbyport((port), (proto))
-#endif /* _OSE_ */
 
 #define sock_recv(s,buf,len,flag)   recv((s),(buf),(len),(flag))
 #define sock_recvfrom(s,buf,blen,flag,addr,alen) \
@@ -402,13 +389,8 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define sock_create_event(d)        ((d)->s) /* return file descriptor */
 #define sock_close_event(e)                  /* do nothing */
 
-#ifdef _OSE_
-#define inet_driver_select(port, e, mode, on) \
-                                    ose_inet_select(port, e, mode, on)
-#else
 #define inet_driver_select(port, e, mode, on) \
                                     driver_select(port, e, mode | (on?ERL_DRV_USE:0), on)
-#endif /* _OSE_ */
 
 #define sock_select(d, flags, onoff) do { \
         (d)->event_mask = (onoff) ? \
@@ -501,13 +483,13 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define INET_REQ_IFGET         22
 #define INET_REQ_IFSET         23
 #define INET_REQ_SUBSCRIBE     24
+#define INET_REQ_GETIFADDRS    25
 /* TCP requests */
 #define TCP_REQ_ACCEPT         40
 #define TCP_REQ_LISTEN         41
 #define TCP_REQ_RECV           42
 #define TCP_REQ_UNRECV         43
 #define TCP_REQ_SHUTDOWN       44
-#define TCP_REQ_MULTI_OP       45
 /* UDP and SCTP requests */
 #define PACKET_REQ_RECV        60 /* Common for UDP and SCTP         */
 #define SCTP_REQ_LISTEN	       61 /* Different from TCP; not for UDP */
@@ -666,15 +648,12 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define IS_BUSY(d) \
   (((d)->state & INET_F_BUSY) == INET_F_BUSY)
 
+#define INET_MAX_OPT_BUFFER (64*1024)
+
 #define INET_DEF_BUFFER     1460        /* default buffer size */
 #define INET_MIN_BUFFER     1           /* internal min buffer */
-#define INET_MAX_BUFFER     (1024*64)   /* internal max buffer */
 
-/* Note: INET_HIGH_WATERMARK MUST be less than 2*INET_MAX_BUFFER */
 #define INET_HIGH_WATERMARK (1024*8) /* 8k pending high => busy  */
-/* Note: INET_LOW_WATERMARK MUST be less than INET_MAX_BUFFER and
-** less than INET_HIGH_WATERMARK
-*/
 #define INET_LOW_WATERMARK  (1024*4) /* 4k pending => allow more */
 
 #define INET_INFINITY  0xffffffff  /* infinity value */
@@ -1085,7 +1064,7 @@ struct erl_drv_entry inet_driver_entry =
 };
 
 /* XXX: is this a driver interface function ??? */
-extern void erl_exit(int n, char*, _DOTS_);
+void erl_exit(int n, char*, ...);
 
 /*
  * Malloc wrapper,
@@ -1285,139 +1264,136 @@ static int load_ip_and_port
 	LOAD_ATOM((spec), (i), (flag) ? am_true : am_false);
 #endif /* HAVE_SCTP */
 
+/* Assume a cache line size of 64 bytes */
+#define INET_DRV_CACHE_LINE_SIZE ((ErlDrvUInt) 64)
+#define INET_DRV_CACHE_LINE_MASK (INET_DRV_CACHE_LINE_SIZE - 1)
+
 /*
 ** Binary Buffer Managment
 ** We keep a stack of usable buffers 
 */
-#define BUFFER_STACK_SIZE 16
-
-static erts_smp_spinlock_t inet_buffer_stack_lock;
-static ErlDrvBinary* buffer_stack[BUFFER_STACK_SIZE];
-static int buffer_stack_pos = 0;
+#define BUFFER_STACK_SIZE 14
+#define BUFFER_STACK_MAX_MEM_SIZE (1024*1024)
 
+ErlDrvTSDKey buffer_stack_key;
 
-/*
- * XXX
- * The erts_smp_spin_* functions should not be used by drivers (but this
- * driver is special). Replace when driver locking api has been implemented.
- * /rickard
- */
-#define BUFSTK_LOCK	erts_smp_spin_lock(&inet_buffer_stack_lock);
-#define BUFSTK_UNLOCK	erts_smp_spin_unlock(&inet_buffer_stack_lock);
-
-#ifdef DEBUG
-static int tot_buf_allocated = 0;  /* memory in use for i_buf */
-static int tot_buf_stacked = 0;   /* memory on stack */
-static int max_buf_allocated = 0; /* max allocated */
-
-#define COUNT_BUF_ALLOC(sz) do { \
-  BUFSTK_LOCK; \
-  tot_buf_allocated += (sz); \
-  if (tot_buf_allocated > max_buf_allocated) \
-    max_buf_allocated = tot_buf_allocated; \
-  BUFSTK_UNLOCK; \
-} while(0)
-
-#define COUNT_BUF_FREE(sz) do { \
- BUFSTK_LOCK; \
- tot_buf_allocated -= (sz); \
- BUFSTK_UNLOCK; \
- } while(0)
-
-#define COUNT_BUF_STACK(sz) do { \
- BUFSTK_LOCK; \
- tot_buf_stacked += (sz); \
- BUFSTK_UNLOCK; \
- } while(0)
+typedef struct {
+    int mem_size;
+    int pos;
+    ErlDrvBinary* stk[BUFFER_STACK_SIZE];
+} InetDrvBufStkBase;
 
-#else
+typedef struct {
+    InetDrvBufStkBase buf;
+    char align[(((sizeof(InetDrvBufStkBase) - 1) / INET_DRV_CACHE_LINE_SIZE) + 1)
+	       * INET_DRV_CACHE_LINE_SIZE];
+} InetDrvBufStk;
+
+static InetDrvBufStk *get_bufstk(void)
+{
+    InetDrvBufStk *bs = erl_drv_tsd_get(buffer_stack_key);
+    if (bs)
+	return bs;
+    bs = driver_alloc(sizeof(InetDrvBufStk)
+		      + INET_DRV_CACHE_LINE_SIZE - 1);
+    if (!bs)
+	return NULL;
+    if ((((ErlDrvUInt) bs) & INET_DRV_CACHE_LINE_MASK) != 0)
+	bs = ((InetDrvBufStk *)
+	      ((((ErlDrvUInt) bs) & ~INET_DRV_CACHE_LINE_MASK)
+	       + INET_DRV_CACHE_LINE_SIZE));
+    erl_drv_tsd_set(buffer_stack_key, bs);
+    bs->buf.pos = 0;
+    bs->buf.mem_size = 0;
 
-#define COUNT_BUF_ALLOC(sz)
-#define COUNT_BUF_FREE(sz)
-#define COUNT_BUF_STACK(sz)
+    ASSERT(bs == erl_drv_tsd_get(buffer_stack_key));
 
-#endif
+    return bs;
+}
 
 static ErlDrvBinary* alloc_buffer(long minsz)
 {
-    ErlDrvBinary* buf = NULL;
+    InetDrvBufStk *bs = get_bufstk();
 
-    BUFSTK_LOCK;
+    DEBUGF(("alloc_buffer: %ld\r\n", minsz));
 
-    DEBUGF(("alloc_buffer: sz = %ld, tot = %d, max = %d\r\n", 
-	    minsz, tot_buf_allocated, max_buf_allocated));
+    if (bs && bs->buf.pos > 0) {
+	long size;
+	ErlDrvBinary* buf = bs->buf.stk[--bs->buf.pos];
+	size = buf->orig_size;
+	bs->buf.mem_size -= size;
+	ASSERT(0 <= bs->buf.mem_size
+	       && bs->buf.mem_size <= BUFFER_STACK_MAX_MEM_SIZE);
+	if (size >= minsz)
+	    return buf;
 
-    if (buffer_stack_pos > 0) {
-	int origsz;
+	driver_free_binary(buf);
+    }
 
-	buf = buffer_stack[--buffer_stack_pos];
-	origsz = buf->orig_size;
-	BUFSTK_UNLOCK;
-	COUNT_BUF_STACK(-origsz);
-	if (origsz < minsz) {
-	    if ((buf = driver_realloc_binary(buf, minsz)) == NULL)
-		return NULL;
-	    COUNT_BUF_ALLOC(buf->orig_size - origsz);
+    ASSERT(!bs || bs->buf.pos != 0 || bs->buf.mem_size == 0);
+
+    return driver_alloc_binary(minsz);
+}
+
+/*#define CHECK_DOUBLE_RELEASE 1*/
+#ifdef CHECK_DOUBLE_RELEASE
+static void
+check_double_release(InetDrvBufStk *bs, ErlDrvBinary* buf)
+{
+#ifdef __GNUC__
+#warning CHECK_DOUBLE_RELEASE is enabled, this is a custom build emulator
+#endif
+    int i;
+    for (i = 0; i < bs->buf.pos; ++i) {
+	if (bs->buf.stk[i] == buf) {
+	    erl_exit(ERTS_ABORT_EXIT,
+		     "Multiple buffer release in inet_drv, this "
+		     "is a bug, save the core and send it to "
+		     "[email protected]!");
 	}
     }
-    else {
-	BUFSTK_UNLOCK;
-	if ((buf = driver_alloc_binary(minsz)) == NULL)
-	    return NULL;
-	COUNT_BUF_ALLOC(buf->orig_size);
-    }
-    return buf;
 }
+#endif
 
-/*
-** Max buffer memory "cached" BUFFER_STACK_SIZE * INET_MAX_BUFFER
-** (16 * 64k ~ 1M)
-*/
-/*#define CHECK_DOUBLE_RELEASE 1*/
 static void release_buffer(ErlDrvBinary* buf)
 {
+    InetDrvBufStk *bs;
+    long size;
+
     DEBUGF(("release_buffer: %ld\r\n", (buf==NULL) ? 0 : buf->orig_size));
-    if (buf == NULL)
+
+    if (!buf)
 	return;
-    BUFSTK_LOCK;
-    if ((buf->orig_size > INET_MAX_BUFFER) || 
-	(buffer_stack_pos >= BUFFER_STACK_SIZE)) {
-	BUFSTK_UNLOCK;
-	COUNT_BUF_FREE(buf->orig_size);
+
+    size = buf->orig_size;
+
+    if (size > BUFFER_STACK_MAX_MEM_SIZE)
+	goto free_binary;
+
+    bs = get_bufstk();
+    if (!bs
+	|| (bs->buf.mem_size + size > BUFFER_STACK_MAX_MEM_SIZE)
+	|| (bs->buf.pos >= BUFFER_STACK_SIZE)) {
+    free_binary:
 	driver_free_binary(buf);
     }
     else {
 #ifdef CHECK_DOUBLE_RELEASE
-#ifdef __GNUC__
-#warning CHECK_DOUBLE_RELEASE is enabled, this is a custom build emulator
-#endif
-	int i;
-	for (i = 0; i < buffer_stack_pos; ++i) {
-	    if (buffer_stack[i] == buf) {
-		erl_exit(1,"Multiple buffer release in inet_drv, this is a "
-			 "bug, save the core and send it to "
-			 "[email protected]!");
-	    }
-	}
+	check_double_release(bs, buf);
 #endif
-	buffer_stack[buffer_stack_pos++] = buf;
-	BUFSTK_UNLOCK;
-	COUNT_BUF_STACK(buf->orig_size);
+	ASSERT(bs->buf.pos != 0 || bs->buf.mem_size == 0);
+
+	bs->buf.mem_size += size;
+	bs->buf.stk[bs->buf.pos++] = buf;
+
+	ASSERT(0 <= bs->buf.mem_size
+	       && bs->buf.mem_size <= BUFFER_STACK_MAX_MEM_SIZE);
     }
 }
 
 static ErlDrvBinary* realloc_buffer(ErlDrvBinary* buf, long newsz)
 {
-    ErlDrvBinary* bin;
-#ifdef DEBUG
-    long orig_size =  buf->orig_size;
-#endif
-
-    if ((bin = driver_realloc_binary(buf,newsz)) != NULL) {
-	COUNT_BUF_ALLOC(newsz - orig_size);
-	;
-    }
-    return bin;
+    return driver_realloc_binary(buf, newsz);
 }
 
 /* use a TRICK, access the refc field to see if any one else has
@@ -1431,10 +1407,8 @@ static void free_buffer(ErlDrvBinary* buf)
     if (buf != NULL) {
 	if (driver_binary_get_refc(buf) == 1)
 	    release_buffer(buf);
-	else {
-	    COUNT_BUF_FREE(buf->orig_size);
+	else
 	    driver_free_binary(buf);
-	}
     }
 }
 
@@ -1788,7 +1762,6 @@ send_async_error(ErlDrvPort port, ErlDrvTermData Port, int Ref,
 			LOAD_INT_CNT + 2*LOAD_TUPLE_CNT];
     int i = 0;
     
-    i = 0;
     i = LOAD_ATOM(spec, i, am_inet_async);
     i = LOAD_PORT(spec, i, Port);
     i = LOAD_INT(spec, i, Ref);
@@ -1975,7 +1948,7 @@ static int http_response_inetdrv(void *arg, int major, int minor,
     tcp_descriptor* desc = (tcp_descriptor*) arg;
     int i = 0;
     ErlDrvTermData spec[27];
-    ErlDrvTermData caller;
+    ErlDrvTermData caller = ERL_DRV_NIL;
     
     if (desc->inet.active == INET_PASSIVE) {
         /* {inet_async,S,Ref,{ok,{http_response,Version,Status,Phrase}}} */
@@ -2068,7 +2041,7 @@ http_request_inetdrv(void* arg, const http_atom_t* meth, const char* meth_ptr,
     tcp_descriptor* desc = (tcp_descriptor*) arg;
     int i = 0;
     ErlDrvTermData spec[43];
-    ErlDrvTermData caller;
+    ErlDrvTermData caller = ERL_DRV_NIL;
     
     if (desc->inet.active == INET_PASSIVE) {
         /* {inet_async, S, Ref, {ok,{http_request,Meth,Uri,Version}}} */
@@ -2119,7 +2092,7 @@ http_header_inetdrv(void* arg, const http_atom_t* name, const char* name_ptr,
     tcp_descriptor* desc = (tcp_descriptor*) arg;
     int i = 0;
     ErlDrvTermData spec[26];
-    ErlDrvTermData caller;
+    ErlDrvTermData caller = ERL_DRV_NIL;
     
     if (desc->inet.active == INET_PASSIVE) {
         /* {inet_async,S,Ref,{ok,{http_header,Bit,Name,IValue,Value}} */
@@ -2208,7 +2181,7 @@ static int http_error_inetdrv(void* arg, const char* buf, int len)
   ErlDrvTermData spec[19];
 
   if (desc->inet.active == INET_PASSIVE) {
-    /* {inet_async,S,Ref,{error,{http_error,Line}}} */
+    /* {inet_async,S,Ref,{ok,{http_error,Line}}} */
     int req;
     int aid;
     ErlDrvTermData caller;
@@ -2218,7 +2191,7 @@ static int http_error_inetdrv(void* arg, const char* buf, int len)
     i = LOAD_ATOM(spec, i,  am_inet_async);
     i = LOAD_PORT(spec, i,  desc->inet.dport);
     i = LOAD_INT(spec, i,   aid);
-    i = LOAD_ATOM(spec, i,  am_error);
+    i = LOAD_ATOM(spec, i,  am_ok);
     i = LOAD_ATOM(spec, i,  am_http_error);
     i = http_load_string(desc, spec, i, buf, len);
     i = LOAD_TUPLE(spec, i, 2);
@@ -2248,7 +2221,7 @@ int ssl_tls_inetdrv(void* arg, unsigned type, unsigned major, unsigned minor,
     tcp_descriptor* desc = (tcp_descriptor*) arg;
     int i = 0;
     ErlDrvTermData spec[28];
-    ErlDrvTermData caller;
+    ErlDrvTermData caller = ERL_DRV_NIL;
     ErlDrvBinary* bin;
     int ret;
 
@@ -3438,20 +3411,14 @@ static int inet_init()
     if (!sock_init())
 	goto error;
 
-    buffer_stack_pos = 0;
-
-    erts_smp_spinlock_init(&inet_buffer_stack_lock, "inet_buffer_stack_lock");
+    if (0 != erl_drv_tsd_key_create("inet_buffer_stack_key", &buffer_stack_key))
+	goto error;
 
     ASSERT(sizeof(struct in_addr) == 4);
 #   if defined(HAVE_IN6) && defined(AF_INET6)
     ASSERT(sizeof(struct in6_addr) == 16);
 #   endif
 
-#ifdef DEBUG
-    tot_buf_allocated = 0;
-    max_buf_allocated = 0;
-    tot_buf_stacked = 0;
-#endif
     INIT_ATOM(ok);
     INIT_ATOM(tcp);
     INIT_ATOM(udp);
@@ -3480,13 +3447,9 @@ static int inet_init()
     INIT_ATOM(scheme);
 
     /* add TCP, UDP and SCTP drivers */
-#ifdef _OSE_
-    add_ose_tcp_drv_entry(&tcp_inet_driver_entry);
-    add_ose_udp_drv_entry(&udp_inet_driver_entry);
-#else
     add_driver_entry(&tcp_inet_driver_entry);
     add_driver_entry(&udp_inet_driver_entry);
-#  ifdef HAVE_SCTP
+#ifdef HAVE_SCTP
     /* Check the size of SCTP AssocID -- currently both this driver and the
        Erlang part require 32 bit: */
     ASSERT(sizeof(sctp_assoc_t)==ASSOC_ID_LEN);
@@ -3501,8 +3464,8 @@ static int inet_init()
 	    add_driver_entry(&sctp_inet_driver_entry);
 	}
     }
-#  endif
-#endif /* _OSE_ */
+#endif
+
     /* remove the dummy inet driver */
     remove_driver_entry(&inet_driver_entry);
     return 0;
@@ -3744,7 +3707,7 @@ static int inet_ctl_fdopen(inet_descriptor* desc, int domain, int type,
     unsigned int sz = sizeof(name);
 
     /* check that it is a socket and that the socket is bound */
-    if (sock_name(s, (struct sockaddr*) &name, &sz) == SOCKET_ERROR)
+    if (IS_SOCKET_ERROR(sock_name(s, (struct sockaddr*) &name, &sz)))
 	return ctl_error(sock_errno(), rbuf, rsize);
     desc->s = s;
     if ((desc->event = sock_create_event(desc)) == INVALID_EVENT)
@@ -3756,7 +3719,7 @@ static int inet_ctl_fdopen(inet_descriptor* desc, int domain, int type,
     desc->state = INET_STATE_BOUND; /* assume bound */
     if (type == SOCK_STREAM) { /* check if connected */
 	sz = sizeof(name);
-	if (sock_peer(s, (struct sockaddr*) &name, &sz) != SOCKET_ERROR)
+	if (!IS_SOCKET_ERROR(sock_peer(s, (struct sockaddr*) &name, &sz)))
 	    desc->state = INET_STATE_CONNECTED;
     }
 
@@ -3862,39 +3825,81 @@ do { if ((end)-(ptr) < (n)) goto error; } while(0)
 static char* sockaddr_to_buf(struct sockaddr* addr, char* ptr, char* end)
 {
     if (addr->sa_family == AF_INET || addr->sa_family == 0) {
-	struct in_addr a;
-	buf_check(ptr,end,sizeof(struct in_addr));
-	a = ((struct sockaddr_in*) addr)->sin_addr;
-	sys_memcpy(ptr, (char*)&a, sizeof(struct in_addr));
-	return ptr + sizeof(struct in_addr);
+	struct in_addr *p = &(((struct sockaddr_in*) addr)->sin_addr);
+	buf_check(ptr, end, 1 + sizeof(struct in_addr));
+	*ptr = INET_AF_INET;
+	sys_memcpy(ptr+1, (char*)p, sizeof(struct in_addr));
+	return ptr + 1 + sizeof(struct in_addr);
     }
 #if defined(HAVE_IN6) && defined(AF_INET6)
     else if (addr->sa_family == AF_INET6) {
-	struct in6_addr a;
-	buf_check(ptr,end,sizeof(struct in6_addr));
-	a = ((struct sockaddr_in6*) addr)->sin6_addr;
-	sys_memcpy(ptr, (char*)&a, sizeof(struct in6_addr));
-	return ptr + sizeof(struct in6_addr);
+	struct in6_addr *p = &(((struct sockaddr_in6*) addr)->sin6_addr);
+	buf_check(ptr, end, 1 + sizeof(struct in6_addr));
+	*ptr = INET_AF_INET6;
+	sys_memcpy(ptr+1, (char*)p, sizeof(struct in6_addr));
+	return ptr + 1 + sizeof(struct in6_addr);
     }
 #endif
+#if defined(AF_LINK)
+    else if (addr->sa_family == AF_LINK) {
+	struct sockaddr_dl *sdl_p = (struct sockaddr_dl*) addr;
+	buf_check(ptr, end, 2 + sdl_p->sdl_alen);
+	put_int16(sdl_p->sdl_alen, ptr); ptr += 2;
+	sys_memcpy(ptr, sdl_p->sdl_data + sdl_p->sdl_nlen, sdl_p->sdl_alen);
+	return ptr + sdl_p->sdl_alen;
+    }
+#endif
+#if defined(AF_PACKET) && defined(HAVE_NETPACKET_PACKET_H)
+    else if(addr->sa_family == AF_PACKET) {
+	struct sockaddr_ll *sll_p = (struct sockaddr_ll*) addr;
+	buf_check(ptr, end, 2 + sll_p->sll_halen);
+	put_int16(sll_p->sll_halen, ptr); ptr += 2;
+	sys_memcpy(ptr, sll_p->sll_addr, sll_p->sll_halen);
+	return ptr + sll_p->sll_halen;
+    }
+#endif
+    return ptr;
  error:
     return NULL;
-
 }
 
 static char* buf_to_sockaddr(char* ptr, char* end, struct sockaddr* addr)
 {
-    buf_check(ptr,end,sizeof(struct in_addr));
-    sys_memcpy((char*) &((struct sockaddr_in*)addr)->sin_addr, ptr,
-	       sizeof(struct in_addr));
-    addr->sa_family = AF_INET;
-    return ptr +  sizeof(struct in_addr);
-
+    buf_check(ptr,end,1);
+    switch (*ptr++) {
+    case INET_AF_INET: {
+	struct in_addr *p = &((struct sockaddr_in*)addr)->sin_addr;
+	buf_check(ptr,end,sizeof(struct in_addr));
+	sys_memcpy((char*) p, ptr, sizeof(struct in_addr));
+	addr->sa_family = AF_INET;
+	return ptr + sizeof(struct in_addr);
+    }
+    case INET_AF_INET6: {
+	struct in6_addr *p = &((struct sockaddr_in6*)addr)->sin6_addr;
+	buf_check(ptr,end,sizeof(struct in6_addr));
+	sys_memcpy((char*) p, ptr, sizeof(struct in6_addr));
+	addr->sa_family = AF_INET6;
+	return ptr + sizeof(struct in6_addr);
+    }
+    }
  error:
     return NULL;
 }
 
 
+#if defined (IFF_POINTOPOINT)
+#define IFGET_FLAGS(cflags) IFGET_FLAGS_P2P(cflags, IFF_POINTOPOINT)
+#elif defined IFF_POINTTOPOINT
+#define IFGET_FLAGS(cflags) IFGET_FLAGS_P2P(cflags, IFF_POINTTOPOINT)
+#endif
+
+#define IFGET_FLAGS_P2P(cflags, iff_ptp)				\
+    ((((cflags) & IFF_UP) ? INET_IFF_UP : 0) |				\
+     (((cflags) & IFF_BROADCAST) ? INET_IFF_BROADCAST : 0) |		\
+     (((cflags) & IFF_LOOPBACK) ? INET_IFF_LOOPBACK : 0) |		\
+     (((cflags) & iff_ptp) ? INET_IFF_POINTTOPOINT : 0) |		\
+     (((cflags) & IFF_UP) ? INET_IFF_RUNNING : 0) |  /* emulate running ? */ \
+     (((cflags) & IFF_MULTICAST) ? INET_IFF_MULTICAST : 0))
 
 #if defined(__WIN32__) && defined(SIO_GET_INTERFACE_LIST)
 
@@ -3932,7 +3937,6 @@ static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize)
     return ctl_reply(INET_REP_OK, sbuf, sptr - sbuf, rbuf, rsize);
 }
 
-
 /* input is an ip-address in string format i.e A.B.C.D 
 ** scan the INTERFACE_LIST to get the options 
 */
@@ -3949,7 +3953,7 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
     INTERFACE_INFO* ifp;
     long namaddr;
 
-    if ((len == 0) || ((namlen = buf[0]) > len))
+    if ((len == 0) || ((namlen = get_int8(buf)) > len))
 	goto error;
     if (parse_addr(buf+1, namlen, &namaddr) < 0)
 	goto error;
@@ -4018,27 +4022,12 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
 	    break;
 
 	case INET_IFOPT_FLAGS: {
-	    long eflags = 0;
 	    int flags = ifp->iiFlags;
 	    /* just enumerate the interfaces (no names) */
 
-	    /* translate flags */
-	    if (flags & IFF_UP)
-		eflags |= INET_IFF_UP;
-	    if (flags & IFF_BROADCAST)
-		eflags |= INET_IFF_BROADCAST;
-	    if (flags & IFF_LOOPBACK)
-		eflags |= INET_IFF_LOOPBACK;
-	    if (flags & IFF_POINTTOPOINT)
-		eflags |= INET_IFF_POINTTOPOINT;
-	    if (flags & IFF_UP) /* emulate runnign ? */
-		eflags |= INET_IFF_RUNNING;
-	    if (flags & IFF_MULTICAST)
-		eflags |= INET_IFF_MULTICAST;
-
 	    buf_check(sptr, s_end, 5);
 	    *sptr++ = INET_IFOPT_FLAGS;
-	    put_int32(eflags, sptr);
+	    put_int32(IFGET_FLAGS(flags), sptr);
 	    sptr += 4;
 	    break;
 	}
@@ -4059,7 +4048,6 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
     return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 }
 
-
 #elif defined(SIOCGIFCONF) && defined(SIOCSIFFLAGS)
 /* cygwin has SIOCGIFCONF but not SIOCSIFFLAGS (Nov 2002) */
 
@@ -4070,69 +4058,81 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
 #define SIZEA(p) (sizeof (p))
 #endif
 
-
-static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize)
-{
-    struct ifconf ifc;
-    struct ifreq *ifr;
-    char *buf;
-    int buflen, ifc_len, i;
-    char *sbuf, *sp;
-    
-    /* Courtesy of Per Bergqvist and W. Richard Stevens */
-    
-    ifc_len = 0;
-    buflen = 100 * sizeof(struct ifreq);
-    buf = ALLOC(buflen);
+static int get_ifconf(SOCKET s, struct ifconf *ifcp) {
+    int ifc_len = 0;
+    int buflen = 100 * sizeof(struct ifreq);
+    char *buf = ALLOC(buflen);
 
     for (;;) {
-	ifc.ifc_len = buflen;
-	ifc.ifc_buf = buf;
-	if (ioctl(desc->s, SIOCGIFCONF, (char *)&ifc) < 0) {
+	ifcp->ifc_len = buflen;
+	ifcp->ifc_buf = buf;
+	if (ioctl(s, SIOCGIFCONF, (char *)ifcp) < 0) {
 	    int res = sock_errno();
 	    if (res != EINVAL || ifc_len) {
 		FREE(buf);
-		return ctl_error(res, rbuf, rsize);
+		return -1;
 	    }
 	} else {
-	    if (ifc.ifc_len == ifc_len) break; /* buf large enough */
-	    ifc_len = ifc.ifc_len;
+	    if (ifcp->ifc_len == ifc_len) break; /* buf large enough */
+	    ifc_len = ifcp->ifc_len;
 	}
 	buflen += 10 * sizeof(struct ifreq);
 	buf = (char *)REALLOC(buf, buflen);
     }
-    
-    sp = sbuf = ALLOC(ifc_len+1);
+    return 0;
+}
+
+static void free_ifconf(struct ifconf *ifcp) {
+    FREE(ifcp->ifc_buf);
+}
+
+static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize)
+{
+    struct ifconf ifc;
+    struct ifreq *ifrp;
+    char *sbuf, *sp;
+    int i;
+
+    /* Courtesy of Per Bergqvist and W. Richard Stevens */
+
+    if (get_ifconf(desc->s, &ifc) < 0) {
+	return ctl_error(sock_errno(), rbuf, rsize);
+    }
+
+    sp = sbuf = ALLOC(ifc.ifc_len+1);
     *sp++ = INET_REP_OK;
     i = 0;
     for (;;) {
 	int n;
-	
-	ifr = (struct ifreq *) VOIDP(buf + i);
-	n = sizeof(ifr->ifr_name) + SIZEA(ifr->ifr_addr);
-	if (n < sizeof(*ifr)) n = sizeof(*ifr);
-	if (i+n > ifc_len) break;
+
+	ifrp = (struct ifreq *) VOIDP(ifc.ifc_buf + i);
+	n = sizeof(ifrp->ifr_name) + SIZEA(ifrp->ifr_addr);
+	if (n < sizeof(*ifrp)) n = sizeof(*ifrp);
+	if (i+n > ifc.ifc_len) break;
 	i += n;
-	
-	switch (ifr->ifr_addr.sa_family) {
+
+	switch (ifrp->ifr_addr.sa_family) {
 #if defined(HAVE_IN6) && defined(AF_INET6)
 	case AF_INET6:
 #endif
 	case AF_INET:
-	    ASSERT(sp+IFNAMSIZ+1 < sbuf+buflen+1)
-	    strncpy(sp, ifr->ifr_name, IFNAMSIZ);
+	    ASSERT(sp+IFNAMSIZ+1 < sbuf+ifc.ifc_len+1)
+	    strncpy(sp, ifrp->ifr_name, IFNAMSIZ);
 	    sp[IFNAMSIZ] = '\0';
 	    sp += strlen(sp), ++sp;
 	}
-	
-	if (i >= ifc_len) break;
+
+	if (i >= ifc.ifc_len) break;
     }
-    FREE(buf);
+    free_ifconf(&ifc);
     *rbuf = sbuf;
     return sp - sbuf;
 }
 
-
+/* FIXME: temporary hack */
+#ifndef IFHWADDRLEN
+#define IFHWADDRLEN 6
+#endif
 
 static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
 			  char** rbuf, int rsize)
@@ -4143,11 +4143,11 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
     struct ifreq ifreq;
     int namlen;
 
-    if ((len == 0) || ((namlen = buf[0]) > len))
+    if ((len == 0) || ((namlen = get_int8(buf)) > len))
 	goto error;
     sys_memset(ifreq.ifr_name, '\0', IFNAMSIZ);
     sys_memcpy(ifreq.ifr_name, buf+1, 
-	       (namlen > IFNAMSIZ) ? IFNAMSIZ : namlen);
+	       (namlen >= IFNAMSIZ) ? IFNAMSIZ-1 : namlen);
     buf += (namlen+1);
     len -= (namlen+1);
     sptr = sbuf;
@@ -4167,11 +4167,52 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
 #ifdef SIOCGIFHWADDR
 	    if (ioctl(desc->s, SIOCGIFHWADDR, (char *)&ifreq) < 0)
 		break;
-	    buf_check(sptr, s_end, 1+IFHWADDRLEN);
+	    buf_check(sptr, s_end, 1+2+IFHWADDRLEN);
 	    *sptr++ = INET_IFOPT_HWADDR;
+	    put_int16(IFHWADDRLEN, sptr); sptr += 2;
 	    /* raw memcpy (fix include autoconf later) */
 	    sys_memcpy(sptr, (char*)(&ifreq.ifr_hwaddr.sa_data), IFHWADDRLEN);
 	    sptr += IFHWADDRLEN;
+#elif defined(SIOCGENADDR)
+	    if (ioctl(desc->s, SIOCGENADDR, (char *)&ifreq) < 0)
+		break;
+	    buf_check(sptr, s_end, 1+2+sizeof(ifreq.ifr_enaddr));
+	    *sptr++ = INET_IFOPT_HWADDR;
+	    put_int16(sizeof(ifreq.ifr_enaddr), sptr); sptr += 2;
+	    /* raw memcpy (fix include autoconf later) */
+	    sys_memcpy(sptr, (char*)(&ifreq.ifr_enaddr),
+		       sizeof(ifreq.ifr_enaddr));
+	    sptr += sizeof(ifreq.ifr_enaddr);
+#elif defined(HAVE_GETIFADDRS) && defined(AF_LINK)
+	    struct ifaddrs *ifa, *ifp;
+	    struct sockaddr_dl *sdlp;
+	    int found = 0;
+
+	    if (getifaddrs(&ifa) == -1)
+		goto error;
+
+	    for (ifp = ifa; ifp; ifp = ifp->ifa_next) {
+		if ((ifp->ifa_addr->sa_family == AF_LINK) &&
+		    (sys_strcmp(ifp->ifa_name, ifreq.ifr_name) == 0)) {
+		    found = 1;
+		    break;
+		}
+	    }
+
+	    if (found == 0) {
+		freeifaddrs(ifa);
+		break;
+	    }
+	    sdlp = (struct sockaddr_dl *)ifp->ifa_addr;
+
+	    buf_check(sptr, s_end, 1+2+sdlp->sdl_alen);
+	    *sptr++ = INET_IFOPT_HWADDR;
+	    put_int16(sdlp->sdl_alen, sptr); sptr += 2;
+	    sys_memcpy(sptr,
+		       sdlp->sdl_data + sdlp->sdl_nlen,
+		       sdlp->sdl_alen);
+	    freeifaddrs(ifa);
+	    sptr += sdlp->sdl_alen;
 #endif
 	    break;
 	}
@@ -4248,29 +4289,15 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
 
 	case INET_IFOPT_FLAGS: {
 	    int flags;
-	    int eflags = 0;
 
 	    if (ioctl(desc->s, SIOCGIFFLAGS, (char*)&ifreq) < 0)
 		flags = 0;
 	    else
 		flags = ifreq.ifr_flags;
-	    /* translate flags */
-	    if (flags & IFF_UP)
-		eflags |= INET_IFF_UP;
-	    if (flags & IFF_BROADCAST)
-		eflags |= INET_IFF_BROADCAST;
-	    if (flags & IFF_LOOPBACK)
-		eflags |= INET_IFF_LOOPBACK;	
-	    if (flags & IFF_POINTOPOINT)
-		eflags |= INET_IFF_POINTTOPOINT;
-	    if (flags & IFF_RUNNING)
-		eflags |= INET_IFF_RUNNING;
-	    if (flags & IFF_MULTICAST)
-		eflags |= INET_IFF_MULTICAST;
 
 	    buf_check(sptr, s_end, 5);
 	    *sptr++ = INET_IFOPT_FLAGS;
-	    put_int32(eflags, sptr);
+	    put_int32(IFGET_FLAGS(flags), sptr);
 	    sptr += 4;
 	    break;
 	}
@@ -4284,10 +4311,6 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
     return ctl_error(EINVAL, rbuf, rsize);
 }
 
-/* FIXME: temporary hack */
-#ifndef IFHWADDRLEN
-#define IFHWADDRLEN 6
-#endif
 
 static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
 			  char** rbuf, int rsize)
@@ -4296,11 +4319,11 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
     int namlen;
     char* b_end = buf + len;
 
-    if ((len == 0) || ((namlen = buf[0]) > len))
+    if ((len == 0) || ((namlen = get_int8(buf)) > len))
 	goto error;
     sys_memset(ifreq.ifr_name, '\0', IFNAMSIZ);
     sys_memcpy(ifreq.ifr_name, buf+1, 
-	       (namlen > IFNAMSIZ) ? IFNAMSIZ : namlen);
+	       (namlen >= IFNAMSIZ) ? IFNAMSIZ-1 : namlen);
     buf += (namlen+1);
     len -= (namlen+1);
 
@@ -4312,17 +4335,22 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
 	    (void) ioctl(desc->s, SIOCSIFADDR, (char*)&ifreq);
 	    break;
 
-	case INET_IFOPT_HWADDR: 
-	    buf_check(buf, b_end, IFHWADDRLEN);
+	case INET_IFOPT_HWADDR: {
+	    unsigned int len;
+	    buf_check(buf, b_end, 2);
+	    len = get_int16(buf); buf += 2;
+	    buf_check(buf, b_end, len);
 #ifdef SIOCSIFHWADDR
 	    /* raw memcpy (fix include autoconf later) */
-	    sys_memcpy((char*)(&ifreq.ifr_hwaddr.sa_data), buf, IFHWADDRLEN);
+	    sys_memset((char*)(&ifreq.ifr_hwaddr.sa_data),
+		       '\0', sizeof(ifreq.ifr_hwaddr.sa_data));
+	    sys_memcpy((char*)(&ifreq.ifr_hwaddr.sa_data), buf, len);
 
 	    (void) ioctl(desc->s, SIOCSIFHWADDR, (char *)&ifreq);
 #endif
-	    buf += IFHWADDRLEN;
+	    buf += len;
 	    break;
-
+	}
 
 	case INET_IFOPT_BROADADDR:
 #ifdef SIOCSIFBRDADDR
@@ -4427,6 +4455,557 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
 
 #endif
 
+
+
+/* Latin-1 to utf8 */
+
+static int utf8_len(const char *c, int m) {
+    int l;
+    for (l = 0;  m;  c++, l++, m--) {
+	if (*c == '\0') break;
+	if ((*c & 0x7f) != *c) l++;
+    }
+    return l;
+}
+
+static void utf8_encode(const char *c, int m, char *p) {
+    for (;  m;  c++, m--) {
+	if (*c == '\0') break;
+	if ((*c & 0x7f) != *c) {
+	    *p++ = (char) (0xC0 | (0x03 & (*c >> 6)));
+	    *p++ = (char) (0x80 | (0x3F & *c));
+	} else {
+	    *p++ = (char) *c;
+	}
+    }
+}
+
+#if defined(__WIN32__)
+
+static void set_netmask_bytes(char *c, int len, int pref_len) {
+    int i, m;
+    for (i = 0, m = pref_len >> 3;  i < m && i < len;  i++) c[i] = '\xFF';
+    if (i < len) c[i++] = 0xFF << (8 - (pref_len & 7));
+    for (;  i < len;  i++) c[i] = '\0';
+}
+
+
+int eq_masked_bytes(char *a, char *b, int pref_len) {
+    int i, m;
+    for (i = 0, m = pref_len >> 3;  i < m;  i++) {
+	if (a[i] != b[i]) return 0;
+    }
+    m = pref_len & 7;
+    if (m) {
+	m = 0xFF & (0xFF << (8 - m));
+	if ((a[i] & m) != (b[i] & m)) return 0;
+    }
+    return !0;
+}
+
+static int inet_ctl_getifaddrs(inet_descriptor* desc_p,
+			       char **rbuf_pp, int rsize)
+{
+    int i;
+    DWORD ret, n;
+    IP_INTERFACE_INFO *info_p;
+    MIB_IPADDRTABLE *ip_addrs_p;
+    IP_ADAPTER_ADDRESSES *ip_adaddrs_p, *ia_p;
+
+    char *buf_p;
+    char *buf_alloc_p;
+    int buf_size =512;
+#   define BUF_ENSURE(Size)						\
+    do {								\
+	int NEED_, GOT_ = buf_p - buf_alloc_p;				\
+	NEED_ = GOT_ + (Size);						\
+	if (NEED_ > buf_size) {						\
+	    buf_size = NEED_ + 512;					\
+	    buf_alloc_p = REALLOC(buf_alloc_p, buf_size);		\
+	    buf_p = buf_alloc_p + GOT_;					\
+	}								\
+    } while(0)
+#   define SOCKADDR_TO_BUF(opt, sa)					\
+    do {								\
+	if (sa) {							\
+	    char *P_;							\
+	    *buf_p++ = (opt);						\
+	    while (! (P_ = sockaddr_to_buf((sa), buf_p,			\
+					   buf_alloc_p+buf_size))) {    \
+		int GOT_ = buf_p - buf_alloc_p;				\
+		buf_size += 512;					\
+		buf_alloc_p = REALLOC(buf_alloc_p, buf_size);		\
+		buf_p = buf_alloc_p + GOT_;				\
+	    }								\
+	    if (P_ == buf_p) {						\
+		buf_p--;						\
+	    } else {							\
+		buf_p = P_;						\
+	    }								\
+	}								\
+    } while (0)
+
+    {
+	/* Try GetAdaptersAddresses, if it is available */
+	unsigned long ip_adaddrs_size = 16 * 1024;
+	ULONG family = AF_UNSPEC;
+	ULONG flags =
+	    GAA_FLAG_INCLUDE_PREFIX | GAA_FLAG_SKIP_ANYCAST |
+	    GAA_FLAG_SKIP_DNS_SERVER | GAA_FLAG_SKIP_FRIENDLY_NAME |
+	    GAA_FLAG_SKIP_MULTICAST;
+	ULONG (WINAPI *fpGetAdaptersAddresses)
+	    (ULONG, ULONG, PVOID, PIP_ADAPTER_ADDRESSES, PULONG);
+	HMODULE iphlpapi = GetModuleHandle("iphlpapi");
+	fpGetAdaptersAddresses = (void *)
+	    (iphlpapi ?
+		GetProcAddress(iphlpapi, "GetAdaptersAddresses") :
+		NULL);
+	if (fpGetAdaptersAddresses) {
+	    ip_adaddrs_p = ALLOC(ip_adaddrs_size);
+	    for (i = 17;  i;  i--) {
+		ret = fpGetAdaptersAddresses(
+		    family, flags, NULL, ip_adaddrs_p, &ip_adaddrs_size);
+		ip_adaddrs_p = REALLOC(ip_adaddrs_p, ip_adaddrs_size);
+		if (ret == NO_ERROR) break;
+		if (ret == ERROR_BUFFER_OVERFLOW) continue;
+		i = 0;
+	    }
+	    if (! i) {
+		FREE(ip_adaddrs_p);
+		ip_adaddrs_p = NULL;
+	    }
+	} else ip_adaddrs_p = NULL;
+    }
+
+    {
+	/* Load the IP_INTERFACE_INFO table (only IPv4 interfaces),
+	 * reliable source of interface names on XP
+	 */
+	unsigned long info_size = 4 * 1024;
+	info_p = ALLOC(info_size);
+	for (i = 17;  i;  i--) {
+	    ret = GetInterfaceInfo(info_p, &info_size);
+	    info_p = REALLOC(info_p, info_size);
+	    if (ret == NO_ERROR) break;
+	    if (ret == ERROR_INSUFFICIENT_BUFFER) continue;
+	    i = 0;
+	}
+	if (! i) {
+	    FREE(info_p);
+	    info_p = NULL;
+	}
+    }
+
+    if (! ip_adaddrs_p) {
+	/* If GetAdaptersAddresses gave nothing we fall back to
+	 * MIB_IPADDRTABLE (only IPv4 interfaces)
+	 */
+	unsigned long ip_addrs_size = 16 * sizeof(*ip_addrs_p);
+	ip_addrs_p = ALLOC(ip_addrs_size);
+	for (i = 17;  i;  i--) {
+	    ret = GetIpAddrTable(ip_addrs_p, &ip_addrs_size, FALSE);
+	    ip_addrs_p = REALLOC(ip_addrs_p, ip_addrs_size);
+	    if (ret == NO_ERROR) break;
+	    if (ret == ERROR_INSUFFICIENT_BUFFER) continue;
+	    i = 0;
+	}
+	if (! i) {
+	    if (info_p) FREE(info_p);
+	    FREE(ip_addrs_p);
+	    return ctl_reply(INET_REP_OK, NULL, 0, rbuf_pp, rsize);
+	}
+    } else ip_addrs_p = NULL;
+
+    buf_p = buf_alloc_p = ALLOC(buf_size);
+    *buf_p++ = INET_REP_OK;
+
+    /* Iterate over MIB_IPADDRTABLE or IP_ADAPTER_ADDRESSES */
+    for (ia_p = NULL, ip_addrs_p ? ((void *)(i = 0)) : (ia_p = ip_adaddrs_p);
+	 ip_addrs_p ? (i < ip_addrs_p->dwNumEntries) : (ia_p != NULL);
+	 ip_addrs_p ? ((void *)(i++)) : (ia_p = ia_p->Next)) {
+	MIB_IPADDRROW *ipaddrrow_p = NULL;
+	DWORD flags = INET_IFF_MULTICAST;
+	DWORD index = 0;
+	WCHAR *wname_p = NULL;
+	MIB_IFROW ifrow;
+
+	if (ip_addrs_p) {
+	    ipaddrrow_p = ip_addrs_p->table + i;
+	    index = ipaddrrow_p->dwIndex;
+	} else {
+	    index = ia_p->IfIndex;
+	    if (ia_p->Flags & IP_ADAPTER_NO_MULTICAST) {
+		flags &= ~INET_IFF_MULTICAST;
+	    }
+	}
+index:
+	if (! index) goto done;
+	sys_memzero(&ifrow, sizeof(ifrow));
+	ifrow.dwIndex = index;
+	if (GetIfEntry(&ifrow) != NO_ERROR) break;
+	/* Find the interface name - first try MIB_IFROW.wzname */
+	if (ifrow.wszName[0] != 0) {
+	    wname_p = ifrow.wszName;
+	} else {
+	    /* Then try IP_ADAPTER_INDEX_MAP.Name (only IPv4 adapters) */
+	    int j;
+	    for (j = 0;  j < info_p->NumAdapters;  j++) {
+		if (info_p->Adapter[j].Index == (ULONG) ifrow.dwIndex) {
+		    if (info_p->Adapter[j].Name[0] != 0) {
+			wname_p = info_p->Adapter[j].Name;
+		    }
+		    break;
+		}
+	    }
+	}
+	if (wname_p) {
+	    int len;
+	    /* Convert interface name to UTF-8 */
+	    len =
+		WideCharToMultiByte(
+		    CP_UTF8, 0, wname_p, -1, NULL, 0, NULL, NULL);
+	    if (! len) break;
+	    BUF_ENSURE(len);
+	    WideCharToMultiByte(
+		CP_UTF8, 0, wname_p, -1, buf_p, len, NULL, NULL);
+	    buf_p += len;
+	} else {
+	    /* Found no name -
+	    * use "MIB_IFROW.dwIndex: MIB_IFROW.bDescr" as name instead */
+	    int l;
+	    l = utf8_len(ifrow.bDescr, ifrow.dwDescrLen);
+	    BUF_ENSURE(9 + l+1);
+	    buf_p +=
+		erts_sprintf(
+		    buf_p, "%lu: ", (unsigned long) ifrow.dwIndex);
+	    utf8_encode(ifrow.bDescr, ifrow.dwDescrLen, buf_p);
+	    buf_p += l;
+	    *buf_p++ = '\0';
+	}
+	/* Interface flags, often make up broadcast and multicast flags */
+	switch (ifrow.dwType) {
+	case IF_TYPE_ETHERNET_CSMACD:
+	    flags |= INET_IFF_BROADCAST;
+	    break;
+	case IF_TYPE_SOFTWARE_LOOPBACK:
+	    flags |= INET_IFF_LOOPBACK;
+	    flags &= ~INET_IFF_MULTICAST;
+	    break;
+	default:
+	    flags &= ~INET_IFF_MULTICAST;
+	    break;
+	}
+	if (ifrow.dwAdminStatus) {
+	    flags |= INET_IFF_UP;
+	    switch (ifrow.dwOperStatus) {
+	    case IF_OPER_STATUS_CONNECTING:
+		flags |= INET_IFF_POINTTOPOINT;
+		break;
+	    case IF_OPER_STATUS_CONNECTED:
+		flags |= INET_IFF_RUNNING | INET_IFF_POINTTOPOINT;
+		break;
+	    case IF_OPER_STATUS_OPERATIONAL:
+		flags |= INET_IFF_RUNNING;
+		break;
+	    }
+	}
+	BUF_ENSURE(1 + 4);
+	*buf_p++ = INET_IFOPT_FLAGS;
+	put_int32(flags, buf_p); buf_p += 4;
+	if (ipaddrrow_p) {
+	    /* Legacy implementation through GetIpAddrTable */
+	    struct sockaddr_in sin;
+	    /* IP Address */
+	    sys_memzero(&sin, sizeof(sin));
+	    sin.sin_family = AF_INET;
+	    sin.sin_addr.s_addr = ipaddrrow_p->dwAddr;
+	    BUF_ENSURE(1);
+	    /* Netmask */
+	    SOCKADDR_TO_BUF(INET_IFOPT_ADDR, (struct sockaddr *) &sin);
+	    sin.sin_addr.s_addr = ipaddrrow_p->dwMask;
+	    BUF_ENSURE(1);
+	    SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, (struct sockaddr *) &sin);
+	    if (flags & INET_IFF_BROADCAST) {
+		/* Broadcast address - fake it*/
+		sin.sin_addr.s_addr = ipaddrrow_p->dwAddr;
+		sin.sin_addr.s_addr |= ~ipaddrrow_p->dwMask;
+		BUF_ENSURE(1);
+		SOCKADDR_TO_BUF(
+		    INET_IFOPT_BROADADDR, (struct sockaddr *) &sin);
+	    }
+	} else {
+	    IP_ADAPTER_UNICAST_ADDRESS *p;
+	    /* IP Address(es) */
+	    for (p = ia_p->FirstUnicastAddress;
+		p;
+		p = p->Next)
+	    {
+		IP_ADAPTER_PREFIX *q;
+		ULONG shortest_length;
+		struct sockaddr *shortest_p, *sa_p = p->Address.lpSockaddr;
+		BUF_ENSURE(1);
+		SOCKADDR_TO_BUF(INET_IFOPT_ADDR, sa_p);
+		shortest_p = NULL;
+		shortest_length = 0;
+		for (q = ia_p->FirstPrefix;
+		     q;
+		     q = q->Next) {
+		    struct sockaddr *sp_p = q->Address.lpSockaddr;
+		    if (sa_p->sa_family != sp_p->sa_family) continue;
+		    switch (sa_p->sa_family) {
+		    case AF_INET: {
+			struct sockaddr_in sin;
+			DWORD sa, sp, mask;
+			sa = ntohl((DWORD)
+				   ((struct sockaddr_in *)
+				    sa_p)->sin_addr.s_addr);
+			sp = ntohl((DWORD)
+				   ((struct sockaddr_in *)
+				    sp_p)->sin_addr.s_addr);
+			mask = 0xFFFFFFFF << (32 - q->PrefixLength);
+			if ((sa & mask) != (sp & mask)) continue;
+			if ((! shortest_p)
+			    || q->PrefixLength < shortest_length) {
+			    shortest_p = sp_p;
+			    shortest_length = q->PrefixLength;
+			}
+		    }   break;
+		    case AF_INET6: {
+			struct sockaddr_in6 sin6;
+			if (!eq_masked_bytes((char *)
+					     &((struct sockaddr_in6 *)
+					       sa_p)->sin6_addr,
+					     (char *)
+					     &((struct sockaddr_in6 *)
+					       sp_p)->sin6_addr,
+					     q->PrefixLength)) {
+			    continue;
+			}
+			if ((! shortest_p)
+			    || q->PrefixLength < shortest_length) {
+			    shortest_p = sp_p;
+			    shortest_length = q->PrefixLength;
+			}
+		    }   break;
+		    }
+		}
+		if (! shortest_p) {
+		    /* Found no shortest prefix */
+		    shortest_p = sa_p;
+		    switch (shortest_p->sa_family) {
+		    case AF_INET: {
+			/* Fall back to old classfull network addresses */
+			DWORD addr = ntohl(((struct sockaddr_in *)shortest_p)
+					   ->sin_addr.s_addr);
+			if (! (addr & 0x800000)) {
+			    /* Class A */
+			    shortest_length = 8;
+			} else if (! (addr & 0x400000)) {
+			    /* Class B */
+			    shortest_length = 16;
+			} else if (! (addr & 0x200000)) {
+			    /* Class C */
+			    shortest_length = 24;
+			} else {
+			    shortest_length = 32;
+			}
+		    }   break;
+		    case AF_INET6: {
+			/* Just play it safe */
+			shortest_length = 128;
+		    }   break;
+		    }
+		}
+		switch (shortest_p->sa_family) {
+		case AF_INET: {
+		    struct sockaddr_in sin;
+		    DWORD mask = 0xFFFFFFFF << (32 - shortest_length);
+		    sys_memzero(&sin, sizeof(sin));
+		    sin.sin_family = shortest_p->sa_family;
+		    sin.sin_addr.s_addr = htonl(mask);
+		    BUF_ENSURE(1);
+		    SOCKADDR_TO_BUF(INET_IFOPT_NETMASK,
+				    (struct sockaddr *) &sin);
+		    if (flags & INET_IFF_BROADCAST) {
+			DWORD sp =
+			    ntohl((DWORD)
+				  ((struct sockaddr_in *)shortest_p)
+				  -> sin_addr.s_addr);
+			sin.sin_addr.s_addr = htonl(sp | ~mask);
+			BUF_ENSURE(1);
+			SOCKADDR_TO_BUF(INET_IFOPT_BROADADDR,
+					(struct sockaddr *) &sin);
+		    }
+		}   break;
+		case AF_INET6: {
+		    struct sockaddr_in6 sin6;
+		    sys_memzero(&sin6, sizeof(sin6));
+		    sin6.sin6_family = shortest_p->sa_family;
+		    set_netmask_bytes((char *) &sin6.sin6_addr,
+				      16,
+				      shortest_length);
+		    BUF_ENSURE(1);
+		    SOCKADDR_TO_BUF(INET_IFOPT_NETMASK,
+				    (struct sockaddr *) &sin6);
+		}   break;
+		}
+	    }
+	}
+	if (ifrow.dwPhysAddrLen) {
+	    /* Hardware Address */
+	    BUF_ENSURE(1 + 2 + ifrow.dwPhysAddrLen);
+	    *buf_p++ = INET_IFOPT_HWADDR;
+	    put_int16(ifrow.dwPhysAddrLen, buf_p); buf_p += 2;
+	    sys_memcpy(buf_p, ifrow.bPhysAddr, ifrow.dwPhysAddrLen);
+	    buf_p += ifrow.dwPhysAddrLen;
+	}
+
+done:
+	/* That is all for this interface */
+	BUF_ENSURE(1);
+	*buf_p++ = '\0';
+	if (ia_p &&
+	    ia_p->Ipv6IfIndex &&
+	    ia_p->Ipv6IfIndex != index)
+	{
+	    /* Oops, there was an other interface for IPv6. Possible? XXX */
+	    index = ia_p->Ipv6IfIndex;
+	    goto index;
+	}
+    }
+
+    if (ip_adaddrs_p) FREE(ip_adaddrs_p);
+    if (info_p) FREE(info_p);
+    if (ip_addrs_p) FREE(ip_addrs_p);
+
+    buf_size = buf_p - buf_alloc_p;
+    buf_alloc_p = REALLOC(buf_alloc_p, buf_size);
+    /* buf_p is now unreliable */
+    *rbuf_pp = buf_alloc_p;
+    return buf_size;
+#   undef BUF_ENSURE
+}
+
+#elif defined(HAVE_GETIFADDRS)
+
+static int inet_ctl_getifaddrs(inet_descriptor* desc_p,
+			       char **rbuf_pp, int rsize)
+{
+    struct ifaddrs *ifa_p, *ifa_free_p;
+
+    int buf_size;
+    char *buf_p;
+    char *buf_alloc_p;
+
+    buf_size = 512;
+    buf_alloc_p = ALLOC(buf_size);
+    buf_p = buf_alloc_p;
+#   define BUF_ENSURE(Size)						\
+    do {								\
+	int NEED_, GOT_ = buf_p - buf_alloc_p;				\
+	NEED_ = GOT_ + (Size);						\
+	if (NEED_ > buf_size) {						\
+	    buf_size = NEED_ + 512;					\
+	    buf_alloc_p = REALLOC(buf_alloc_p, buf_size);		\
+	    buf_p = buf_alloc_p + GOT_;					\
+	}								\
+    } while (0)
+#   define SOCKADDR_TO_BUF(opt, sa)				        \
+    do {    						                \
+	if (sa) {                                                       \
+	    char *P_;							\
+	    *buf_p++ = (opt);						\
+	    while (! (P_ = sockaddr_to_buf((sa), buf_p,			\
+					   buf_alloc_p+buf_size))) {	\
+		int GOT_ = buf_p - buf_alloc_p;				\
+		buf_size += 512;					\
+		buf_alloc_p = REALLOC(buf_alloc_p, buf_size);		\
+		buf_p = buf_alloc_p + GOT_;				\
+	    }								\
+	    if (P_ == buf_p) {						\
+		buf_p--;						\
+	    } else {							\
+		buf_p = P_;						\
+	    }								\
+	}                                                               \
+    } while (0)
+
+    if (getifaddrs(&ifa_p) < 0) {
+	return ctl_error(sock_errno(), rbuf_pp, rsize);
+    }
+    ifa_free_p = ifa_p;
+    *buf_p++ = INET_REP_OK;
+    for (;  ifa_p;  ifa_p = ifa_p->ifa_next) {
+	int len = utf8_len(ifa_p->ifa_name, -1);
+	BUF_ENSURE(len+1 + 1+4 + 1);
+	utf8_encode(ifa_p->ifa_name, -1, buf_p);
+	buf_p += len;
+	*buf_p++ = '\0';
+	*buf_p++ = INET_IFOPT_FLAGS;
+	put_int32(IFGET_FLAGS(ifa_p->ifa_flags), buf_p); buf_p += 4;
+	if (ifa_p->ifa_addr) {
+	    if (ifa_p->ifa_addr->sa_family == AF_INET
+#if defined(AF_INET6)
+		|| ifa_p->ifa_addr->sa_family == AF_INET6
+#endif
+		) {
+		SOCKADDR_TO_BUF(INET_IFOPT_ADDR, ifa_p->ifa_addr);
+		if (ifa_p->ifa_netmask) {
+		    BUF_ENSURE(1);
+		    SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, ifa_p->ifa_netmask);
+		}
+		if (ifa_p->ifa_dstaddr &&
+		    (ifa_p->ifa_flags & IFF_POINTOPOINT)) {
+		    BUF_ENSURE(1);
+		    SOCKADDR_TO_BUF(INET_IFOPT_DSTADDR, ifa_p->ifa_dstaddr);
+		} else if (ifa_p->ifa_broadaddr &&
+			   (ifa_p->ifa_flags & IFF_BROADCAST)) {
+		    BUF_ENSURE(1);
+		    SOCKADDR_TO_BUF(INET_IFOPT_BROADADDR, ifa_p->ifa_broadaddr);
+		}
+	    }
+#if defined(AF_LINK) || defined(AF_PACKET)
+	    else if (
+#if defined(AF_LINK)
+		     ifa_p->ifa_addr->sa_family == AF_LINK
+#else
+		     0
+#endif
+#if defined(AF_PACKET)
+		     || ifa_p->ifa_addr->sa_family == AF_PACKET
+#endif
+		     ) {
+		char *bp = buf_p;
+		BUF_ENSURE(1);
+		SOCKADDR_TO_BUF(INET_IFOPT_HWADDR, ifa_p->ifa_addr);
+		if (buf_p - bp < 4) buf_p = bp; /* Empty hwaddr */
+	    }
+#endif
+	}
+	BUF_ENSURE(1);
+	*buf_p++ = '\0';
+    }
+    buf_size = buf_p - buf_alloc_p;
+    buf_alloc_p = REALLOC(buf_alloc_p, buf_size);
+    /* buf_p is now unreliable */
+    freeifaddrs(ifa_free_p);
+    *rbuf_pp = buf_alloc_p;
+    return buf_size;
+#   undef BUF_ENSURE
+}
+
+#else
+
+static int inet_ctl_getifaddrs(inet_descriptor* desc_p,
+			       char **rbuf_pp, int rsize)
+{
+    return ctl_error(ENOTSUP, rbuf_pp, rsize);
+}
+
+#endif
+
+
+
 #ifdef VXWORKS
 /*
 ** THIS is a terrible creature, a bug in the TCP part
@@ -4469,9 +5048,17 @@ static STATUS wrap_sockopt(STATUS (*function)() /* Yep, no parameter
 }
 #endif
 
+/* Per H @ Tail-f: The original code here had problems that possibly
+   only occur if you abuse it for non-INET sockets, but anyway:
+   a) If the getsockopt for SO_PRIORITY or IP_TOS failed, the actual
+      requested setsockopt was never even attempted.
+   b) If {get,set}sockopt for one of IP_TOS and SO_PRIORITY failed,
+      but ditto for the other worked and that was actually the requested
+      option, failure was still reported to erlang.                  */
+
 #if  defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY)
 static int setopt_prio_tos_trick
-	   (int fd, int proto, int type, char* arg_ptr, int arg_sz)
+	(int fd, int proto, int type, char* arg_ptr, int arg_sz, int propagate)
 {
     /* The relations between SO_PRIORITY, TOS and other options
        is not what you (or at least I) would expect...:
@@ -4484,6 +5071,8 @@ static int setopt_prio_tos_trick
     int          tmp_ival_prio;
     int          tmp_ival_tos;
     int          res;
+    int          res_prio;
+    int          res_tos;
 #ifdef HAVE_SOCKLEN_T
 	    socklen_t
 #else
@@ -4492,28 +5081,35 @@ static int setopt_prio_tos_trick
 		tmp_arg_sz_prio = sizeof(tmp_ival_prio),
 		tmp_arg_sz_tos  = sizeof(tmp_ival_tos);
 
-    res = sock_getopt(fd, SOL_SOCKET, SO_PRIORITY,
+    res_prio = sock_getopt(fd, SOL_SOCKET, SO_PRIORITY,
 		      (char *) &tmp_ival_prio, &tmp_arg_sz_prio);
-    if (res == 0) {
-	res = sock_getopt(fd, SOL_IP, IP_TOS, 
+    res_tos = sock_getopt(fd, SOL_IP, IP_TOS, 
 		      (char *) &tmp_ival_tos, &tmp_arg_sz_tos);
-	if (res == 0) {
 	    res = sock_setopt(fd, proto, type, arg_ptr, arg_sz);
 	    if (res == 0) {
 		if (type != SO_PRIORITY) {
-		    if (type != IP_TOS) {
-			res = sock_setopt(fd, 
+	    if (type != IP_TOS && res_tos == 0) {
+		res_tos = sock_setopt(fd, 
 					  SOL_IP, 
 					  IP_TOS,
 					  (char *) &tmp_ival_tos, 
 					  tmp_arg_sz_tos);
+		if (propagate)
+		    res = res_tos;
 		    }
-		    if (res == 0) {
-			res =  sock_setopt(fd, 
+	    if (res == 0 && res_prio == 0) {
+		res_prio = sock_setopt(fd, 
 					   SOL_SOCKET, 
 					   SO_PRIORITY,
 					   (char *) &tmp_ival_prio, 
 					   tmp_arg_sz_prio);
+		if (propagate) {		
+		    /* Some kernels set a SO_PRIORITY by default that you are not permitted to reset,
+		       silently ignore this error condition */
+		    if (res_prio != 0 && sock_errno() == EPERM) {
+			res = 0;
+		    } else {
+			res = res_prio;
 		    }
 		}
 	    }
@@ -4588,8 +5184,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	case INET_LOPT_BUFFER:
 	    DEBUGF(("inet_set_opts(%ld): s=%d, BUFFER=%d\r\n",
 		    (long)desc->port, desc->s, ival));
-	    if (ival > INET_MAX_BUFFER)  ival = INET_MAX_BUFFER;
-	    else if (ival < INET_MIN_BUFFER) ival = INET_MIN_BUFFER;
+	    if (ival < INET_MIN_BUFFER) ival = INET_MIN_BUFFER;
 	    desc->bufsz = ival;
 	    continue;
 
@@ -4654,7 +5249,6 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    if (desc->stype == SOCK_STREAM) {
 		tcp_descriptor* tdesc = (tcp_descriptor*) desc;
 		if (ival < 0) ival = 0;
-		else if (ival > INET_MAX_BUFFER*2) ival = INET_MAX_BUFFER*2;
 		if (tdesc->low > ival)
 		    tdesc->low = ival;
 		tdesc->high = ival;
@@ -4665,7 +5259,6 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    if (desc->stype == SOCK_STREAM) {
 		tcp_descriptor* tdesc = (tcp_descriptor*) desc;
 		if (ival < 0) ival = 0;
-		else if (ival > INET_MAX_BUFFER) ival = INET_MAX_BUFFER;
 		if (tdesc->high < ival)
 		    tdesc->high = ival;
 		tdesc->low = ival;
@@ -4862,7 +5455,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    return -1;
 	}
 #if  defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY)
-	res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz);
+	res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz, propagate);
 #else
 	res = sock_setopt	    (desc->s, proto, type, arg_ptr, arg_sz);
 #endif
@@ -5011,9 +5604,6 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	case INET_LOPT_BUFFER:
 	    desc->bufsz  = get_int32(curr);		curr += 4;
 
-            if (desc->bufsz > INET_MAX_BUFFER)
-		desc->bufsz = INET_MAX_BUFFER;
-            else
 	    if (desc->bufsz < INET_MIN_BUFFER)
 		desc->bufsz = INET_MIN_BUFFER;
 	    res = 0;	  /* This does not affect the kernel buffer size */
@@ -5076,8 +5666,8 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	}
 	case INET_OPT_LINGER:
 	{
-	    CHKLEN(curr, ASSOC_ID_LEN + 2 + 4);
-	    arg.lin.l_onoff  = get_int16 (curr);  curr += 2;
+	    CHKLEN(curr, 2*4);
+	    arg.lin.l_onoff  = get_int32 (curr);  curr += 4;
 	    arg.lin.l_linger = get_int32 (curr);  curr += 4;
 
 	    proto   = SOL_SOCKET;
@@ -5254,9 +5844,12 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    char *after;
 #	    ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_FLAGS
 	    int eflags, cflags, hb_enable, hb_disable,
-		pmtud_enable, pmtud_disable,
+		pmtud_enable, pmtud_disable;
+#	    ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY
+	    int
 		sackdelay_enable, sackdelay_disable;
 #           endif
+#           endif
 	    
 	    CHKLEN(curr, ASSOC_ID_LEN);
 	    arg.pap.spp_assoc_id = GET_ASSOC_ID(curr);	curr += ASSOC_ID_LEN;
@@ -5305,12 +5898,15 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    if (pmtud_enable)			cflags |= SPP_PMTUD_ENABLE;
 	    if (pmtud_disable)			cflags |= SPP_PMTUD_DISABLE;
 
+#	    ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY
+	    /* The followings are missing in FreeBSD 7.1 */
 	    sackdelay_enable =eflags& SCTP_FLAG_SACDELAY_ENABLE;
 	    sackdelay_disable=eflags& SCTP_FLAG_SACDELAY_DISABLE;
 	    if (sackdelay_enable && sackdelay_disable)
 		return -1;
 	    if (sackdelay_enable)		cflags |= SPP_SACKDELAY_ENABLE;
 	    if (sackdelay_disable)		cflags |= SPP_SACKDELAY_DISABLE;
+#           endif
 
 	    arg.pap.spp_flags  = cflags;
 #	    endif
@@ -5389,7 +5985,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    return -1;
 	}
 #if  defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY)
-	res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz);
+	res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz, 1);
 #else
 	res = sock_setopt	    (desc->s, proto, type, arg_ptr, arg_sz);
 #endif
@@ -5448,7 +6044,7 @@ static int inet_fill_opts(inet_descriptor* desc,
 #define PLACE_FOR(Size,Ptr)						   \
     do {								   \
 	int need = dest_used + (Size);					   \
-	if (need > INET_MAX_BUFFER) {					   \
+	if (need > INET_MAX_OPT_BUFFER) {				   \
 	    RETURN_ERROR();						   \
 	}								   \
 	if (need > dest_allocated) {					   \
@@ -5672,7 +6268,7 @@ static int inet_fill_opts(inet_descriptor* desc,
 		buf += 4;
 		data_provided = (int) *buf++;
 		arg_sz = get_int32(buf);
-		if (arg_sz > INET_MAX_BUFFER) {	
+		if (arg_sz > INET_MAX_OPT_BUFFER) {
 		    RETURN_ERROR();
 		}
 		buf += 4;
@@ -5687,8 +6283,8 @@ static int inet_fill_opts(inet_descriptor* desc,
 		    buf += arg_sz;
 		    len -= arg_sz;
 		}
-		if (sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz) == 
-		    SOCKET_ERROR) {
+		if (IS_SOCKET_ERROR(sock_getopt(desc->s,proto,type,
+						arg_ptr,&arg_sz))) {
 		    TRUNCATE_TO(0,ptr); 
 		    continue;
 		}
@@ -5705,7 +6301,7 @@ static int inet_fill_opts(inet_descriptor* desc,
 	    RETURN_ERROR();
 	}
 	/* We have 5 bytes allocated to ptr */
-	if (sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz) == SOCKET_ERROR) {
+	if (IS_SOCKET_ERROR(sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz))) {
 	    TRUNCATE_TO(0,ptr);
 	    continue;
 	}
@@ -5786,7 +6382,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 		     "miscalculated buffer size");              \
 	}                                                       \
 	need = (Index) + (N);                                   \
-	if (need > INET_MAX_BUFFER/sizeof(ErlDrvTermData)) {    \
+	if (need > INET_MAX_OPT_BUFFER/sizeof(ErlDrvTermData)) {\
 	    RETURN_ERROR((Spec), -ENOMEM);                      \
 	}                                                       \
 	if (need > spec_allocated) {                            \
@@ -6211,13 +6807,15 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 	    
 	    if (ap.spp_flags & SPP_PMTUD_DISABLE)
 		{ i = LOAD_ATOM (spec, i, am_pmtud_disable);         n++; }
-	    
+#	    ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY
+	    /* SPP_SACKDELAY_* not in FreeBSD 7.1 */
 	    if (ap.spp_flags & SPP_SACKDELAY_ENABLE)
 		{ i = LOAD_ATOM (spec, i, am_sackdelay_enable);      n++; }
 	    
 	    if (ap.spp_flags & SPP_SACKDELAY_DISABLE)
 		{ i = LOAD_ATOM (spec, i, am_sackdelay_disable);     n++; }
 #	    endif
+#	    endif
 	    
 	    PLACE_FOR(spec, i,
 		      LOAD_NIL_CNT + LOAD_LIST_CNT + 2*LOAD_TUPLE_CNT);
@@ -6237,6 +6835,10 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 	    struct       sctp_sndrcvinfo sri;
 	    unsigned int sz  = sizeof(sri);
 	    
+	    if (buflen < ASSOC_ID_LEN) RETURN_ERROR(spec, -EINVAL);
+	    sri.sinfo_assoc_id = GET_ASSOC_ID(buf);
+	    buf += ASSOC_ID_LEN;
+	    buflen -= ASSOC_ID_LEN;
 	    if (sock_getopt(desc->s, IPPROTO_SCTP, SCTP_DEFAULT_SEND_PARAM,
 			    &sri, &sz) < 0) continue;
 	    /* Fill in the response: */
@@ -6633,7 +7235,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	      }
 	  }
 	  DEBUGF(("inet_ctl(%ld): GETSTAT\r\n", (long) desc->port)); 
-	  if (dstlen > INET_MAX_BUFFER) /* sanity check */
+	  if (dstlen > INET_MAX_OPT_BUFFER) /* sanity check */
 	      return 0;
 	  if (dstlen > rsize) {
 	      if ((dst = (char*) ALLOC(dstlen)) == NULL)
@@ -6649,7 +7251,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	  char* dst;
 	  int dstlen = 1 /* Reply code */ + len*5;
 	  DEBUGF(("inet_ctl(%ld): INET_REQ_SUBSCRIBE\r\n", (long) desc->port)); 
-	  if (dstlen > INET_MAX_BUFFER) /* sanity check */
+	  if (dstlen > INET_MAX_OPT_BUFFER) /* sanity check */
 	      return 0;
 	  if (dstlen > rsize) {
 	      if ((dst = (char*) ALLOC(dstlen)) == NULL)
@@ -6684,6 +7286,13 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	return inet_ctl_getiflist(desc, rbuf, rsize);
     }
 
+    case INET_REQ_GETIFADDRS: {
+	DEBUGF(("inet_ctl(%ld): GETIFADDRS\r\n", (long)desc->port));
+	if (!IS_OPEN(desc))
+	    return ctl_xerror(EXBADPORT, rbuf, rsize);
+	return inet_ctl_getifaddrs(desc, rbuf, rsize);
+    }
+
     case INET_REQ_IFGET: {
 	DEBUGF(("inet_ctl(%ld): IFGET\r\n", (long)desc->port)); 	
 	if (!IS_OPEN(desc))
@@ -6771,7 +7380,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	if (len != 0)
 	    return ctl_error(EINVAL, rbuf, rsize);
 
-	if (sock_hostname(tbuf, MAXHOSTNAMELEN) == SOCKET_ERROR)
+	if (IS_SOCKET_ERROR(sock_hostname(tbuf, MAXHOSTNAMELEN)))
 	    return ctl_error(sock_errno(), rbuf, rsize);
 	return ctl_reply(INET_REP_OK, tbuf, strlen(tbuf), rbuf, rsize);
     }
@@ -6788,7 +7397,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	    return ctl_error(ENOTCONN, rbuf, rsize);
 	if ((ptr = desc->peer_ptr) == NULL) {
 	    ptr = &peer;
-	    if (sock_peer(desc->s, (struct sockaddr*)ptr,&sz) == SOCKET_ERROR)
+	    if (IS_SOCKET_ERROR(sock_peer(desc->s, (struct sockaddr*)ptr,&sz)))
 		return ctl_error(sock_errno(), rbuf, rsize);
 	}
 	if (inet_get_address(desc->sfamily, tbuf, ptr, &sz) < 0)
@@ -6825,7 +7434,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 
 	if ((ptr = desc->name_ptr) == NULL) {
 	    ptr = &name;
-	    if (sock_name(desc->s, (struct sockaddr*)ptr, &sz) == SOCKET_ERROR)
+	    if (IS_SOCKET_ERROR(sock_name(desc->s, (struct sockaddr*)ptr, &sz)))
 		return ctl_error(sock_errno(), rbuf, rsize);
 	}
 	if (inet_get_address(desc->sfamily, tbuf, ptr, &sz) < 0)
@@ -6864,7 +7473,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	if (inet_set_address(desc->sfamily, &local, buf, &len) == NULL)
 	    return ctl_error(EINVAL, rbuf, rsize);
 
-	if (sock_bind(desc->s,(struct sockaddr*) &local, len) == SOCKET_ERROR)
+	if (IS_SOCKET_ERROR(sock_bind(desc->s,(struct sockaddr*) &local, len)))
 	    return ctl_error(sock_errno(), rbuf, rsize);
 
 	desc->state = INET_STATE_BOUND;
@@ -6891,13 +7500,13 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 
 	if (len < 2)
 	    return ctl_error(EINVAL, rbuf, rsize);
-	n = buf[0]; buf++; len--;
+	n = get_int8(buf); buf++; len--;
 	if (n >= len) /* the = sign makes the test inklude next length byte */
 	    return ctl_error(EINVAL, rbuf, rsize);
 	memcpy(namebuf, buf, n);
 	namebuf[n] = '\0';
 	len -= n; buf += n;
-	n = buf[0]; buf++; len--;
+	n = get_int8(buf); buf++; len--;
 	if (n > len)
 	    return ctl_error(EINVAL, rbuf, rsize);
 	memcpy(protobuf, buf, n);
@@ -6920,7 +7529,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	port = get_int16(buf);
 	port = sock_htons(port);
 	buf += 2;
-	n = buf[0]; buf++; len -= 3;
+	n = get_int8(buf); buf++; len -= 3;
 	if (n > len)
 	    return ctl_error(EINVAL, rbuf, rsize);
 	memcpy(protobuf, buf, n);
@@ -7297,7 +7906,7 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	if (len != 2)
 	    return ctl_error(EINVAL, rbuf, rsize);
 	backlog = get_int16(buf);
-	if (sock_listen(desc->inet.s, backlog) == SOCKET_ERROR)
+	if (IS_SOCKET_ERROR(sock_listen(desc->inet.s, backlog)))
 	    return ctl_error(sock_errno(), rbuf, rsize);
 	desc->inet.state = TCP_STATE_LISTEN;
 	return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
@@ -7331,7 +7940,7 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	
 	code = sock_connect(desc->inet.s, 
 			    (struct sockaddr*) &desc->inet.remote, len);
-	if ((code == SOCKET_ERROR) && 
+	if (IS_SOCKET_ERROR(code) &&
 		((sock_errno() == ERRNO_BLOCK) ||  /* Winsock2 */
 		 (sock_errno() == EINPROGRESS))) {	/* Unix & OSE!! */
           sock_select(INETP(desc), FD_CONNECT, 1);
@@ -7371,11 +7980,11 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	timeout = get_int32(buf);
 
 	if (desc->inet.state == TCP_STATE_ACCEPTING) {
-	    unsigned long time_left;
-	    int oid;
-	    ErlDrvTermData ocaller;
-	    int oreq;
-	    unsigned otimeout;
+	    unsigned long time_left = 0;
+	    int oid = 0;
+	    ErlDrvTermData ocaller = ERL_DRV_NIL;
+	    int oreq = 0;
+	    unsigned otimeout = 0;
 	    ErlDrvTermData caller = driver_caller(desc->inet.port);
 	    MultiTimerData *mtd = NULL,*omtd = NULL;
 	    ErlDrvMonitor monitor, omonitor;
@@ -7518,7 +8127,6 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	    tcp_deliver(desc, 0);
 	return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
     }
-#ifndef _OSE_
     case TCP_REQ_SHUTDOWN: {
 	int how;
 	DEBUGF(("tcp_inet_ctl(%ld): FDOPEN\r\n", (long)desc->inet.port)); 
@@ -7535,7 +8143,6 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	    return ctl_error(sock_errno(), rbuf, rsize);
 	}
     }
-#endif
     default:
 	DEBUGF(("tcp_inet_ctl(%ld): %u\r\n", (long)desc->inet.port, cmd)); 
 	return inet_ctl(INETP(desc), cmd, buf, len, rbuf, rsize);
@@ -7941,7 +8548,9 @@ static int tcp_deliver(tcp_descriptor* desc, int len)
 	len = 0;
 
 	if (!desc->inet.active) {
-	    driver_cancel_timer(desc->inet.port);
+	    if (!desc->busy_on_send) {
+		driver_cancel_timer(desc->inet.port);
+	    }
 	    sock_select(INETP(desc),(FD_READ|FD_CLOSE),0);
 	    if (desc->i_buf != NULL)
 		tcp_restart_input(desc);
@@ -8009,7 +8618,7 @@ static int tcp_recv(tcp_descriptor* desc, int request_len)
 
     n = sock_recv(desc->inet.s, desc->i_ptr, nread, 0);
 
-    if (n == SOCKET_ERROR) {
+    if (IS_SOCKET_ERROR(n)) {
 	int err = sock_errno();
 	if (err == ECONNRESET) {
 	    DEBUGF((" => detected close (connreset)\r\n"));
@@ -8511,8 +9120,8 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev)
 		(long)desc->inet.port, desc->inet.s, h_len, len));
 	if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) {
 	    n = 0;
-	} else if (sock_sendv(desc->inet.s, ev->iov, vsize, &n, 0) 
-		   == SOCKET_ERROR) {
+	} else if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s, ev->iov,
+					      vsize, &n, 0))) {
 	    if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) {
 		int err = sock_errno();
 		DEBUGF(("tcp_sendv(%ld): s=%d, "
@@ -8605,7 +9214,7 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, int len)
 	if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) {
 	    sock_send(desc->inet.s, buf, 0, 0);
 	    n = 0;
-	} else 	if (sock_sendv(desc->inet.s,iov,2,&n,0) == SOCKET_ERROR) {
+	} else 	if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s,iov,2,&n,0))) {
 	    if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) {
 		int err = sock_errno();
 		DEBUGF(("tcp_send(%ld): s=%d,sock_sendv(size=2) errno = %d\r\n",
@@ -8678,7 +9287,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 	    int code = sock_peer(desc->inet.s,
 				 (struct sockaddr*) &desc->inet.remote, &sz);
 
-	    if (code == SOCKET_ERROR) {
+	    if (IS_SOCKET_ERROR(code)) {
 		desc->inet.state = TCP_STATE_BOUND;  /* restore state */
 		ret =  async_error(INETP(desc), sock_errno());
 		goto done;
@@ -8719,7 +9328,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 	    vsize = vsize > MAX_VSIZE ? MAX_VSIZE : vsize;
 	    DEBUGF(("tcp_inet_output(%ld): s=%d, About to send %d items\r\n", 
 		    (long)desc->inet.port, desc->inet.s, vsize));
-	    if (sock_sendv(desc->inet.s, iov, vsize, &n, 0)==SOCKET_ERROR) {
+	    if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s, iov, vsize, &n, 0))) {
 		if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) {
 		    DEBUGF(("tcp_inet_output(%ld): sock_sendv(%d) errno = %d\r\n",
 			    (long)desc->inet.port, vsize, sock_errno()));
@@ -8988,7 +9597,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	    sock_select(desc, FD_CONNECT, 1);
 	    code = sock_connect(desc->s, &remote.sa, len);
 
-	    if ((code == SOCKET_ERROR) && (sock_errno() == EINPROGRESS)) {
+	    if (IS_SOCKET_ERROR(code) && (sock_errno() == EINPROGRESS)) {
 		/* XXX: Unix only -- WinSock would have a different cond! */
 		desc->state = SCTP_STATE_CONNECTING;
 		if (timeout != INET_INFINITY)
@@ -9028,7 +9637,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	    
 	    code = sock_connect(desc->s,
 				(struct sockaddr*) &desc->remote, len);
-	    if (code == SOCKET_ERROR) {
+	    if (IS_SOCKET_ERROR(code)) {
 		sock_connect(desc->s, (struct sockaddr*) NULL, 0);
 		desc->state &= ~INET_F_ACTIVE;
 		return ctl_error(sock_errno(), rbuf, rsize);
@@ -9062,7 +9671,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 		return ctl_error(EINVAL, rbuf, rsize);
 	    flag = get_int8(buf);
 
-	    if (sock_listen(desc->s, flag) == SOCKET_ERROR)
+	    if (IS_SOCKET_ERROR(sock_listen(desc->s, flag)))
 		return ctl_error(sock_errno(), rbuf, rsize);
 
 	    desc->state = SCTP_STATE_LISTEN;   /* XXX: not used? */
@@ -9267,7 +9876,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, int len)
  check_result_code:
     /* "code" analysis is the same for both SCTP and UDP cases above: */
 #endif
-    if (code == SOCKET_ERROR) {
+    if (IS_SOCKET_ERROR(code)) {
 	int err = sock_errno();
 	inet_reply_error(desc, err);
     }
@@ -9366,7 +9975,7 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
     check_result:
 #endif
 	/* Analyse the result: */
-	if (n == SOCKET_ERROR
+	if (IS_SOCKET_ERROR(n)
 #ifdef HAVE_SCTP
 	    || (short_recv = (IS_SCTP(desc) && !(mhdr.msg_flags & MSG_EOR)))
 	    /* NB: here we check for EOR not being set -- this is an error as
@@ -9379,11 +9988,13 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 	    if (err != ERRNO_BLOCK) {
 		if (!desc->active) {
 #ifdef HAVE_SCTP
-		    if (short_recv)
+		    if (short_recv) {
 			async_error_am(desc, am_short_recv);
-		    else
-#else
+		    } else {
 			async_error(desc, err);
+		    }
+#else
+		    async_error(desc, err);
 #endif
 		    driver_cancel_timer(desc->port);
 		    sock_select(desc,FD_READ,0);
@@ -9481,7 +10092,7 @@ static int packet_inet_output(udp_descriptor* udesc, HANDLE event)
 	    int code = sock_peer(desc->s,
 				 (struct sockaddr*) &desc->remote, &sz);
 
-	    if (code == SOCKET_ERROR) {
+	    if (IS_SOCKET_ERROR(code)) {
 		desc->state = PACKET_STATE_BOUND;  /* restore state */
 		ret =  async_error(desc, sock_errno());
 		goto done;
@@ -9922,23 +10533,26 @@ int erts_sock_connect(erts_sock_t socket, byte *ip_addr, int len, Uint16 port)
     if (!inet_set_address(AF_INET, &addr, buf, &blen))
 	return 0;
 
-    if (SOCKET_ERROR == sock_connect(s,
+    if (IS_SOCKET_ERROR(sock_connect(s,
 				     (struct sockaddr *) &addr,
-				     sizeof(struct sockaddr_in)))
+				     sizeof(struct sockaddr_in))))
 	return 0;
     return 1;
 }
 
 Sint erts_sock_send(erts_sock_t socket, const void *buf, Sint len)
 {
-    return (Sint) sock_send((SOCKET) socket, buf, (size_t) len, 0);
+    Sint result = (Sint) sock_send((SOCKET) socket, buf, (size_t) len, 0);
+    if (IS_SOCKET_ERROR(result))
+	return SOCKET_ERROR;
+    return result;
 }
 
 
 int erts_sock_gethostname(char *buf, int bufsz)
 {
-    if (sock_hostname(buf, bufsz) == SOCKET_ERROR)
-	return -1;
+    if (IS_SOCKET_ERROR(sock_hostname(buf, bufsz)))
+	return SOCKET_ERROR;
     return 0;
 }
 
diff --git a/erts/emulator/drivers/common/ram_file_drv.c b/erts/emulator/drivers/common/ram_file_drv.c
index 4a39a156e6..abedcc933a 100644
--- a/erts/emulator/drivers/common/ram_file_drv.c
+++ b/erts/emulator/drivers/common/ram_file_drv.c
@@ -35,6 +35,7 @@
 #define RAM_FILE_TRUNCATE      14
 #define RAM_FILE_PREAD         17
 #define RAM_FILE_PWRITE        18
+#define RAM_FILE_FDATASYNC     19
 
 /* other operations */
 #define RAM_FILE_GET           30
@@ -45,6 +46,8 @@
 #define RAM_FILE_UUENCODE      35  /* uuencode file */
 #define RAM_FILE_UUDECODE      36  /* uudecode file */
 #define RAM_FILE_SIZE          37  /* get file size */
+#define RAM_FILE_ADVISE        38  /* predeclare the access
+                                    * pattern for file data */
 /* possible new operations include:
    DES_ENCRYPT
    DES_DECRYPT
@@ -558,6 +561,13 @@ static void rfile_command(ErlDrvData e, char* buf, int count)
 	    numeric_reply(f, 0); /* 0 is not used */
 	break;
 
+    case RAM_FILE_FDATASYNC:
+	if (f->flags == 0)
+	    error_reply(f, EBADF);
+	else
+	    reply(f, 1, 0);
+	break;
+
     case RAM_FILE_FSYNC:
 	if (f->flags == 0)
 	    error_reply(f, EBADF);
@@ -685,6 +695,13 @@ static void rfile_command(ErlDrvData e, char* buf, int count)
     case RAM_FILE_UUDECODE:   /* uudecode file */
 	ram_file_uudecode(f);
 	break;
+
+    case RAM_FILE_ADVISE:
+	if (f->flags == 0)
+	    error_reply(f, EBADF);
+	else
+	    reply(f, 1, 0);
+	break;
     }
     /*
      * Ignore anything else -- let the caller hang.
diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c
index 723efeaa13..f50899a730 100644
--- a/erts/emulator/drivers/common/zlib_drv.c
+++ b/erts/emulator/drivers/common/zlib_drv.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -115,7 +115,7 @@ typedef struct {
 static int zlib_inflate(ZLibData* d, int flush);
 static int zlib_deflate(ZLibData* d, int flush);
 
-#if defined(_OSE_) || defined(__WIN32__)
+#if defined(__WIN32__)
 static int i32(char* buf)
 #else
 static inline int i32(char* buf)
diff --git a/erts/emulator/drivers/unix/mem_drv.c b/erts/emulator/drivers/unix/mem_drv.c
deleted file mode 100644
index 1417ca1121..0000000000
--- a/erts/emulator/drivers/unix/mem_drv.c
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-/* Purpose: Access to elib memory statistics */
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "sys.h"
-#include "erl_driver.h"
-#include "elib_stat.h"
-
-#define MAP_BUF_SIZE    1000   /* Max map size */
-#define HISTO_BUF_SIZE  100    /* Max histogram buckets */
-
-static ErlDrvData mem_start(ErlDrvPort);
-static int mem_init(void); 
-static void mem_stop(ErlDrvData);
-static void mem_command(ErlDrvData, char*, int);
-
-const struct driver_entry mem_driver_entry = {
-    mem_init,
-    mem_start,
-    mem_stop,
-    mem_command,
-    NULL,
-    NULL,
-    "mem_drv"
-};
-
-static int mem_init(void)
-{
-    return 0;
-}
-
-static ErlDrvData mem_start(ErlDrvPort port, char* buf)
-{
-    return (ErlDrvData)port;
-}
-
-static void mem_stop(ErlDrvData port)
-{
-}
-
-void putint32(p, v)
-byte* p; int v;
-{
-    p[0] = (v >> 24) & 0xff;
-    p[1] = (v >> 16) & 0xff;
-    p[2] = (v >> 8) & 0xff;
-    p[3] = (v) & 0xff;
-}
-
-int getint16(p)
-byte* p;
-{
-    return (p[0] << 8) | p[1];
-}
-
-/*
-** Command:
-**   m L1 L0  -> a heap map of length L1*256 + L0 is returned
-**   s        -> X3 X2 X1 X0 Y3 Y2 Y1 Y0 Z3 Z2 Z1 Z0
-**               X == Total heap size bytes
-**               Y == Total free bytes
-**               Z == Size of largest free block in bytes
-**
-**   h L1 L0 B0 -> Generate a logarithm historgram base B with L buckets
-**   l L1 L0 S0 -> Generate a linear histogram with step S with L buckets
-*/
-unsigned char outbuf[HISTO_BUF_SIZE*2*4];
-
-static void mem_command(ErlDrvData port, char* buf, int count)
-{
-    if ((count == 1) && buf[0] == 's') {
-	struct elib_stat info;
-	char v[3*4];
-
-	elib_stat(&info);
-
-	putint32(v, info.mem_total*4);
-	putint32(v+4, info.mem_free*4);
-	putint32(v+8, info.max_free*4);
-	driver_output((ErlDrvPort)port, v, 12);
-	return;
-    }
-    else if ((count == 3) && buf[0] == 'm') {
-	char w[MAP_BUF_SIZE];
-	int n = getint16(buf+1);
-
-	if (n > MAP_BUF_SIZE)
-	    n = MAP_BUF_SIZE;
-	elib_heap_map(w, n);
-	driver_output((ErlDrvPort)port, w, n);
-	return;
-    }
-    else if ((count == 4) && (buf[0] == 'h' || buf[0] == 'l')) {
-	unsigned long vf[HISTO_BUF_SIZE];
-	unsigned long va[HISTO_BUF_SIZE];
-	int n = getint16(buf+1);
-	int base = (unsigned char) buf[3];
-
-	if (n >= HISTO_BUF_SIZE)
-	    n = HISTO_BUF_SIZE;
-	if (buf[0] == 'l')
-	    base = -base;
-	if (elib_histo(vf, va, n, base) < 0) {
-	    driver_failure((ErlDrvPort)port, -1);
-	    return;
-	}
-	else {
-	    char* p = outbuf;
-	    int i;
-
-	    for (i = 0; i < n; i++) {
-		putint32(p, vf[i]);
-		p += 4;
-	    }
-	    for (i = 0; i < n; i++) {
-		putint32(p, va[i]);
-		p += 4;
-	    }
-	    driver_output((ErlDrvPort)port, outbuf, n*8);
-	}
-	return;
-    }
-    driver_failure((ErlDrvPort)port, -1);
-}
diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c
index 4cd54c073f..d782b044a9 100644
--- a/erts/emulator/drivers/unix/ttsl_drv.c
+++ b/erts/emulator/drivers/unix/ttsl_drv.c
@@ -314,7 +314,7 @@ static ErlDrvData ttysl_start(ErlDrvPort port, char* buf)
     sys_sigset(SIGCONT, cont);
     sys_sigset(SIGWINCH, winch);
 
-    driver_select(port, (ErlDrvEvent)(Uint)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 1);
+    driver_select(port, (ErlDrvEvent)(UWord)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 1);
     ttysl_port = port;
 
     /* we need to know this when we enter the break handler */
@@ -394,7 +394,7 @@ static void ttysl_stop(ErlDrvData ttysl_data)
 	stop_lbuf();
 	stop_termcap();
 	tty_reset(ttysl_fd);
-	driver_select(ttysl_port, (ErlDrvEvent)(Uint)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 0);
+	driver_select(ttysl_port, (ErlDrvEvent)(UWord)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 0);
 	sys_sigset(SIGCONT, SIG_DFL);
 	sys_sigset(SIGWINCH, SIG_DFL);
     }
@@ -685,7 +685,7 @@ static void ttysl_from_tty(ErlDrvData ttysl_data, ErlDrvEvent fd)
 	utf8buf_size = 0;
     }
     
-    if ((i = read((int)(Sint)fd, (char *) p, left)) >= 0) {
+    if ((i = read((int)(SWord)fd, (char *) p, left)) >= 0) {
 	if (p != b) {
 	    i += (p - b);
 	}
diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c
index d395b68691..4b3934657c 100644
--- a/erts/emulator/drivers/unix/unix_efile.c
+++ b/erts/emulator/drivers/unix/unix_efile.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1997-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -34,17 +34,6 @@
 #include <sys/uio.h>
 #endif
 
-#ifdef _OSE_
-#include "efs.h"
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-#ifdef _OSE_SFK_
-#include <string.h>
-#endif
-#endif /* _OSE_ */
-
 #if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
 #define DARWIN 1
 #endif
@@ -88,23 +77,6 @@ extern STATUS copy(char *, char *);
  * Macros for testing file types.
  */
 
-#ifdef _OSE_
-
-#define ISDIR(st) S_ISDIR(((st).st_mode))
-#define ISREG(st) S_ISREG(((st).st_mode))
-#define ISDEV(st) (S_ISCHR(((st).st_mode)) || S_ISBLK(((st).st_mode)))
-#define ISLNK(st) S_ISLNK(((st).st_mode))
-#ifdef NO_UMASK
-#define FILE_MODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)
-#define DIR_MODE  (S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
-#else
-#define FILE_MODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
-#define DIR_MODE  (S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | \
-                   S_IWOTH | S_IXOTH)
-#endif 
-
-#else  /* !_OSE_ */
-
 #define ISDIR(st) (((st).st_mode & S_IFMT) == S_IFDIR)
 #define ISREG(st) (((st).st_mode & S_IFMT) == S_IFREG)
 #define ISDEV(st) \
@@ -118,8 +90,6 @@ extern STATUS copy(char *, char *);
 #define DIR_MODE  0777
 #endif
 
-#endif /* _OSE_ */
-
 #ifdef VXWORKS /* Currently only used on vxworks */
 
 #define EF_ALLOC(S)		driver_alloc((S))
@@ -128,7 +98,7 @@ extern STATUS copy(char *, char *);
 #define EF_SAFE_REALLOC(P, S)	ef_safe_realloc((P), (S))
 #define EF_FREE(P)		do { if((P)) driver_free((P)); } while(0)
 
-extern void erl_exit(int n, char *fmt, _DOTS_);
+void erl_exit(int n, char *fmt, ...);
 
 static void *ef_safe_alloc(Uint s)
 {
@@ -157,7 +127,7 @@ static void *ef_safe_realloc(void *op, Uint s)
     (s[0] == '.' && (s[1] == '\0' || (s[1] == '.' && s[2] == '\0')))
 
 #ifdef VXWORKS
-static FUNCTION(int, vxworks_to_posix, (int vx_errno));
+static int vxworks_to_posix(int vx_errno);
 #endif
 
 /*
@@ -176,7 +146,7 @@ static FUNCTION(int, vxworks_to_posix, (int vx_errno));
 #define CHECK_PATHLEN(X,Y) /* Nothing */
 #endif
 
-static FUNCTION(int, check_error, (int result, Efile_error* errInfo));
+static int check_error(int result, Efile_error* errInfo);
 
 static int
 check_error(int result, Efile_error *errInfo)
@@ -361,15 +331,6 @@ path_size(char *pathname)
     
 #endif /* VXWORKS */
 
-#ifdef _OSE_
-static int 
-ose_enotsup(Efile_error *errInfo) 
-{
-    errInfo->posix_errno = errInfo->os_errno = ENOTSUP;
-    return 0;
-}
-#endif /* _OSE_ */
-
 int
 efile_mkdir(Efile_error* errInfo,	/* Where to return error codes. */
 	    char* name)			/* Name of directory to create. */
@@ -446,18 +407,12 @@ efile_delete_file(Efile_error* errInfo,	/* Where to return error codes. */
 		  char* name)		/* Name of file to delete. */
 {
     CHECK_PATHLEN(name,errInfo);
-#ifdef _OSE_
-    if (remove(name) == 0) {
-	return 1;
-    }
-#else
     if (unlink(name) == 0) {
 	return 1;
     }
     if (errno == EISDIR) {	/* Linux sets the wrong error code. */
 	errno = EPERM;
     }
-#endif
     return check_error(-1, errInfo);
 }
 
@@ -524,7 +479,7 @@ efile_rename(Efile_error* errInfo,	/* Where to return error codes. */
     if (errno == ENOTEMPTY) {
 	errno = EEXIST;
     }
-#if defined (sparc) && !defined(VXWORKS) && !defined(_OSE_)
+#if defined (sparc) && !defined(VXWORKS)
     /*
      * SunOS 4.1.4 reports overwriting a non-empty directory with a
      * directory as EINVAL instead of EEXIST (first rule out the correct
@@ -632,7 +587,8 @@ efile_readdir(Efile_error* errInfo,	/* Where to return error codes. */
 						   open directory.*/
 	      char* buffer,		/* Pointer to buffer for 
 					   one filename. */
-	      size_t size)		/* Size of buffer. */
+	      size_t *size)		/* in-out Size of buffer, length
+					   of name. */
 {
     DIR *dp;			/* Pointer to directory structure. */
     struct dirent* dirp;	/* Pointer to directory entry. */
@@ -664,7 +620,8 @@ efile_readdir(Efile_error* errInfo,	/* Where to return error codes. */
 	if (IS_DOT_OR_DOTDOT(dirp->d_name))
 	    continue;
 	buffer[0] = '\0';
-	strncat(buffer, dirp->d_name, size-1);
+	strncat(buffer, dirp->d_name, (*size)-1);
+	*size = strlen(dirp->d_name);
 	return 1;
     }
 }
@@ -751,6 +708,9 @@ efile_openfile(Efile_error* errInfo,	/* Where to return error codes. */
 #endif
     }
 
+    if (flags & EFILE_MODE_EXCL) {
+	mode |= O_EXCL;
+    }
 
 #ifdef VXWORKS
     if (*name != '/') {
@@ -819,6 +779,17 @@ efile_closefile(int fd)
 }
 
 int
+efile_fdatasync(Efile_error *errInfo, /* Where to return error codes. */
+	    int fd)               /* File descriptor for file to sync data. */
+{
+#ifdef HAVE_FDATASYNC
+    return check_error(fdatasync(fd), errInfo);
+#else
+    return efile_fsync(errInfo, fd);
+#endif
+}
+
+int
 efile_fsync(Efile_error *errInfo, /* Where to return error codes. */
 	    int fd)               /* File descriptor for file to sync. */
 {
@@ -855,7 +826,7 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
     CHECK_PATHLEN(name, errInfo);
 
     if (info_for_link) {
-#if (defined(VXWORKS) || defined(_OSE_))
+#if (defined(VXWORKS))
 	result = stat(name, &statbuf);
 #else
 	result = lstat(name, &statbuf);
@@ -939,11 +910,7 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
     pInfo->mode = statbuf.st_mode;
     pInfo->links = statbuf.st_nlink;
     pInfo->major_device = statbuf.st_dev;
-#ifdef _OSE_
-    pInfo->minor_device = 0;
-#else
     pInfo->minor_device = statbuf.st_rdev;
-#endif
     pInfo->inode = statbuf.st_ino;
     pInfo->uid = statbuf.st_uid;
     pInfo->gid = statbuf.st_gid;
@@ -989,11 +956,9 @@ efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name)
      * you don't try to chown a file to someone besides youself.
      */
 
-#ifndef _OSE_    
     if (chown(name, pInfo->uid, pInfo->gid) && errno != EPERM) {
 	return check_error(-1, errInfo);
     }
-#endif
 
     if (pInfo->mode != -1) {
 	mode_t newMode = pInfo->mode & (S_ISUID | S_ISGID |
@@ -1008,8 +973,6 @@ efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name)
 
 #endif /* !VXWORKS */
 
-#ifndef _OSE_
-
     if (pInfo->accessTime.year != -1 && pInfo->modifyTime.year != -1) {
 	struct utimbuf tval;
 	struct tm timebuf;
@@ -1041,7 +1004,6 @@ efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name)
 	return check_error(utime(name, &tval), errInfo);
 #endif
     }
-#endif /* !_OSE_ */
     return 1;
 }
 
@@ -1451,9 +1413,6 @@ efile_truncate_file(Efile_error* errInfo, int *fd, int flags)
 int
 efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size)
 {
-#ifdef _OSE_
-    return ose_enotsup(errInfo);
-#else
 #ifdef VXWORKS
     return vxworks_enotsup(errInfo);
 #else
@@ -1466,7 +1425,6 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size)
     buffer[len] = '\0';
     return 1;
 #endif
-#endif
 }
 
 int
@@ -1479,27 +1437,30 @@ efile_altname(Efile_error* errInfo, char* name, char* buffer, size_t size)
 int
 efile_link(Efile_error* errInfo, char* old, char* new)
 {
-#ifdef _OSE_
-    return ose_enotsup(errInfo);
-#else
 #ifdef VXWORKS
     return vxworks_enotsup(errInfo);
 #else
     return check_error(link(old, new), errInfo);
 #endif
-#endif
 }
 
 int
 efile_symlink(Efile_error* errInfo, char* old, char* new)
 {
-#ifdef _OSE_
-    return ose_enotsup(errInfo);
-#else
 #ifdef VXWORKS
     return vxworks_enotsup(errInfo);
 #else
     return check_error(symlink(old, new), errInfo);
 #endif
+}
+
+int
+efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset,
+		  Sint64 length, int advise)
+{
+#ifdef HAVE_POSIX_FADVISE
+    return check_error(posix_fadvise(fd, offset, length, advise), errInfo);
+#else
+    return check_error(0, errInfo);
 #endif
 }
diff --git a/erts/emulator/drivers/win32/mem_drv.c b/erts/emulator/drivers/win32/mem_drv.c
deleted file mode 100644
index fa7c46eca8..0000000000
--- a/erts/emulator/drivers/win32/mem_drv.c
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-/* Purpose: Access to elib memory statistics */
-
-#include "sys.h"
-#include "erl_driver.h"
-#include "elib_stat.h"
-
-#define MAP_BUF_SIZE    1000   /* Max map size */
-#define HISTO_BUF_SIZE  100    /* Max histogram buckets */
-
-static ErlDrvData mem_start(ErlDrvPort, char*);
-static int mem_init(void); 
-static void mem_stop(ErlDrvData);
-static void mem_command(ErlDrvData);
-
-ErlDrvEntry mem_driver_entry = {
-    mem_init,
-    mem_start,
-    mem_stop,
-    mem_command,
-    NULL,
-    NULL,
-    "mem_drv"
-};
-
-static int mem_init(void)
-{
-    return 0;
-}
-
-static ErlDrvData mem_start(ErlDrvPort port, char* buf)
-{
-    return (ErlDrvData)port;
-}
-
-static void mem_stop(ErlDrvData port)
-{
-}
-
-void putint32(p, v)
-byte* p; int v;
-{
-    p[0] = (v >> 24) & 0xff;
-    p[1] = (v >> 16) & 0xff;
-    p[2] = (v >> 8) & 0xff;
-    p[3] = (v) & 0xff;
-}
-
-int getint16(p)
-byte* p;
-{
-    return (p[0] << 8) | p[1];
-}
-
-/*
-** Command:
-**   m L1 L0  -> a heap map of length L1*256 + L0 is returned
-**   s        -> X3 X2 X1 X0 Y3 Y2 Y1 Y0 Z3 Z2 Z1 Z0
-**               X == Total heap size bytes
-**               Y == Total free bytes
-**               Z == Size of largest free block in bytes
-**
-**   h L1 L0 B0 -> Generate a logarithm histogram base B with L buckets
-**   l L1 L0 S0 -> Generate a linear histogram with step S with L buckets
-*/
-unsigned char outbuf[HISTO_BUF_SIZE*2*4];
-
-static void mem_command(ErlDrvData port, char* buf, int count)
-{
-    if ((count == 1) && buf[0] == 's') {
-	struct elib_stat info;
-	char v[3*4];
-
-	elib_stat(&info);
-
-	putint32(v, info.mem_total*4);
-	putint32(v+4, info.mem_free*4);
-	putint32(v+8, info.max_free*4);
-	driver_output((ErlDrvPort)port, v, 12);
-	return;
-    }
-    else if ((count == 3) && buf[0] == 'm') {
-	char w[MAP_BUF_SIZE];
-	int n = getint16(buf+1);
-
-	if (n > MAP_BUF_SIZE)
-	    n = MAP_BUF_SIZE;
-	elib_heap_map(w, n);
-	driver_output((ErlDrvPort)port, w, n);
-	return;
-    }
-    else if ((count == 4) && (buf[0] == 'h' || buf[0] == 'l')) {
-	unsigned long vf[HISTO_BUF_SIZE];
-	unsigned long va[HISTO_BUF_SIZE];
-	int n = getint16(buf+1);
-	int base = (unsigned char) buf[3];
-
-	if (n >= HISTO_BUF_SIZE)
-	    n = HISTO_BUF_SIZE;
-	if (buf[0] == 'l')
-	    base = -base;
-	if (elib_histo(vf, va, n, base) < 0) {
-	    driver_failure((ErlDrvPort)port, -1);
-	    return;
-	}
-	else {
-	    char* p = outbuf;
-	    int i;
-
-	    for (i = 0; i < n; i++) {
-		putint32(p, vf[i]);
-		p += 4;
-	    }
-	    for (i = 0; i < n; i++) {
-		putint32(p, va[i]);
-		p += 4;
-	    }
-	    driver_output((ErlDrvPort)port, outbuf, n*8);
-	}
-	return;
-    }
-    driver_failure((ErlDrvPort)port, -1);
-}
-
diff --git a/erts/emulator/drivers/win32/win_con.c b/erts/emulator/drivers/win32/win_con.c
index 2202ca655f..c788ad409d 100644
--- a/erts/emulator/drivers/win32/win_con.c
+++ b/erts/emulator/drivers/win32/win_con.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -704,6 +704,18 @@ FrameWndProc(HWND hwnd, UINT iMsg, WPARAM wParam, LPARAM lParam)
 	}
         write_inbuf(&c, 1);
 	return 0;
+    case WM_MOUSEWHEEL:
+      {
+	int delta = GET_WHEEL_DELTA_WPARAM(wParam);
+	if (delta < 0) {
+	  PostMessage(hClientWnd, WM_VSCROLL, MAKELONG(SB_THUMBTRACK,
+						       (iVscrollPos + 5)),0);
+	} else {
+	  WORD pos = ((iVscrollPos - 5) < 0) ? 0 : (iVscrollPos - 5);
+	  PostMessage(hClientWnd, WM_VSCROLL, MAKELONG(SB_THUMBTRACK,pos),0);
+	}
+      return 0;
+      }
     case WM_CHAR:
 	c = (TCHAR)wParam;
         write_inbuf(&c,1);
diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c
index 89aaad31da..3d59564f7b 100644..100755
--- a/erts/emulator/drivers/win32/win_efile.c
+++ b/erts/emulator/drivers/win32/win_efile.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -23,20 +23,20 @@
 #include <windows.h>
 #include "sys.h"
 #include <ctype.h>
-
+#include <wchar.h>
 #include "erl_efile.h"
 
 /*
  * Microsoft-specific function to map a WIN32 error code to a Posix errno.
  */
 
-#define ISSLASH(a)  ((a) == '\\' || (a) == '/')
+#define ISSLASH(a)  ((a) == L'\\' || (a) == L'/')
 
 #define ISDIR(st) (((st).st_mode&S_IFMT) == S_IFDIR)
 #define ISREG(st) (((st).st_mode&S_IFMT) == S_IFREG)
 
 #define IS_DOT_OR_DOTDOT(s) \
-    (s[0] == '.' && (s[1] == '\0' || (s[1] == '.' && s[2] == '\0')))
+    ((s)[0] == L'.' && ((s)[1] == L'\0' || ((s)[1] == L'.' && (s)[2] == L'\0')))
 
 #ifndef INVALID_FILE_ATTRIBUTES
 #define INVALID_FILE_ATTRIBUTES ((DWORD) 0xFFFFFFFF)
@@ -44,9 +44,9 @@
 
 static int check_error(int result, Efile_error* errInfo);
 static int set_error(Efile_error* errInfo);
-static int IsRootUNCName(const char* path);
-static int extract_root(char* name);
-static unsigned short dos_to_posix_mode(int attr, const char *name);
+static int is_root_unc_name(const WCHAR *path);
+static int extract_root(WCHAR *name);
+static unsigned short dos_to_posix_mode(int attr, const WCHAR *name);
 
 static int errno_map(DWORD last_error) {
 
@@ -196,27 +196,26 @@ win_writev(Efile_error* errInfo,
 
 
 int
-efile_mkdir(errInfo, name)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of directory to create. */
+efile_mkdir(Efile_error* errInfo,	/* Where to return error codes. */
+	    char* name)			/* Name of directory to create. */
 {
-    return check_error(mkdir(name), errInfo);
+    return check_error(_wmkdir((WCHAR *) name), errInfo);
 }
 
 int
-efile_rmdir(errInfo, name)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of directory to delete. */
+efile_rmdir(Efile_error* errInfo,	/* Where to return error codes. */
+	    char* name)			/* Name of directory to delete. */
 {
     OSVERSIONINFO os;
     DWORD attr;
+    WCHAR *wname = (WCHAR *) name;
 
-    if (RemoveDirectory(name) != FALSE) {
+    if (RemoveDirectoryW(wname) != FALSE) {
 	return 1;
     }
     errno = errno_map(GetLastError());
     if (errno == EACCES) {
-	attr = GetFileAttributes(name);
+	attr = GetFileAttributesW(wname);
 	if (attr != (DWORD) -1) {
 	    if ((attr & FILE_ATTRIBUTE_DIRECTORY) == 0) {
 		/* 
@@ -238,21 +237,21 @@ char* name;			/* Name of directory to delete. */
 	    GetVersionEx(&os);
 	    if (os.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) {
 		HANDLE handle;
-		WIN32_FIND_DATA data;
-		char buffer[2*MAX_PATH];
+		WIN32_FIND_DATAW data;
+		WCHAR buffer[2*MAX_PATH];
 		int len;
 
-		len = strlen(name);
-		strcpy(buffer, name);
-		if (buffer[0] && buffer[len-1] != '\\' && buffer[len-1] != '/') {
-		    strcat(buffer, "\\");
+		len = wcslen(wname);
+		wcscpy(buffer, wname);
+		if (buffer[0] && buffer[len-1] != L'\\' && buffer[len-1] != L'/') {
+		    wcscat(buffer, L"\\");
 		}
-		strcat(buffer, "*.*");
-		handle = FindFirstFile(buffer, &data);
+		wcscat(buffer, L"*.*");
+		handle = FindFirstFileW(buffer, &data);
 		if (handle != INVALID_HANDLE_VALUE) {
 		    while (1) {
-			if ((strcmp(data.cFileName, ".") != 0)
-				&& (strcmp(data.cFileName, "..") != 0)) {
+			if ((wcscmp(data.cFileName, L".") != 0)
+				&& (wcscmp(data.cFileName, L"..") != 0)) {
 			    /*
 			     * Found something in this directory.
 			     */
@@ -260,7 +259,7 @@ char* name;			/* Name of directory to delete. */
 			    errno = EEXIST;
 			    break;
 			}
-			if (FindNextFile(handle, &data) == FALSE) {
+			if (FindNextFileW(handle, &data) == FALSE) {
 			    break;
 			}
 		    }
@@ -284,19 +283,19 @@ char* name;			/* Name of directory to delete. */
 }
 
 int
-efile_delete_file(errInfo, name)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of file to delete. */
+efile_delete_file(Efile_error* errInfo,		/* Where to return error codes. */
+		  char* name)			/* Name of file to delete. */
 {
     DWORD attr;
+    WCHAR *wname = (WCHAR *) name;
 
-    if (DeleteFile(name) != FALSE) {
+    if (DeleteFileW(wname) != FALSE) {
 	return 1;
     }
 
     errno = errno_map(GetLastError());
     if (errno == EACCES) {
-        attr = GetFileAttributes(name);
+        attr = GetFileAttributesW(wname);
 	if (attr != (DWORD) -1) {
 	    if (attr & FILE_ATTRIBUTE_DIRECTORY) {
 		/*
@@ -308,7 +307,7 @@ char* name;			/* Name of file to delete. */
 	    }
 	}
     } else if (errno == ENOENT) {
-        attr = GetFileAttributes(name);
+        attr = GetFileAttributesW(wname);
 	if (attr != (DWORD) -1) {
 	    if (attr & FILE_ATTRIBUTE_DIRECTORY) {
 	    	/*
@@ -362,20 +361,21 @@ char* name;			/* Name of file to delete. */
  */
 
 int
-efile_rename(errInfo, src, dst)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* src;			/* Original name. */
-char* dst;			/* New name. */
+efile_rename(Efile_error* errInfo,	/* Where to return error codes. */
+	     char* src,			/* Original name. */
+	     char* dst)			/* New name. */
 {
     DWORD srcAttr, dstAttr;
+    WCHAR *wsrc = (WCHAR *) src;
+    WCHAR *wdst = (WCHAR *) dst;
     
-    if (MoveFile(src, dst) != FALSE) {
+    if (MoveFileW(wsrc, wdst) != FALSE) {
 	return 1;
     }
 
     errno = errno_map(GetLastError());
-    srcAttr = GetFileAttributes(src);
-    dstAttr = GetFileAttributes(dst);
+    srcAttr = GetFileAttributesW(wsrc);
+    dstAttr = GetFileAttributesW(wdst);
     if (srcAttr == (DWORD) -1) {
 	srcAttr = 0;
     }
@@ -390,22 +390,22 @@ char* dst;			/* New name. */
     if (errno == EACCES) {
 	decode:
 	if (srcAttr & FILE_ATTRIBUTE_DIRECTORY) {
-	    char srcPath[MAX_PATH], dstPath[MAX_PATH];
-	    char *srcRest, *dstRest;
+	    WCHAR srcPath[MAX_PATH], dstPath[MAX_PATH];
+	    WCHAR *srcRest, *dstRest;
 	    int size;
 
-	    size = GetFullPathName(src, sizeof(srcPath), srcPath, &srcRest);
-	    if ((size == 0) || (size > sizeof(srcPath))) {
+	    size = GetFullPathNameW(wsrc, MAX_PATH, srcPath, &srcRest);
+	    if ((size == 0) || (size > MAX_PATH)) {
 		return check_error(-1, errInfo);
 	    }
-	    size = GetFullPathName(dst, sizeof(dstPath), dstPath, &dstRest);
-	    if ((size == 0) || (size > sizeof(dstPath))) {
+	    size = GetFullPathNameW(wdst, MAX_PATH, dstPath, &dstRest);
+	    if ((size == 0) || (size > MAX_PATH)) {
 		return check_error(-1, errInfo);
 	    }
 	    if (srcRest == NULL) {
-		srcRest = srcPath + strlen(srcPath);
+		srcRest = srcPath + wcslen(srcPath);
 	    }
-	    if (strnicmp(srcPath, dstPath, srcRest - srcPath) == 0) {
+	    if (_wcsnicmp(srcPath, dstPath, srcRest - srcPath) == 0) {
 		/*
 		 * Trying to move a directory into itself.
 		 */
@@ -420,14 +420,14 @@ char* dst;			/* New name. */
 	    }
 
 	    (void) extract_root(dstPath);
-	    if (dstPath[0] == '\0') {
+	    if (dstPath[0] == L'\0') {
 		/*
 		 * The filename was invalid.  (Don't know why,
 		 * but play it safe.)
 		 */
 		errno = EINVAL;
 	    }
-	    if (stricmp(srcPath, dstPath) != 0) {
+	    if (_wcsicmp(srcPath, dstPath) != 0) {
 		/*
 		 * If src is a directory and dst filesystem != src
 		 * filesystem, errno should be EXDEV.  It is very
@@ -463,14 +463,14 @@ char* dst;			/* New name. */
 		 * fails, it's because it wasn't empty.
 		 */
 
-		if (RemoveDirectory(dst)) {
+		if (RemoveDirectoryW(wdst)) {
 		    /*
 		     * Now that that empty directory is gone, we can try
 		     * renaming again.  If that fails, we'll put this empty
 		     * directory back, for completeness.
 		     */
 
-		    if (MoveFile(src, dst) != FALSE) {
+		    if (MoveFileW(wsrc, wdst) != FALSE) {
 			return 1;
 		    }
 
@@ -480,8 +480,8 @@ char* dst;			/* New name. */
 		     */
 
 		    errno = errno_map(GetLastError());
-		    CreateDirectory(dst, NULL);
-		    SetFileAttributes(dst, dstAttr);
+		    CreateDirectoryW(wdst, NULL);
+		    SetFileAttributesW(wdst, dstAttr);
 		    if (errno == EACCES) {
 			/*
 			 * Decode the EACCES to a more meaningful error.
@@ -506,17 +506,17 @@ char* dst;			/* New name. */
 		 *    put temp file back to old name.
 		 */
 
-		char tempName[MAX_PATH];
+		WCHAR tempName[MAX_PATH];
 		int result, size;
-		char *rest;
+		WCHAR *rest;
 		
-		size = GetFullPathName(dst, sizeof(tempName), tempName, &rest);
-		if ((size == 0) || (size > sizeof(tempName)) || (rest == NULL)) {
+		size = GetFullPathNameW(wdst, MAX_PATH, tempName, &rest);
+		if ((size == 0) || (size > MAX_PATH) || (rest == NULL)) {
 		    return check_error(-1, errInfo);
 		}
-		*rest = '\0';
+		*rest = L'\0';
 		result = -1;
-		if (GetTempFileName(tempName, "erlr", 0, tempName) != 0) {
+		if (GetTempFileNameW(tempName, L"erlr", 0, tempName) != 0) {
 		    /*
 		     * Strictly speaking, need the following DeleteFile and
 		     * MoveFile to be joined as an atomic operation so no
@@ -524,15 +524,15 @@ char* dst;			/* New name. */
 		     * same temp file.
 		     */
 		     
-		    DeleteFile(tempName);
-		    if (MoveFile(dst, tempName) != FALSE) {
-			if (MoveFile(src, dst) != FALSE) {
-			    SetFileAttributes(tempName, FILE_ATTRIBUTE_NORMAL);
-			    DeleteFile(tempName);
+		    DeleteFileW(tempName);
+		    if (MoveFileW(wdst, tempName) != FALSE) {
+			if (MoveFileW(wsrc, wdst) != FALSE) {
+			    SetFileAttributesW(tempName, FILE_ATTRIBUTE_NORMAL);
+			    DeleteFileW(tempName);
 			    return 1;
 			} else {
-			    DeleteFile(dst);
-			    MoveFile(tempName, dst);
+			    DeleteFileW(wdst);
+			    MoveFileW(tempName, wdst);
 			}
 		    } 
 
@@ -558,11 +558,10 @@ char* dst;			/* New name. */
 }
 
 int
-efile_chdir(errInfo, name)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of directory to make current. */
+efile_chdir(Efile_error* errInfo,	/* Where to return error codes. */
+	    char* name)			/* Name of directory to make current. */
 {
-    int success = check_error(chdir(name), errInfo);
+    int success = check_error(_wchdir((WCHAR *) name), errInfo);
     if (!success && errInfo->posix_errno == EINVAL)
 	/* POSIXification of errno */
 	errInfo->posix_errno = ENOENT;
@@ -570,59 +569,65 @@ char* name;			/* Name of directory to make current. */
 }
 
 int
-efile_getdcwd(errInfo, drive, buffer, size)
-Efile_error* errInfo;		/* Where to return error codes. */
-int drive;			/* 0 - current, 1 - A, 2 - B etc. */
-char* buffer;			/* Where to return the current directory. */
-size_t size;			/* Size of buffer. */
+efile_getdcwd(Efile_error* errInfo,		/* Where to return error codes. */
+	      int drive,			/* 0 - current, 1 - A, 2 - B etc. */
+	      char* buffer,			/* Where to return the current directory. */
+	      size_t size)			/* Size of buffer. */
 {
-    if (_getdcwd(drive, buffer, size) == NULL)
+    WCHAR *wbuffer = (WCHAR *) buffer;
+    size_t wbuffer_size = size / 2; 
+    if (_wgetdcwd(drive, wbuffer, wbuffer_size) == NULL)
 	return check_error(-1, errInfo);
-    for ( ; *buffer; buffer++) 
-	if (*buffer == '\\')
-	    *buffer = '/';
+    for ( ; *wbuffer; wbuffer++) 
+	if (*wbuffer == L'\\')
+	    *wbuffer = L'/';
     return 1;
 }
 
 int
-efile_readdir(errInfo, name, dir_handle, buffer, size)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of directory to open. */
-EFILE_DIR_HANDLE* dir_handle;	/* Directory handle of open directory. */
-char* buffer;			/* Pointer to buffer for one filename. */
-size_t size;			/* Size of buffer. */
+efile_readdir(Efile_error* errInfo, /* Where to return error codes. */
+	      char* name,           /* Name of directory to list */
+	      EFILE_DIR_HANDLE* dir_handle, /* Handle of opened directory or NULL */
+	      char* buffer,                 /* Buffer to put one filename in */ 
+	      size_t *size)                 /* in-out size of buffer/size of filename excluding zero
+					       termination in bytes*/
 {
     HANDLE dir;			/* Handle to directory. */
-    char wildcard[MAX_PATH];	/* Wildcard to search for. */
-    WIN32_FIND_DATA findData;	/* Data found by FindFirstFile() or FindNext(). */
+    WCHAR wildcard[MAX_PATH];	/* Wildcard to search for. */
+    WIN32_FIND_DATAW findData;	/* Data found by FindFirstFile() or FindNext(). */
+    /* Alignment is not honored, this works on x86 because of alignment fixup by processor.
+       Not perfect, but faster than alinging by hand (really) */
+    WCHAR *wname = (WCHAR *) name;
+    WCHAR *wbuffer = (WCHAR *) buffer;
 
     /*
      * First time we must setup everything.
      */
 
     if (*dir_handle == NULL) {
-	int length = strlen(name);
-	char* s;
+	int length = wcslen(wname);
+	WCHAR* s;
 
 	if (length+3 >= MAX_PATH) {
 	    errno = ENAMETOOLONG;
 	    return check_error(-1, errInfo);
 	}
 
-	strcpy(wildcard, name);
+	wcscpy(wildcard, wname);
 	s = wildcard+length-1;
-	if (*s != '/' && *s != '\\')
-	    *++s = '\\';
-	*++s = '*';
-	*++s = '\0';
-	DEBUGF(("Reading %s\n", wildcard));
-	dir = FindFirstFile(wildcard, &findData);
+	if (*s != L'/' && *s != L'\\')
+	    *++s = L'\\';
+	*++s = L'*';
+	*++s = L'\0';
+	DEBUGF(("Reading %ws\n", wildcard));
+	dir = FindFirstFileW(wildcard, &findData);
 	if (dir == INVALID_HANDLE_VALUE)
 	    return set_error(errInfo);
 	*dir_handle = (EFILE_DIR_HANDLE) dir;
 
 	if (!IS_DOT_OR_DOTDOT(findData.cFileName)) {
-	    strcpy(buffer, findData.cFileName);
+	    wcscpy(wbuffer, findData.cFileName);
+	    *size = wcslen(wbuffer)*2;
 	    return 1;
 	}
     }
@@ -635,10 +640,11 @@ size_t size;			/* Size of buffer. */
     dir = (HANDLE) *dir_handle;
 
     for (;;) {
-	if (FindNextFile(dir, &findData)) {
+	if (FindNextFileW(dir, &findData)) {
 	    if (IS_DOT_OR_DOTDOT(findData.cFileName))
 		continue;
-	    strcpy(buffer, findData.cFileName);
+	    wcscpy(wbuffer, findData.cFileName);
+	    *size = wcslen(wbuffer)*2;
 	    return 1;
 	}
 
@@ -655,17 +661,17 @@ size_t size;			/* Size of buffer. */
 }
 
 int
-efile_openfile(errInfo, name, flags, pfd, pSize)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of directory to open. */
-int flags;			/* Flags to use for opening. */
-int* pfd;			/* Where to store the file descriptor. */
-Sint64* pSize;			/* Where to store the size of the file. */
+efile_openfile(Efile_error* errInfo,		/* Where to return error codes. */
+	       char* name,			/* Name of directory to open. */
+	       int flags,			/* Flags to use for opening. */
+	       int* pfd,			/* Where to store the file descriptor. */
+	       Sint64* pSize)			/* Where to store the size of the file. */
 {
     BY_HANDLE_FILE_INFORMATION fileInfo; /* File information from a handle. */
     HANDLE fd;			/* Handle to open file. */
     DWORD access;		/* Access mode: GENERIC_READ, GENERIC_WRITE. */
     DWORD crFlags;
+    WCHAR *wname = (WCHAR *) name;
 
     switch (flags & (EFILE_MODE_READ|EFILE_MODE_WRITE)) {
     case EFILE_MODE_READ:
@@ -689,7 +695,11 @@ Sint64* pSize;			/* Where to store the size of the file. */
     if (flags & EFILE_MODE_APPEND) {
 	crFlags = OPEN_ALWAYS;
     }
-    fd = CreateFile(name, access, FILE_SHARE_READ | FILE_SHARE_WRITE,
+    if (flags & EFILE_MODE_EXCL) {
+	crFlags = CREATE_NEW;
+    }
+    fd = CreateFileW(wname, access,
+		    FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
 		    NULL, crFlags, FILE_ATTRIBUTE_NORMAL, NULL);
 
     /*
@@ -707,7 +717,7 @@ Sint64* pSize;			/* Where to store the size of the file. */
 	 * to EISDIR.
 	 */
 	if (errInfo->posix_errno && 
-	    (attr = GetFileAttributes(name)) != INVALID_FILE_ATTRIBUTES && 
+	    (attr = GetFileAttributesW(wname)) != INVALID_FILE_ATTRIBUTES && 
 	    (attr & FILE_ATTRIBUTE_DIRECTORY)) {
 	    errInfo->posix_errno = EISDIR;
 	}
@@ -731,9 +741,10 @@ Sint64* pSize;			/* Where to store the size of the file. */
 
 int 
 efile_may_openfile(Efile_error* errInfo, char *name) {
+    WCHAR *wname = (WCHAR *) name;
     DWORD attr;
 
-    if ((attr = GetFileAttributes(name)) == INVALID_FILE_ATTRIBUTES) {
+    if ((attr = GetFileAttributesW(wname)) == INVALID_FILE_ATTRIBUTES) {
 	return check_error(-1, errInfo);
     }
 
@@ -742,18 +753,6 @@ efile_may_openfile(Efile_error* errInfo, char *name) {
 	return check_error(-1, errInfo);
     }
     return 1;
-#if 0
-    struct stat statbuf;
-    
-    if (stat(name, &statbuf)) {
-	return check_error(-1, errInfo);
-    }
-    if (ISDIR(statbuf)) {
-	errno = EISDIR;
-	return check_error(-1, errInfo);
-    }
-    return 1;
-#endif
 }
 
 void
@@ -764,6 +763,15 @@ int fd;				/* File descriptor for file to close. */
 }
 
 int
+efile_fdatasync(errInfo, fd)
+Efile_error* errInfo;		/* Where to return error codes. */
+int fd;				/* File descriptor for file to sync. */
+{
+    /* Not available in Windows, just call regular fsync */
+    return efile_fsync(errInfo, fd);
+}
+
+int
 efile_fsync(errInfo, fd)
 Efile_error* errInfo;		/* Where to return error codes. */
 int fd;				/* File descriptor for file to sync. */
@@ -779,16 +787,17 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
 	       char* orig_name, int info_for_link)
 {
     HANDLE findhandle;		/* Handle returned by FindFirstFile(). */
-    WIN32_FIND_DATA findbuf;	/* Data return by FindFirstFile(). */
-    char name[_MAX_PATH];
+    WIN32_FIND_DATAW findbuf;	/* Data return by FindFirstFile(). */
+    WCHAR name[_MAX_PATH];
     int name_len;
-    char* path;
-    char pathbuf[_MAX_PATH];
+    WCHAR *path;
+    WCHAR pathbuf[_MAX_PATH];
     int drive;			/* Drive for filename (1 = A:, 2 = B: etc). */
+    WCHAR *worig_name = (WCHAR *) orig_name; 
 
     /* Don't allow wildcards to be interpreted by system */
 
-    if (strpbrk(orig_name, "?*")) {
+    if (wcspbrk(worig_name, L"?*")) {
     enoent:
 	errInfo->posix_errno = ENOENT;
 	errInfo->os_errno = ERROR_FILE_NOT_FOUND;
@@ -800,25 +809,25 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
      * slash, because it causes FindFirstFile() to fail on Win95.
      */
 
-    if ((name_len = strlen(orig_name)) >= _MAX_PATH) {
+    if ((name_len = wcslen(worig_name)) >= _MAX_PATH) {
 	goto enoent;
     } else {
-	strcpy(name, orig_name);
+	wcscpy(name, worig_name);
 	if (name_len > 2 && ISSLASH(name[name_len-1]) &&
-	    name[name_len-2] != ':') {
-	    name[name_len-1] = '\0';
+	    name[name_len-2] != L':') {
+	    name[name_len-1] = L'\0';
 	}
     }
     
     /* Try to get disk from name.  If none, get current disk.  */
 
-    if (name[1] != ':') {
+    if (name[1] != L':') {
         drive = 0;
-        if (GetCurrentDirectory(sizeof(pathbuf), pathbuf) &&
-	    pathbuf[1] == ':') {
-	    drive = tolower(pathbuf[0]) - 'a' + 1;
+        if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) &&
+	    pathbuf[1] == L':') {
+	    drive = towlower(pathbuf[0]) - L'a' + 1;
 	}
-    } else if (*name && name[2] == '\0') {
+    } else if (*name && name[2] == L'\0') {
 	/*
 	 * X: and nothing more is an error.
 	 */
@@ -826,15 +835,15 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
 	errInfo->os_errno = ERROR_FILE_NOT_FOUND;
 	return 0;
     } else
-        drive = tolower(*name) - 'a' + 1;
+        drive = towlower(*name) - L'a' + 1;
 
-    findhandle = FindFirstFile(name, &findbuf);
+    findhandle = FindFirstFileW(name, &findbuf);
     if (findhandle == INVALID_HANDLE_VALUE) {
-        if (!(strpbrk(name, "./\\") &&
-	      (path = _fullpath(pathbuf, name, _MAX_PATH)) &&
+        if (!(wcspbrk(name, L"./\\") &&
+	      (path = _wfullpath(pathbuf, name, _MAX_PATH)) &&
 	      /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */
-	      ((strlen(path) == 3) || IsRootUNCName(path)) &&
-	      (GetDriveType(path) > 1)   ) ) {
+	      ((wcslen(path) == 3) || is_root_unc_name(path)) &&
+	      (GetDriveTypeW(path) > 1)   ) ) {
 	    errInfo->posix_errno = ENOENT;
 	    errInfo->os_errno = ERROR_FILE_NOT_FOUND;
 	    return 0;
@@ -847,8 +856,9 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
         findbuf.dwFileAttributes = FILE_ATTRIBUTE_DIRECTORY;
         findbuf.nFileSizeHigh = 0;
         findbuf.nFileSizeLow = 0;
-        findbuf.cFileName[0] = '\0';
+        findbuf.cFileName[0] = L'\0';
 
+	pInfo->links = 1;
 	pInfo->modifyTime.year = 1980;
 	pInfo->modifyTime.month = 1;
 	pInfo->modifyTime.day = 1;
@@ -861,6 +871,35 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
 	SYSTEMTIME SystemTime;
         FILETIME LocalFTime;
 
+	/*first check if we are a symlink */
+	if (!info_for_link && (findbuf.dwFileAttributes &
+			       FILE_ATTRIBUTE_REPARSE_POINT)){
+	    /*
+	     * given that we know this is a symlink,
+	     we should be able to find its target */
+	    WCHAR target_name[_MAX_PATH];
+	    if (efile_readlink(errInfo, (char *) name, 
+			       (char *) target_name,256) == 1) {
+		FindClose(findhandle);
+		return efile_fileinfo(errInfo, pInfo,
+				      (char *) target_name, info_for_link);
+	    }
+	}
+
+	/* number of links: */
+	{
+	    HANDLE handle;	/* Handle returned by CreateFile() */
+	    BY_HANDLE_FILE_INFORMATION fileInfo; /* from  CreateFile() */
+	    if (handle = CreateFileW(name, GENERIC_READ, 0,NULL,
+				    OPEN_EXISTING, 0, NULL)) {
+		GetFileInformationByHandle(handle, &fileInfo);
+		pInfo->links = fileInfo.nNumberOfLinks;
+		CloseHandle(handle);
+	    } else {
+		pInfo->links = 1;
+	    }	
+	}
+
 #define GET_TIME(dst, src) \
 if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \
     !FileTimeToSystemTime(&LocalFTime, &SystemTime)) { \
@@ -895,7 +934,10 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \
     pInfo->size_low = findbuf.nFileSizeLow;
     pInfo->size_high = findbuf.nFileSizeHigh;
 	
-    if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+    if (info_for_link && (findbuf.dwFileAttributes &
+			  FILE_ATTRIBUTE_REPARSE_POINT))
+	pInfo->type = FT_SYMLINK;
+    else if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
 	pInfo->type = FT_DIRECTORY;
     else
 	pInfo->type = FT_REGULAR;
@@ -906,7 +948,6 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \
 	pInfo->access = FA_READ|FA_WRITE;
 
     pInfo->mode = dos_to_posix_mode(findbuf.dwFileAttributes, name);
-    pInfo->links = 1;
     pInfo->major_device = drive;
     pInfo->minor_device = 0;
     pInfo->inode = 0;
@@ -917,10 +958,9 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \
 }
 
 int
-efile_write_info(errInfo, pInfo, name)
-Efile_error* errInfo;
-Efile_info* pInfo;
-char* name;
+efile_write_info(Efile_error* errInfo,
+		 Efile_info* pInfo,
+		 char* name)
 {
     SYSTEMTIME timebuf;
     FILETIME LocalFileTime;
@@ -934,12 +974,13 @@ char* name;
     DWORD attr;
     DWORD tempAttr;
     BOOL modifyTime = FALSE;
+    WCHAR *wname = (WCHAR *) name;
 
     /*
      * Get the attributes for the file.
      */
 
-    tempAttr = attr = GetFileAttributes((LPTSTR)name);
+    tempAttr = attr = GetFileAttributesW(wname);
     if (attr == 0xffffffff) {
 	return set_error(errInfo);
     }
@@ -975,8 +1016,8 @@ char* name;
      } \
     }
 
-    MKTIME(ModifyFileTime, pInfo->accessTime, mtime);
-    MKTIME(AccessFileTime, pInfo->modifyTime, atime);
+    MKTIME(ModifyFileTime, pInfo->modifyTime, mtime);
+    MKTIME(AccessFileTime, pInfo->accessTime, atime);
     MKTIME(CreationFileTime, pInfo->cTime, ctime);
 #undef MKTIME
 
@@ -993,12 +1034,12 @@ char* name;
 
 	if (tempAttr & FILE_ATTRIBUTE_READONLY) {
 	    tempAttr &= ~FILE_ATTRIBUTE_READONLY;
-	    if (!SetFileAttributes((LPTSTR) name, tempAttr)) {
+	    if (!SetFileAttributesW(wname, tempAttr)) {
 		return set_error(errInfo);
 	    }
 	}
 
-	fd = CreateFile(name, GENERIC_READ|GENERIC_WRITE,
+	fd = CreateFileW(wname, GENERIC_READ|GENERIC_WRITE,
 			FILE_SHARE_READ | FILE_SHARE_WRITE,
 			NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
 	if (fd != INVALID_HANDLE_VALUE) {
@@ -1016,7 +1057,7 @@ char* name;
      */
 
     if (tempAttr != attr) {
-	if (!SetFileAttributes((LPTSTR) name, attr)) {
+	if (!SetFileAttributesW(wname, attr)) {
 	    return set_error(errInfo);
 	}
     }
@@ -1069,12 +1110,17 @@ char* buf;			/* Buffer to write. */
 size_t count;			/* Number of bytes to write. */
 {
     DWORD written;		/* Bytes written in last operation. */
+    OVERLAPPED overlapped;
+    OVERLAPPED* pOverlapped = NULL;
 
     if (flags & EFILE_MODE_APPEND) {
-	(void) SetFilePointer((HANDLE) fd, 0, NULL, FILE_END);
+	memset(&overlapped, 0, sizeof(overlapped));
+	overlapped.Offset = 0xffffffff;
+	overlapped.OffsetHigh = 0xffffffff;
+	pOverlapped = &overlapped;
     }
     while (count > 0) {
-	if (!WriteFile((HANDLE) fd, buf, count, &written, NULL))
+	if (!WriteFile((HANDLE) fd, buf, count, &written, pOverlapped))
 	    return set_error(errInfo);
 	buf += written;
 	count -= written;
@@ -1094,11 +1140,16 @@ efile_writev(Efile_error* errInfo,   /* Where to return error codes */
 	     size_t size)            /* Number of bytes to write */
 {
     int cnt;                         /* Buffers so far written */
+    OVERLAPPED overlapped;
+    OVERLAPPED* pOverlapped = NULL;
 
     ASSERT(iovcnt >= 0);
     
     if (flags & EFILE_MODE_APPEND) {
-	(void) SetFilePointer((HANDLE) fd, 0, NULL, FILE_END);
+	memset(&overlapped, 0, sizeof(overlapped));
+	overlapped.Offset = 0xffffffff;
+	overlapped.OffsetHigh = 0xffffffff;
+	pOverlapped = &overlapped;
     }
     for (cnt = 0; cnt < iovcnt; cnt++) {
 	if (iov[cnt].iov_base && iov[cnt].iov_len > 0) {
@@ -1110,7 +1161,7 @@ efile_writev(Efile_error* errInfo,   /* Where to return error codes */
 			       iov[cnt].iov_base + p, 
 			       iov[cnt].iov_len - p, 
 			       &w, 
-			       NULL))
+			       pOverlapped))
 		    return set_error(errInfo);
 	    }
 	}
@@ -1182,7 +1233,7 @@ int flags;
 
 
 /*
- * IsRootUNCName - returns TRUE if the argument is a UNC name specifying
+ * is_root_unc_name - returns TRUE if the argument is a UNC name specifying
  *      a root share.  That is, if it is of the form \\server\share\.
  *      This routine will also return true if the argument is of the
  *      form \\server\share (no trailing slash) but Win32 currently
@@ -1192,16 +1243,16 @@ int flags;
  */
 
 static int
-IsRootUNCName(const char* path)
+is_root_unc_name(const WCHAR *path)
 {
     /*
      * If a root UNC name, path will start with 2 (but not 3) slashes
      */
 
-    if ((strlen(path) >= 5) /* minimum string is "//x/y" */
+    if ((wcslen(path) >= 5) /* minimum string is "//x/y" */
 	&& ISSLASH(path[0]) && ISSLASH(path[1]))
     {
-        const char * p = path + 2 ;
+        const WCHAR *p = path + 2;
 
         /*
          * find the slash between the server name and share name
@@ -1244,19 +1295,19 @@ IsRootUNCName(const char* path)
  */
 
 static int
-extract_root(char* name)
+extract_root(WCHAR* name)
 {
-    int len = strlen(name);
+    int len = wcslen(name);
 
-    if (isalpha(name[0]) && name[1] == ':' && ISSLASH(name[2])) {
-	int c = name[3];
-	name[3] = '\0';
-	return c == '\0';
+    if (iswalpha(name[0]) && name[1] == L':' && ISSLASH(name[2])) {
+	WCHAR c = name[3];
+	name[3] = L'\0';
+	return c == L'\0';
     } else if (len < 5 || !ISSLASH(name[0]) || !ISSLASH(name[1])) {
 	goto error;
     } else {			/* Try to find the end of the UNC name. */
-	char* p;
-	int c;
+	WCHAR* p;
+	WCHAR c;
 
         /*
          * Find the slash between the server name and share name.
@@ -1265,7 +1316,7 @@ extract_root(char* name)
 	for (p = name + 2; *p; p++)
             if (ISSLASH(*p))
                 break;
-	if (*p == '\0')
+	if (*p == L'\0')
 	    goto error;
 
 	/*
@@ -1276,24 +1327,24 @@ extract_root(char* name)
             if (ISSLASH(*p))
                 break;
 	c = *p;
-	*p = '\0';
-	return c == '\0' || p[1] == '\0';
+	*p = L'\0';
+	return c == L'\0' || p[1] == L'\0';
     }
 
  error:
-    *name = '\0';
+    *name = L'\0';
     return 1;
 }
 
 static unsigned short
-dos_to_posix_mode(int attr, const char *name)
+dos_to_posix_mode(int attr, const WCHAR *name)
 {
     register unsigned short uxmode;
     unsigned dosmode;
-    register const char *p;
+    register const WCHAR *p;
 
     dosmode = attr & 0xff;
-    if ((p = name)[1] == ':')
+    if ((p = name)[1] == L':')
 	p += 2;
 
     /* check to see if this is a directory - note we must make a special
@@ -1302,7 +1353,7 @@ dos_to_posix_mode(int attr, const char *name)
 
     uxmode = (unsigned short)
 	(((ISSLASH(*p) && !p[1]) || (dosmode & FILE_ATTRIBUTE_DIRECTORY) ||
-	       *p == '\0') ? _S_IFDIR|_S_IEXEC : _S_IFREG);
+	       *p == L'\0') ? _S_IFDIR|_S_IEXEC : _S_IFREG);
 
     /* If attribute byte does not have read-only bit, it is read-write */
 
@@ -1311,11 +1362,11 @@ dos_to_posix_mode(int attr, const char *name)
 
     /* see if file appears to be executable - check extension of name */
 
-    if (p = strrchr(name, '.')) {
-        if (!stricmp(p, ".exe") ||
-	    !stricmp(p, ".cmd") ||
-	    !stricmp(p, ".bat") ||
-	    !stricmp(p, ".com"))
+    if (p = wcsrchr(name, L'.')) {
+        if (!_wcsicmp(p, L".exe") ||
+	    !_wcsicmp(p, L".cmd") ||
+	    !_wcsicmp(p, L".bat") ||
+	    !_wcsicmp(p, L".com"))
             uxmode |= _S_IEXEC;
     }
 
@@ -1330,6 +1381,60 @@ dos_to_posix_mode(int attr, const char *name)
 int
 efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size)
 {
+    /*
+     * load dll and see if we have CreateSymbolicLink at runtime:
+     * (Vista only)
+     */
+    HINSTANCE hModule = NULL;
+    WCHAR *wname = (WCHAR *) name;
+    WCHAR *wbuffer = (WCHAR *) buffer;
+    if ((hModule = LoadLibrary("kernel32.dll")) != NULL) {
+	typedef DWORD (WINAPI * GETFINALPATHNAMEBYHANDLEPTR)(
+							     HANDLE hFile,
+							     LPCWSTR lpFilePath,
+							     DWORD cchFilePath,
+							     DWORD dwFlags);
+
+	GETFINALPATHNAMEBYHANDLEPTR pGetFinalPathNameByHandle =
+	    (GETFINALPATHNAMEBYHANDLEPTR)GetProcAddress(hModule, "GetFinalPathNameByHandleW");
+
+	if (pGetFinalPathNameByHandle == NULL) {
+	    FreeLibrary(hModule);
+	} else {
+	    /* first check if file is a symlink; {error, einval} otherwise */
+	    DWORD fileAttributes =  GetFileAttributesW(wname);
+	    if ((fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
+		BOOLEAN success = 0;
+		HANDLE h = CreateFileW(wname, GENERIC_READ, 0,NULL, OPEN_EXISTING, 0, NULL);
+		int len;
+		if(h != INVALID_HANDLE_VALUE) {
+		    success = pGetFinalPathNameByHandle(h, wbuffer, size,0);
+		    /* GetFinalPathNameByHandle prepends path with "\\?\": */
+		    len = wcslen(wbuffer);
+		    wmemmove(wbuffer,wbuffer+4,len-3);
+		    if (len - 4 >= 2 && wbuffer[1] == L':' && wbuffer[0] >= L'A' &&
+			wbuffer[0] <= L'Z') {
+			wbuffer[0] = wbuffer[0] + L'a' - L'A';
+		    }
+			
+		    for ( ; *wbuffer; wbuffer++) 
+			if (*wbuffer == L'\\')
+			    *wbuffer = L'/';
+		    CloseHandle(h);
+		}
+		FreeLibrary(hModule);
+		if (success) {
+		    return 1;
+		} else {
+		    return set_error(errInfo);
+		}
+	    } else {
+		FreeLibrary(hModule);
+		errno = EINVAL;
+		return check_error(-1, errInfo);
+	    }
+	}
+    }
     errno = ENOTSUP;
     return check_error(-1, errInfo);
 }
@@ -1338,17 +1443,20 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size)
 int
 efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size)
 {
-    WIN32_FIND_DATA wfd;
+    WIN32_FIND_DATAW wfd;
     HANDLE fh;
-    char name[_MAX_PATH];
+    WCHAR name[_MAX_PATH+1];
     int name_len;
-    char* path;
-    char pathbuf[_MAX_PATH];
+    WCHAR* path;
+    WCHAR pathbuf[_MAX_PATH+1]; /* Unclear weather GetCurrentDirectory will access one char after
+				   _MAX_PATH */
+    WCHAR *worig_name = (WCHAR *) orig_name;
+    WCHAR *wbuffer = (WCHAR *) buffer;
     int drive;			/* Drive for filename (1 = A:, 2 = B: etc). */
 
     /* Don't allow wildcards to be interpreted by system */
 
-    if (strpbrk(orig_name, "?*")) {
+    if (wcspbrk(worig_name, L"?*")) {
     enoent:
 	errInfo->posix_errno = ENOENT;
 	errInfo->os_errno = ERROR_FILE_NOT_FOUND;
@@ -1360,67 +1468,114 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size)
      * slash, because it causes FindFirstFile() to fail on Win95.
      */
 
-    if ((name_len = strlen(orig_name)) >= _MAX_PATH) {
+    if ((name_len = wcslen(worig_name)) >= _MAX_PATH) {
 	goto enoent;
     } else {
-	strcpy(name, orig_name);
+	wcscpy(name, worig_name);
 	if (name_len > 2 && ISSLASH(name[name_len-1]) &&
-	    name[name_len-2] != ':') {
-	    name[name_len-1] = '\0';
+	    name[name_len-2] != L':') {
+	    name[name_len-1] = L'\0';
 	}
     }
     
     /* Try to get disk from name.  If none, get current disk.  */
 
-    if (name[1] != ':') {
+    if (name[1] != L':') {
         drive = 0;
-        if (GetCurrentDirectory(sizeof(pathbuf), pathbuf) &&
-	    pathbuf[1] == ':') {
-	    drive = tolower(pathbuf[0]) - 'a' + 1;
+        if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) &&
+	    pathbuf[1] == L':') {
+	    drive = towlower(pathbuf[0]) - L'a' + 1;
 	}
-    } else if (*name && name[2] == '\0') {
+    } else if (*name && name[2] == L'\0') {
 	/*
 	 * X: and nothing more is an error.
 	 */
 	goto enoent;
     } else {
-        drive = tolower(*name) - 'a' + 1;
+        drive = towlower(*name) - L'a' + 1;
     }
-    fh = FindFirstFile(name,&wfd);
+    fh = FindFirstFileW(name,&wfd);
     if (fh == INVALID_HANDLE_VALUE) {
-        if (!(strpbrk(name, "./\\") &&
-	      (path = _fullpath(pathbuf, name, _MAX_PATH)) &&
+        if (!(wcspbrk(name, L"./\\") &&
+	      (path = _wfullpath(pathbuf, name, _MAX_PATH)) &&
 	      /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */
-	      ((strlen(path) == 3) || IsRootUNCName(path)) &&
-	      (GetDriveType(path) > 1)   ) ) {
+	      ((wcslen(path) == 3) || is_root_unc_name(path)) &&
+	      (GetDriveTypeW(path) > 1)   ) ) {
 	    errno = errno_map(GetLastError());
 	    return check_error(-1, errInfo);
 	}
         /*
          * Root directories (such as C:\ or \\server\share\ are fabricated.
          */
-	strcpy(buffer,name);
+	wcscpy(wbuffer,name);
 	return 1;
     }
 	
-    strcpy(buffer,wfd.cAlternateFileName);
-    if (!*buffer) {
-	strcpy(buffer,wfd.cFileName);
+    wcscpy(wbuffer,wfd.cAlternateFileName);
+    if (!*wbuffer) {
+	wcscpy(wbuffer,wfd.cFileName);
     }
-
+    FindClose(fh);
     return 1;
 }
 
+
 int
 efile_link(Efile_error* errInfo, char* old, char* new)
 {
-    errno = ENOTSUP;
-    return check_error(-1, errInfo);
+    WCHAR *wold = (WCHAR *) old;
+    WCHAR *wnew = (WCHAR *) new;
+    if(!CreateHardLinkW(wnew, wold, NULL)) {
+	return set_error(errInfo);
+    }
+    return 1;
 }
 
 int
 efile_symlink(Efile_error* errInfo, char* old, char* new)
 {
+    /*
+     * Load dll and see if we have CreateSymbolicLink at runtime:
+     * (Vista only)
+     */
+    HINSTANCE hModule = NULL;
+    WCHAR *wold = (WCHAR *) old;
+    WCHAR *wnew = (WCHAR *) new;
+    if ((hModule = LoadLibrary("kernel32.dll")) != NULL) {
+	typedef BOOLEAN (WINAPI  * CREATESYMBOLICLINKFUNCPTR) (
+	     LPCWSTR lpSymlinkFileName,
+	     LPCWSTR lpTargetFileName,
+	     DWORD dwFlags);
+
+	CREATESYMBOLICLINKFUNCPTR pCreateSymbolicLink =
+	    (CREATESYMBOLICLINKFUNCPTR) GetProcAddress(hModule,
+						       "CreateSymbolicLinkW");
+	/* A for MBCS, W for UNICODE... char* above implies 'W'! */
+	if (pCreateSymbolicLink != NULL) {
+	    DWORD attr = GetFileAttributesW(wold);
+	    int flag = (attr != INVALID_FILE_ATTRIBUTES &&
+			attr & FILE_ATTRIBUTE_DIRECTORY) ? 1 : 0;
+	    /*  SYMBOLIC_LINK_FLAG_DIRECTORY = 1 */
+	    BOOLEAN success = pCreateSymbolicLink(wnew, wold, flag);
+	    FreeLibrary(hModule);
+
+	    if (success) {
+		return 1;
+	    } else {
+		return set_error(errInfo);
+	    }
+	} else
+	    FreeLibrary(hModule);
+    }
     errno = ENOTSUP;
     return check_error(-1, errInfo);
 }
+
+int
+efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset,
+	    Sint64 length, int advise)
+{
+    /* posix_fadvise is not available on Windows, do nothing */
+    errno = ERROR_SUCCESS;
+    return check_error(0, errInfo);
+}
diff --git a/erts/emulator/hipe/hipe_abi.txt b/erts/emulator/hipe/hipe_abi.txt
index aea30d262d..d0ec162342 100644
--- a/erts/emulator/hipe/hipe_abi.txt
+++ b/erts/emulator/hipe/hipe_abi.txt
@@ -2,7 +2,7 @@
  %CopyrightBegin%
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE ABI
 ========
diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index ff87492f4d..b5dff06987 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include <stddef.h>	/* offsetof() */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -334,43 +334,3 @@ void hipe_arch_print_pcb(struct hipe_process_state *p)
     U("narity     ", narity);
 #undef U
 }
-
-/*
- * XXX: The following should really be moved to a generic hipe_bifs_64 file.
- */
-
-#if 0 /* unused */
-static int term_to_Sint64(Eterm term, Sint64 *sp)
-{
-    return term_to_Sint(term, sp);
-}
-
-BIF_RETTYPE hipe_bifs_write_s64_2(BIF_ALIST_2)
-{
-    Sint64 *address;
-    Sint64 value;
-
-    address = term_to_address(BIF_ARG_1);
-    if (!address || !hipe_word64_address_ok(address))
-	BIF_ERROR(BIF_P, BADARG);
-    if (!term_to_Sint64(BIF_ARG_2, &value))
-	BIF_ERROR(BIF_P, BADARG);
-    *address = value;
-    BIF_RET(NIL);
-}
-#endif
-
-BIF_RETTYPE hipe_bifs_write_u64_2(BIF_ALIST_2)
-{
-    Uint64 *address;
-    Uint64 value;
-
-    address = term_to_address(BIF_ARG_1);
-    if (!address || !hipe_word64_address_ok(address))
-	BIF_ERROR(BIF_P, BADARG);
-    if (!term_to_Uint(BIF_ARG_2, &value))
-	BIF_ERROR(BIF_P, BADARG);
-    *address = value;
-    hipe_flush_icache_word(address);
-    BIF_RET(NIL);
-}
diff --git a/erts/emulator/hipe/hipe_amd64.h b/erts/emulator/hipe/hipe_amd64.h
index 532d47c092..bf41d238dd 100644
--- a/erts/emulator/hipe/hipe_amd64.h
+++ b/erts/emulator/hipe/hipe_amd64.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_AMD64_H
 #define HIPE_AMD64_H
 
diff --git a/erts/emulator/hipe/hipe_amd64.tab b/erts/emulator/hipe/hipe_amd64.tab
index 3787bbf23b..e039d74525 100644
--- a/erts/emulator/hipe/hipe_amd64.tab
+++ b/erts/emulator/hipe/hipe_amd64.tab
@@ -1,28 +1,24 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2004-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
 # AMD64-specific atoms and bifs
 
 atom amd64
 atom handle_fp_exception
 atom inc_stack_0
 atom sse2_fnegate_mask
-
-# bif hipe_bifs:write_s64/2
-bif hipe_bifs:write_u64/2
diff --git a/erts/emulator/hipe/hipe_amd64_abi.txt b/erts/emulator/hipe/hipe_amd64_abi.txt
index 27beff4ea2..8a34bfa67f 100644
--- a/erts/emulator/hipe/hipe_amd64_abi.txt
+++ b/erts/emulator/hipe/hipe_amd64_abi.txt
@@ -2,7 +2,7 @@
  %CopyrightBegin%
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE AMD64 ABI
 ==============
diff --git a/erts/emulator/hipe/hipe_amd64_asm.m4 b/erts/emulator/hipe/hipe_amd64_asm.m4
index 7f563c35d8..7c81040b8b 100644
--- a/erts/emulator/hipe/hipe_amd64_asm.m4
+++ b/erts/emulator/hipe/hipe_amd64_asm.m4
@@ -2,7 +2,7 @@ changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -17,9 +17,8 @@ changecom(`/*', `*/')dnl
  *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 `#ifndef HIPE_AMD64_ASM_H
 #define HIPE_AMD64_ASM_H'
 
diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index f7c9604e2b..0ba763cbea 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -2,7 +2,7 @@ changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -17,9 +17,7 @@ changecom(`/*', `*/')dnl
  *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 include(`hipe/hipe_amd64_asm.m4')
 #`include' "hipe_literals.h"
diff --git a/erts/emulator/hipe/hipe_amd64_gc.h b/erts/emulator/hipe/hipe_amd64_gc.h
index 56650901d6..c5a6fee6fe 100644
--- a/erts/emulator/hipe/hipe_amd64_gc.h
+++ b/erts/emulator/hipe/hipe_amd64_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  */
 #ifndef HIPE_AMD64_GC_H
diff --git a/erts/emulator/hipe/hipe_amd64_glue.S b/erts/emulator/hipe/hipe_amd64_glue.S
index 83b7b0397b..8816906870 100644
--- a/erts/emulator/hipe/hipe_amd64_glue.S
+++ b/erts/emulator/hipe/hipe_amd64_glue.S
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -16,9 +16,7 @@
  *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 #include "hipe_amd64_asm.h"
 #include "hipe_literals.h"
@@ -346,6 +344,8 @@ nbif_3_gc_after_bif:
 	subq	$(16-8), %rsp
 	movq	P, %rdi
 	movq	%rax, %rsi
+	xorl	%edx, %edx		# Pass NULL in regs
+	xorl	%ecx, %ecx		# Pass 0 in arity
 	call	CSYM(erts_gc_after_bif_call)
 	addq	$(16-8), %rsp
 	movl	$0, P_NARITY(P)		# Note: narity is a 32-bit field
@@ -400,7 +400,7 @@ nbif_3_simple_exception:
 	 * - the native heap/stack/reds registers are saved in P
 	 */
 .handle_trap:
-	movq	%rax, P_NARITY(P)
+	movl	%eax, P_NARITY(P)	# Note: narity is a 32-bit field
 	movl	$HIPE_MODE_SWITCH_RES_TRAP, %eax
 	jmp	.nosave_exit
 
diff --git a/erts/emulator/hipe/hipe_amd64_glue.h b/erts/emulator/hipe/hipe_amd64_glue.h
index c92eb842cb..36508467fa 100644
--- a/erts/emulator/hipe/hipe_amd64_glue.h
+++ b/erts/emulator/hipe/hipe_amd64_glue.h
@@ -1,24 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 #ifndef HIPE_AMD64_GLUE_H
 #define HIPE_AMD64_GLUE_H
 
diff --git a/erts/emulator/hipe/hipe_amd64_primops.h b/erts/emulator/hipe/hipe_amd64_primops.h
index dcfa8be92a..e3c7111997 100644
--- a/erts/emulator/hipe/hipe_amd64_primops.h
+++ b/erts/emulator/hipe/hipe_amd64_primops.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 PRIMOP_LIST(am_inc_stack_0, &nbif_inc_stack_0)
 PRIMOP_LIST(am_handle_fp_exception, &nbif_handle_fp_exception)
 PRIMOP_LIST(am_sse2_fnegate_mask, &sse2_fnegate_mask)
diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h
index 7803543ef1..04ed980126 100644
--- a/erts/emulator/hipe/hipe_arch.h
+++ b/erts/emulator/hipe/hipe_arch.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_ARCH_H
 #define HIPE_ARCH_H
 
diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c
index b70b32947b..d52f429a9b 100644
--- a/erts/emulator/hipe/hipe_arm.c
+++ b/erts/emulator/hipe/hipe_arm.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include <stddef.h>	/* offsetof() */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
diff --git a/erts/emulator/hipe/hipe_arm.h b/erts/emulator/hipe/hipe_arm.h
index 84f58a681f..19f2a986cf 100644
--- a/erts/emulator/hipe/hipe_arm.h
+++ b/erts/emulator/hipe/hipe_arm.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_ARM_H
 #define HIPE_ARM_H
 
diff --git a/erts/emulator/hipe/hipe_arm.tab b/erts/emulator/hipe/hipe_arm.tab
index 81626796a7..49b89d6748 100644
--- a/erts/emulator/hipe/hipe_arm.tab
+++ b/erts/emulator/hipe/hipe_arm.tab
@@ -1,22 +1,22 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2005-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2005-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
+
 # ARM-specific atoms and bifs
 
 atom arm
diff --git a/erts/emulator/hipe/hipe_arm_abi.txt b/erts/emulator/hipe/hipe_arm_abi.txt
index 6868704d62..6778ff6663 100644
--- a/erts/emulator/hipe/hipe_arm_abi.txt
+++ b/erts/emulator/hipe/hipe_arm_abi.txt
@@ -2,7 +2,7 @@
  %CopyrightBegin%
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE ARM ABI
 ================
diff --git a/erts/emulator/hipe/hipe_arm_asm.m4 b/erts/emulator/hipe/hipe_arm_asm.m4
index b9a696ffff..85dc84973d 100644
--- a/erts/emulator/hipe/hipe_arm_asm.m4
+++ b/erts/emulator/hipe/hipe_arm_asm.m4
@@ -1,25 +1,24 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 `#ifndef HIPE_ARM_ASM_H
 #define HIPE_ARM_ASM_H'
 
diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4
index 4d8636e711..3664fb6502 100644
--- a/erts/emulator/hipe/hipe_arm_bifs.m4
+++ b/erts/emulator/hipe/hipe_arm_bifs.m4
@@ -1,25 +1,23 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 include(`hipe/hipe_arm_asm.m4')
 #`include' "hipe_literals.h"
diff --git a/erts/emulator/hipe/hipe_arm_gc.h b/erts/emulator/hipe/hipe_arm_gc.h
index a2a919e3d7..787c6fef3e 100644
--- a/erts/emulator/hipe/hipe_arm_gc.h
+++ b/erts/emulator/hipe/hipe_arm_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  * ARM version.
  */
diff --git a/erts/emulator/hipe/hipe_arm_glue.S b/erts/emulator/hipe/hipe_arm_glue.S
index 5d626a5f69..2e2b8604a6 100644
--- a/erts/emulator/hipe/hipe_arm_glue.S
+++ b/erts/emulator/hipe/hipe_arm_glue.S
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include "hipe_arm_asm.h"
 #include "hipe_literals.h"
 #define ASM
@@ -311,6 +311,8 @@ nbif_3_gc_after_bif:
 	str	TEMP_LR, [P, #P_NRA]
 	str	NSP, [P, #P_NSP]
 	mov	TEMP_LR, lr
+	mov	r3, #0			/* Pass 0 in arity */
+	mov	r2, #0			/* Pass NULL in regs */
 	mov	r1, r0
 	mov	r0, P
 	bl	erts_gc_after_bif_call
diff --git a/erts/emulator/hipe/hipe_arm_glue.h b/erts/emulator/hipe/hipe_arm_glue.h
index e840c3dc0f..165f73320d 100644
--- a/erts/emulator/hipe/hipe_arm_glue.h
+++ b/erts/emulator/hipe/hipe_arm_glue.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_ARM_GLUE_H
 #define HIPE_ARM_GLUE_H
 
diff --git a/erts/emulator/hipe/hipe_arm_primops.h b/erts/emulator/hipe/hipe_arm_primops.h
index a28b509eee..2a1a87b862 100644
--- a/erts/emulator/hipe/hipe_arm_primops.h
+++ b/erts/emulator/hipe/hipe_arm_primops.h
@@ -1,21 +1,21 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 PRIMOP_LIST(am_inc_stack_0, &hipe_arm_inc_stack)
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index b0abfd2310..e7fb850530 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -440,59 +440,23 @@ BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2)
 	 align != sizeof(long) && align != sizeof(double)))
 	BIF_ERROR(BIF_P, BADARG);
     nrbytes = unsigned_val(BIF_ARG_2);
+    if (nrbytes == 0)
+	BIF_RET(make_small(0));
     block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes);
     if ((unsigned long)block & (align-1))
-	fprintf(stderr, "Yikes! erts_alloc() returned misaligned address %p\r\n", block);
+	fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n",
+		__FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align);
     BIF_RET(address_to_term(block, BIF_P));
 }
 
 /*
- * Memory area for constant Erlang terms.
- *
- * These constants must not be forwarded by the gc.
- * Therefore, the gc needs to be able to distinguish between
- * collectible objects and constants. Unfortunately, an Erlang
- * process' collectible objects are scattered around in two
- * heaps and a list of message buffers, so testing "is X a
- * collectible object?" can be expensive.
- *
- * Instead, constants are placed in a single contiguous area,
- * which allows for an inexpensive "is X a constant?" test.
- *
- * XXX: Allow this area to be grown.
+ * Statistics on hipe constants: size of HiPE constants, in words.
  */
-
-/* not static, needed by garbage collector */
-Eterm *hipe_constants_start = NULL;
-Eterm *hipe_constants_next = NULL;
-static unsigned constants_avail_words = 0;
-#define CONSTANTS_BYTES	(1536*1024*sizeof(Eterm))  /* 1.5 M words */
-
-static Eterm *constants_alloc(unsigned nwords)
-{
-    Eterm *next;
-
-    /* initialise at the first call */
-    if ((next = hipe_constants_next) == NULL) {
-	next = (Eterm*)erts_alloc(ERTS_ALC_T_HIPE, CONSTANTS_BYTES);
-	hipe_constants_start = next;
-	hipe_constants_next = next;
-	constants_avail_words = CONSTANTS_BYTES / sizeof(Eterm);
-    }
-    if (nwords > constants_avail_words) {
-	fprintf(stderr, "Native code constants pool depleted!\r\n");
-	/* Must terminate immediately. erl_exit() seems to
-	   continue running some code which then SIGSEGVs. */
-	exit(1);
-    }
-    constants_avail_words -= nwords;
-    hipe_constants_next = next + nwords;
-    return next;
-}
+unsigned int hipe_constants_size = 0;
 
 BIF_RETTYPE hipe_bifs_constants_size_0(BIF_ALIST_0)
 {
-    BIF_RET(make_small(hipe_constants_next - hipe_constants_start));
+    BIF_RET(make_small(hipe_constants_size));
 }
 
 /*
@@ -523,14 +487,17 @@ static void *const_term_alloc(void *tmpl)
 {
     Eterm obj;
     Uint size;
+    Uint alloc_size;
     Eterm *hp;
     struct const_term *p;
 
     obj = (Eterm)tmpl;
     ASSERT(is_not_immed(obj));
     size = size_object(obj);
+    alloc_size = size + (offsetof(struct const_term, mem)/sizeof(Eterm));
+    hipe_constants_size += alloc_size;
 
-    p = (struct const_term*)constants_alloc(size + (offsetof(struct const_term, mem)/sizeof(Eterm)));
+    p = (struct const_term*)erts_alloc(ERTS_ALC_T_HIPE, alloc_size * sizeof(Eterm));
 
     /* I have absolutely no idea if having a private 'off_heap'
        works or not. _Some_ off_heap object is required for
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index ed27d5616a..f02e8862dc 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -1,22 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_bif0.h
  *
  * Compiler and linker support.
@@ -26,10 +27,6 @@
 
 extern Uint *hipe_bifs_find_pc_from_mfa(Eterm mfa);
 
-/* shared with ggc.c -- NOT an official API */
-extern Eterm *hipe_constants_start;
-extern Eterm *hipe_constants_next;
-
 extern void hipe_mfa_info_table_init(void);
 extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a);
 extern Eterm hipe_find_na_or_make_stub(Process*, Eterm, Eterm, Eterm);
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 46c0a3d67d..b6c6bede23 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -1,22 +1,22 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2001-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2001-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
+#
 # HiPE level 0 bifs: compiler and linker support
 #
 # bif hipe_bifs:name/arity
diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c
index 5188950e17..87cdfb8c7a 100644
--- a/erts/emulator/hipe/hipe_bif1.c
+++ b/erts/emulator/hipe/hipe_bif1.c
@@ -1,22 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_bif1.c
  *
  * Performance analysis support.
@@ -876,22 +877,44 @@ BIF_RETTYPE hipe_bifs_misc_timer_clear_0(BIF_ALIST_0)
  * + The fallback, which is the same as {X,_} = runtime(statistics).
  */
 
+static double fallback_get_hrvtime(void)
+{
+    unsigned long ms_user;
+
+    elapsed_time_both(&ms_user, NULL, NULL, NULL);
+    return (double)ms_user;
+}
+
 #if USE_PERFCTR
 
 #include "hipe_perfctr.h"
-static int hrvtime_is_open;
-#define hrvtime_is_started()	hrvtime_is_open
+static int hrvtime_started;	/* 0: closed, +1: perfctr, -1: fallback */
+#define hrvtime_is_started()	(hrvtime_started != 0)
 
 static void start_hrvtime(void)
 {
     if (hipe_perfctr_hrvtime_open() >= 0)
-	hrvtime_is_open = 1;
+	hrvtime_started = 1;
+    else
+	hrvtime_started = -1;
+}
+
+static void stop_hrvtime(void)
+{
+    if (hrvtime_started > 0)
+	hipe_perfctr_hrvtime_close();
+    hrvtime_started = 0;
 }
 
-#define get_hrvtime()		hipe_perfctr_hrvtime_get()
-#define stop_hrvtime()		hipe_perfctr_hrvtime_close()
+static double get_hrvtime(void)
+{
+    if (hrvtime_started > 0)
+	return hipe_perfctr_hrvtime_get();
+    else
+	return fallback_get_hrvtime();
+}
 
-#else
+#else	/* !USE_PERFCTR */
 
 /*
  * Fallback, if nothing better exists.
@@ -902,15 +925,9 @@ static void start_hrvtime(void)
 #define hrvtime_is_started()	1
 #define start_hrvtime()		do{}while(0)
 #define stop_hrvtime()		do{}while(0)
+#define get_hrvtime()		fallback_get_hrvtime()
 
-static double get_hrvtime(void)
-{
-    unsigned long ms_user;
-    elapsed_time_both(&ms_user, NULL, NULL, NULL);
-    return (double)ms_user;
-}
-
-#endif	/* hrvtime support */
+#endif	/* !USE_PERFCTR */
 
 BIF_RETTYPE hipe_bifs_get_hrvtime_0(BIF_ALIST_0)
 {
@@ -918,11 +935,8 @@ BIF_RETTYPE hipe_bifs_get_hrvtime_0(BIF_ALIST_0)
     Eterm res;
     FloatDef f;
 
-    if (!hrvtime_is_started()) {
+    if (!hrvtime_is_started())
 	start_hrvtime();
-	if (!hrvtime_is_started())
-	    BIF_RET(NIL); /* arity 0 BIFs may not fail */
-    }
     f.fd = get_hrvtime();
     hp = HAlloc(BIF_P, FLOAT_SIZE_OBJECT);
     res = make_float(hp);
diff --git a/erts/emulator/hipe/hipe_bif1.h b/erts/emulator/hipe/hipe_bif1.h
index c3b607565d..89241fb835 100644
--- a/erts/emulator/hipe/hipe_bif1.h
+++ b/erts/emulator/hipe/hipe_bif1.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_bif1.h
  *
  * Performance analysis support.
diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c
index f992b758be..2660f74a82 100644
--- a/erts/emulator/hipe/hipe_bif2.c
+++ b/erts/emulator/hipe/hipe_bif2.c
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_bif2.c
  *
  * Miscellaneous add-ons.
@@ -33,18 +33,28 @@
 #include "big.h"
 #include "hipe_debug.h"
 #include "hipe_mode_switch.h"
-#include "hipe_bif0.h" /* hipe_constants_{start,next} */
 #include "hipe_arch.h"
 #include "hipe_stack.h"
 
-BIF_RETTYPE hipe_bifs_show_estack_1(BIF_ALIST_1)
+static void proc_unlock(Process* c_p, Process* rp)
 {
+    ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL;
+    if (rp == c_p) {
+	locks &= ~ERTS_PROC_LOCK_MAIN;
+    }
+    if (rp && locks) {
+	erts_smp_proc_unlock(rp, locks);
+    }
+}
+
+BIF_RETTYPE hipe_bifs_show_estack_1(BIF_ALIST_1)
+{    
     Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
 				BIF_ARG_1, ERTS_PROC_LOCKS_ALL);
     if (!rp)
 	BIF_ERROR(BIF_P, BADARG);
     hipe_print_estack(rp);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
+    proc_unlock(BIF_P, rp);
     BIF_RET(am_true);
 }
 
@@ -55,7 +65,7 @@ BIF_RETTYPE hipe_bifs_show_heap_1(BIF_ALIST_1)
     if (!rp)
 	BIF_ERROR(BIF_P, BADARG);
     hipe_print_heap(rp);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
+    proc_unlock(BIF_P, rp);
     BIF_RET(am_true);
 }
 
@@ -66,7 +76,7 @@ BIF_RETTYPE hipe_bifs_show_nstack_1(BIF_ALIST_1)
     if (!rp)
 	BIF_ERROR(BIF_P, BADARG);
     hipe_print_nstack(rp);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
+    proc_unlock(BIF_P, rp);
     BIF_RET(am_true);
 }
 
@@ -82,7 +92,7 @@ BIF_RETTYPE hipe_bifs_show_pcb_1(BIF_ALIST_1)
     if (!rp)
 	BIF_ERROR(BIF_P, BADARG);
     hipe_print_pcb(rp);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
+    proc_unlock(BIF_P, rp);
     BIF_RET(am_true);
 }
 
@@ -124,18 +134,6 @@ BIF_RETTYPE hipe_bifs_show_term_1(BIF_ALIST_1)
     BIF_RET(am_true);
 }
 
-BIF_RETTYPE hipe_bifs_show_literals_0(BIF_ALIST_0)
-{
-    Eterm *p;
-
-    p = hipe_constants_start;
-    for (; p < hipe_constants_next; ++p)
-	printf("0x%0*lx: 0x%0*lx\r\n",
-	       2*(int)sizeof(long), (unsigned long)p,
-	       2*(int)sizeof(long), *p);
-    BIF_RET(am_true);
-}
-
 BIF_RETTYPE hipe_bifs_in_native_0(BIF_ALIST_0)
 {
     BIF_RET(am_false);
diff --git a/erts/emulator/hipe/hipe_bif2.tab b/erts/emulator/hipe/hipe_bif2.tab
index d8d627e370..51323ce7af 100644
--- a/erts/emulator/hipe/hipe_bif2.tab
+++ b/erts/emulator/hipe/hipe_bif2.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2001-2009. All Rights Reserved.
+# Copyright Ericsson AB 2001-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -26,7 +26,6 @@ bif hipe_bifs:show_nstack/1
 bif hipe_bifs:nstack_used_size/0
 bif hipe_bifs:show_pcb/1
 bif hipe_bifs:show_term/1
-bif hipe_bifs:show_literals/0
 bif hipe_bifs:in_native/0
 bif hipe_bifs:modeswitch_debug_on/0
 bif hipe_bifs:modeswitch_debug_off/0
diff --git a/erts/emulator/hipe/hipe_bif64.c b/erts/emulator/hipe/hipe_bif64.c
new file mode 100644
index 0000000000..baaf5af2cd
--- /dev/null
+++ b/erts/emulator/hipe/hipe_bif64.c
@@ -0,0 +1,68 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/* hipe_bif_64.c
+ *
+ * Compiler and linker support. 64-bit specific.
+ */
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "global.h"
+#include "error.h"
+#include "bif.h"
+#include "big.h"	/* term_to_Sint() */
+#include "hipe_arch.h"
+#include "hipe_bif0.h"
+#include "hipe_bif64.h"
+
+#if 0 /* unused */
+static int term_to_Sint64(Eterm term, Sint64 *sp)
+{
+    return term_to_Sint(term, sp);
+}
+
+BIF_RETTYPE hipe_bifs_write_s64_2(BIF_ALIST_2)
+{
+    Sint64 *address;
+    Sint64 value;
+
+    address = term_to_address(BIF_ARG_1);
+    if (!address || !hipe_word64_address_ok(address))
+	BIF_ERROR(BIF_P, BADARG);
+    if (!term_to_Sint64(BIF_ARG_2, &value))
+	BIF_ERROR(BIF_P, BADARG);
+    *address = value;
+    BIF_RET(NIL);
+}
+#endif
+
+BIF_RETTYPE hipe_bifs_write_u64_2(BIF_ALIST_2)
+{
+    Uint64 *address;
+    Uint64 value;
+
+    address = term_to_address(BIF_ARG_1);
+    if (!address || !hipe_word64_address_ok(address))
+	BIF_ERROR(BIF_P, BADARG);
+    if (!term_to_Uint(BIF_ARG_2, &value))
+	BIF_ERROR(BIF_P, BADARG);
+    *address = value;
+    hipe_flush_icache_word(address);
+    BIF_RET(NIL);
+}
diff --git a/erts/emulator/hipe/hipe_bif64.h b/erts/emulator/hipe/hipe_bif64.h
new file mode 100644
index 0000000000..6d494886ec
--- /dev/null
+++ b/erts/emulator/hipe/hipe_bif64.h
@@ -0,0 +1,26 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/* hipe_bif64.c
+ *
+ * Compiler and linker support. 64-bit specific.
+ */
+#ifndef HIPE_BIF64_H
+#define HIPE_BIF64_H
+
+#endif /* HIPE_BIF64_H */
diff --git a/erts/emulator/hipe/hipe_bif64.tab b/erts/emulator/hipe/hipe_bif64.tab
new file mode 100644
index 0000000000..228318af39
--- /dev/null
+++ b/erts/emulator/hipe/hipe_bif64.tab
@@ -0,0 +1,22 @@
+#
+# %CopyrightBegin%
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+#
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+#
+# %CopyrightEnd%
+#
+# HiPE 64-bit specific bifs
+
+# bif hipe_bifs:write_s64/2
+bif hipe_bifs:write_u64/2
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index c92d94ed9d..083788997b 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  *
  * List all non architecture-specific BIFs and primops, and
  * classify each as belonging to one of the classes below.
diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index 548998b7b7..7ca11f8c6c 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_debug.c
  *
  * TODO:
@@ -51,7 +51,7 @@ static const char stars[2*sizeof(long)+5] = {
 
 extern Uint beam_apply[];
 
-static void print_beam_pc(Uint *pc)
+static void print_beam_pc(BeamInstr *pc)
 {
     if (pc == hipe_beam_pc_return) {
 	printf("return-to-native");
@@ -60,7 +60,7 @@ static void print_beam_pc(Uint *pc)
     } else if (pc == &beam_apply[1]) {
 	printf("normal-process-exit");
     } else {
-	Eterm *mfa = find_function_from_pc(pc);
+	BeamInstr *mfa = find_function_from_pc(pc);
 	if (mfa)
 	    erts_printf("%T:%T/%bpu + 0x%bpx",
 			mfa[0], mfa[1], mfa[2], pc - &mfa[3]);
@@ -71,7 +71,7 @@ static void print_beam_pc(Uint *pc)
 
 static void catch_slot(Eterm *pos, Eterm val)
 {
-    Uint *pc = catch_pc(val);
+    BeamInstr *pc = catch_pc(val);
     printf(" | 0x%0*lx | 0x%0*lx | CATCH 0x%0*lx (BEAM ",
 	   2*(int)sizeof(long), (unsigned long)pos,
 	   2*(int)sizeof(long), (unsigned long)val,
diff --git a/erts/emulator/hipe/hipe_debug.h b/erts/emulator/hipe/hipe_debug.h
index 3980bc8230..a28597000a 100644
--- a/erts/emulator/hipe/hipe_debug.h
+++ b/erts/emulator/hipe/hipe_debug.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_debug.h
  */
 #ifndef HIPE_DEBUG_H
diff --git a/erts/emulator/hipe/hipe_gbif_list.h b/erts/emulator/hipe/hipe_gbif_list.h
index 659f74b5e5..69dbab7ab9 100644
--- a/erts/emulator/hipe/hipe_gbif_list.h
+++ b/erts/emulator/hipe/hipe_gbif_list.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * GBIF_LIST(FunctionAtom,Arity,CFun)
  * manually maintained for now -- expand when necessary
  */
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index e57e293547..0199dea99e 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * GC support procedures
  */
 #ifdef HAVE_CONFIG_H
@@ -28,7 +28,6 @@
 
 #include "hipe_stack.h"
 #include "hipe_gc.h"
-#include "hipe_bif0.h"		/* for hipe_constants_{start,next} */
 
 Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 {
@@ -86,7 +85,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 		if (is_boxed(gval)) {
 		    Eterm *ptr = boxed_val(gval);
 		    Eterm val = *ptr;
-		    if (IS_MOVED(val)) {
+		    if (IS_MOVED_BOXED(val)) {
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
 		    } else if (in_area(ptr, src, src_size) ||
@@ -96,7 +95,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 		} else if (is_list(gval)) {
 		    Eterm *ptr = list_val(gval);
 		    Eterm val = *ptr;
-		    if (is_non_value(val)) {
+		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
 		    } else if (in_area(ptr, src, src_size) ||
 			       in_area(ptr, oh, oh_size)) {
@@ -193,7 +192,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 		if (is_boxed(gval)) {
 		    Eterm *ptr = boxed_val(gval);
 		    Eterm val = *ptr;
-		    if (IS_MOVED(val)) {
+		    if (IS_MOVED_BOXED(val)) {
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
 		    } else if (in_area(ptr, heap, mature_size)) {
@@ -205,7 +204,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 		} else if (is_list(gval)) {
 		    Eterm *ptr = list_val(gval);
 		    Eterm val = *ptr;
-		    if (is_non_value(val)) {
+		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
 		    } else if (in_area(ptr, heap, mature_size)) {
 			MOVE_CONS(ptr, val, old_htop, nsp_i);
diff --git a/erts/emulator/hipe/hipe_gc.h b/erts/emulator/hipe/hipe_gc.h
index 712d0ffa78..0d5614c9cf 100644
--- a/erts/emulator/hipe/hipe_gc.h
+++ b/erts/emulator/hipe/hipe_gc.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_GC_H
 #define HIPE_GC_H
 
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index a77aec7919..bced90785d 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -1,24 +1,24 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
@@ -261,7 +261,7 @@ static const struct literal {
     /* Field offsets in a process struct */
     { "P_HP", offsetof(struct process, htop) },
     { "P_HP_LIMIT", offsetof(struct process, stop) },
-    { "P_OFF_HEAP_MSO", offsetof(struct process, off_heap.mso) },
+    { "P_OFF_HEAP_FIRST", offsetof(struct process, off_heap.first) },
     { "P_MBUF", offsetof(struct process, mbuf) },
     { "P_ID", offsetof(struct process, id) },
     { "P_FLAGS", offsetof(struct process, flags) },
@@ -456,7 +456,7 @@ static const struct rts_param {
 } rts_params[] = {
     { 1, "P_OFF_HEAP_FUNS",
 #if !defined(HYBRID)
-      1, offsetof(struct process, off_heap.funs)
+      1, offsetof(struct process, off_heap.first)
 #endif
     },
 
@@ -587,9 +587,9 @@ static void print_params(FILE *fp, void (*print_param)(FILE*,const struct rts_pa
 	(*print_param)(fp, &rts_params[i]);
 }
 
-static int do_c(FILE *fp)
+static int do_c(FILE *fp, const char* this_exe)
 {
-    fprintf(fp, "/* File: hipe_literals.h, generated by hipe_mkliterals */\n");
+    fprintf(fp, "/* File: hipe_literals.h, generated by %s */\n", this_exe);
     fprintf(fp, "#ifndef __HIPE_LITERALS_H__\n");
     fprintf(fp, "#define __HIPE_LITERALS_H__\n\n");
     print_literals(fp, c_define_literal);
@@ -603,9 +603,9 @@ static int do_c(FILE *fp)
     return 0;
 }
 
-static int do_e(FILE *fp)
+static int do_e(FILE *fp, const char* this_exe)
 {
-    fprintf(fp, "%%%% File: hipe_literals.hrl, generated by hipe_mkliterals");
+    fprintf(fp, "%%%% File: hipe_literals.hrl, generated by %s", this_exe);
     fprintf(fp, "\n\n");
     print_literals(fp, e_define_literal);
     fprintf(fp, "\n");
@@ -622,9 +622,9 @@ int main(int argc, const char **argv)
     compute_crc();
     if (argc == 2) {
 	if (strcmp(argv[1], "-c") == 0)
-	    return do_c(stdout);
+	    return do_c(stdout, argv[0]);
 	if (strcmp(argv[1], "-e") == 0)
-	    return do_e(stdout);
+	    return do_e(stdout, argv[0]);
     }
     fprintf(stderr, "usage: %s [-c | -e] > output-file\n", argv[0]);
     return 1;
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index e5de244d25..16f8fb1347 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -1,22 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_mode_switch.c
  */
 #ifdef HAVE_CONFIG_H
@@ -208,6 +209,8 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 #endif
 
     p->i = NULL;
+    /* Set current_function to undefined. stdlib hibernate tests rely on it. */
+    p->current = NULL;
 
     DPRINTF("cmd == %#x (%s)", cmd, code_str(cmd));
     HIPE_CHECK_PCB(p);
@@ -322,20 +325,31 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 	   * We need to remove the BIF's parameters from the native
 	   * stack: to this end hipe_${ARCH}_glue.S stores the BIF's
 	   * arity in p->hipe.narity.
+	   *
+	   * If the BIF emptied the stack (typically hibernate), p->hipe.nsp is
+	   * NULL and there is no need to get rid of stacked parameters.
 	   */
-	  unsigned int i, is_recursive, callee_arity;
+	  unsigned int i, is_recursive = 0;
 
 	  /* Save p->arity, then update it with the original BIF's arity.
 	     Get rid of any stacked parameters in that call. */
 	  /* XXX: hipe_call_from_native_is_recursive() copies data to
 	     reg[], which is useless in the TRAP case. Maybe write a
 	     specialised hipe_trap_from_native_is_recursive() later. */
-	  callee_arity = p->arity;
-	  p->arity = p->hipe.narity; /* caller's arity */
-	  is_recursive = hipe_call_from_native_is_recursive(p, reg);
-
-	  p->i = (Eterm *)(p->def_arg_reg[3]);
-	  p->arity = callee_arity;
+          if (p->hipe.nsp != NULL) {
+	      unsigned int callee_arity;
+	      callee_arity = p->arity;
+	      p->arity = p->hipe.narity; /* caller's arity */
+	      is_recursive = hipe_call_from_native_is_recursive(p, reg);
+
+	      p->i = (Eterm *)(p->def_arg_reg[3]);
+	      p->arity = callee_arity;
+          }
+
+	  /* If process is in P_WAITING state, we schedule the next process */
+	  if (p->status == P_WAITING) {
+	      goto do_schedule;
+	  }
 
 	  for (i = 0; i < p->arity; ++i)
 	      reg[i] = p->def_arg_reg[i];
@@ -592,6 +606,17 @@ void hipe_inc_nstack(Process *p)
 }
 #endif
 
+void hipe_empty_nstack(Process *p)
+{
+    if (p->hipe.nstack) {
+	erts_free(ERTS_ALC_T_HIPE, p->hipe.nstack);
+    }
+    p->hipe.nstgraylim = NULL;
+    p->hipe.nsp = NULL;
+    p->hipe.nstack = NULL;
+    p->hipe.nstend = NULL;
+}
+
 static void hipe_check_nstack(Process *p, unsigned nwords)
 {
     while (hipe_nstack_avail(p) < nwords)
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index 187b9145e2..dbc2386e14 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -1,22 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_mode_switch.h
  */
 #ifndef HIPE_MODE_SWITCH_H
@@ -54,6 +55,7 @@ void hipe_mode_switch_init(void);
 void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure);
 Process *hipe_mode_switch(Process*, unsigned, Eterm*);
 void hipe_inc_nstack(Process *p);
+void hipe_empty_nstack(Process *p);
 void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free);
 Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s);
 
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index f8c2502522..8d31348496 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_native_bif.c
  */
 #ifdef HAVE_CONFIG_H
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index 3b55b64a41..13a02b84a2 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_native_bif.h
  */
  
diff --git a/erts/emulator/hipe/hipe_ops.tab b/erts/emulator/hipe/hipe_ops.tab
index eb6f824d1c..50c3a4ae2f 100644
--- a/erts/emulator/hipe/hipe_ops.tab
+++ b/erts/emulator/hipe/hipe_ops.tab
@@ -1,19 +1,19 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2001-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2001-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 
diff --git a/erts/emulator/hipe/hipe_perfctr.c b/erts/emulator/hipe/hipe_perfctr.c
index 69bb648854..371b3fb097 100644
--- a/erts/emulator/hipe/hipe_perfctr.c
+++ b/erts/emulator/hipe/hipe_perfctr.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
diff --git a/erts/emulator/hipe/hipe_perfctr.h b/erts/emulator/hipe/hipe_perfctr.h
index 7b20c68cac..8fbf9ecf35 100644
--- a/erts/emulator/hipe/hipe_perfctr.h
+++ b/erts/emulator/hipe/hipe_perfctr.h
@@ -1,23 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
 
 extern int hipe_perfctr_hrvtime_open(void);
 extern void hipe_perfctr_hrvtime_close(void);
diff --git a/erts/emulator/hipe/hipe_perfctr.tab b/erts/emulator/hipe/hipe_perfctr.tab
index 663522f85e..eaecea4651 100644
--- a/erts/emulator/hipe/hipe_perfctr.tab
+++ b/erts/emulator/hipe/hipe_perfctr.tab
@@ -1,22 +1,21 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2004-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
 
 bif hipe_bifs:vperfctr_open/0
 bif hipe_bifs:vperfctr_close/0
diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c
index 3a0beedb68..bc25061a16 100644
--- a/erts/emulator/hipe/hipe_ppc.c
+++ b/erts/emulator/hipe/hipe_ppc.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include <stddef.h>	/* offsetof() */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -87,48 +87,6 @@ static struct segment {
 #define MAP_ANONYMOUS MAP_ANON
 #endif
 
-#if defined(__powerpc64__)
-static void *new_code_mapping(void)
-{
-    char *map_hint, *map_start;
-
-    /*
-     * Allocate a new 32MB code segment in the low 2GB of the address space.
-     *
-     * This is problematic for several reasons:
-     * - Linux/ppc64 lacks the MAP_32BIT flag that Linux/x86-64 has.
-     * - The address space hint to mmap is only respected if that
-     *   area is available. If it isn't, then mmap falls back to its
-     *   defaults, which (according to testing) results in very high
-     *   (and thus useless for us) addresses being returned.
-     * - Another mapping, presumably the brk, also occupies low addresses.
-     *
-     * As initial implementation, simply start allocating at the 0.5GB
-     * boundary. This leaves plenty of space for the brk before malloc
-     * needs to switch to mmap, while allowing for 1.5GB of code.
-     *
-     * A more robust implementation would be to parse /proc/self/maps,
-     * reserve all available space between (say) 0.5GB and 2GB with
-     * PROT_NONE MAP_NORESERVE mappings, and then allocate by releasing
-     * 32MB segments and re-mapping them properly. This would work on
-     * Linux/ppc64, I have no idea how things should be done on Darwin64.
-     */
-    if (curseg.base)
-	map_hint = (char*)curseg.base + SEGMENT_NRBYTES;
-    else
-	map_hint = (char*)(512*1024*1024); /* 0.5GB */
-    map_start = mmap(map_hint, SEGMENT_NRBYTES,
-		     PROT_EXEC|PROT_READ|PROT_WRITE,
-		     MAP_PRIVATE|MAP_ANONYMOUS,
-		     -1, 0);
-    if (map_start != MAP_FAILED &&
-	(((unsigned long)map_start + (SEGMENT_NRBYTES-1)) & ~0x7FFFFFFFUL)) {
-	fprintf(stderr, "mmap with hint %p returned code memory %p\r\n", map_hint, map_start);
-	abort();
-    }
-    return map_start;
-}
-#else
 static void *new_code_mapping(void)
 {
     return mmap(0, SEGMENT_NRBYTES,
@@ -136,7 +94,6 @@ static void *new_code_mapping(void)
 		MAP_PRIVATE|MAP_ANONYMOUS,
 		-1, 0);
 }
-#endif
 
 static int check_callees(Eterm callees)
 {
@@ -182,20 +139,30 @@ static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsig
 	unsigned int a = unsigned_val(tuple_val(mfa)[3]);
 	unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a);
 	if (!in_area(trampoline, base, SEGMENT_NRBYTES)) {
+#if defined(__powerpc64__)
+	    if (nrfreewords < 7)
+		return NULL;
+	    nrfreewords -= 7;
+	    tramp_pos = trampoline = tramp_pos - 7;
+	    trampoline[0] = 0x3D600000; /* addis r11,r0,0 */
+	    trampoline[1] = 0x616B0000; /* ori r11,r11,0 */
+	    trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */
+	    trampoline[3] = 0x656B0000; /* oris r11,r11,0 */
+	    trampoline[4] = 0x616B0000; /* ori r11,r11,0 */
+	    trampoline[5] = 0x7D6903A6; /* mtctr r11 */
+	    trampoline[6] = 0x4E800420; /* bctr */
+	    hipe_flush_icache_range(trampoline, 7*sizeof(int));
+#else
 	    if (nrfreewords < 4)
 		return NULL;
 	    nrfreewords -= 4;
 	    tramp_pos = trampoline = tramp_pos - 4;
-#if defined(__powerpc64__)
-	    trampoline[0] = 0x3D600000; /* addis r11,0,0 */
-	    trampoline[1] = 0x616B0000; /* ori r11,r11,0 */
-#else
 	    trampoline[0] = 0x39600000; /* addi r11,r0,0 */
 	    trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */
-#endif
 	    trampoline[2] = 0x7D6903A6; /* mtctr r11 */
 	    trampoline[3] = 0x4E800420; /* bctr */
 	    hipe_flush_icache_range(trampoline, 4*sizeof(int));
+#endif
 	    hipe_mfa_set_trampoline(m, f, a, trampoline);
 	}
 	trampvec[trampnr-1] = trampoline;
@@ -281,21 +248,22 @@ static void patch_imm16(Uint32 *address, unsigned int imm16)
 }
 
 #if defined(__powerpc64__)
+/*
+ * To load a 64-bit immediate value 'val' into Rd (Rd != R0):
+ *
+ * addis Rd, 0, val@highest // (val >> 48) & 0xFFFF
+ * ori Rd, Rd, val@higher   // (val >> 32) & 0xFFFF
+ * rldicr Rd, Rd, 32, 31
+ * oris Rd, Rd, val@h       // (val >> 16) & 0xFFFF
+ * ori Rd, Rd, val@l        // val & 0xFFFF
+ */
 static void patch_li64(Uint32 *address, Uint64 value)
 {
-    patch_imm16(address+0, value >> 48);/* addis r,0,value@highest */
-    patch_imm16(address+1, value >> 32);/* ori r,r,value@higher */
-    /* sldi r,r,32 */
-    patch_imm16(address+3, value >> 16);/* oris r,r,value@h */
-    patch_imm16(address+4, value);	/* ori r,r,value@l */
-}
-
-static int patch_li31(Uint32 *address, Uint32 value)
-{
-    if ((value >> 31) != 0)
-	return -1;
-    patch_imm16(address, value >> 16);	/* addis r,0,value@h */
-    patch_imm16(address+1, value);	/* ori r,r,value@l */
+    patch_imm16(address+0, value >> 48);
+    patch_imm16(address+1, value >> 32);
+    /* rldicr Rd, Rd, 32, 31 */
+    patch_imm16(address+3, value >> 16);
+    patch_imm16(address+4, value);
 }
 
 void hipe_patch_load_fe(Uint *address, Uint value)
@@ -308,11 +276,10 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type)
     switch (type) {
       case am_closure:
       case am_constant:
-	patch_li64((Uint32*)address, value);
-	return 0;
       case am_atom:
       case am_c_const:
-	return patch_li31((Uint32*)address, value);
+	patch_li64((Uint32*)address, value);
+	return 0;
       default:
 	return -1;
     }
@@ -442,34 +409,33 @@ static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA)
 
 int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
 {
-    if ((Uint32)destAddress == ((Uint32)destAddress & 0x01FFFFFC)) {
+    if ((UWord)destAddress == ((UWord)destAddress & 0x01FFFFFC)) {
 	/* The destination is in the [0,32MB[ range.
 	   We can reach it with a ba/bla instruction.
 	   This is the typical case for BIFs and primops.
 	   It's also common for trap-to-BEAM stubs (on ppc32). */
-	patch_b((Uint32*)callAddress, (Uint32)destAddress >> 2, 2);
+	patch_b((Uint32*)callAddress, (Sint32)destAddress >> 2, 2);
     } else {
-	Sint32 destOffset = ((Sint32)destAddress - (Sint32)callAddress) >> 2;
+	SWord destOffset = ((SWord)destAddress - (SWord)callAddress) >> 2;
 	if (destOffset >= -0x800000 && destOffset <= 0x7FFFFF) {
 	    /* The destination is within a [-32MB,+32MB[ range from us.
 	       We can reach it with a b/bl instruction.
 	       This is typical for nearby Erlang code. */
-	    patch_b((Uint32*)callAddress, destOffset, 0);
+	    patch_b((Uint32*)callAddress, (Sint32)destOffset, 0);
 	} else {
 	    /* The destination is too distant for b/bl/ba/bla.
 	       Must do a b/bl to the trampoline. */
-	    Sint32 trampOffset = ((Sint32)trampoline - (Sint32)callAddress) >> 2;
+	    SWord trampOffset = ((SWord)trampoline - (SWord)callAddress) >> 2;
 	    if (trampOffset >= -0x800000 && trampOffset <= 0x7FFFFF) {
 		/* Update the trampoline's address computation.
 		   (May be redundant, but we can't tell.) */
 #if defined(__powerpc64__)
-		/* This relies on the fact that we allocate code below 2GB. */
-		patch_li31((Uint32*)trampoline, (Uint32)destAddress);
+		patch_li64((Uint32*)trampoline, (Uint64)destAddress);
 #else
 		patch_li((Uint32*)trampoline, (Uint32)destAddress);
 #endif
 		/* Update this call site. */
-		patch_b((Uint32*)callAddress, trampOffset, 0);
+		patch_b((Uint32*)callAddress, (Sint32)trampOffset, 0);
 	    } else
 		return -1;
 	}
diff --git a/erts/emulator/hipe/hipe_ppc.h b/erts/emulator/hipe/hipe_ppc.h
index e30ce30ed2..66000c1846 100644
--- a/erts/emulator/hipe/hipe_ppc.h
+++ b/erts/emulator/hipe/hipe_ppc.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_PPC_H
 #define HIPE_PPC_H
 
@@ -44,12 +44,20 @@ static __inline__ int hipe_word32_address_ok(void *address)
     return ((unsigned long)address & 0x3) == 0;
 }
 
+#if defined(__powerpc64__)
+/* for hipe_bifs_{read,write}_{s,u}64 */
+static __inline__ int hipe_word64_address_ok(void *address)
+{
+    return ((unsigned long)address & 0x7) == 0;
+}
+#endif
+
 /* Native stack growth direction. */
 #define HIPE_NSTACK_GROWS_DOWN
 
 #if defined(__powerpc64__)
 #define hipe_arch_name	am_ppc64
-#define AEXTERN(RET,NAME,PROTO)	extern const int NAME
+#define AEXTERN(RET,NAME,PROTO)	extern const int NAME[]
 AEXTERN(void,hipe_ppc_inc_stack,(void));
 #else
 #define hipe_arch_name	am_powerpc
diff --git a/erts/emulator/hipe/hipe_ppc.tab b/erts/emulator/hipe/hipe_ppc.tab
index a32dd820e7..38b7f46d3a 100644
--- a/erts/emulator/hipe/hipe_ppc.tab
+++ b/erts/emulator/hipe/hipe_ppc.tab
@@ -1,22 +1,22 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2004-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
+
 # PowerPC-specific atoms
 
 atom fconv_constant
diff --git a/erts/emulator/hipe/hipe_ppc64.tab b/erts/emulator/hipe/hipe_ppc64.tab
index 513182721c..0a390a3bb8 100644
--- a/erts/emulator/hipe/hipe_ppc64.tab
+++ b/erts/emulator/hipe/hipe_ppc64.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2005-2009. All Rights Reserved.
+# Copyright Ericsson AB 2005-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -16,7 +16,7 @@
 # 
 # %CopyrightEnd%
 #
-# $Id$
+
 # PPC64-specific atoms
 
 atom inc_stack_0
diff --git a/erts/emulator/hipe/hipe_ppc_abi.txt b/erts/emulator/hipe/hipe_ppc_abi.txt
index 4bf41e02b2..be0ef98b0d 100644
--- a/erts/emulator/hipe/hipe_ppc_abi.txt
+++ b/erts/emulator/hipe/hipe_ppc_abi.txt
@@ -2,7 +2,7 @@
  %CopyrightBegin%
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE PowerPC ABI
 ================
diff --git a/erts/emulator/hipe/hipe_ppc_asm.m4 b/erts/emulator/hipe/hipe_ppc_asm.m4
index a0f8b78679..0eb5c441e6 100644
--- a/erts/emulator/hipe/hipe_ppc_asm.m4
+++ b/erts/emulator/hipe/hipe_ppc_asm.m4
@@ -1,25 +1,24 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 `#ifndef HIPE_PPC_ASM_H
 #define HIPE_PPC_ASM_H'
 
@@ -63,15 +62,31 @@ ifelse(OPSYS,darwin,``
 #define SEMI		@
 #define SET_SIZE(NAME)	/*empty*/
 #define TYPE_FUNCTION(NAME)	/*empty*/
+#define OPD(NAME)	/*empty*/
 '',``
 /* Not Darwin */''
 `ifelse(ARCH,ppc64,``
 /* 64-bit */
+/*
+ * The 64-bit PowerPC ABI requires us to setup Official Procedure Descriptors
+ * for functions called from C. These are exported as "func", while the entry
+ * point should is exported as ".func". A function pointer in C points to the
+ * function descriptor in the opd rather than to the function entry point.
+ */
 #define JOIN(X,Y)	X##Y
 #define CSYM(NAME)	JOIN(.,NAME)
+#define OPD(NAME)       			\
+	.pushsection .opd, "aw";		\
+	.align 3;				\
+	.global NAME;				\
+NAME:						\
+	.quad CSYM(NAME), .TOC.@tocbase, 0;	\
+	.type NAME, @function;			\
+	.popsection
 '',``
 /* 32-bit */
 #define CSYM(NAME)	NAME
+#define OPD(NAME)	/*empty*/
 '')'
 ``#define ASYM(NAME)	NAME
 #define GLOBAL(NAME)	.global NAME
diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4
index 3849d9113a..203fefe1a1 100644
--- a/erts/emulator/hipe/hipe_ppc_bifs.m4
+++ b/erts/emulator/hipe/hipe_ppc_bifs.m4
@@ -1,25 +1,23 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 include(`hipe/hipe_ppc_asm.m4')
 #`include' "hipe_literals.h"
diff --git a/erts/emulator/hipe/hipe_ppc_gc.h b/erts/emulator/hipe/hipe_ppc_gc.h
index 796ebeb20a..823ba0ad06 100644
--- a/erts/emulator/hipe/hipe_ppc_gc.h
+++ b/erts/emulator/hipe/hipe_ppc_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  * PowerPC version.
  */
diff --git a/erts/emulator/hipe/hipe_ppc_glue.S b/erts/emulator/hipe/hipe_ppc_glue.S
index 97b07353f9..6f0217c738 100644
--- a/erts/emulator/hipe/hipe_ppc_glue.S
+++ b/erts/emulator/hipe/hipe_ppc_glue.S
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include "hipe_ppc_asm.h"
 #include "hipe_literals.h"
 #define ASM
@@ -198,6 +198,7 @@
  * int hipe_ppc_call_to_native(Process *p);
  * Emulated code recursively calls native code.
  */
+	OPD(hipe_ppc_call_to_native)
 	GLOBAL(CSYM(hipe_ppc_call_to_native))
 CSYM(hipe_ppc_call_to_native):
 	/* save C context */
@@ -229,6 +230,7 @@ ASYM(nbif_return):
  * int hipe_ppc_return_to_native(Process *p);
  * Emulated code returns to its native code caller.
  */
+	OPD(hipe_ppc_return_to_native)
 	GLOBAL(CSYM(hipe_ppc_return_to_native))
 CSYM(hipe_ppc_return_to_native):
 	/* save C context */
@@ -252,6 +254,7 @@ CSYM(hipe_ppc_return_to_native):
  * int hipe_ppc_tailcall_to_native(Process *p);
  * Emulated code tailcalls native code.
  */
+	OPD(hipe_ppc_tailcall_to_native)
 	GLOBAL(CSYM(hipe_ppc_tailcall_to_native))
 CSYM(hipe_ppc_tailcall_to_native):
 	/* save C context */
@@ -274,6 +277,7 @@ CSYM(hipe_ppc_tailcall_to_native):
  * int hipe_ppc_throw_to_native(Process *p);
  * Emulated code throws an exception to its native code caller.
  */
+	OPD(hipe_ppc_throw_to_native)
 	GLOBAL(CSYM(hipe_ppc_throw_to_native))
 CSYM(hipe_ppc_throw_to_native):
 	/* save C context */
@@ -455,6 +459,10 @@ ASYM(nbif_fail):
 	li	r3, HIPE_MODE_SWITCH_RES_THROW
 	b	.flush_exit	/* no need to save RA */
 
+	OPD(nbif_0_gc_after_bif)
+	OPD(nbif_1_gc_after_bif)
+	OPD(nbif_2_gc_after_bif)
+	OPD(nbif_3_gc_after_bif)
 	GLOBAL(CSYM(nbif_0_gc_after_bif))
 	GLOBAL(CSYM(nbif_1_gc_after_bif))
 	GLOBAL(CSYM(nbif_2_gc_after_bif))
@@ -476,6 +484,8 @@ CSYM(nbif_3_gc_after_bif):
 	STORE	TEMP_LR, P_NRA(P)
 	STORE	NSP, P_NSP(P)
 	mflr	TEMP_LR
+	li	r6, 0			/* Pass 0 in arity */
+	li	r5, 0			/* Pass NULL in regs */
 	mr	r4, r3
 	mr	r3, P
 	bl	CSYM(erts_gc_after_bif_call)
@@ -491,18 +501,22 @@ CSYM(nbif_3_gc_after_bif):
  * The heap pointer was just read from P.
  * TEMP_LR contains a copy of LR
  */
+	OPD(nbif_0_simple_exception)
 	GLOBAL(CSYM(nbif_0_simple_exception))
 CSYM(nbif_0_simple_exception):
 	li	r4, 0
 	b	.nbif_simple_exception
+	OPD(nbif_1_simple_exception)
 	GLOBAL(CSYM(nbif_1_simple_exception))
 CSYM(nbif_1_simple_exception):
 	li	r4, 1
 	b	.nbif_simple_exception
+	OPD(nbif_2_simple_exception)
 	GLOBAL(CSYM(nbif_2_simple_exception))
 CSYM(nbif_2_simple_exception):
 	li	r4, 2
 	b	.nbif_simple_exception
+	OPD(nbif_3_simple_exception)
 	GLOBAL(CSYM(nbif_3_simple_exception))
 CSYM(nbif_3_simple_exception):
 	li	r4, 3
@@ -539,7 +553,7 @@ CSYM(nbif_3_simple_exception):
 .handle_trap:
 	li	r3, HIPE_MODE_SWITCH_RES_TRAP
 	STORE	NSP, P_NSP(P)
-	STORE	r4, P_NARITY(P)
+	stw	r4, P_NARITY(P)	/* Note: narity is a 32-bit field */
 	STORE	TEMP_LR, P_NRA(P)
 	b	.nosave_exit
 
diff --git a/erts/emulator/hipe/hipe_ppc_glue.h b/erts/emulator/hipe/hipe_ppc_glue.h
index dcf5ec7644..f9c4460e60 100644
--- a/erts/emulator/hipe/hipe_ppc_glue.h
+++ b/erts/emulator/hipe/hipe_ppc_glue.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_PPC_GLUE_H
 #define HIPE_PPC_GLUE_H
 
diff --git a/erts/emulator/hipe/hipe_ppc_primops.h b/erts/emulator/hipe/hipe_ppc_primops.h
index 67205fe1d1..7dba0afc88 100644
--- a/erts/emulator/hipe/hipe_ppc_primops.h
+++ b/erts/emulator/hipe/hipe_ppc_primops.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #if !defined(__powerpc64__)
 PRIMOP_LIST(am_fconv_constant, &fconv_constant)
 #endif
diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h
index cc2fc425d5..94113ffcd8 100644
--- a/erts/emulator/hipe/hipe_primops.h
+++ b/erts/emulator/hipe/hipe_primops.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_PRIMOPS_H
 #define HIPE_PRIMOPS_H
 
diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h
index 5528e68826..5effacb398 100644
--- a/erts/emulator/hipe/hipe_process.h
+++ b/erts/emulator/hipe/hipe_process.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * HiPE-specific process fields
  */
 #ifndef HIPE_PROCESS_H
diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h
index 4a9a7878f0..947eb5956b 100644
--- a/erts/emulator/hipe/hipe_risc_gc.h
+++ b/erts/emulator/hipe/hipe_risc_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  * Generic RISC version.
  */
diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index 3b2d6498d3..e74023e3e9 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_RISC_GLUE_H
 #define HIPE_RISC_GLUE_H
 
diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c
index 976ca0b85d..1183856c7e 100644
--- a/erts/emulator/hipe/hipe_risc_stack.c
+++ b/erts/emulator/hipe/hipe_risc_stack.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
diff --git a/erts/emulator/hipe/hipe_signal.h b/erts/emulator/hipe/hipe_signal.h
index 3c3c844d52..4eacf52b5d 100644
--- a/erts/emulator/hipe/hipe_signal.h
+++ b/erts/emulator/hipe/hipe_signal.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_signal.h
  *
  * Architecture-specific initialisation of Unix signals.
diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c
index 661b42130a..49d4da7bab 100644
--- a/erts/emulator/hipe/hipe_sparc.c
+++ b/erts/emulator/hipe/hipe_sparc.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include <stddef.h>	/* offsetof() */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
diff --git a/erts/emulator/hipe/hipe_sparc.h b/erts/emulator/hipe/hipe_sparc.h
index 53cb18ee45..1134b86004 100644
--- a/erts/emulator/hipe/hipe_sparc.h
+++ b/erts/emulator/hipe/hipe_sparc.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_SPARC_H
 #define HIPE_SPARC_H
 
diff --git a/erts/emulator/hipe/hipe_sparc.tab b/erts/emulator/hipe/hipe_sparc.tab
index f192e1f81c..c620c73c67 100644
--- a/erts/emulator/hipe/hipe_sparc.tab
+++ b/erts/emulator/hipe/hipe_sparc.tab
@@ -1,22 +1,22 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2004-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
+
 # SPARC-specific atoms
 
 atom inc_stack_0
diff --git a/erts/emulator/hipe/hipe_sparc_abi.txt b/erts/emulator/hipe/hipe_sparc_abi.txt
index d016a96c1c..cb5cda310b 100644
--- a/erts/emulator/hipe/hipe_sparc_abi.txt
+++ b/erts/emulator/hipe/hipe_sparc_abi.txt
@@ -2,7 +2,7 @@
  %CopyrightBegin%
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE SPARC ABI
 ==============
diff --git a/erts/emulator/hipe/hipe_sparc_asm.m4 b/erts/emulator/hipe/hipe_sparc_asm.m4
index 7a4403ac09..227d10ed80 100644
--- a/erts/emulator/hipe/hipe_sparc_asm.m4
+++ b/erts/emulator/hipe/hipe_sparc_asm.m4
@@ -1,25 +1,24 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 `#ifndef HIPE_SPARC_ASM_H
 #define HIPE_SPARC_ASM_H'
 
diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4
index f3753b3847..03db7f3413 100644
--- a/erts/emulator/hipe/hipe_sparc_bifs.m4
+++ b/erts/emulator/hipe/hipe_sparc_bifs.m4
@@ -1,25 +1,23 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 include(`hipe/hipe_sparc_asm.m4')
 #`include' "hipe_literals.h"
diff --git a/erts/emulator/hipe/hipe_sparc_gc.h b/erts/emulator/hipe/hipe_sparc_gc.h
index 9035f5baee..b870ddd59e 100644
--- a/erts/emulator/hipe/hipe_sparc_gc.h
+++ b/erts/emulator/hipe/hipe_sparc_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  * SPARC version.
  */
diff --git a/erts/emulator/hipe/hipe_sparc_glue.S b/erts/emulator/hipe/hipe_sparc_glue.S
index d1af5c43f5..44bdf1bc7e 100644
--- a/erts/emulator/hipe/hipe_sparc_glue.S
+++ b/erts/emulator/hipe/hipe_sparc_glue.S
@@ -1,23 +1,24 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include "hipe_sparc_asm.h"
 #include "hipe_literals.h"
 #define ASM
@@ -333,6 +334,8 @@ nbif_3_gc_after_bif:
 	st	TEMP_RA, [P+P_NRA]
 	st	NSP, [P+P_NSP]
 	mov	RA, TEMP_RA
+	mov	0, %o3			/* Pass 0 in arity */
+	mov	0, %o2			/* Pass NULL in regs */
 	mov	%o0, %o1
 	call	erts_gc_after_bif_call
 	mov	P, %o0			/* delay slot */
diff --git a/erts/emulator/hipe/hipe_sparc_glue.h b/erts/emulator/hipe/hipe_sparc_glue.h
index 3f881d2140..1404c0d4c0 100644
--- a/erts/emulator/hipe/hipe_sparc_glue.h
+++ b/erts/emulator/hipe/hipe_sparc_glue.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_SPARC_GLUE_H
 #define HIPE_SPARC_GLUE_H
 
diff --git a/erts/emulator/hipe/hipe_sparc_primops.h b/erts/emulator/hipe/hipe_sparc_primops.h
index 1fbb261c67..413371e5f0 100644
--- a/erts/emulator/hipe/hipe_sparc_primops.h
+++ b/erts/emulator/hipe/hipe_sparc_primops.h
@@ -1,21 +1,21 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 PRIMOP_LIST(am_inc_stack_0, &hipe_sparc_inc_stack)
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index 82f7f022b6..da462a64e1 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 354ac81b4c..4c14b4a519 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_STACK_H
 #define HIPE_STACK_H
 
diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c
index f79a2d53f4..24d232c968 100644
--- a/erts/emulator/hipe/hipe_x86.c
+++ b/erts/emulator/hipe/hipe_x86.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include <stddef.h>	/* offsetof() */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
diff --git a/erts/emulator/hipe/hipe_x86.h b/erts/emulator/hipe/hipe_x86.h
index 94ca39fc4f..f0f3c158af 100644
--- a/erts/emulator/hipe/hipe_x86.h
+++ b/erts/emulator/hipe/hipe_x86.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_X86_H
 #define HIPE_X86_H
 
diff --git a/erts/emulator/hipe/hipe_x86.tab b/erts/emulator/hipe/hipe_x86.tab
index a38fe49156..fb33d0a6b9 100644
--- a/erts/emulator/hipe/hipe_x86.tab
+++ b/erts/emulator/hipe/hipe_x86.tab
@@ -1,22 +1,22 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2004-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
+
 # x86-specific atoms
 
 atom handle_fp_exception
diff --git a/erts/emulator/hipe/hipe_x86_abi.txt b/erts/emulator/hipe/hipe_x86_abi.txt
index 62a704eef3..aa04a12611 100644
--- a/erts/emulator/hipe/hipe_x86_abi.txt
+++ b/erts/emulator/hipe/hipe_x86_abi.txt
@@ -1,7 +1,7 @@
 
  %CopyrightBegin%
  
- Copyright Ericsson AB 2001-2009. All Rights Reserved.
+ Copyright Ericsson AB 2001-2011. All Rights Reserved.
  
  The contents of this file are subject to the Erlang Public License,
  Version 1.1, (the "License"); you may not use this file except in
@@ -16,7 +16,7 @@
  
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE x86 ABI
 ============
diff --git a/erts/emulator/hipe/hipe_x86_asm.m4 b/erts/emulator/hipe/hipe_x86_asm.m4
index 4c1d612ccd..020ccf8d4b 100644
--- a/erts/emulator/hipe/hipe_x86_asm.m4
+++ b/erts/emulator/hipe/hipe_x86_asm.m4
@@ -1,25 +1,24 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 `#ifndef HIPE_X86_ASM_H
 #define HIPE_X86_ASM_H'
 
diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4
index 80be74f7b2..1bb6488b00 100644
--- a/erts/emulator/hipe/hipe_x86_bifs.m4
+++ b/erts/emulator/hipe/hipe_x86_bifs.m4
@@ -1,25 +1,23 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 include(`hipe/hipe_x86_asm.m4')
 #`include' "hipe_literals.h"
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index 4f17f767df..e4607ad27d 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  */
 #ifndef HIPE_X86_GC_H
diff --git a/erts/emulator/hipe/hipe_x86_glue.S b/erts/emulator/hipe/hipe_x86_glue.S
index 2f7dff39f5..88b86f4de7 100644
--- a/erts/emulator/hipe/hipe_x86_glue.S
+++ b/erts/emulator/hipe/hipe_x86_glue.S
@@ -1,24 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 #include "hipe_x86_asm.h"
 #include "hipe_literals.h"
@@ -320,11 +319,13 @@ nbif_3_gc_after_bif:
 	.align	4
 .gc_after_bif:
 	movl	%edx, P_NARITY(P)
-	subl	$(16-4), %esp
+	subl	$(32-4), %esp
 	movl	P, (%esp)
 	movl	%eax, 4(%esp)
+	movl	$0, 8(%esp)		# Pass NULL in regs
+	movl	$0, 12(%esp)		# Pass 0 in arity
 	call	CSYM(erts_gc_after_bif_call)
-	addl	$(16-4), %esp
+	addl	$(32-4), %esp
 	movl	$0, P_NARITY(P)
 	ret
 
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index 4c9c92c52f..a7b0f164be 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -1,24 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 #ifndef HIPE_X86_GLUE_H
 #define HIPE_X86_GLUE_H
 
diff --git a/erts/emulator/hipe/hipe_x86_primops.h b/erts/emulator/hipe/hipe_x86_primops.h
index 757da484ad..96d2336bc5 100644
--- a/erts/emulator/hipe/hipe_x86_primops.h
+++ b/erts/emulator/hipe/hipe_x86_primops.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 PRIMOP_LIST(am_inc_stack_0, &nbif_inc_stack_0)
 PRIMOP_LIST(am_handle_fp_exception, &nbif_handle_fp_exception)
diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c
index a4fff4ce31..64c0e0da3e 100644
--- a/erts/emulator/hipe/hipe_x86_signal.c
+++ b/erts/emulator/hipe/hipe_x86_signal.c
@@ -1,22 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_x86_signal.c
  *
  * Erlang code compiled to x86 native code uses the x86 %esp as its
@@ -195,7 +196,7 @@ static void do_init(void)
 #define INIT()	do { if (!init_done()) do_init(); } while (0)
 #endif /* __DARWIN__ */
 
-#if !defined(__GLIBC__) && !defined(__DARWIN__)
+#if !defined(__GLIBC__) && !defined(__DARWIN__) && !defined(__NetBSD__)
 /*
  * Assume Solaris/x86 2.8.
  * There is a number of sigaction() procedures in libc:
@@ -231,6 +232,7 @@ static void do_init(void)
 #define INIT()	do { if (!init_done()) do_init(); } while (0)
 #endif	/* not glibc or darwin */
 
+#if !defined(__NetBSD__)
 /*
  * This is our wrapper for sigaction(). sigaction() can be called before
  * hipe_signal_init() has been executed, especially when threads support
@@ -253,7 +255,7 @@ static int my_sigaction(int signum, const struct sigaction *act, struct sigactio
     }
     return __next_sigaction(signum, act, oldact);
 }
-
+#endif
 /*
  * This overrides the C library's core sigaction() procedure, catching
  * all its internal calls.
@@ -268,7 +270,7 @@ int __SIGACTION(int signum, const struct sigaction *act, struct sigaction *oldac
 /*
  * This catches the application's own sigaction() calls.
  */
-#if !defined(__DARWIN__)
+#if !defined(__DARWIN__) && !defined(__NetBSD__)
 int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact)
 {
     return my_sigaction(signum, act, oldact);
@@ -326,7 +328,9 @@ void hipe_signal_init(void)
     struct sigaction sa;
     int i;
 
+#ifndef __NetBSD__
     INIT();
+#endif
 
     hipe_sigaltstack_init();
 
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index b459593883..9ad3fa9d31 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
diff --git a/erts/emulator/internal_doc/dec.dat b/erts/emulator/internal_doc/dec.dat
new file mode 100644
index 0000000000..771ef51baa
--- /dev/null
+++ b/erts/emulator/internal_doc/dec.dat
@@ -0,0 +1,942 @@
+{[59],894}.
+{[96],8175}.
+{[180],8189}.
+{[183],903}.
+{[198],1236}.
+{[230],1237}.
+{[399],1240}.
+{[415],1256}.
+{[439],1248}.
+{[601],1241}.
+{[629],1257}.
+{[658],1249}.
+{[697],884}.
+{[768],832}.
+{[768,65],192}.
+{[768,69],200}.
+{[768,73],204}.
+{[768,79],210}.
+{[768,85],217}.
+{[768,87],7808}.
+{[768,89],7922}.
+{[768,97],224}.
+{[768,101],232}.
+{[768,105],236}.
+{[768,111],242}.
+{[768,117],249}.
+{[768,119],7809}.
+{[768,121],7923}.
+{[768,168],8173}.
+{[768,770,65],7846}.
+{[768,770,69],7872}.
+{[768,770,79],7890}.
+{[768,770,97],7847}.
+{[768,770,101],7873}.
+{[768,770,111],7891}.
+{[768,772,69],7700}.
+{[768,772,79],7760}.
+{[768,772,101],7701}.
+{[768,772,111],7761}.
+{[768,774,65],7856}.
+{[768,774,97],7857}.
+{[768,776,85],475}.
+{[768,776,117],476}.
+{[768,776,953],8146}.
+{[768,776,965],8162}.
+{[768,787,837,913],8074}.
+{[768,787,837,919],8090}.
+{[768,787,837,937],8106}.
+{[768,787,837,945],8066}.
+{[768,787,837,951],8082}.
+{[768,787,837,969],8098}.
+{[768,787,913],7946}.
+{[768,787,917],7962}.
+{[768,787,919],7978}.
+{[768,787,921],7994}.
+{[768,787,927],8010}.
+{[768,787,937],8042}.
+{[768,787,945],7938}.
+{[768,787,949],7954}.
+{[768,787,951],7970}.
+{[768,787,953],7986}.
+{[768,787,959],8002}.
+{[768,787,965],8018}.
+{[768,787,969],8034}.
+{[768,788,837,913],8075}.
+{[768,788,837,919],8091}.
+{[768,788,837,937],8107}.
+{[768,788,837,945],8067}.
+{[768,788,837,951],8083}.
+{[768,788,837,969],8099}.
+{[768,788,913],7947}.
+{[768,788,917],7963}.
+{[768,788,919],7979}.
+{[768,788,921],7995}.
+{[768,788,927],8011}.
+{[768,788,933],8027}.
+{[768,788,937],8043}.
+{[768,788,945],7939}.
+{[768,788,949],7955}.
+{[768,788,951],7971}.
+{[768,788,953],7987}.
+{[768,788,959],8003}.
+{[768,788,965],8019}.
+{[768,788,969],8035}.
+{[768,795,79],7900}.
+{[768,795,85],7914}.
+{[768,795,111],7901}.
+{[768,795,117],7915}.
+{[768,837,945],8114}.
+{[768,837,951],8130}.
+{[768,837,969],8178}.
+{[768,913],8122}.
+{[768,917],8136}.
+{[768,919],8138}.
+{[768,921],8154}.
+{[768,927],8184}.
+{[768,933],8170}.
+{[768,937],8186}.
+{[768,945],8048}.
+{[768,949],8050}.
+{[768,951],8052}.
+{[768,953],8054}.
+{[768,959],8056}.
+{[768,965],8058}.
+{[768,969],8060}.
+{[768,8127],8141}.
+{[768,8190],8157}.
+{[769],833}.
+{[769,65],193}.
+{[769,67],262}.
+{[769,69],201}.
+{[769,71],500}.
+{[769,73],205}.
+{[769,75],7728}.
+{[769,76],313}.
+{[769,77],7742}.
+{[769,78],323}.
+{[769,79],211}.
+{[769,80],7764}.
+{[769,82],340}.
+{[769,83],346}.
+{[769,85],218}.
+{[769,87],7810}.
+{[769,89],221}.
+{[769,90],377}.
+{[769,97],225}.
+{[769,99],263}.
+{[769,101],233}.
+{[769,103],501}.
+{[769,105],237}.
+{[769,107],7729}.
+{[769,108],314}.
+{[769,109],7743}.
+{[769,110],324}.
+{[769,111],243}.
+{[769,112],7765}.
+{[769,114],341}.
+{[769,115],347}.
+{[769,117],250}.
+{[769,119],7811}.
+{[769,121],253}.
+{[769,122],378}.
+{[769,168],8174}.
+{[769,198],508}.
+{[769,216],510}.
+{[769,230],509}.
+{[769,248],511}.
+{[769,770,65],7844}.
+{[769,770,69],7870}.
+{[769,770,79],7888}.
+{[769,770,97],7845}.
+{[769,770,101],7871}.
+{[769,770,111],7889}.
+{[769,771,79],7756}.
+{[769,771,85],7800}.
+{[769,771,111],7757}.
+{[769,771,117],7801}.
+{[769,772,69],7702}.
+{[769,772,79],7762}.
+{[769,772,101],7703}.
+{[769,772,111],7763}.
+{[769,774,65],7854}.
+{[769,774,97],7855}.
+{[769,776,73],7726}.
+{[769,776,85],471}.
+{[769,776,105],7727}.
+{[769,776,117],472}.
+{[769,776,953],8147}.
+{[769,776,965],8163}.
+{[769,778,65],506}.
+{[769,778,97],507}.
+{[769,787,837,913],8076}.
+{[769,787,837,919],8092}.
+{[769,787,837,937],8108}.
+{[769,787,837,945],8068}.
+{[769,787,837,951],8084}.
+{[769,787,837,969],8100}.
+{[769,787,913],7948}.
+{[769,787,917],7964}.
+{[769,787,919],7980}.
+{[769,787,921],7996}.
+{[769,787,927],8012}.
+{[769,787,937],8044}.
+{[769,787,945],7940}.
+{[769,787,949],7956}.
+{[769,787,951],7972}.
+{[769,787,953],7988}.
+{[769,787,959],8004}.
+{[769,787,965],8020}.
+{[769,787,969],8036}.
+{[769,788,837,913],8077}.
+{[769,788,837,919],8093}.
+{[769,788,837,937],8109}.
+{[769,788,837,945],8069}.
+{[769,788,837,951],8085}.
+{[769,788,837,969],8101}.
+{[769,788,913],7949}.
+{[769,788,917],7965}.
+{[769,788,919],7981}.
+{[769,788,921],7997}.
+{[769,788,927],8013}.
+{[769,788,933],8029}.
+{[769,788,937],8045}.
+{[769,788,945],7941}.
+{[769,788,949],7957}.
+{[769,788,951],7973}.
+{[769,788,953],7989}.
+{[769,788,959],8005}.
+{[769,788,965],8021}.
+{[769,788,969],8037}.
+{[769,795,79],7898}.
+{[769,795,85],7912}.
+{[769,795,111],7899}.
+{[769,795,117],7913}.
+{[769,807,67],7688}.
+{[769,807,99],7689}.
+{[769,837,945],8116}.
+{[769,837,951],8132}.
+{[769,837,959],8180}.
+{[769,913],8123}.
+{[769,917],8137}.
+{[769,919],8139}.
+{[769,921],8155}.
+{[769,927],8185}.
+{[769,933],8171}.
+{[769,937],8187}.
+{[769,945],8049}.
+{[769,949],8051}.
+{[769,951],8053}.
+{[769,953],8055}.
+{[769,959],8057}.
+{[769,965],8059}.
+{[769,969],8061}.
+{[769,1043],1027}.
+{[769,1050],1036}.
+{[769,1075],1107}.
+{[769,1082],1116}.
+{[769,8127],8142}.
+{[769,8190],8158}.
+{[770,65],194}.
+{[770,67],264}.
+{[770,69],202}.
+{[770,71],284}.
+{[770,72],292}.
+{[770,73],206}.
+{[770,74],308}.
+{[770,79],212}.
+{[770,83],348}.
+{[770,85],219}.
+{[770,87],372}.
+{[770,89],374}.
+{[770,90],7824}.
+{[770,97],226}.
+{[770,99],265}.
+{[770,101],234}.
+{[770,103],285}.
+{[770,104],293}.
+{[770,105],238}.
+{[770,106],309}.
+{[770,111],244}.
+{[770,115],349}.
+{[770,117],251}.
+{[770,119],373}.
+{[770,121],375}.
+{[770,122],7825}.
+{[770,803,65],7852}.
+{[770,803,69],7878}.
+{[770,803,79],7896}.
+{[770,803,97],7853}.
+{[770,803,101],7879}.
+{[770,803,111],7897}.
+{[771,65],195}.
+{[771,69],7868}.
+{[771,73],296}.
+{[771,78],209}.
+{[771,79],213}.
+{[771,85],360}.
+{[771,86],7804}.
+{[771,89],7928}.
+{[771,97],227}.
+{[771,101],7869}.
+{[771,105],297}.
+{[771,110],241}.
+{[771,111],245}.
+{[771,117],361}.
+{[771,118],7805}.
+{[771,121],7929}.
+{[771,770,65],7850}.
+{[771,770,69],7876}.
+{[771,770,79],7894}.
+{[771,770,97],7851}.
+{[771,770,101],7877}.
+{[771,770,111],7895}.
+{[771,774,65],7860}.
+{[771,774,97],7861}.
+{[771,795,79],7904}.
+{[771,795,85],7918}.
+{[771,795,111],7905}.
+{[771,795,117],7919}.
+{[772,65],256}.
+{[772,69],274}.
+{[772,71],7712}.
+{[772,73],298}.
+{[772,79],332}.
+{[772,85],362}.
+{[772,97],257}.
+{[772,101],275}.
+{[772,103],7713}.
+{[772,105],299}.
+{[772,111],333}.
+{[772,117],363}.
+{[772,198],482}.
+{[772,230],483}.
+{[772,775,65],480}.
+{[772,775,97],481}.
+{[772,776,65],478}.
+{[772,776,85],469}.
+{[772,776,97],479}.
+{[772,776,117],470}.
+{[772,803,76],7736}.
+{[772,803,82],7772}.
+{[772,803,108],7737}.
+{[772,803,114],7773}.
+{[772,808,79],492}.
+{[772,808,111],493}.
+{[772,913],8121}.
+{[772,921],8153}.
+{[772,933],8169}.
+{[772,945],8113}.
+{[772,953],8145}.
+{[772,965],8161}.
+{[772,1048],1250}.
+{[772,1059],1262}.
+{[772,1080],1251}.
+{[772,1091],1263}.
+{[774,65],258}.
+{[774,69],276}.
+{[774,71],286}.
+{[774,73],300}.
+{[774,79],334}.
+{[774,85],364}.
+{[774,97],259}.
+{[774,101],277}.
+{[774,103],287}.
+{[774,105],301}.
+{[774,111],335}.
+{[774,117],365}.
+{[774,803,65],7862}.
+{[774,803,97],7863}.
+{[774,807,69],7708}.
+{[774,807,101],7709}.
+{[774,913],8120}.
+{[774,921],8152}.
+{[774,933],8168}.
+{[774,945],8112}.
+{[774,953],8144}.
+{[774,965],8160}.
+{[774,1040],1232}.
+{[774,1045],1238}.
+{[774,1046],1217}.
+{[774,1048],1049}.
+{[774,1059],1038}.
+{[774,1072],1233}.
+{[774,1077],1239}.
+{[774,1078],1218}.
+{[774,1080],1081}.
+{[774,1091],1118}.
+{[775,66],7682}.
+{[775,67],266}.
+{[775,68],7690}.
+{[775,69],278}.
+{[775,70],7710}.
+{[775,71],288}.
+{[775,72],7714}.
+{[775,73],304}.
+{[775,77],7744}.
+{[775,78],7748}.
+{[775,80],7766}.
+{[775,82],7768}.
+{[775,83],7776}.
+{[775,84],7786}.
+{[775,87],7814}.
+{[775,88],7818}.
+{[775,89],7822}.
+{[775,90],379}.
+{[775,98],7683}.
+{[775,99],267}.
+{[775,100],7691}.
+{[775,101],279}.
+{[775,102],7711}.
+{[775,103],289}.
+{[775,104],7715}.
+{[775,109],7745}.
+{[775,110],7749}.
+{[775,112],7767}.
+{[775,114],7769}.
+{[775,115],7777}.
+{[775,116],7787}.
+{[775,119],7815}.
+{[775,120],7819}.
+{[775,121],7823}.
+{[775,122],380}.
+{[775,383],7835}.
+{[775,769,83],7780}.
+{[775,769,115],7781}.
+{[775,774],784}.
+{[775,780,83],7782}.
+{[775,780,115],7783}.
+{[775,803,83],7784}.
+{[775,803,115],7785}.
+{[776,65],196}.
+{[776,69],203}.
+{[776,72],7718}.
+{[776,73],207}.
+{[776,79],214}.
+{[776,85],220}.
+{[776,87],7812}.
+{[776,88],7820}.
+{[776,89],376}.
+{[776,97],228}.
+{[776,101],235}.
+{[776,104],7719}.
+{[776,105],239}.
+{[776,111],246}.
+{[776,116],7831}.
+{[776,117],252}.
+{[776,119],7813}.
+{[776,120],7821}.
+{[776,121],255}.
+{[776,399],1242}.
+{[776,415],1258}.
+{[776,601],1243}.
+{[776,629],1259}.
+{[776,771,79],7758}.
+{[776,771,111],7759}.
+{[776,772,85],7802}.
+{[776,772,117],7803}.
+{[776,921],938}.
+{[776,933],939}.
+{[776,953],970}.
+{[776,965],971}.
+{[776,978],980}.
+{[776,1030],1031}.
+{[776,1040],1234}.
+{[776,1045],1025}.
+{[776,1046],1244}.
+{[776,1047],1246}.
+{[776,1048],1252}.
+{[776,1054],1254}.
+{[776,1059],1264}.
+{[776,1063],1268}.
+{[776,1067],1272}.
+{[776,1072],1235}.
+{[776,1077],1105}.
+{[776,1078],1245}.
+{[776,1079],1247}.
+{[776,1080],1253}.
+{[776,1086],1255}.
+{[776,1091],1265}.
+{[776,1095],1269}.
+{[776,1099],1273}.
+{[776,1110],1111}.
+{[777,65],7842}.
+{[777,69],7866}.
+{[777,73],7880}.
+{[777,79],7886}.
+{[777,85],7910}.
+{[777,89],7926}.
+{[777,97],7843}.
+{[777,101],7867}.
+{[777,105],7881}.
+{[777,111],7887}.
+{[777,117],7911}.
+{[777,121],7927}.
+{[777,770,65],7848}.
+{[777,770,69],7874}.
+{[777,770,79],7892}.
+{[777,770,97],7849}.
+{[777,770,101],7875}.
+{[777,770,111],7893}.
+{[777,774,65],7858}.
+{[777,774,97],7859}.
+{[777,795,79],7902}.
+{[777,795,85],7916}.
+{[777,795,111],7903}.
+{[777,795,117],7917}.
+{[778,65],197}.
+{[778,85],366}.
+{[778,97],229}.
+{[778,117],367}.
+{[778,119],7832}.
+{[778,121],7833}.
+{[779,79],336}.
+{[779,85],368}.
+{[779,111],337}.
+{[779,117],369}.
+{[779,1059],1266}.
+{[779,1091],1267}.
+{[780,65],461}.
+{[780,67],268}.
+{[780,68],270}.
+{[780,69],282}.
+{[780,71],486}.
+{[780,73],463}.
+{[780,75],488}.
+{[780,76],317}.
+{[780,78],327}.
+{[780,79],465}.
+{[780,82],344}.
+{[780,83],352}.
+{[780,84],356}.
+{[780,85],467}.
+{[780,90],381}.
+{[780,97],462}.
+{[780,99],269}.
+{[780,100],271}.
+{[780,101],283}.
+{[780,103],487}.
+{[780,105],464}.
+{[780,106],496}.
+{[780,107],489}.
+{[780,108],318}.
+{[780,110],328}.
+{[780,111],466}.
+{[780,114],345}.
+{[780,115],353}.
+{[780,116],357}.
+{[780,117],468}.
+{[780,122],382}.
+{[780,439],494}.
+{[780,658],495}.
+{[780,776,85],473}.
+{[780,776,117],474}.
+{[781,168],901}.
+{[781,776],836}.
+{[781,776,953],912}.
+{[781,776,965],944}.
+{[781,913],902}.
+{[781,917],904}.
+{[781,919],905}.
+{[781,921],906}.
+{[781,927],908}.
+{[781,933],910}.
+{[781,937],911}.
+{[781,945],940}.
+{[781,949],941}.
+{[781,951],942}.
+{[781,953],943}.
+{[781,959],972}.
+{[781,965],973}.
+{[781,969],974}.
+{[781,978],979}.
+{[783,65],512}.
+{[783,69],516}.
+{[783,73],520}.
+{[783,79],524}.
+{[783,82],528}.
+{[783,85],532}.
+{[783,97],513}.
+{[783,101],517}.
+{[783,105],521}.
+{[783,111],525}.
+{[783,114],529}.
+{[783,117],533}.
+{[783,1140],1142}.
+{[783,1141],1143}.
+{[785,65],514}.
+{[785,69],518}.
+{[785,73],522}.
+{[785,79],526}.
+{[785,82],530}.
+{[785,85],534}.
+{[785,97],515}.
+{[785,101],519}.
+{[785,105],523}.
+{[785,111],527}.
+{[785,114],531}.
+{[785,117],535}.
+{[787],835}.
+{[787,837,913],8072}.
+{[787,837,919],8088}.
+{[787,837,937],8104}.
+{[787,837,945],8064}.
+{[787,837,951],8080}.
+{[787,837,969],8096}.
+{[787,913],7944}.
+{[787,917],7960}.
+{[787,919],7976}.
+{[787,921],7992}.
+{[787,927],8008}.
+{[787,937],8040}.
+{[787,945],7936}.
+{[787,949],7952}.
+{[787,951],7968}.
+{[787,953],7984}.
+{[787,959],8000}.
+{[787,961],8164}.
+{[787,965],8016}.
+{[787,969],8032}.
+{[788,837,913],8073}.
+{[788,837,919],8089}.
+{[788,837,937],8105}.
+{[788,837,945],8065}.
+{[788,837,951],8081}.
+{[788,837,969],8097}.
+{[788,913],7945}.
+{[788,917],7961}.
+{[788,919],7977}.
+{[788,921],7993}.
+{[788,927],8009}.
+{[788,929],8172}.
+{[788,933],8025}.
+{[788,937],8041}.
+{[788,945],7937}.
+{[788,949],7953}.
+{[788,951],7969}.
+{[788,953],7985}.
+{[788,959],8001}.
+{[788,961],8165}.
+{[788,965],8017}.
+{[788,969],8033}.
+{[795,79],416}.
+{[795,85],431}.
+{[795,111],417}.
+{[795,117],432}.
+{[803,65],7840}.
+{[803,66],7684}.
+{[803,68],7692}.
+{[803,69],7864}.
+{[803,72],7716}.
+{[803,73],7882}.
+{[803,75],7730}.
+{[803,76],7734}.
+{[803,77],7746}.
+{[803,78],7750}.
+{[803,79],7884}.
+{[803,82],7770}.
+{[803,83],7778}.
+{[803,84],7788}.
+{[803,85],7908}.
+{[803,86],7806}.
+{[803,87],7816}.
+{[803,89],7924}.
+{[803,90],7826}.
+{[803,97],7841}.
+{[803,98],7685}.
+{[803,100],7693}.
+{[803,101],7865}.
+{[803,104],7717}.
+{[803,105],7883}.
+{[803,107],7731}.
+{[803,108],7735}.
+{[803,109],7747}.
+{[803,110],7751}.
+{[803,111],7885}.
+{[803,114],7771}.
+{[803,115],7779}.
+{[803,116],7789}.
+{[803,117],7909}.
+{[803,118],7807}.
+{[803,119],7817}.
+{[803,121],7925}.
+{[803,122],7827}.
+{[803,795,79],7906}.
+{[803,795,85],7920}.
+{[803,795,111],7907}.
+{[803,795,117],7921}.
+{[804,85],7794}.
+{[804,117],7795}.
+{[805,65],7680}.
+{[805,97],7681}.
+{[807,67],199}.
+{[807,68],7696}.
+{[807,71],290}.
+{[807,72],7720}.
+{[807,75],310}.
+{[807,76],315}.
+{[807,78],325}.
+{[807,82],342}.
+{[807,83],350}.
+{[807,84],354}.
+{[807,99],231}.
+{[807,100],7697}.
+{[807,103],291}.
+{[807,104],7721}.
+{[807,107],311}.
+{[807,108],316}.
+{[807,110],326}.
+{[807,114],343}.
+{[807,115],351}.
+{[807,116],355}.
+{[808,65],260}.
+{[808,69],280}.
+{[808,73],302}.
+{[808,79],490}.
+{[808,85],370}.
+{[808,97],261}.
+{[808,101],281}.
+{[808,105],303}.
+{[808,111],491}.
+{[808,117],371}.
+{[813,68],7698}.
+{[813,69],7704}.
+{[813,76],7740}.
+{[813,78],7754}.
+{[813,84],7792}.
+{[813,85],7798}.
+{[813,100],7699}.
+{[813,101],7705}.
+{[813,108],7741}.
+{[813,110],7755}.
+{[813,116],7793}.
+{[813,117],7799}.
+{[814,72],7722}.
+{[814,104],7723}.
+{[816,69],7706}.
+{[816,73],7724}.
+{[816,85],7796}.
+{[816,101],7707}.
+{[816,105],7725}.
+{[816,117],7797}.
+{[817,66],7686}.
+{[817,68],7694}.
+{[817,75],7732}.
+{[817,76],7738}.
+{[817,78],7752}.
+{[817,82],7774}.
+{[817,84],7790}.
+{[817,90],7828}.
+{[817,98],7687}.
+{[817,100],7695}.
+{[817,104],7830}.
+{[817,107],7733}.
+{[817,108],7739}.
+{[817,110],7753}.
+{[817,114],7775}.
+{[817,116],7791}.
+{[817,122],7829}.
+{[834,168],8129}.
+{[834,776,953],8151}.
+{[834,776,965],8167}.
+{[834,787,837,913],8078}.
+{[834,787,837,919],8094}.
+{[834,787,837,937],8110}.
+{[834,787,837,945],8070}.
+{[834,787,837,951],8086}.
+{[834,787,837,969],8102}.
+{[834,787,913],7950}.
+{[834,787,919],7982}.
+{[834,787,921],7998}.
+{[834,787,937],8046}.
+{[834,787,945],7942}.
+{[834,787,951],7974}.
+{[834,787,953],7990}.
+{[834,787,965],8022}.
+{[834,787,969],8038}.
+{[834,788,837,913],8079}.
+{[834,788,837,919],8095}.
+{[834,788,837,937],8111}.
+{[834,788,837,945],8071}.
+{[834,788,837,951],8087}.
+{[834,788,837,969],8103}.
+{[834,788,913],7951}.
+{[834,788,919],7983}.
+{[834,788,921],7999}.
+{[834,788,933],8031}.
+{[834,788,937],8047}.
+{[834,788,945],7943}.
+{[834,788,951],7975}.
+{[834,788,953],7991}.
+{[834,788,965],8023}.
+{[834,788,969],8039}.
+{[834,837,945],8119}.
+{[834,837,951],8135}.
+{[834,837,969],8183}.
+{[834,945],8118}.
+{[834,951],8134}.
+{[834,953],8150}.
+{[834,965],8166}.
+{[834,969],8182}.
+{[834,8127],8143}.
+{[834,8190],8159}.
+{[837,913],8124}.
+{[837,919],8140}.
+{[837,937],8188}.
+{[837,945],8115}.
+{[837,951],8131}.
+{[837,969],8179}.
+{[953],8126}.
+{[1463,1488],64302}.
+{[1463,1522],64287}.
+{[1464,1488],64303}.
+{[1465,1493],64331}.
+{[1468,1488],64304}.
+{[1468,1489],64305}.
+{[1468,1490],64306}.
+{[1468,1491],64307}.
+{[1468,1492],64308}.
+{[1468,1493],64309}.
+{[1468,1494],64310}.
+{[1468,1496],64312}.
+{[1468,1497],64313}.
+{[1468,1498],64314}.
+{[1468,1499],64315}.
+{[1468,1500],64316}.
+{[1468,1502],64318}.
+{[1468,1504],64320}.
+{[1468,1505],64321}.
+{[1468,1507],64323}.
+{[1468,1508],64324}.
+{[1468,1510],64326}.
+{[1468,1511],64327}.
+{[1468,1512],64328}.
+{[1468,1513],64329}.
+{[1468,1514],64330}.
+{[1471,1489],64332}.
+{[1471,1499],64333}.
+{[1471,1508],64334}.
+{[1473,1468,1513],64300}.
+{[1473,1513],64298}.
+{[1474,1468,1513],64301}.
+{[1474,1513],64299}.
+{[2364,2325],2392}.
+{[2364,2326],2393}.
+{[2364,2327],2394}.
+{[2364,2332],2395}.
+{[2364,2337],2396}.
+{[2364,2338],2397}.
+{[2364,2344],2345}.
+{[2364,2347],2398}.
+{[2364,2351],2399}.
+{[2364,2352],2353}.
+{[2364,2355],2356}.
+{[2492,2465],2524}.
+{[2492,2466],2525}.
+{[2492,2476],2480}.
+{[2492,2479],2527}.
+{[2494,2503],2507}.
+{[2519,2503],2508}.
+{[2620,2582],2649}.
+{[2620,2583],2650}.
+{[2620,2588],2651}.
+{[2620,2593],2652}.
+{[2620,2603],2654}.
+{[2876,2849],2908}.
+{[2876,2850],2909}.
+{[2876,2863],2911}.
+{[2878,2887],2891}.
+{[2902,2887],2888}.
+{[2903,2887],2892}.
+{[3006,3014],3018}.
+{[3006,3015],3019}.
+{[3031,2962],2964}.
+{[3031,3014],3020}.
+{[3158,3142],3144}.
+{[3266,3270],3274}.
+{[3285,3263],3264}.
+{[3285,3266,3270],3275}.
+{[3285,3270],3271}.
+{[3286,3270],3272}.
+{[3390,3398],3402}.
+{[3390,3399],3403}.
+{[3415,3398],3404}.
+{[3634,3661],3635}.
+{[3762,3789],3763}.
+{[3953,3954],3955}.
+{[3953,3956],3957}.
+{[3953,3968],3969}.
+{[3953,3968,4018],3959}.
+{[3953,3968,4019],3961}.
+{[3968,4018],3958}.
+{[3968,4019],3960}.
+{[4021,3904],3945}.
+{[4021,3984],4025}.
+{[4023,3906],3907}.
+{[4023,3916],3917}.
+{[4023,3921],3922}.
+{[4023,3926],3927}.
+{[4023,3931],3932}.
+{[4023,3986],3987}.
+{[4023,3996],3997}.
+{[4023,4001],4002}.
+{[4023,4006],4007}.
+{[4023,4011],4012}.
+{[12441,12358],12436}.
+{[12441,12363],12364}.
+{[12441,12365],12366}.
+{[12441,12367],12368}.
+{[12441,12369],12370}.
+{[12441,12371],12372}.
+{[12441,12373],12374}.
+{[12441,12375],12376}.
+{[12441,12377],12378}.
+{[12441,12379],12380}.
+{[12441,12381],12382}.
+{[12441,12383],12384}.
+{[12441,12385],12386}.
+{[12441,12388],12389}.
+{[12441,12390],12391}.
+{[12441,12392],12393}.
+{[12441,12399],12400}.
+{[12441,12402],12403}.
+{[12441,12405],12406}.
+{[12441,12408],12409}.
+{[12441,12411],12412}.
+{[12441,12445],12446}.
+{[12441,12454],12532}.
+{[12441,12459],12460}.
+{[12441,12461],12462}.
+{[12441,12463],12464}.
+{[12441,12465],12466}.
+{[12441,12467],12468}.
+{[12441,12469],12470}.
+{[12441,12471],12472}.
+{[12441,12473],12474}.
+{[12441,12475],12476}.
+{[12441,12477],12478}.
+{[12441,12479],12480}.
+{[12441,12481],12482}.
+{[12441,12484],12485}.
+{[12441,12486],12487}.
+{[12441,12488],12489}.
+{[12441,12495],12496}.
+{[12441,12498],12499}.
+{[12441,12501],12502}.
+{[12441,12504],12505}.
+{[12441,12507],12508}.
+{[12441,12527],12535}.
+{[12441,12528],12536}.
+{[12441,12529],12537}.
+{[12441,12530],12538}.
+{[12441,12541],12542}.
+{[12442,12399],12401}.
+{[12442,12402],12404}.
+{[12442,12405],12407}.
+{[12442,12408],12410}.
+{[12442,12411],12413}.
+{[12442,12495],12497}.
+{[12442,12498],12500}.
+{[12442,12501],12503}.
+{[12442,12504],12506}.
+{[12442,12507],12509}.
diff --git a/erts/emulator/internal_doc/dec.erl b/erts/emulator/internal_doc/dec.erl
new file mode 100644
index 0000000000..0315f2a52d
--- /dev/null
+++ b/erts/emulator/internal_doc/dec.erl
@@ -0,0 +1,237 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2000-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%% This program is used to generate a header file with data for
+%% normalizing denormalized unicode.
+
+%% The C header is generated from a text file containing tuples in the 
+%% following format:
+%% {RevList,Translation}
+%% Where 'RevList' is a reversed list of the denormalized repressentation of
+%% the character 'Translation'. An example would be the swedish character 
+%% '�', which would be represented in the file as:
+%% {[776,111],246}, as the denormalized representation of codepoint 246
+%% is [111,776] (i.e an 'o' followed by the "double dot accent character 776),
+%% while '�' instead is represented as {[776,97],228}, as the denormalized 
+%% form would be [97,776] (same accent but an 'a' instead).
+%% The datafile is generated from the table on Apple's developer connection
+%% http://developer.apple.com/library/mac/#technotes/tn/tn1150table.html
+%% The generating is done whenever new data is present (i.e. dec.dat has 
+%% to be changed) and not for every build. The product (the C header) is copied
+%% to $ERL_TOP/erts/beam after generation and checked in.
+%% The program and the data file is included for reference.
+
+-module(dec).
+
+-compile(export_all).
+
+-define(HASH_SIZE_FACTOR,2).
+-define(BIG_PREFIX_SIZE,392).
+
+-define(INPUT_FILE_NAME,"dec.dat").
+-define(OUTPUT_FILE_NAME,"erl_unicode_normalize.h").
+
+read(FName) ->
+    {ok,L} = file:consult(FName),
+    [{A,B} || {A,B} <- L,
+	      length(A) > 1% , hd(A) < 769
+		 ].
+
+dec() ->
+    L = read(?INPUT_FILE_NAME),
+    G = group(L),
+    {ok,Out} = file:open(?OUTPUT_FILE_NAME,[write]),
+    io:format
+      (Out,
+       "/*~n"
+       "* %CopyrightBegin%~n"
+       "*~n"
+       "* Copyright Ericsson AB 1999-2010. All Rights Reserved.~n"
+       "*~n"
+       "* The contents of this file are subject to the Erlang Public License,~n"
+       "* Version 1.1, (the \"License\"); you may not use this file except in~n"
+       "* compliance with the License. You should have received a copy of the~n"
+       "* Erlang Public License along with this software. If not, it can be~n"
+       "* retrieved online at http://www.erlang.org/.~n"
+       "*~n"
+       "* Software distributed under the License is distributed on an "
+       "\"AS IS\"~n"
+       "* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See~n"
+       "* the License for the specific language governing rights and "
+       "limitations~n"
+       "* under the License.~n"
+       "*~n"
+       "* %CopyrightEnd%~n"
+       "*/~n"
+       "/*~n"
+       "* This file is automatically generated by ~p.erl, "
+       "do not edit manually~n"
+       "*/~n",
+       [?MODULE]),
+
+    io:format(Out,
+	      "#define HASH_SIZE_FACTOR ~w~n"
+	      "typedef struct _compose_entry {~n"
+	      "    Uint16 c;~n"
+	      "    Uint16 res;~n"
+	      "    Uint16 num_subs;~n"
+	      "    struct _compose_entry *subs;~n"
+	      "    int *hash;~n"
+	      "} CompEntry;~n~n"
+	      "static int compose_tab_size = ~p;~n", 
+	      [?HASH_SIZE_FACTOR,length(G)]),
+    d(Out,G,[],0),
+    PreTab = tuple_to_list(make_prefix_table(G,erlang:make_tuple(102,0))),
+    dump_prefixes(Out,PreTab),
+%% Using this cuts down on the searching in the
+%% actual implementation, but wastes memory with little real gain..
+%%    LL = lists:flatten([PartList || {PartList,_} <- L]),
+%%    BigPreTab = tuple_to_list(
+%%		  make_big_prefixes(LL,
+%%				    erlang:make_tuple(?BIG_PREFIX_SIZE,0))),
+%%    dump_big_prefixes(Out,BigPreTab),
+    file:close(Out),
+    ok.
+    
+   
+
+d(Out,List,D,C) ->
+    d_sub(Out,List,D,C),
+    d_top_hash(Out,List,D,C),
+    d_top(Out,List,D,C).
+d_sub(_Out,[],_D,_C) ->
+    ok;
+d_sub(Out,[{_CP,[],_Res}|T],D,C) ->
+    d_sub(Out,T,D,C+1);
+d_sub(Out,[{_CP,Subs,_Res0}|T],D,C) ->
+    d(Out,Subs,[C|D],0),
+    d_sub(Out,T,D,C+1).
+d_top(Out,L,D,C) ->
+    io:format(Out,"static CompEntry ~s[] = {~n",[format_depth(D)]),
+    d_top_1(Out,L,D,C),
+    io:format(Out,"}; /* ~s */ ~n",[format_depth(D)]).
+    
+d_top_1(_Out,[],_D,_C) ->
+    ok;
+d_top_1(Out,[{CP,[],Res}|T],D,C) ->
+    io:format(Out,
+	          "{~w, ~w, 0, NULL, NULL}",[CP,Res]),
+    if 
+	T =:= [] ->
+	    io:format(Out,"~n",[]);
+	true ->
+	    io:format(Out,",~n",[])
+    end,
+    d_top_1(Out,T,D,C+1);
+d_top_1(Out,[{CP,Subs,_Res}|T],D,C) ->
+    io:format(Out,
+	          "{~w, 0, ~w, ~s, ~s}",[CP,length(Subs),
+					  format_depth([C|D]),
+					 "hash_"++format_depth([C|D])]),
+    if 
+	T =:= [] ->
+	    io:format(Out,"~n",[]);
+	true ->
+	    io:format(Out,",~n",[])
+    end,
+    d_top_1(Out,T,D,C+1).
+
+
+d_top_hash(Out,List,D,_C) ->
+     HSize = length(List)*?HASH_SIZE_FACTOR,
+     io:format(Out,"static int ~s[~p] = ~n",["hash_"++format_depth(D),HSize]),
+     Tup = d_top_hash_1(List,0,erlang:make_tuple(HSize,-1),HSize),
+     io:format(Out,"~p; /* ~s */ ~n",[Tup,"hash_"++format_depth(D)]).
+
+d_top_hash_1([],_,Hash,_HSize) -> 
+    Hash;
+d_top_hash_1([{CP,_,_}|T],Index,Hash,HSize) ->
+    Bucket = hash_search(Hash,HSize,CP rem HSize),
+    d_top_hash_1(T,Index+1,erlang:setelement(Bucket+1,Hash,Index),HSize).
+
+hash_search(Hash,_HSize,Bucket) when element(Bucket+1,Hash) =:= -1 ->
+    Bucket;
+hash_search(Hash,HSize,Bucket) ->
+    hash_search(Hash,HSize,(Bucket + 1) rem HSize). 
+
+format_depth(D) ->
+    lists:reverse(tl(lists:reverse(lists:flatten(["compose_tab_",[ integer_to_list(X) ++ "_" || X <- lists:reverse(D) ]])))).
+
+
+
+
+make_prefix_table([],Table) ->
+    Table;
+make_prefix_table([{C,_,_}|T],Table) when C =< 4023 ->
+    Index = (C div 32) + 1 - 24,
+    Pos = C rem 32,
+    X = element(Index,Table),
+    Y = X bor (1 bsl Pos),
+    NewTab = setelement(Index,Table,Y),
+    make_prefix_table(T,NewTab);
+make_prefix_table([_|T],Tab) ->
+    make_prefix_table(T,Tab).
+
+dump_prefixes(Out,L) ->
+    io:format(Out,"#define COMP_CANDIDATE_MAP_OFFSET 24~n",[]),
+    io:format(Out,"static Uint32 comp_candidate_map[] = {~n",[]),
+    dump_prefixes_1(Out,L).
+dump_prefixes_1(Out,[H]) ->
+    io:format(Out,"    0x~8.16.0BU~n",[H]),
+    io:format(Out,"};~n",[]);
+dump_prefixes_1(Out,[H|T]) ->
+    io:format(Out,"    0x~8.16.0BU,~n",[H]),
+    dump_prefixes_1(Out,T).
+
+%% make_big_prefixes([],Table) ->
+%%     Table;
+%% make_big_prefixes([C|T],Table) ->
+%%     Index = (C div 32) + 1,
+%%     Pos = C rem 32,
+%%     X = element(Index,Table),
+%%     Y = X bor (1 bsl Pos),
+%%     NewTab = setelement(Index,Table,Y),
+%%     make_big_prefixes(T,NewTab).
+
+%% dump_big_prefixes(Out,L) ->
+%%     io:format(Out,"#define BIG_COMP_CANDIDATE_SIZE ~w~n", [?BIG_PREFIX_SIZE]),
+%%     io:format(Out,"static Uint32 big_comp_candidate_map[] = {~n",[]),
+%%     dump_prefixes_1(Out,L).
+   
+pick([],_,Acc) ->
+    {lists:reverse(Acc),[]};
+pick([{[H|TT],N}|T],H,Acc) ->
+    pick(T,H,[{TT,N}|Acc]);
+pick([{[H|_],_}|_]=L,M,Acc) when H =/= M ->
+    {lists:reverse(Acc),L}.
+    
+
+group([]) ->
+    [];
+group([{[H],N}|T]) ->
+    {Part,Rest} = pick(T,H,[]),
+    [{H,group(Part),N}| group(Rest)];
+group([{[H|_],_}|_]=L) ->
+    {Part,Rest} = pick(L,H,[]),
+    [{H,group(Part),0}| group(Rest)].
+    
+    
+
+
+    
diff --git a/erts/emulator/obsolete/driver.h b/erts/emulator/obsolete/driver.h
deleted file mode 100644
index 708fe68e1a..0000000000
--- a/erts/emulator/obsolete/driver.h
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-/*
- * OLD, OBSOLETE include file for erlang driver writers.
- * New drivers should use erl_driver.h instead.
- */
-
-#ifndef __DRIVER_H__
-#define __DRIVER_H__
-
-#include <stdlib.h>
-#include "driver_int.h"
-
-#undef _ANSI_ARGS_
-#undef CONST
-
-#if ((defined(__STDC__) || defined(SABER)) && !defined(NO_PROTOTYPE)) || defined(__cplusplus) || defined(USE_PROTOTYPE)
-#   define _USING_PROTOTYPES_ 1
-#   define _ANSI_ARGS_(x)	x
-#   define CONST const
-#else
-#   define _ANSI_ARGS_(x)	()
-#   define CONST
-#endif
-
-#ifdef __cplusplus
-#   define EXTERN extern "C"
-#else
-#   define EXTERN extern
-#endif
-
-/* Values for mode arg to driver_select() */
-
-#define DO_READ	 (1 << 0)
-#define DO_WRITE (1 << 1)
-
-/* Flags for set_port_control_flags() */
-#define PORT_CONTROL_FLAG_BINARY	1
-#define PORT_CONTROL_FLAG_HEAVY		2
-
-/* This macro is used to name a dynamic driver's init function in */
-/* a way that doesn't lead to conflicts. This is crucial when using */
-/* operating systems that has one namespace for all symbols */
-/* (e.g. VxWorks). Example: if you have an dynamic driver C source */
-/* file named echo_drv.c, you use the macro like this: */
-/* int DRIVER_INIT(echo_drv)(void *handle) */
-#if defined(VXWORKS)
-#  define DRIVER_INIT(DRIVER_NAME) DRIVER_NAME  ## _init
-#elif defined(__WIN32__)
-#  define DRIVER_INIT(DRIVER_NAME) __declspec(dllexport) driver_init
-#else 
-#  define DRIVER_INIT(DRIVER_NAME)  driver_init
-#endif
-
-typedef int (*F_PTR)();    /* a function pointer */
-typedef long (*L_PTR)();   /* pointer to a function returning long */
-
-extern int null_func();
-
-/* This structure MUST match Binary in global.h exactly!!! */
-typedef struct driver_binary {
-    int orig_size;        /* total length of binary */
-    char orig_bytes[1];   /* the data (char instead of byte!) */
-} DriverBinary;
-
-typedef struct {
-    int vsize;     /* length of vectors */
-    int size;      /* total size in bytes */
-    SysIOVec* iov;
-    DriverBinary**  binv;
-} ErlIOVec;
-
-/*
- * OLD, OBSOLETE driver entry structure.
- */
-
-typedef struct driver_entry {
-    F_PTR init;          /* called at system start up (no args) */
-    L_PTR start;         /* called when some one does an open_port
-			 args: port, command (nul-terminated),
-			 additional/alternate args for fd/vanilla/spawn driver.
-			 return value -1 means failure, other
-			 is saved and passed to the other funcs */
-    F_PTR stop;          /* called when port is closed, and when the
-			    emulator is halted - arg: start_return */
-    F_PTR output;	 /* called when we have output from erlang to the port
-		         args: start_return, buf, buflen */
-    F_PTR ready_input;   /* called when we have input from one of the driver's
-			 file descriptors - args: start_return, fd */
-    F_PTR ready_output;  /* called when output is possible to one of the driver's
-			 file descriptors - args: start_return, fd */
-    char *driver_name;   /* name supplied as {driver,Name,Args} to open_port */
-
-    F_PTR finish;        /* called before unloading (DYNAMIC DRIVERS ONLY) */
-    void *handle;        /* file handle             (DYNAMIC DRIVERS ONLY) */
-    F_PTR control;	 /* "ioctl" for drivers (invoked by port_command/3) */
-    F_PTR timeout;       /* Reserved */
-    F_PTR outputv;       /* Reserved */
-    F_PTR ready_async;   /* Completion routine for driver_async */
-    F_PTR padding1[3];   /* pad to match size of modern driver struct */
-    int padding2[4];     /* more pad */
-    F_PTR padding3[3];   /* even more padding */
-} DriverEntry;
-
-
-/* These are the kernel functions available for driver writers */
-
-EXTERN int driver_select _ANSI_ARGS_((int,int,int,int));
-
-EXTERN int driver_output _ANSI_ARGS_((int, char*, int));
-EXTERN int driver_output2 _ANSI_ARGS_((int, char*, int, char*, int));
-EXTERN int driver_output_binary _ANSI_ARGS_((int, char*, int,
-					     DriverBinary*, int, int));
-EXTERN int driver_outputv _ANSI_ARGS_((int, char*,int,ErlIOVec*,int));
-
-EXTERN int driver_vec_to_buf _ANSI_ARGS_((ErlIOVec*, char*, int));
-
-EXTERN int driver_set_timer _ANSI_ARGS_((int, unsigned long));
-EXTERN int driver_cancel_timer _ANSI_ARGS_((int));
-
-/*
- * The following functions are used to initiate a close of a port
- * from a driver.
- */
-EXTERN int driver_failure_eof _ANSI_ARGS_((int));
-EXTERN int driver_failure_atom _ANSI_ARGS_((int, char *));
-EXTERN int driver_failure_posix _ANSI_ARGS_((int, int));
-EXTERN int driver_failure _ANSI_ARGS_((int, int));
-EXTERN int driver_exit _ANSI_ARGS_ ((int, int));
-
-EXTERN char* erl_errno_id _ANSI_ARGS_((int error));
-EXTERN void set_busy_port _ANSI_ARGS_((int, int));
-EXTERN void add_driver_entry _ANSI_ARGS_((DriverEntry *));
-EXTERN int remove_driver_entry _ANSI_ARGS_((DriverEntry *));
-EXTERN void set_port_control_flags _ANSI_ARGS_((int, int));
-
-/* Binary interface */
-/* NOTE: DO NOT overwrite a binary with new data (if the data is delivered);
-** since the binary is a shared object it MUST be written once.
-*/
-
-EXTERN DriverBinary* driver_alloc_binary _ANSI_ARGS_((int));
-EXTERN DriverBinary* driver_realloc_binary _ANSI_ARGS_((DriverBinary*, int));
-EXTERN void driver_free_binary _ANSI_ARGS_((DriverBinary*));
-
-
-/* Queue interface */
-EXTERN int driver_enqv _ANSI_ARGS_((int, ErlIOVec*, int));
-EXTERN int driver_pushqv _ANSI_ARGS_((int, ErlIOVec*, int));
-EXTERN int driver_deq _ANSI_ARGS_((int, int));
-EXTERN SysIOVec* driver_peekq _ANSI_ARGS_((int, int*));
-EXTERN int driver_sizeq _ANSI_ARGS_((int));
-EXTERN int driver_enq_bin _ANSI_ARGS_((int, DriverBinary*, int, int));
-EXTERN int driver_enq _ANSI_ARGS_((int, char*, int));
-EXTERN int driver_pushq_bin _ANSI_ARGS_((int, DriverBinary*, int, int));
-EXTERN int driver_pushq _ANSI_ARGS_((int, char*, int));
-
-/* Memory management */
-EXTERN void *driver_alloc _ANSI_ARGS_((size_t));
-EXTERN void *driver_realloc _ANSI_ARGS_((void*, size_t));
-EXTERN void driver_free _ANSI_ARGS_((void*));
-
-/* Shared / dynamic link libraries */
-EXTERN void *driver_dl_open _ANSI_ARGS_((char *));
-EXTERN void *driver_dl_sym _ANSI_ARGS_((void *, char *));
-EXTERN int driver_dl_close _ANSI_ARGS_((void *));
-EXTERN char *driver_dl_error _ANSI_ARGS_((void));
-
-/* Async IO functions */
-EXTERN long driver_async _ANSI_ARGS_((int,
-				      unsigned int*,
-				      void (*)(void*), 
-				      void *,
-				      void (*)(void*)));
-EXTERN int driver_async_cancel _ANSI_ARGS_((long));
-
-EXTERN int driver_lock_driver _ANSI_ARGS_((int));
-
-/* Threads */
-typedef void* erl_mutex_t;
-typedef void* erl_cond_t;
-typedef void* erl_thread_t;
-
-EXTERN erl_mutex_t erts_mutex_create _ANSI_ARGS_((void));
-EXTERN int erts_mutex_destroy _ANSI_ARGS_((erl_mutex_t));
-EXTERN int erts_mutex_lock _ANSI_ARGS_((erl_mutex_t));
-EXTERN int erts_mutex_unlock _ANSI_ARGS_((erl_mutex_t));
-
-EXTERN erl_cond_t erts_cond_create _ANSI_ARGS_((void));
-EXTERN int erts_cond_destroy _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_signal _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_broadcast _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_wait _ANSI_ARGS_((erl_cond_t, erl_mutex_t));
-EXTERN int erts_cond_timedwait _ANSI_ARGS_((erl_cond_t, erl_mutex_t, long));
-
-EXTERN int erts_thread_create _ANSI_ARGS_((erl_thread_t*,
-					 void* (*func)(void*),
-					 void* arg,
-					 int detached));
-EXTERN erl_thread_t erts_thread_self _ANSI_ARGS_((void));
-EXTERN void erts_thread_exit _ANSI_ARGS_((void*));
-EXTERN int  erts_thread_join _ANSI_ARGS_((erl_thread_t, void**));
-EXTERN int  erts_thread_kill _ANSI_ARGS_((erl_thread_t));
-
-
-typedef unsigned long DriverTermData;
-
-#define TERM_DATA(x) ((DriverTermData) (x))
-
-/* Possible types to send from driver          Argument type */
-#define ERL_DRV_NIL         ((DriverTermData) 1)  /* None */
-#define ERL_DRV_ATOM        ((DriverTermData) 2)  /* driver_mk_atom(string) */
-#define ERL_DRV_INT         ((DriverTermData) 3)  /* int */
-#define ERL_DRV_PORT        ((DriverTermData) 4)  /* driver_mk_port(ix) */
-#define ERL_DRV_BINARY      ((DriverTermData) 5)  /* ErlDriverBinary*, int */
-#define ERL_DRV_STRING      ((DriverTermData) 6)  /* char*, int */
-#define ERL_DRV_TUPLE       ((DriverTermData) 7)  /* int */
-#define ERL_DRV_LIST        ((DriverTermData) 8)  /* int */
-#define ERL_DRV_STRING_CONS ((DriverTermData) 9)  /* char*, int */
-#define ERL_DRV_PID         ((DriverTermData) 10) /* driver_connected,... */
-
-/* DriverTermData is the type to use for casts when building 
- * terms that should be sent to connected process,
- * for instance a tuple on the form {tcp, Port, [Tag|Binary]}
- *
- * DriverTermData spec[] = {
- *    ERL_DRV_ATOM, driver_mk_atom("tcp"),
- *    ERL_DRV_PORT, driver_mk_port(drv->ix),
- *             ERL_DRV_INT, REPLY_TAG,
- *             ERL_DRV_BIN, 50, TERM_DATA(buffer),
- *             ERL_DRV_LIST, 2,
- *    ERL_DRV_TUPLE, 3,
- *  }       
- *             
- */
-
-EXTERN DriverTermData driver_mk_atom _ANSI_ARGS_ ((char*));
-EXTERN DriverTermData driver_mk_port _ANSI_ARGS_ ((int));
-EXTERN DriverTermData driver_connected _ANSI_ARGS_((int));
-EXTERN DriverTermData driver_caller _ANSI_ARGS_((int));
-
-EXTERN int driver_output_term _ANSI_ARGS_((int, DriverTermData *, int));
-EXTERN int driver_send_term _ANSI_ARGS_((int, DriverTermData, DriverTermData *, int));
-
-#endif
-
-
diff --git a/erts/emulator/pcre/pcre_compile.c b/erts/emulator/pcre/pcre_compile.c
index 29743362d4..9508c5a697 100644
--- a/erts/emulator/pcre/pcre_compile.c
+++ b/erts/emulator/pcre/pcre_compile.c
@@ -92,6 +92,11 @@ is 4 there is plenty of room. */
 
 #define COMPILE_WORK_SIZE (4096)
 
+/* The overrun tests check for a slightly smaller size so that they detect the
+overrun before it actually does run off the end of the data block. */
+
+#define WORK_SIZE_CHECK (COMPILE_WORK_SIZE - 100)
+
 
 /* Table for handling escaped characters in the range '0'-'z'. Positive returns
 are simple data values; negative values are for special things like \d and so
@@ -2445,7 +2450,7 @@ for (;; ptr++)
 #ifdef DEBUG
     if (code > cd->hwm) cd->hwm = code;                 /* High water info */
 #endif
-    if (code > cd->start_workspace + COMPILE_WORK_SIZE) /* Check for overrun */
+    if (code > cd->start_workspace + WORK_SIZE_CHECK)   /* Check for overrun */
       {
       *errorcodeptr = ERR52;
       goto FAILED;
@@ -2494,7 +2499,7 @@ for (;; ptr++)
   /* In the real compile phase, just check the workspace used by the forward
   reference list. */
 
-  else if (cd->hwm > cd->start_workspace + COMPILE_WORK_SIZE)
+  else if (cd->hwm > cd->start_workspace + WORK_SIZE_CHECK)
     {
     *errorcodeptr = ERR52;
     goto FAILED;
diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c
index 51625130c3..3fe13ca32e 100644
--- a/erts/emulator/pcre/pcre_exec.c
+++ b/erts/emulator/pcre/pcre_exec.c
@@ -5191,7 +5191,6 @@ for(;;)
       EDEBUGF(("Loop limit break detected"));
       return PCRE_ERROR_LOOP_LIMIT;
   RESTART_INTERRUPTED:
-      md->match_call_count = 0; 
       md->loop_limit = extra_data->loop_limit;
       rc = match(NULL,NULL,NULL,0,md,0,NULL,0,0);
       *extra_data->loop_counter_return = 
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 218bd79584..cd4de21d65 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -1137,6 +1137,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 
  restart:
 
+#ifdef ERTS_BREAK_REQUESTED
+    if (ERTS_BREAK_REQUESTED)
+	erts_do_break_handling();
+#endif
+
     /* Figure out timeout value */
     if (do_wait) {
 	erts_time_remaining(&wait_time);
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 5dfd66bd7c..eaef6680dd 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -35,6 +35,7 @@
 #include "global.h"
 #include "erl_threads.h"
 #include "erl_mtrace.h"
+#include "erl_time.h"
 #include "big.h"
 
 #if HAVE_ERTS_MSEG
@@ -76,17 +77,29 @@ static int atoms_initialized;
 
 static Uint cache_check_interval;
 
+typedef struct mem_kind_t MemKind;
+
 static void check_cache(void *unused);
-static void mseg_clear_cache(void);
+static void mseg_clear_cache(MemKind*);
 static int is_cache_check_scheduled;
 #ifdef ERTS_THREADS_NO_SMP
 static int is_cache_check_requested;
 #endif
 
+#if HALFWORD_HEAP
+static int initialize_pmmap(void);
+static void *pmmap(size_t size);
+static int pmunmap(void *p, size_t size);
+static void *pmremap(void *old_address, size_t old_size,
+		     size_t new_size);
+#endif
+
 #if HAVE_MMAP
 /* Mmap ... */
 
 #define MMAP_PROT		(PROT_READ|PROT_WRITE)
+
+
 #ifdef MAP_ANON
 #  define MMAP_FLAGS		(MAP_ANON|MAP_PRIVATE)
 #  define MMAP_FD		(-1)
@@ -102,19 +115,35 @@ static int mmap_fd;
 #  define HAVE_MSEG_RECREATE 0
 #endif
 
+#if HALFWORD_HEAP
+#define CAN_PARTLY_DESTROY 0
+#else
 #define CAN_PARTLY_DESTROY 1
+#endif
 #else  /* #if HAVE_MMAP */
 #define CAN_PARTLY_DESTROY 0
 #error "Not supported"
 #endif /* #if HAVE_MMAP */
 
+#if defined(ERTS_MSEG_FAKE_SEGMENTS) && HALFWORD_HEAP
+#  warning "ERTS_MSEG_FAKE_SEGMENTS will only be used for high memory segments" 
+#endif 
 
 #if defined(ERTS_MSEG_FAKE_SEGMENTS)
 #undef CAN_PARTLY_DESTROY
 #define CAN_PARTLY_DESTROY 0
 #endif
 
-static const ErtsMsegOpt_t default_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER;
+const ErtsMsegOpt_t erts_mseg_default_opt = {
+    1,			/* Use cache		     */
+    1,			/* Preserv data		     */
+    0,			/* Absolute shrink threshold */
+    0			/* Relative shrink threshold */
+#if HALFWORD_HEAP
+    ,0                  /* need low memory */
+#endif
+};
+
 
 typedef struct cache_desc_t_ {
     void *seg;
@@ -145,14 +174,43 @@ static struct {
     CallCounter check_cache;
 } calls;
 
-static cache_desc_t cache_descs[MAX_CACHE_SIZE];
-static cache_desc_t *free_cache_descs;
-static cache_desc_t *cache;
-static cache_desc_t *cache_end;
-static Uint cache_hits;
-static Uint cache_size;
-static Uint min_cached_seg_size;
-static Uint max_cached_seg_size;
+struct mem_kind_t {
+    cache_desc_t cache_descs[MAX_CACHE_SIZE];
+    cache_desc_t *free_cache_descs;
+    cache_desc_t *cache;
+    cache_desc_t *cache_end;
+
+    Uint cache_size;
+    Uint min_cached_seg_size;
+    Uint max_cached_seg_size;
+    Uint cache_hits;
+
+    struct {
+	struct {
+	    Uint watermark;
+	    Uint no;
+	    Uint sz;
+	} current;
+	struct {
+	    Uint no;
+	    Uint sz;
+	} max;
+	struct {
+	    Uint no;
+	    Uint sz;
+	} max_ever;
+    } segments;
+
+    const char* name;
+    MemKind* next;
+};/*MemKind*/
+
+#if HALFWORD_HEAP
+static MemKind low_mem, hi_mem;
+#else
+static MemKind the_mem;
+#endif
+static MemKind* mk_list = NULL;
 
 static Uint max_cache_size;
 static Uint abs_max_cache_bad_fit;
@@ -162,47 +220,32 @@ static Uint rel_max_cache_bad_fit;
 static Uint min_seg_size;
 #endif
 
-struct {
-    struct {
-	Uint watermark;
-	Uint no;
-	Uint sz;
-    } current;
-    struct {
-	Uint no;
-	Uint sz;
-    } max;
-    struct {
-	Uint no;
-	Uint sz;
-    } max_ever;
-} segments;
 
-#define ERTS_MSEG_ALLOC_STAT(SZ)					\
+#define ERTS_MSEG_ALLOC_STAT(C,SZ)					\
 do {									\
-    segments.current.no++;						\
-    if (segments.max.no < segments.current.no)				\
-	segments.max.no = segments.current.no;				\
-    if (segments.current.watermark < segments.current.no)		\
-	segments.current.watermark = segments.current.no;		\
-    segments.current.sz += (SZ);					\
-    if (segments.max.sz < segments.current.sz)				\
-	segments.max.sz = segments.current.sz;				\
+    C->segments.current.no++;						\
+    if (C->segments.max.no < C->segments.current.no)			\
+	C->segments.max.no = C->segments.current.no;			\
+    if (C->segments.current.watermark < C->segments.current.no)		\
+	C->segments.current.watermark = C->segments.current.no;		\
+    C->segments.current.sz += (SZ);					\
+    if (C->segments.max.sz < C->segments.current.sz)			\
+	C->segments.max.sz = C->segments.current.sz;			\
 } while (0)
 
-#define ERTS_MSEG_DEALLOC_STAT(SZ)					\
+#define ERTS_MSEG_DEALLOC_STAT(C,SZ)					\
 do {									\
-    ASSERT(segments.current.no > 0);					\
-    segments.current.no--;						\
-    ASSERT(segments.current.sz >= (SZ));				\
-    segments.current.sz -= (SZ);					\
+    ASSERT(C->segments.current.no > 0);					\
+    C->segments.current.no--;						\
+    ASSERT(C->segments.current.sz >= (SZ));				\
+    C->segments.current.sz -= (SZ);					\
 } while (0) 
 
-#define ERTS_MSEG_REALLOC_STAT(OSZ, NSZ)				\
+#define ERTS_MSEG_REALLOC_STAT(C,OSZ, NSZ)				\
 do {									\
-    ASSERT(segments.current.sz >= (OSZ));				\
-    segments.current.sz -= (OSZ);					\
-    segments.current.sz += (NSZ);					\
+    ASSERT(C->segments.current.sz >= (OSZ));				\
+    C->segments.current.sz -= (OSZ);					\
+    C->segments.current.sz += (NSZ);					\
 } while (0)
 
 #define ONE_GIGA (1000000000)
@@ -232,6 +275,7 @@ static void thread_safe_init(void)
 {
     erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
     erts_mtx_init(&mseg_mutex, "mseg");
+
 #ifdef ERTS_THREADS_NO_SMP
     main_tid = erts_thr_self();
 #endif
@@ -256,7 +300,7 @@ schedule_cache_check(void)
 #endif
 	{
 	    cache_check_timer.active = 0;
-	    erl_set_timer(&cache_check_timer,
+	    erts_set_timer(&cache_check_timer,
 			  check_cache,
 			  NULL,
 			  NULL,
@@ -287,28 +331,45 @@ check_schedule_cache_check(void)
 static void
 mseg_shutdown(void)
 {
+    MemKind* mk;
     erts_mtx_lock(&mseg_mutex);
-    mseg_clear_cache();
+    for (mk=mk_list; mk; mk=mk->next) {
+	mseg_clear_cache(mk);
+    }
     erts_mtx_unlock(&mseg_mutex);
 }
 
 static ERTS_INLINE void *
-mseg_create(Uint size)
+mseg_create(MemKind* mk, Uint size)
 {
     void *seg;
 
     ASSERT(size % page_size == 0);
 
+#if HALFWORD_HEAP
+    if (mk == &low_mem) {
+	seg = pmmap(size);
+	if ((unsigned long) seg & CHECK_POINTER_MASK) {
+	    erts_fprintf(stderr,"Pointer mask failure (0x%08lx)\n",(unsigned long) seg);
+	    return NULL;
+	}
+    }
+    else
+#endif
+    {
 #if defined(ERTS_MSEG_FAKE_SEGMENTS)
-    seg = erts_sys_alloc(ERTS_ALC_N_INVALID, NULL, size);
+	seg = erts_sys_alloc(ERTS_ALC_N_INVALID, NULL, size);
 #elif HAVE_MMAP
-    seg = (void *) mmap((void *) 0, (size_t) size,
-			MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0);
-    if (seg == (void *) MAP_FAILED)
-	seg = NULL;
+	{
+	    seg = (void *) mmap((void *) 0, (size_t) size,
+				MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0);
+	    if (seg == (void *) MAP_FAILED)
+		seg = NULL;
+	}
 #else
-#error "Missing mseg_create() implementation"
+# error "Missing mseg_create() implementation"
 #endif
+    }
 
     INC_CC(create);
 
@@ -316,23 +377,29 @@ mseg_create(Uint size)
 }
 
 static ERTS_INLINE void
-mseg_destroy(void *seg, Uint size)
+mseg_destroy(MemKind* mk, void *seg, Uint size)
 {
-#if defined(ERTS_MSEG_FAKE_SEGMENTS)
-    erts_sys_free(ERTS_ALC_N_INVALID, NULL, seg);
-#elif HAVE_MMAP
+    int res;
 
-#ifdef DEBUG
-    int res =
+#if HALFWORD_HEAP
+    if (mk == &low_mem) {
+	res = pmunmap((void *) seg, size);
+    }
+    else
 #endif
-
-	munmap((void *) seg, size);
+    {
+#ifdef ERTS_MSEG_FAKE_SEGMENTS
+	erts_sys_free(ERTS_ALC_N_INVALID, NULL, seg);
+	res = 0;
+#elif HAVE_MMAP
+	res = munmap((void *) seg, size);
+#else
+# error "Missing mseg_destroy() implementation"
+#endif
+    }
 
     ASSERT(size % page_size == 0);
     ASSERT(res == 0);
-#else
-#error "Missing mseg_destroy() implementation"
-#endif
 
     INC_CC(destroy);
 
@@ -341,25 +408,44 @@ mseg_destroy(void *seg, Uint size)
 #if HAVE_MSEG_RECREATE
 
 static ERTS_INLINE void *
-mseg_recreate(void *old_seg, Uint old_size, Uint new_size)
+mseg_recreate(MemKind* mk, void *old_seg, Uint old_size, Uint new_size)
 {
     void *new_seg;
 
     ASSERT(old_size % page_size == 0);
     ASSERT(new_size % page_size == 0);
 
+#if HALFWORD_HEAP
+    if (mk == &low_mem) {
+	new_seg = (void *) pmremap((void *) old_seg,
+				   (size_t) old_size,
+				   (size_t) new_size);
+    }
+    else
+#endif
+    {
 #if defined(ERTS_MSEG_FAKE_SEGMENTS)
-    new_seg = erts_sys_realloc(ERTS_ALC_N_INVALID, NULL, old_seg, new_size);
+	new_seg = erts_sys_realloc(ERTS_ALC_N_INVALID, NULL, old_seg, new_size);
 #elif HAVE_MREMAP
-    new_seg = (void *) mremap((void *) old_seg,
-			      (size_t) old_size,
-			      (size_t) new_size,
-			      MREMAP_MAYMOVE);
-    if (new_seg == (void *) MAP_FAILED)
-	new_seg = NULL;
+
+    #if defined(__NetBSD__)
+	new_seg = (void *) mremap((void *) old_seg,
+				  (size_t) old_size,
+				  NULL,
+				  (size_t) new_size,
+				  0);
+    #else
+	new_seg = (void *) mremap((void *) old_seg,
+				  (size_t) old_size,
+				  (size_t) new_size,
+				  MREMAP_MAYMOVE);
+    #endif
+	if (new_seg == (void *) MAP_FAILED)
+	    new_seg = NULL;
 #else
 #error "Missing mseg_recreate() implementation"
 #endif
+    }
 
     INC_CC(recreate);
 
@@ -370,134 +456,142 @@ mseg_recreate(void *old_seg, Uint old_size, Uint new_size)
 
 
 static ERTS_INLINE cache_desc_t * 
-alloc_cd(void)
+alloc_cd(MemKind* mk)
 {    
-    cache_desc_t *cd = free_cache_descs;
+    cache_desc_t *cd = mk->free_cache_descs;
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
     if (cd)
-	free_cache_descs = cd->next;
+	mk->free_cache_descs = cd->next;
     return cd;
 }
 
 static ERTS_INLINE void
-free_cd(cache_desc_t *cd)
+free_cd(MemKind* mk, cache_desc_t *cd)
 {
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
-    cd->next = free_cache_descs;
-    free_cache_descs = cd;
+    cd->next = mk->free_cache_descs;
+    mk->free_cache_descs = cd;
 }
 
 
 static ERTS_INLINE void
-link_cd(cache_desc_t *cd)
+link_cd(MemKind* mk, cache_desc_t *cd)
 {
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
-    if (cache)
-	cache->prev = cd;
-    cd->next = cache;
+    if (mk->cache)
+	mk->cache->prev = cd;
+    cd->next = mk->cache;
     cd->prev = NULL;
-    cache = cd;
+    mk->cache = cd;
 
-    if (!cache_end) {
+    if (!mk->cache_end) {
 	ASSERT(!cd->next);
-	cache_end = cd;
+	mk->cache_end = cd;
     }
 
-    cache_size++;
+    mk->cache_size++;
 }
 
+#if CAN_PARTLY_DESTROY
 static ERTS_INLINE void
-end_link_cd(cache_desc_t *cd)
+end_link_cd(MemKind* mk, cache_desc_t *cd)
 {
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
-    if (cache_end)
-	cache_end->next = cd;
+    if (mk->cache_end)
+	mk->cache_end->next = cd;
     cd->next = NULL;
-    cd->prev = cache_end;
-    cache_end = cd;
+    cd->prev = mk->cache_end;
+    mk->cache_end = cd;
 
-    if (!cache) {
+    if (!mk->cache) {
 	ASSERT(!cd->prev);
-	cache = cd;
+	mk->cache = cd;
     }
 
-    cache_size++;
+    mk->cache_size++;
 }
+#endif
 
 static ERTS_INLINE void
-unlink_cd(cache_desc_t *cd)
+unlink_cd(MemKind* mk, cache_desc_t *cd)
 {
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
     if (cd->next)
 	cd->next->prev = cd->prev;
     else
-	cache_end = cd->prev;
+	mk->cache_end = cd->prev;
     
     if (cd->prev)
 	cd->prev->next = cd->next;
     else
-	cache = cd->next;
-    ASSERT(cache_size > 0);
-    cache_size--;
+	mk->cache = cd->next;
+    ASSERT(mk->cache_size > 0);
+    mk->cache_size--;
 }
 
 static ERTS_INLINE void
-check_cache_limits(void)
+check_cache_limits(MemKind* mk)
 {
     cache_desc_t *cd;
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
-    max_cached_seg_size = 0;
-    min_cached_seg_size = ~((Uint) 0);
-    for (cd = cache; cd; cd = cd->next) {
-	if (cd->size < min_cached_seg_size)
-	    min_cached_seg_size = cd->size;
-	if (cd->size > max_cached_seg_size)
-	    max_cached_seg_size = cd->size;
+    mk->max_cached_seg_size = 0;
+    mk->min_cached_seg_size = ~((Uint) 0);
+    for (cd = mk->cache; cd; cd = cd->next) {
+	if (cd->size < mk->min_cached_seg_size)
+	    mk->min_cached_seg_size = cd->size;
+	if (cd->size > mk->max_cached_seg_size)
+	    mk->max_cached_seg_size = cd->size;
     }
-
 }
 
 static ERTS_INLINE void
-adjust_cache_size(int force_check_limits)
+adjust_cache_size(MemKind* mk, int force_check_limits)
 {
     cache_desc_t *cd;
     int check_limits = force_check_limits;
-    Sint max_cached = ((Sint) segments.current.watermark
-		       - (Sint) segments.current.no);
+    Sint max_cached = ((Sint) mk->segments.current.watermark
+		       - (Sint) mk->segments.current.no);
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
-    while (((Sint) cache_size) > max_cached && ((Sint) cache_size) > 0) {
-	ASSERT(cache_end);
-	cd = cache_end;
+    while (((Sint) mk->cache_size) > max_cached && ((Sint) mk->cache_size) > 0) {
+	ASSERT(mk->cache_end);
+	cd = mk->cache_end;
 	if (!check_limits &&
-	    !(min_cached_seg_size < cd->size
-	      && cd->size < max_cached_seg_size)) {
+	    !(mk->min_cached_seg_size < cd->size
+	      && cd->size < mk->max_cached_seg_size)) {
 	    check_limits = 1;
 	}
 	if (erts_mtrace_enabled)
 	    erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg);
-	mseg_destroy(cd->seg, cd->size);
-	unlink_cd(cd);
-	free_cd(cd);
+	mseg_destroy(mk, cd->seg, cd->size);
+	unlink_cd(mk,cd);
+	free_cd(mk,cd);
     }
 
     if (check_limits)
-	check_cache_limits();
-
+	check_cache_limits(mk);
 }
 
 static void
-check_cache(void *unused)
+check_one_cache(MemKind* mk)
+{
+    if (mk->segments.current.watermark > mk->segments.current.no)
+	mk->segments.current.watermark--;
+    adjust_cache_size(mk, 0);
+
+    if (mk->cache_size)
+	schedule_cache_check();
+}
+
+static void check_cache(void* unused)
 {
+    MemKind* mk;
     erts_mtx_lock(&mseg_mutex);
 
     is_cache_check_scheduled = 0;
 
-    if (segments.current.watermark > segments.current.no)
-	segments.current.watermark--;
-    adjust_cache_size(0);
-
-    if (cache_size)
-	schedule_cache_check();
+    for (mk=mk_list; mk; mk=mk->next) {
+	check_one_cache(mk);
+    }
 
     INC_CC(check_cache);
 
@@ -505,28 +599,37 @@ check_cache(void *unused)
 }
 
 static void
-mseg_clear_cache(void)
+mseg_clear_cache(MemKind* mk)
 {
-    segments.current.watermark = 0;
+    mk->segments.current.watermark = 0;
 
-    adjust_cache_size(1);
+    adjust_cache_size(mk, 1);
 
-    ASSERT(!cache);
-    ASSERT(!cache_end);
-    ASSERT(!cache_size);
+    ASSERT(!mk->cache);
+    ASSERT(!mk->cache_end);
+    ASSERT(!mk->cache_size);
 
-    segments.current.watermark = segments.current.no;
+    mk->segments.current.watermark = mk->segments.current.no;
 
     INC_CC(clear_cache);
 }
 
+static ERTS_INLINE MemKind* memkind(const ErtsMsegOpt_t *opt)
+{
+#if HALFWORD_HEAP
+    return opt->low_mem ? &low_mem : &hi_mem;
+#else
+    return &the_mem;
+#endif
+}
+
 static void *
 mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
 {
-
     Uint max, min, diff_size, size;
     cache_desc_t *cd, *cand_cd;
     void *seg;
+    MemKind* mk = memkind(opt);
 
     INC_CC(alloc);
 
@@ -539,11 +642,11 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
 
     if (!opt->cache) {
     create_seg:
-	adjust_cache_size(0);
-	seg = mseg_create(size);
+	adjust_cache_size(mk,0);
+	seg = mseg_create(mk, size);
 	if (!seg) {
-	    mseg_clear_cache();
-	    seg = mseg_create(size);
+	    mseg_clear_cache(mk);
+	    seg = mseg_create(mk, size);
 	    if (!seg)
 		size = 0;
 	}
@@ -552,17 +655,17 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
 	if (seg) {
 	    if (erts_mtrace_enabled)
 		erts_mtrace_crr_alloc(seg, atype, ERTS_MTRACE_SEGMENT_ID, size);
-	    ERTS_MSEG_ALLOC_STAT(size);
+	    ERTS_MSEG_ALLOC_STAT(mk,size);
 	}
 	return seg;
     }
 
-    if (size > max_cached_seg_size)
+    if (size > mk->max_cached_seg_size)
 	goto create_seg;
 
-    if (size < min_cached_seg_size) {
+    if (size < mk->min_cached_seg_size) {
 
-	diff_size = min_cached_seg_size - size;
+	diff_size = mk->min_cached_seg_size - size;
 
 	if (diff_size > abs_max_cache_bad_fit)
 	    goto create_seg;
@@ -576,7 +679,7 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
     min = ~((Uint) 0);
     cand_cd = NULL;
 
-    for (cd = cache; cd; cd = cd->next) {
+    for (cd = mk->cache; cd; cd = cd->next) {
 	if (cd->size >= size) {
 	    if (!cand_cd) {
 		cand_cd = cd;
@@ -597,8 +700,8 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
 	    min = cd->size;
     }
 
-    min_cached_seg_size = min;
-    max_cached_seg_size = max;
+    mk->min_cached_seg_size = min;
+    mk->max_cached_seg_size = max;
 
     if (!cand_cd)
 	goto create_seg;
@@ -607,20 +710,20 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
 
     if (diff_size > abs_max_cache_bad_fit
 	|| 100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size)) {
-	if (max_cached_seg_size < cand_cd->size)	
-	    max_cached_seg_size = cand_cd->size;
-	if (min_cached_seg_size > cand_cd->size)
-	    min_cached_seg_size = cand_cd->size;
+	if (mk->max_cached_seg_size < cand_cd->size)
+	    mk->max_cached_seg_size = cand_cd->size;
+	if (mk->min_cached_seg_size > cand_cd->size)
+	    mk->min_cached_seg_size = cand_cd->size;
 	goto create_seg;
     }
 
-    cache_hits++;
+    mk->cache_hits++;
 
     size = cand_cd->size;
     seg = cand_cd->seg;
 
-    unlink_cd(cand_cd);
-    free_cd(cand_cd);
+    unlink_cd(mk,cand_cd);
+    free_cd(mk,cand_cd);
 
     *size_p = size;
 
@@ -630,7 +733,8 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
     }
 
     if (seg)
-	ERTS_MSEG_ALLOC_STAT(size);
+	ERTS_MSEG_ALLOC_STAT(mk,size);
+
     return seg;
 }
 
@@ -639,41 +743,42 @@ static void
 mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size,
 	     const ErtsMsegOpt_t *opt)
 {
+    MemKind* mk = memkind(opt);
     cache_desc_t *cd;
 
-    ERTS_MSEG_DEALLOC_STAT(size);
+    ERTS_MSEG_DEALLOC_STAT(mk,size);
 
     if (!opt->cache || max_cache_size == 0) {
 	if (erts_mtrace_enabled)
 	    erts_mtrace_crr_free(atype, SEGTYPE, seg);
-	mseg_destroy(seg, size);
+	mseg_destroy(mk, seg, size);
     }
     else {
 	int check_limits = 0;
 	
-	if (size < min_cached_seg_size)
-	    min_cached_seg_size = size;
-	if (size > max_cached_seg_size)
-	    max_cached_seg_size = size;
-
-	if (!free_cache_descs) {
-	    cd = cache_end;
-	    if (!(min_cached_seg_size < cd->size
-		  && cd->size < max_cached_seg_size)) {
+	if (size < mk->min_cached_seg_size)
+	    mk->min_cached_seg_size = size;
+	if (size > mk->max_cached_seg_size)
+	    mk->max_cached_seg_size = size;
+
+	if (!mk->free_cache_descs) {
+	    cd = mk->cache_end;
+	    if (!(mk->min_cached_seg_size < cd->size
+		  && cd->size < mk->max_cached_seg_size)) {
 		check_limits = 1;
 	    }
 	    if (erts_mtrace_enabled)
 		erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg);
-	    mseg_destroy(cd->seg, cd->size);
-	    unlink_cd(cd);
-	    free_cd(cd);
+	    mseg_destroy(mk, cd->seg, cd->size);
+	    unlink_cd(mk,cd);
+	    free_cd(mk,cd);
 	}
 
-	cd = alloc_cd();
+	cd = alloc_cd(mk);
 	ASSERT(cd);
 	cd->seg = seg;
 	cd->size = size;
-	link_cd(cd);
+	link_cd(mk,cd);
 
 	if (erts_mtrace_enabled) {
 	    erts_mtrace_crr_free(atype, SEGTYPE, seg);
@@ -683,7 +788,7 @@ mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size,
 	/* ASSERT(segments.current.watermark >= segments.current.no + cache_size); */
 
 	if (check_limits)
-	    check_cache_limits();
+	    check_cache_limits(mk);
 
 	schedule_cache_check();
 
@@ -696,6 +801,7 @@ static void *
 mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
 	     const ErtsMsegOpt_t *opt)
 {
+    MemKind* mk = memkind(opt);
     void *new_seg;
     Uint new_size;
 
@@ -733,15 +839,15 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
 #if CAN_PARTLY_DESTROY
 
 	    if (shrink_sz > min_seg_size
-		&& free_cache_descs
+		&& mk->free_cache_descs
 		&& opt->cache) {
 		cache_desc_t *cd;
 
-		cd = alloc_cd();
+		cd = alloc_cd(mk);
 		ASSERT(cd);
 		cd->seg = ((char *) seg) + new_size;
 		cd->size = shrink_sz;
-		end_link_cd(cd);
+		end_link_cd(mk,cd);
 
 		if (erts_mtrace_enabled) {
 		    erts_mtrace_crr_realloc(new_seg,
@@ -760,7 +866,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
 					    SEGTYPE,
 					    seg,
 					    new_size);
-		mseg_destroy(((char *) seg) + new_size, shrink_sz);
+		mseg_destroy(mk, ((char *) seg) + new_size, shrink_sz);
 	    }
 
 #elif HAVE_MSEG_RECREATE
@@ -794,7 +900,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
 #if !CAN_PARTLY_DESTROY
 	do_recreate:
 #endif
-	    new_seg = mseg_recreate((void *) seg, old_size, new_size);
+	    new_seg = mseg_recreate(mk, (void *) seg, old_size, new_size);
 	    if (erts_mtrace_enabled)
 		erts_mtrace_crr_realloc(new_seg, atype, SEGTYPE, seg, new_size);
 	    if (!new_seg)
@@ -817,7 +923,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
 
     *new_size_p = new_size;
 
-    ERTS_MSEG_REALLOC_STAT(old_size, new_size);
+    ERTS_MSEG_REALLOC_STAT(mk, old_size, new_size);
 
     return new_seg;
 }
@@ -833,6 +939,8 @@ static struct {
     Eterm mcs;
     Eterm cci;
 
+    Eterm memkind;
+    Eterm name;
     Eterm status;
     Eterm cached_segments;
     Eterm cache_hits;
@@ -882,6 +990,8 @@ init_atoms(void)
 #endif
 
 	AM_INIT(version);
+	AM_INIT(memkind);
+	AM_INIT(name);
 
 	AM_INIT(options);
 	AM_INIT(amcbf);
@@ -983,10 +1093,10 @@ info_options(char *prefix,
     if (print_to_p) {
 	int to = *print_to_p;
 	void *arg = print_to_arg;
-	erts_print(to, arg, "%samcbf: %bpu\n", prefix, abs_max_cache_bad_fit);
-	erts_print(to, arg, "%srmcbf: %bpu\n", prefix, rel_max_cache_bad_fit);
-	erts_print(to, arg, "%smcs: %bpu\n", prefix, max_cache_size);
-	erts_print(to, arg, "%scci: %bpu\n", prefix, cache_check_interval);
+	erts_print(to, arg, "%samcbf: %beu\n", prefix, abs_max_cache_bad_fit);
+	erts_print(to, arg, "%srmcbf: %beu\n", prefix, rel_max_cache_bad_fit);
+	erts_print(to, arg, "%smcs: %beu\n", prefix, max_cache_size);
+	erts_print(to, arg, "%scci: %beu\n", prefix, cache_check_interval);
     }
 
     if (hpp || szp) {
@@ -1022,9 +1132,9 @@ info_calls(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
 
 #define PRINT_CC(TO, TOA, CC)						\
     if (calls.CC.giga_no == 0)						\
-	erts_print(TO, TOA, "mseg_%s calls: %bpu\n", #CC, calls.CC.no);	\
+	erts_print(TO, TOA, "mseg_%s calls: %b32u\n", #CC, calls.CC.no);	\
     else								\
-	erts_print(TO, TOA, "mseg_%s calls: %bpu%09bpu\n", #CC,		\
+	erts_print(TO, TOA, "mseg_%s calls: %b32u%09b32u\n", #CC,		\
 		   calls.CC.giga_no, calls.CC.no)
 
 	int to = *print_to_p;
@@ -1092,65 +1202,88 @@ info_calls(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
 }
 
 static Eterm
-info_status(int *print_to_p,
-	    void *print_to_arg,
-	    int begin_new_max_period,
-	    Uint **hpp,
-	    Uint *szp)
+info_status(MemKind* mk, int *print_to_p, void *print_to_arg,
+	    int begin_new_max_period, Uint **hpp, Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
     
-    if (segments.max_ever.no < segments.max.no)
-	segments.max_ever.no = segments.max.no;
-    if (segments.max_ever.sz < segments.max.sz)
-	segments.max_ever.sz = segments.max.sz;
+    if (mk->segments.max_ever.no < mk->segments.max.no)
+	mk->segments.max_ever.no = mk->segments.max.no;
+    if (mk->segments.max_ever.sz < mk->segments.max.sz)
+	mk->segments.max_ever.sz = mk->segments.max.sz;
 
     if (print_to_p) {
 	int to = *print_to_p;
 	void *arg = print_to_arg;
 
-	erts_print(to, arg, "cached_segments: %bpu\n", cache_size);
-	erts_print(to, arg, "cache_hits: %bpu\n", cache_hits);
-	erts_print(to, arg, "segments: %bpu %bpu %bpu\n",
-		   segments.current.no, segments.max.no, segments.max_ever.no);
-	erts_print(to, arg, "segments_size: %bpu %bpu %bpu\n",
-		   segments.current.sz, segments.max.sz, segments.max_ever.sz);
-	erts_print(to, arg, "segments_watermark: %bpu\n",
-		   segments.current.watermark);
+	erts_print(to, arg, "cached_segments: %beu\n", mk->cache_size);
+	erts_print(to, arg, "cache_hits: %beu\n", mk->cache_hits);
+	erts_print(to, arg, "segments: %beu %beu %beu\n",
+		   mk->segments.current.no, mk->segments.max.no, mk->segments.max_ever.no);
+	erts_print(to, arg, "segments_size: %beu %beu %beu\n",
+		   mk->segments.current.sz, mk->segments.max.sz, mk->segments.max_ever.sz);
+	erts_print(to, arg, "segments_watermark: %beu\n",
+		   mk->segments.current.watermark);
     }
 
     if (hpp || szp) {
 	res = NIL;
 	add_2tup(hpp, szp, &res,
 		 am.segments_watermark,
-		 bld_unstable_uint(hpp, szp, segments.current.watermark));
+		 bld_unstable_uint(hpp, szp, mk->segments.current.watermark));
 	add_4tup(hpp, szp, &res,
 		 am.segments_size,
-		 bld_unstable_uint(hpp, szp, segments.current.sz),
-		 bld_unstable_uint(hpp, szp, segments.max.sz),
-		 bld_unstable_uint(hpp, szp, segments.max_ever.sz));
+		 bld_unstable_uint(hpp, szp, mk->segments.current.sz),
+		 bld_unstable_uint(hpp, szp, mk->segments.max.sz),
+		 bld_unstable_uint(hpp, szp, mk->segments.max_ever.sz));
 	add_4tup(hpp, szp, &res,
 		 am.segments,
-		 bld_unstable_uint(hpp, szp, segments.current.no),
-		 bld_unstable_uint(hpp, szp, segments.max.no),
-		 bld_unstable_uint(hpp, szp, segments.max_ever.no));
+		 bld_unstable_uint(hpp, szp, mk->segments.current.no),
+		 bld_unstable_uint(hpp, szp, mk->segments.max.no),
+		 bld_unstable_uint(hpp, szp, mk->segments.max_ever.no));
 	add_2tup(hpp, szp, &res,
 		 am.cache_hits,
-		 bld_unstable_uint(hpp, szp, cache_hits));
+		 bld_unstable_uint(hpp, szp, mk->cache_hits));
 	add_2tup(hpp, szp, &res,
 		 am.cached_segments,
-		 bld_unstable_uint(hpp, szp, cache_size));
+		 bld_unstable_uint(hpp, szp, mk->cache_size));
 
     }
 
     if (begin_new_max_period) {
-	segments.max.no = segments.current.no;
-	segments.max.sz = segments.current.sz;
+	mk->segments.max.no = mk->segments.current.no;
+	mk->segments.max.sz = mk->segments.current.sz;
     }
 
     return res;
 }
 
+static Eterm info_memkind(MemKind* mk, int *print_to_p, void *print_to_arg,
+			  int begin_max_per, Uint **hpp, Uint *szp)
+{
+    Eterm res = THE_NON_VALUE;
+    Eterm atoms[3];
+    Eterm values[3];
+
+    if (print_to_p) {
+	erts_print(*print_to_p, print_to_arg, "memory kind: %s\n", mk->name);
+    }
+    if (hpp || szp) {
+	atoms[0] = am.name;
+	atoms[1] = am.status;
+	atoms[2] = am.calls;
+	values[0] = erts_bld_string(hpp, szp, mk->name);
+    }
+    values[1] = info_status(mk, print_to_p, print_to_arg, begin_max_per, hpp, szp);
+    values[2] = info_calls(print_to_p, print_to_arg, hpp, szp);
+
+    if (hpp || szp)
+	res = bld_2tup_list(hpp, szp, 3, atoms, values);
+
+    return res;
+}
+
+
 static Eterm
 info_version(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
 {
@@ -1197,6 +1330,7 @@ erts_mseg_info(int *print_to_p,
     Eterm res = THE_NON_VALUE;
     Eterm atoms[4];
     Eterm values[4];
+    Uint n = 0;
 
     erts_mtx_lock(&mseg_mutex);
 
@@ -1207,17 +1341,19 @@ erts_mseg_info(int *print_to_p,
 
 	atoms[0] = am.version;
 	atoms[1] = am.options;
-	atoms[2] = am.status;
-	atoms[3] = am.calls;
+	atoms[2] = am.memkind;
+	atoms[3] = am.memkind;
     }
-
-    values[0] = info_version(print_to_p, print_to_arg, hpp, szp);
-    values[1] = info_options("option ", print_to_p, print_to_arg, hpp, szp);
-    values[2] = info_status(print_to_p, print_to_arg, begin_max_per, hpp, szp);
-    values[3] = info_calls(print_to_p, print_to_arg, hpp, szp);
-
+    values[n++] = info_version(print_to_p, print_to_arg, hpp, szp);
+    values[n++] = info_options("option ", print_to_p, print_to_arg, hpp, szp);
+#if HALFWORD_HEAP
+    values[n++] = info_memkind(&low_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp);
+    values[n++] = info_memkind(&hi_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp);
+#else
+    values[n++] = info_memkind(&the_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp);
+#endif
     if (hpp || szp)
-	res = bld_2tup_list(hpp, szp, 4, atoms, values);
+	res = bld_2tup_list(hpp, szp, n, atoms, values);
 
     erts_mtx_unlock(&mseg_mutex);
 
@@ -1237,7 +1373,7 @@ erts_mseg_alloc_opt(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
 void *
 erts_mseg_alloc(ErtsAlcType_t atype, Uint *size_p)
 {
-    return erts_mseg_alloc_opt(atype, size_p, &default_opt);
+    return erts_mseg_alloc_opt(atype, size_p, &erts_mseg_default_opt);
 }
 
 void
@@ -1252,7 +1388,7 @@ erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, Uint size,
 void
 erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size)
 {
-    erts_mseg_dealloc_opt(atype, seg, size, &default_opt);
+    erts_mseg_dealloc_opt(atype, seg, size, &erts_mseg_default_opt);
 }
 
 void *
@@ -1270,23 +1406,29 @@ void *
 erts_mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size,
 		  Uint *new_size_p)
 {
-    return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, &default_opt);
+    return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, &erts_mseg_default_opt);
 }
 
 void
 erts_mseg_clear_cache(void)
 {
+    MemKind* mk;
     erts_mtx_lock(&mseg_mutex);
-    mseg_clear_cache();
+    for (mk=mk_list; mk; mk=mk->next) {
+	mseg_clear_cache(mk);
+    }
     erts_mtx_unlock(&mseg_mutex);
 }
 
 Uint
 erts_mseg_no(void)
 {
-    Uint n;
+    MemKind* mk;
+    Uint n = 0;
     erts_mtx_lock(&mseg_mutex);
-    n = segments.current.no;
+    for (mk=mk_list; mk; mk=mk->next) {
+	n += mk->segments.current.no;
+    }
     erts_mtx_unlock(&mseg_mutex);
     return n;
 }
@@ -1297,11 +1439,43 @@ erts_mseg_unit_size(void)
     return page_size;
 }
 
-void
-erts_mseg_init(ErtsMsegInit_t *init)
+static void mem_kind_init(MemKind* mk, const char* name)
 {
     unsigned i;
 
+    mk->cache = NULL;
+    mk->cache_end = NULL;
+    mk->max_cached_seg_size = 0;
+    mk->min_cached_seg_size = ~((Uint) 0);
+    mk->cache_size = 0;
+    mk->cache_hits = 0;
+
+    if (max_cache_size > 0) {
+	for (i = 0; i < max_cache_size - 1; i++)
+	    mk->cache_descs[i].next = &mk->cache_descs[i + 1];
+	mk->cache_descs[max_cache_size - 1].next = NULL;
+	mk->free_cache_descs = &mk->cache_descs[0];
+    }
+    else
+	mk->free_cache_descs = NULL;
+
+    mk->segments.current.watermark = 0;
+    mk->segments.current.no = 0;
+    mk->segments.current.sz = 0;
+    mk->segments.max.no = 0;
+    mk->segments.max.sz = 0;
+    mk->segments.max_ever.no = 0;
+    mk->segments.max_ever.sz = 0;
+
+    mk->name = name;
+    mk->next = mk_list;
+    mk_list = mk;
+}
+
+
+void
+erts_mseg_init(ErtsMsegInit_t *init)
+{
     atoms_initialized = 0;
     is_init_done = 0;
 
@@ -1324,13 +1498,17 @@ erts_mseg_init(ErtsMsegInit_t *init)
 	erl_exit(ERTS_ABORT_EXIT, "erts_mseg: unable to open /dev/zero\n");
 #endif
 
+#if HAVE_MMAP && HALFWORD_HEAP
+    initialize_pmmap();
+#endif
+
     page_size = GET_PAGE_SIZE;
 
     page_shift = 1;
     while ((page_size >> page_shift) != 1) {
 	if ((page_size & (1 << (page_shift - 1))) != 0)
 	    erl_exit(ERTS_ABORT_EXIT,
-		     "erts_mseg: Unexpected page_size %bpu\n", page_size);
+		     "erts_mseg: Unexpected page_size %beu\n", page_size);
 	page_shift++;
     }
 
@@ -1340,40 +1518,33 @@ erts_mseg_init(ErtsMsegInit_t *init)
     min_seg_size = ~((Uint) 0);
 #endif
 
-    cache = NULL;
-    cache_end = NULL;
-    cache_hits = 0;
-    max_cached_seg_size = 0;
-    min_cached_seg_size = ~((Uint) 0);
-    cache_size = 0;
+    if (max_cache_size > MAX_CACHE_SIZE)
+	max_cache_size = MAX_CACHE_SIZE;
+
+#if HALFWORD_HEAP
+    mem_kind_init(&low_mem, "low memory");
+    mem_kind_init(&hi_mem, "high memory");
+#else
+    mem_kind_init(&the_mem, "all memory");
+#endif
 
     is_cache_check_scheduled = 0;
 #ifdef ERTS_THREADS_NO_SMP
     is_cache_check_requested = 0;
 #endif
+}
 
-    if (max_cache_size > MAX_CACHE_SIZE)
-	max_cache_size = MAX_CACHE_SIZE;
 
-    if (max_cache_size > 0) {
-	for (i = 0; i < max_cache_size - 1; i++)
-	    cache_descs[i].next = &cache_descs[i + 1];
-	cache_descs[max_cache_size - 1].next = NULL;
-	free_cache_descs = &cache_descs[0];
+static ERTS_INLINE Uint tot_cache_size(void)
+{
+    MemKind* mk;
+    Uint sz = 0;
+    for (mk=mk_list; mk; mk=mk->next) {
+	sz += mk->cache_size;
     }
-    else
-	free_cache_descs = NULL;
-
-    segments.current.watermark = 0;
-    segments.current.no = 0;
-    segments.current.sz = 0;
-    segments.max.no = 0;
-    segments.max.sz = 0;
-    segments.max_ever.no = 0;
-    segments.max_ever.sz = 0;
+    return sz;
 }
 
-
 /*
  * erts_mseg_late_init() have to be called after all allocators,
  * threads and timers have been initialized.
@@ -1391,7 +1562,7 @@ erts_mseg_late_init(void)
 #ifdef ERTS_THREADS_NO_SMP
     async_handle = handle;
 #endif
-    if (cache_size)
+    if (tot_cache_size())
 	schedule_cache_check();
     erts_mtx_unlock(&mseg_mutex);
 }
@@ -1432,7 +1603,7 @@ erts_mseg_test(unsigned long op,
     case 0x406: {
 	unsigned long res;
 	erts_mtx_lock(&mseg_mutex);
-	res = (unsigned long) cache_size;
+	res = (unsigned long) tot_cache_size();
 	erts_mtx_unlock(&mseg_mutex);
 	return res;
     }
@@ -1446,3 +1617,432 @@ erts_mseg_test(unsigned long op,
 }
 
 
+#if HALFWORD_HEAP
+/*
+ * Very simple page oriented mmap replacer. Works in the lower
+ * 32 bit address range of a 64bit program.
+ * Implements anonymous mmap mremap and munmap with address order first fit.
+ * The free list is expected to be very short...
+ * To be used for compressed pointers in Erlang halfword emulator
+ * implementation. The MacOS X version is more of a toy, it's not really
+ * for production as the halfword erlang VM relies on Linux specific memory
+ * mapping tricks.
+ */
+
+/*#define HARDDEBUG 1*/
+
+#ifdef __APPLE__
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+
+#define INIT_LOCK() do {erts_mtx_init(&pmmap_mutex, "pmmap");} while(0)
+
+#define TAKE_LOCK()  do {erts_mtx_lock(&pmmap_mutex);} while(0)
+
+#define RELEASE_LOCK() do {erts_mtx_unlock(&pmmap_mutex);} while(0)
+
+static erts_mtx_t pmmap_mutex; /* Also needed when !USE_THREADS */
+
+typedef struct _free_block {
+    unsigned long num; /*pages*/
+    struct _free_block *next;
+} FreeBlock;
+
+/* Assigned once and for all */
+static size_t pagsz;
+
+/* Protect with lock */
+static FreeBlock *first;
+
+static size_t round_up_to_pagesize(size_t size)
+{
+    size_t x  = size / pagsz;
+
+    if ((size % pagsz)) {
+	++x;
+    }
+
+    return pagsz * x;
+}
+
+static size_t round_down_to_pagesize(size_t size)
+{
+    size_t x  = size / pagsz;
+
+    return pagsz * x;
+}
+
+static void *do_map(void *ptr, size_t sz)
+{
+    void *res;
+
+    if (round_up_to_pagesize(sz) != sz) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld "
+		"does not map complete pages\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return NULL;
+    }
+
+    if (((unsigned long) ptr) % pagsz) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld "
+		"is not page aligned\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return NULL;
+    }
+
+#if HAVE_MMAP
+    res = mmap(ptr, sz,
+	       PROT_READ | PROT_WRITE, MAP_PRIVATE |
+	       MAP_ANONYMOUS | MAP_FIXED,
+	       -1 , 0);
+#else
+#  error "Missing mmap support"
+#endif
+
+    if (res == MAP_FAILED) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld failed!\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return NULL;
+    }
+
+    return res;
+}
+
+static int do_unmap(void *ptr, size_t sz)
+{
+    void *res;
+
+    if (round_up_to_pagesize(sz) != sz) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld "
+		"does not map complete pages\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return 1;
+    }
+
+    if (((unsigned long) ptr) % pagsz) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld "
+		"is not page aligned\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return 1;
+    }
+
+
+    res = mmap(ptr, sz,
+	       PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE
+	       | MAP_FIXED,
+	       -1 , 0);
+
+    if (res == MAP_FAILED) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld failed!\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return 1;
+    }
+
+    return 0;
+}
+
+#ifdef __APPLE__
+/*
+ * The first 4 gig's are protected on Macos X for 64bit processes :(
+ * The range 0x1000000000 - 0x10FFFFFFFF is selected as an arbitrary
+ * value of a normally unused range... Real MMAP's will avoid
+ * it and all 32bit compressed pointers can be in that range...
+ * More expensive than on Linux where expansion of compressed
+ * poiters involves no masking (as they are in the first 4 gig's).
+ * It's also very uncertain if the MAP_NORESERVE flag really has
+ * any effect in MacOS X. Swap space may always be allocated...
+ */
+#define SET_RANGE_MIN() /* nothing */
+#define RANGE_MIN 0x1000000000UL
+#define RANGE_MAX 0x1100000000UL
+#define RANGE_MASK (RANGE_MIN)
+#define EXTRA_MAP_FLAGS (MAP_FIXED)
+#else
+static size_t range_min;
+#define SET_RANGE_MIN() do { range_min = (size_t) sbrk(0); } while (0)
+#define RANGE_MIN range_min
+#define RANGE_MAX 0x100000000UL
+#define RANGE_MASK 0UL
+#define EXTRA_MAP_FLAGS (0)
+#endif
+
+static int initialize_pmmap(void)
+{
+    char *p,*q,*rptr;
+    size_t rsz;
+    FreeBlock *initial;
+
+
+    pagsz = getpagesize();
+    SET_RANGE_MIN();
+    if (sizeof(void *) != 8) {
+	erl_exit(1,"Halfword emulator cannot be run in 32bit mode");
+    }
+
+    p = (char *) RANGE_MIN;
+    q = (char *) RANGE_MAX;
+
+    rsz = round_down_to_pagesize(q - p);
+
+    rptr = mmap((void *) p, rsz,
+		PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS |
+		MAP_NORESERVE | EXTRA_MAP_FLAGS,
+		-1 , 0);
+#ifdef HARDDEBUG
+    printf("p=%p, rsz = %ld, pages = %ld, got range = %p -> %p\r\n",
+	   p, (unsigned long) rsz, (unsigned long) (rsz / pagsz),
+	   (void *) rptr, (void*)(rptr + rsz));
+#endif
+    if ((UWord)(rptr + rsz) > RANGE_MAX) {
+	size_t rsz_trunc = RANGE_MAX - (UWord)rptr;
+#ifdef HARDDEBUG
+	printf("Reducing mmap'ed memory from %lu to %lu Mb, reduced range = %p -> %p\r\n",
+		rsz/(1024*1024), rsz_trunc/(1024*1024), rptr, rptr+rsz_trunc);
+#endif
+	munmap((void*)RANGE_MAX, rsz - rsz_trunc);
+	rsz = rsz_trunc;
+    }
+    if (!do_map(rptr,pagsz)) {
+	erl_exit(1,"Could not actually mmap first page for halfword emulator...\n");
+    }
+    initial = (FreeBlock *) rptr;
+    initial->num = (rsz / pagsz);
+    initial->next = NULL;
+    first = initial;
+    INIT_LOCK();
+    return 0;
+}
+
+#ifdef HARDDEBUG
+static void dump_freelist(void)
+{
+    FreeBlock *p = first;
+
+    while (p) {
+	printf("p = %p\r\np->num = %ld\r\np->next = %p\r\n\r\n",
+	       (void *) p, (unsigned long) p->num, (void *) p->next);
+	p = p->next;
+    }
+}
+#endif
+
+
+static void *pmmap(size_t size)
+{
+    size_t real_size = round_up_to_pagesize(size);
+    size_t num_pages = real_size / pagsz;
+    FreeBlock **block;
+    FreeBlock *tail;
+    FreeBlock *res;
+    TAKE_LOCK();
+    for (block = &first;
+	 *block != NULL && (*block)->num < num_pages;
+	 block = &((*block)->next))
+	;
+    if (!(*block)) {
+	RELEASE_LOCK();
+	return NULL;
+    }
+    if ((*block)->num == num_pages) {
+	/* nice, perfect fit */
+	res = *block;
+	*block = (*block)->next;
+    } else {
+	tail = (FreeBlock *) (((char *) ((void *) (*block))) + real_size);
+	if (!do_map(tail,pagsz)) {
+#ifdef HARDDEBUG
+	    fprintf(stderr, "Could not actually allocate page at %p...\r\n",
+		    (void *) tail);
+#endif
+	    RELEASE_LOCK();
+	    return NULL;
+	}
+	tail->num = (*block)->num - num_pages;
+	tail->next = (*block)->next;
+	res = *block;
+	*block = tail;
+    }
+    RELEASE_LOCK();
+    if (!do_map(res,real_size)) {
+#ifdef HARDDEBUG
+	fprintf(stderr, "Could not actually allocate %ld at %p...\r\n",
+		(unsigned long) real_size, (void *) res);
+#endif
+	return NULL;
+    }
+
+    return (void *) res;
+}
+
+static int pmunmap(void *p, size_t size)
+{
+    size_t real_size = round_up_to_pagesize(size);
+    size_t num_pages = real_size / pagsz;
+    FreeBlock *block;
+    FreeBlock *last;
+    FreeBlock *nb = (FreeBlock *) p;
+
+    ASSERT(((unsigned long)p & CHECK_POINTER_MASK)==0);
+    if (real_size > pagsz) {
+	if (do_unmap(((char *) p) + pagsz,real_size - pagsz)) {
+	    return 1;
+	}
+    }
+
+    TAKE_LOCK();
+
+    last = NULL;
+    block = first;
+    while(block != NULL && ((void *) block) < p) {
+	last = block;
+	block = block->next;
+    }
+
+    if (block != NULL &&
+	((void *) block) == ((void *) (((char *) p) + real_size))) {
+	/* Merge new free block with following */
+	nb->num = block->num + num_pages;
+	nb->next = block->next;
+	if (do_unmap(block,pagsz)) {
+	    RELEASE_LOCK();
+	    return 1;
+	}
+    } else {
+	/* just link in */
+	nb->num = num_pages;
+	nb->next = block;
+    }
+    if (last != NULL) {
+	if (p == ((void *) (((char *) last) + (last->num * pagsz)))) {
+	    /* Merge with previous */
+	    last->num += nb->num;
+	    last->next = nb->next;
+	    if (do_unmap(nb,pagsz)) {
+		RELEASE_LOCK();
+		return 1;
+	    }
+	} else {
+	    last->next = nb;
+	}
+    } else {
+	first = nb;
+    }
+    RELEASE_LOCK();
+    return 0;
+}
+
+static void *pmremap(void *old_address, size_t old_size,
+	      size_t new_size)
+{
+    size_t new_real_size = round_up_to_pagesize(new_size);
+    size_t new_num_pages = new_real_size / pagsz;
+    size_t old_real_size = round_up_to_pagesize(old_size);
+    size_t old_num_pages = old_real_size / pagsz;
+    if (new_num_pages == old_num_pages) {
+	return old_address;
+    } else if (new_num_pages < old_num_pages) { /* Shrink */
+	size_t nfb_pages = old_num_pages - new_num_pages;
+	size_t nfb_real_size = old_real_size - new_real_size;
+	void *vnfb = (void *) (((char *)old_address) + new_real_size);
+	FreeBlock *nfb = (FreeBlock *) vnfb;
+	FreeBlock **block;
+	TAKE_LOCK();
+	for (block = &first;
+	     *block != NULL && (*block) < nfb;
+	     block = &((*block)->next))
+	;
+	if (!(*block) ||
+	    (*block) > ((FreeBlock *)(((char *) vnfb) + nfb_real_size))) {
+	    /* Normal link in */
+	    if (nfb_pages > 1) {
+		if (do_unmap((void *)(((char *) vnfb) + pagsz),
+			     (nfb_pages - 1)*pagsz)) {
+		    return NULL;
+		}
+	    }
+	    nfb->next = (*block);
+	    nfb->num = nfb_pages;
+	    (*block) = nfb;
+	} else { /* block merge */
+	    nfb->next = (*block)->next;
+	    nfb->num = nfb_pages + (*block)->num;
+	    /* unmap also the first page of the next freeblock */
+	    (*block) = nfb;
+	    if (do_unmap((void *)(((char *) vnfb) + pagsz),
+			 nfb_pages*pagsz)) {
+		return NULL;
+	    }
+	}
+	RELEASE_LOCK();
+	return old_address;
+    } else { /* Enlarge */
+	FreeBlock **block;
+	void *old_end = (void *) (((char *)old_address) + old_real_size);
+	TAKE_LOCK();
+	for (block = &first;
+	     *block != NULL && (*block) < (FreeBlock *) old_address;
+	     block = &((*block)->next))
+	    ;
+	if ((*block) == NULL || old_end > ((void *) RANGE_MAX) ||
+	    (*block) != old_end ||
+	    (*block)->num < (new_num_pages - old_num_pages)) {
+	    /* cannot extend */
+	    void *result;
+	    RELEASE_LOCK();
+	    result = pmmap(new_size);
+	    if (result == NULL) {
+		return NULL;
+	    }
+	    memcpy(result,old_address,old_size);
+	    if (pmunmap(old_address,old_size)) {
+		/* Oups... */
+		pmunmap(result,new_size);
+		return NULL;
+	    }
+	    return result;
+	} else { /* extend */
+	    size_t remaining_pages = (*block)->num -
+		(new_num_pages - old_num_pages);
+	    if (!remaining_pages) {
+		void *p = (void *) (((char *) (*block)) + pagsz);
+		void *n = (*block)->next;
+		size_t x = ((*block)->num - 1) * pagsz;
+		if (x > 0) {
+		    if (do_map(p,x) == NULL) {
+			RELEASE_LOCK();
+			return NULL;
+		    }
+		}
+		(*block) = n;
+	    } else {
+		FreeBlock *nfb = (FreeBlock *) ((void *)
+						(((char *) old_address) +
+						 new_real_size));
+		void *p = (void *) (((char *) (*block)) + pagsz);
+		if (do_map(p,new_real_size - old_real_size) == NULL) {
+		    RELEASE_LOCK();
+		    return NULL;
+		}
+		nfb->num = remaining_pages;
+		nfb->next = (*block)->next;
+		(*block) = nfb;
+	    }
+	    RELEASE_LOCK();
+	    return old_address;
+	}
+    }
+}
+
+#endif /* HALFWORD_HEAP */
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index 1c5aa63e90..8f116030a8 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -58,17 +58,14 @@ typedef struct {
 typedef struct {
     int  cache;
     int  preserv;
-    Uint abs_shrink_th;
-    Uint rel_shrink_th;
+    UWord abs_shrink_th;
+    UWord rel_shrink_th;
+#if HALFWORD_HEAP
+    int low_mem;
+#endif
 } ErtsMsegOpt_t;
 
-#define ERTS_MSEG_DEFAULT_OPT_INITIALIZER				\
-{									\
-    1,			/* Use cache				*/	\
-    1,			/* Preserv data				*/	\
-    0,			/* Absolute shrink threshold		*/	\
-    0			/* Relative shrink threshold		*/	\
-}
+extern const ErtsMsegOpt_t erts_mseg_default_opt;
 
 void *erts_mseg_alloc(ErtsAlcType_t, Uint *);
 void *erts_mseg_alloc_opt(ErtsAlcType_t, Uint *, const ErtsMsegOpt_t *);
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 5cca33d7eb..f5c785d683 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -124,20 +124,11 @@
   erts_smp_mtx_unlock(&(PS)->mtx)
 
 #define ERTS_POLLSET_SET_POLLED_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->polled, (long) 1))
+  ((int) erts_atomic32_xchg(&(PS)->polled, (erts_aint32_t) 1))
 #define ERTS_POLLSET_UNSET_POLLED(PS) \
-  erts_smp_atomic_set(&(PS)->polled, (long) 0)
+  erts_atomic32_set(&(PS)->polled, (erts_aint32_t) 0)
 #define ERTS_POLLSET_IS_POLLED(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->polled))
-
-#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->woken, (long) 1))
-#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \
-  erts_smp_atomic_set(&(PS)->woken, (long) 1)
-#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \
-  erts_smp_atomic_set(&(PS)->woken, (long) 0)
-#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->woken))
+  ((int) erts_atomic32_read(&(PS)->polled))
 
 #else
 
@@ -147,64 +138,21 @@
 #define ERTS_POLLSET_UNSET_POLLED(PS)
 #define ERTS_POLLSET_IS_POLLED(PS) 0
 
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
-
-/*
- * Ideally, the ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) operation would
- * be atomic. This operation isn't, but we will do okay anyway. The
- * "woken check" is only an optimization. The only requirement we have:
- * If (PS)->woken is set to a value != 0 when interrupting, we have to
- * write on the the wakeup pipe at least once. Multiple writes are okay.
- */
-#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) ((PS)->woken++)
-#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) ((PS)->woken = 1, (void) 0)
-#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) ((PS)->woken = 0, (void) 0)
-#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) ((PS)->woken)
-
-#else
-
-#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) 1
-#define ERTS_POLLSET_SET_POLLER_WOKEN(PS)
-#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS)
-#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) 1
-
-#endif
-
 #endif
 
 #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
 #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \
-  erts_smp_atomic_set(&(PS)->have_update_requests, (long) 1)
+  erts_smp_atomic32_set(&(PS)->have_update_requests, (erts_aint32_t) 1)
 #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \
-  erts_smp_atomic_set(&(PS)->have_update_requests, (long) 0)
+  erts_smp_atomic32_set(&(PS)->have_update_requests, (erts_aint32_t) 0)
 #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->have_update_requests))
+  ((int) erts_smp_atomic32_read(&(PS)->have_update_requests))
 #else
 #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS)
 #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS)
 #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) 0
 #endif
 
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
-
-#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) unset_interrupted_chk((PS))
-#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) ((PS)->interrupt = 0, (void) 0)
-#define ERTS_POLLSET_SET_INTERRUPTED(PS) ((PS)->interrupt = 1, (void) 0)
-#define ERTS_POLLSET_IS_INTERRUPTED(PS) ((PS)->interrupt)
-
-#else
-
-#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->interrupt, (long) 0))
-#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \
-  erts_smp_atomic_set(&(PS)->interrupt, (long) 0)
-#define ERTS_POLLSET_SET_INTERRUPTED(PS) \
-  erts_smp_atomic_set(&(PS)->interrupt, (long) 1)
-#define ERTS_POLLSET_IS_INTERRUPTED(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->interrupt))
-
-#endif
-
 #if ERTS_POLL_USE_FALLBACK
 #  if ERTS_POLL_USE_POLL
 #    define ERTS_POLL_NEED_FALLBACK(PS) ((PS)->no_poll_fds > 1)
@@ -276,7 +224,7 @@ struct ErtsPollSet_ {
     ErtsPollSet next;
     int internal_fd_limit;
     ErtsFdStatus *fds_status;
-    int no_of_user_fds;
+    erts_smp_atomic_t no_of_user_fds;
     int fds_status_len;
 #if ERTS_POLL_USE_KERNEL_POLL
     int kp_fd;
@@ -308,14 +256,12 @@ struct ErtsPollSet_ {
 #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
     ErtsPollSetUpdateRequestsBlock update_requests;
     ErtsPollSetUpdateRequestsBlock *curr_upd_req_block;
-    erts_smp_atomic_t have_update_requests;
+    erts_smp_atomic32_t have_update_requests;
 #endif
 #ifdef ERTS_SMP
-    erts_smp_atomic_t polled;
-    erts_smp_atomic_t woken;
+    erts_atomic32_t polled;
     erts_smp_mtx_t mtx;
 #elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
-    volatile int woken;
 #endif
 #if ERTS_POLL_USE_WAKEUP_PIPE
     int wake_fds[2];
@@ -323,12 +269,12 @@ struct ErtsPollSet_ {
 #if ERTS_POLL_USE_FALLBACK
     int fallback_used;
 #endif
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
-    volatile int interrupt;
-#else
-    erts_smp_atomic_t interrupt;
+#ifdef ERTS_SMP
+    erts_atomic32_t wakeup_state;
+#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    volatile int wakeup_state;
 #endif
-    erts_smp_atomic_t timeout;
+    erts_smp_atomic32_t timeout;
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
     erts_smp_atomic_t no_avoided_wakeups;
     erts_smp_atomic_t no_avoided_interrupts;
@@ -336,20 +282,6 @@ struct ErtsPollSet_ {
 #endif
 };
 
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
-
-static ERTS_INLINE int
-unset_interrupted_chk(ErtsPollSet ps)
-{
-    /* This operation isn't atomic, but we have no need at all for an
-       atomic operation here... */
-    int res = ps->interrupt;
-    ps->interrupt = 0;
-    return res;
-}
-
-#endif
-
 void erts_silence_warn_unused_result(long unused);
 static void fatal_error(char *format, ...);
 static void fatal_error_async_signal_safe(char *error_str);
@@ -406,6 +338,64 @@ static void check_poll_status(ErtsPollSet ps);
 static void print_misc_debug_info(void);
 #endif
 
+#define ERTS_POLL_NOT_WOKEN	0
+#define ERTS_POLL_WOKEN		-1
+#define ERTS_POLL_WOKEN_INTR	1
+
+static ERTS_INLINE void
+reset_wakeup_state(ErtsPollSet ps)
+{
+#ifdef ERTS_SMP
+    erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
+    ERTS_THR_MEMORY_BARRIER;
+#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    ps->wakeup_state = 0;
+#endif
+}
+
+static ERTS_INLINE int
+is_woken(ErtsPollSet ps)
+{
+#ifdef ERTS_SMP
+    return erts_atomic32_read_acqb(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN;
+#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    return ps->wakeup_state != ERTS_POLL_NOT_WOKEN;
+#else
+    return 0;
+#endif
+}
+
+static ERTS_INLINE int
+is_interrupted_reset(ErtsPollSet ps)
+{
+#ifdef ERTS_SMP
+    return (erts_atomic32_xchg(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN)
+	    == ERTS_POLL_WOKEN_INTR);
+#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    int res = ps->wakeup_state == ERTS_POLL_WOKEN_INTR;
+    ps->wakeup_state = ERTS_POLL_NOT_WOKEN;
+    return res;
+#else
+    return 0;
+#endif
+}
+
+static ERTS_INLINE void
+woke_up(ErtsPollSet ps)
+{
+#ifdef ERTS_SMP
+    erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+    if (wakeup_state == ERTS_POLL_NOT_WOKEN)
+	(void) erts_atomic32_cmpxchg(&ps->wakeup_state,
+				     ERTS_POLL_WOKEN,
+				     ERTS_POLL_NOT_WOKEN);
+    ASSERT(erts_atomic32_read(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN);
+#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    if (ps->wakeup_state == ERTS_POLL_NOT_WOKEN)
+	ps->wakeup_state = ERTS_POLL_WOKEN;
+#endif
+}
+
 /*
  * --- Wakeup pipe -----------------------------------------------------------
  */
@@ -413,14 +403,34 @@ static void print_misc_debug_info(void);
 #if ERTS_POLL_USE_WAKEUP_PIPE
 
 static ERTS_INLINE void
-wake_poller(ErtsPollSet ps)
+wake_poller(ErtsPollSet ps, int interrupted)
 {
+    int wake;
+#ifdef ERTS_SMP
+    erts_aint32_t wakeup_state;
+    if (!interrupted)
+	wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state,
+						  ERTS_POLL_WOKEN,
+						  ERTS_POLL_NOT_WOKEN);
+    else {
+	/*
+	 * We might unnecessarily write to the pipe, however,
+	 * that isn't problematic.
+	 */
+	wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+	erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR);
+    }
+    wake = wakeup_state == ERTS_POLL_NOT_WOKEN;
+#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    wake = ps->wakeup_state == ERTS_POLL_NOT_WOKEN;
+    ps->wakeup_state = interrupted ? ERTS_POLL_WOKEN_INTR : ERTS_POLL_NOT_WOKEN;
+#endif
     /*
      * NOTE: This function might be called from signal handlers in the
      *       non-smp case; therefore, it has to be async-signal safe in
      *       the non-smp case.
      */
-    if (!ERTS_POLLSET_SET_POLLER_WOKEN_CHK(ps)) {
+    if (wake) {
 	ssize_t res;
 	if (ps->wake_fds[1] < 0)
 	    return; /* Not initialized yet */
@@ -756,7 +766,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 		short filter;
 		int fd = (int) ebuf[i].ident;
 
-		switch ((int) ebuf[i].udata) {
+		switch ((int) (long) ebuf[i].udata) {
 
 		    /*
 		     * Since we use a lazy update approach EV_DELETE will
@@ -795,7 +805,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 			if (fd == (int) ebuf[j].ident) {
 			    ebuf[j].udata = (void *) ERTS_POLL_KQ_OP_HANDLED;
 			    if (!(ebuf[j].flags & EV_ERROR)) {
-				switch ((int) ebuf[j].udata) {
+				switch ((int) (long) ebuf[j].udata) {
 				case ERTS_POLL_KQ_OP_ADD2_W:
 				    filter = EVFILT_WRITE;
 				    goto rm_add_fb;
@@ -813,7 +823,8 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 			}
 		    }
 		    /* The other add succeded... */
-		    filter = (((int) ebuf[i].udata == ERTS_POLL_KQ_OP_ADD2_W)
+		    filter = ((((int) (long) ebuf[i].udata)
+			       == ERTS_POLL_KQ_OP_ADD2_W)
 			      ? EVFILT_READ
 			      : EVFILT_WRITE);
 		rm_add_fb:
@@ -828,7 +839,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 		    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK;
 		    ASSERT(ps->fds_status[fd].used_events);
 		    ps->fds_status[fd].used_events = 0;
-		    ps->no_of_user_fds--;
+		    erts_smp_atomic_dec(&ps->no_of_user_fds);
 		    update_fallback_pollset(ps, fd);
 		    ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
 		    break;
@@ -878,11 +889,11 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp)
     events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events);
     if (!events) {
 	buf[buf_len].events = POLLREMOVE;
-	ps->no_of_user_fds--;
+	erts_smp_atomic_dec(&ps->no_of_user_fds);
     }
     else if (!ps->fds_status[fd].used_events) {
 	buf[buf_len].events = events;
-	ps->no_of_user_fds++;
+	erts_smp_atomic_inc(&ps->no_of_user_fds);
     }
     else {
 	if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)
@@ -972,12 +983,12 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp)
 	}
 	if (used_events) {
 	    if (!events) {
-		ps->no_of_user_fds--;
+		erts_smp_atomic_dec(&ps->no_of_user_fds);
 	    }
 	}
 	else {
 	    if (events)
-		ps->no_of_user_fds++;
+		erts_smp_atomic_inc(&ps->no_of_user_fds);
 	}
 	ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0);
 	ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0);
@@ -1051,7 +1062,7 @@ update_pollset(ErtsPollSet ps, int fd)
 	    epe.data.fd = epe_templ.data.fd;
 	    res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe);
 	} while (res != 0 && errno == EINTR);
-	ps->no_of_user_fds--;
+	erts_smp_atomic_dec(&ps->no_of_user_fds);
 	ps->fds_status[fd].used_events = 0;
     }
 
@@ -1059,11 +1070,11 @@ update_pollset(ErtsPollSet ps, int fd)
 	/* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9
 	   need a non-NULL event pointer even though it is ignored... */
 	op = EPOLL_CTL_DEL;
-	ps->no_of_user_fds--;
+	erts_smp_atomic_dec(&ps->no_of_user_fds);
     }
     else if (!ps->fds_status[fd].used_events) {
 	op = EPOLL_CTL_ADD;
-	ps->no_of_user_fds++;
+	erts_smp_atomic_inc(&ps->no_of_user_fds);
     }
     else {
 	op = EPOLL_CTL_MOD;
@@ -1113,7 +1124,7 @@ update_pollset(ErtsPollSet ps, int fd)
 	/* Fall through ... */
 	case EPOLL_CTL_ADD: {
 	    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK;
-	    ps->no_of_user_fds--;
+	    erts_smp_atomic_dec(&ps->no_of_user_fds);
 #if ERTS_POLL_USE_CONCURRENT_UPDATE
 	    if (!*update_fallback) {
 		*update_fallback = 1;
@@ -1201,7 +1212,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 #if ERTS_POLL_USE_FALLBACK
 	ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
 #endif
-	ps->no_of_user_fds--;
+	erts_smp_atomic_dec(&ps->no_of_user_fds);
 	last_pix = --ps->no_poll_fds;
 	if (pix != last_pix) {
 	/* Move last pix to this pix */
@@ -1228,7 +1239,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 	    ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)
 		    || fd == ps->kp_fd);
 #endif
-	    ps->no_of_user_fds++;
+	    erts_smp_atomic_inc(&ps->no_of_user_fds);
 	    ps->fds_status[fd].pix = pix = ps->no_poll_fds++;
 	    if (pix >= ps->poll_fds_len)
 		grow_poll_fds(ps, pix);
@@ -1279,7 +1290,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 
 	if (!ps->fds_status[fd].used_events) {
 	    ASSERT(events);
-	    ps->no_of_user_fds++;
+	    erts_smp_atomic_inc(&ps->no_of_user_fds);
 #if ERTS_POLL_USE_FALLBACK
 	    ps->no_select_fds++;
 	    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK;
@@ -1287,7 +1298,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 	}
 	else if (!events) {
 	    ASSERT(ps->fds_status[fd].used_events);
-	    ps->no_of_user_fds--;
+	    erts_smp_atomic_dec(&ps->no_of_user_fds);
 	    ps->fds_status[fd].events = events;
 #if ERTS_POLL_USE_FALLBACK
 	    ps->no_select_fds--;
@@ -1363,9 +1374,7 @@ handle_update_requests(ErtsPollSet ps)
 #endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */
 
 static ERTS_INLINE ErtsPollEvents
-poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on,
-	     int *have_set_have_update_requests,
-	     int *do_wake)
+poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake)
 {
     ErtsPollEvents new_events;
 
@@ -1469,7 +1478,6 @@ ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps,
 				     int len)
 {
     int i;
-    int hshur = 0;
     int do_wake;
     int final_do_wake = 0;
 
@@ -1481,17 +1489,17 @@ ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps,
 				      pcev[i].fd,
 				      pcev[i].events,
 				      pcev[i].on,
-				      &hshur,
 				      &do_wake);
 	final_do_wake |= do_wake;
     }
 
+    ERTS_POLLSET_UNLOCK(ps);
+
 #ifdef ERTS_SMP
     if (final_do_wake)
-	wake_poller(ps);
+	wake_poller(ps, 0);
 #endif /* ERTS_SMP */
 
-    ERTS_POLLSET_UNLOCK(ps);
 }
 
 ErtsPollEvents
@@ -1502,20 +1510,20 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps,
 				    int* do_wake) /* In: Wake up polling thread */
 				                  /* Out: Poller is woken */
 {
-    int hshur = 0;
     ErtsPollEvents res;
 
     ERTS_POLLSET_LOCK(ps);
 
-    res = poll_control(ps, fd, events, on, &hshur, do_wake);
+    res = poll_control(ps, fd, events, on, do_wake);
+
+    ERTS_POLLSET_UNLOCK(ps);
 
 #ifdef ERTS_SMP
     if (*do_wake) {
-	wake_poller(ps);
+	wake_poller(ps, 0);
     }
 #endif /* ERTS_SMP */
 
-    ERTS_POLLSET_UNLOCK(ps);
     return res;
 }
 
@@ -1888,14 +1896,17 @@ static ERTS_INLINE int
 check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked)
 {
     ASSERT(!*ps_locked);
-    if (ps->no_of_user_fds == 0 && tv->tv_usec == 0 && tv->tv_sec == 0) {
+    if (erts_smp_atomic_read(&ps->no_of_user_fds) == 0
+	&& tv->tv_usec == 0 && tv->tv_sec == 0) {
 	/* Nothing to poll and zero timeout; done... */
 	return 0;
     }
     else {
 	long timeout = tv->tv_sec*1000 + tv->tv_usec/1000;
+	if (timeout > ERTS_AINT32_T_MAX)
+	    timeout = ERTS_AINT32_T_MAX;
 	ASSERT(timeout >= 0);
-	erts_smp_atomic_set(&ps->timeout, timeout);
+	erts_smp_atomic32_set_relb(&ps->timeout, (erts_aint32_t) timeout);
 #if ERTS_POLL_USE_FALLBACK
 	if (!(ps->fallback_used = ERTS_POLL_NEED_FALLBACK(ps))) {
 
@@ -1926,7 +1937,7 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked)
 	     * the maximum number of file descriptors in the poll set.
 	     */
 	    struct dvpoll poll_res;
-	    int nfds = ps->no_of_user_fds;
+	    int nfds = (int) erts_smp_atomic_read(&ps->no_of_user_fds);
 #ifdef ERTS_SMP
 	    nfds++; /* Wakeup pipe */
 #endif
@@ -2017,15 +2028,14 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 		 (int) tv->tv_sec*1000 + tv->tv_usec/1000);
 #endif
 
-    ERTS_POLLSET_UNSET_POLLER_WOKEN(ps);
     if (ERTS_POLLSET_SET_POLLED_CHK(ps)) {
 	res = EINVAL; /* Another thread is in erts_poll_wait()
 			 on this pollset... */
 	goto done;
     }
 
-    if (ERTS_POLLSET_IS_INTERRUPTED(ps)) {
-	/* Interrupt use zero timeout */
+    if (is_woken(ps)) {
+	/* Use zero timeout */
 	itv.tv_sec = 0;
 	itv.tv_usec = 0;
 	tvp = &itv;
@@ -2042,7 +2052,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
     ps_locked = 0;
     res = check_fd_events(ps, tvp, no_fds, &ps_locked);
 
-    ERTS_POLLSET_SET_POLLER_WOKEN(ps);
+    woke_up(ps);
 
     if (res == 0) {
 	res = ETIMEDOUT;
@@ -2074,9 +2084,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 	check_poll_result(pr, no_fds);
 #endif
 
-	res = (no_fds == 0
-	       ? (ERTS_POLLSET_UNSET_INTERRUPTED_CHK(ps) ? EINTR : EAGAIN)
-	       : 0);
+	res = (no_fds == 0 ? (is_interrupted_reset(ps) ? EINTR : EAGAIN) : 0);
 	*len = no_fds;
     }
 
@@ -2087,7 +2095,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 #endif
 
  done:
-    erts_smp_atomic_set(&ps->timeout, LONG_MAX);
+    erts_smp_atomic32_set_relb(&ps->timeout, ERTS_AINT32_T_MAX);
 #ifdef ERTS_POLL_DEBUG_PRINT
     erts_printf("Leaving %s = erts_poll_wait()\n",
 		 res == 0 ? "0" : erl_errno_id(res));
@@ -2103,20 +2111,17 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 void
 ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set)
 {
+#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP)
     /*
      * NOTE: This function might be called from signal handlers in the
      *       non-smp case; therefore, it has to be async-signal safe in
      *       the non-smp case.
      */
-    if (set) {
-	ERTS_POLLSET_SET_INTERRUPTED(ps);
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP)
-	wake_poller(ps);
+    if (!set)
+	reset_wakeup_state(ps);
+    else
+	wake_poller(ps, 1);
 #endif
-    }
-    else {
-	ERTS_POLLSET_UNSET_INTERRUPTED(ps);
-    }
 }
 
 /*
@@ -2125,15 +2130,16 @@ ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set)
  * is not guaranteed that it will timeout before 'msec' milli seconds.
  */
 void
-ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, int set, long msec)
+ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps,
+					    int set,
+					    long msec)
 {
-    if (set) {
-	if (erts_smp_atomic_read(&ps->timeout) > msec) {
-	    ERTS_POLLSET_SET_INTERRUPTED(ps);
 #if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP)
-	    wake_poller(ps);
-#endif
-	}
+    if (!set)
+	reset_wakeup_state(ps);
+    else {
+	if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec)
+	    wake_poller(ps, 1);
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
 	else {
 	    if (ERTS_POLLSET_IS_POLLED(ps))
@@ -2143,9 +2149,7 @@ ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, int set, long msec)
 	erts_smp_atomic_inc(&ps->no_interrupt_timed);
 #endif
     }
-    else {
-	ERTS_POLLSET_UNSET_INTERRUPTED(ps);
-    }
+#endif
 }
 
 int
@@ -2204,7 +2208,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
     ps->internal_fd_limit = 0;
     ps->fds_status = NULL;
     ps->fds_status_len = 0;
-    ps->no_of_user_fds = 0;
+    erts_smp_atomic_init(&ps->no_of_user_fds, 0);
 #if ERTS_POLL_USE_KERNEL_POLL
     ps->kp_fd = -1;
 #if ERTS_POLL_USE_EPOLL
@@ -2256,14 +2260,16 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
     ps->update_requests.next = NULL;
     ps->update_requests.len = 0;
     ps->curr_upd_req_block = &ps->update_requests;
-    erts_smp_atomic_init(&ps->have_update_requests, 0);
+    erts_smp_atomic32_init(&ps->have_update_requests, 0);
 #endif
 #ifdef ERTS_SMP
-    erts_smp_atomic_init(&ps->polled, 0);
-    erts_smp_atomic_init(&ps->woken, 0);
+    erts_atomic32_init(&ps->polled, 0);
     erts_smp_mtx_init(&ps->mtx, "pollset");
+#endif
+#ifdef ERTS_SMP
+    erts_atomic32_init(&ps->wakeup_state, (erts_aint32_t) 0);
 #elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
-    ps->woken = 0;
+    ps->wakeup_state = 0;
 #endif
 #if ERTS_POLL_USE_WAKEUP_PIPE
     create_wakeup_pipe(ps);
@@ -2285,12 +2291,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 	ps->internal_fd_limit = kp_fd + 1;
     ps->kp_fd = kp_fd;
 #endif
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
-    ps->interrupt = 0;
-#else
-    erts_smp_atomic_init(&ps->interrupt, 0);
-#endif
-    erts_smp_atomic_init(&ps->timeout, LONG_MAX);
+    erts_smp_atomic32_init(&ps->timeout, ERTS_AINT32_T_MAX);
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
     erts_smp_atomic_init(&ps->no_avoided_wakeups, 0);
     erts_smp_atomic_init(&ps->no_avoided_interrupts, 0);
@@ -2302,7 +2303,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 #if ERTS_POLL_USE_FALLBACK
     ps->fallback_used = 0;
 #endif
-    ps->no_of_user_fds = 0; /* Don't count wakeup pipe and fallback fd */
+    erts_smp_atomic_set(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */
 
     erts_smp_spin_lock(&pollsets_lock);
     ps->next = pollsets;
@@ -2405,6 +2406,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 	while (urqbp) {
 	    size += sizeof(ErtsPollSetUpdateRequestsBlock);
 	    pending_updates += urqbp->len;
+	    urqbp = urqbp->next;
 	}
     }
 #endif
@@ -2447,7 +2449,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 
     pip->memory_size = size;
 
-    pip->poll_set_size = ps->no_of_user_fds;
+    pip->poll_set_size = (int) erts_smp_atomic_read(&ps->no_of_user_fds);
 #ifdef ERTS_SMP
     pip->poll_set_size++; /* Wakeup pipe */
 #endif
diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
new file mode 100644
index 0000000000..461e763f03
--- /dev/null
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -0,0 +1,107 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2006-2010. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+
+
+/*
+ * Darwin needs conversion!
+ * http://developer.apple.com/library/mac/#qa/qa2001/qa1235.html
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "global.h"
+
+#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
+#define __DARWIN__ 1
+#endif
+
+#if !defined(__WIN32__)
+#include <locale.h>
+#if !defined(HAVE_SETLOCALE) || !defined(HAVE_NL_LANGINFO) || !defined(HAVE_LANGINFO_H)
+#define PRIMITIVE_UTF8_CHECK 1
+#else
+#include <langinfo.h>
+#endif
+#endif
+
+/* Written once and only once */
+
+static int filename_encoding = ERL_FILENAME_UNKNOWN;
+#if defined(__WIN32__) || defined(__DARWIN__)
+static int user_filename_encoding = ERL_FILENAME_UTF8; /* Default unicode on windows */
+#else
+static int user_filename_encoding = ERL_FILENAME_LATIN1;
+#endif
+void erts_set_user_requested_filename_encoding(int encoding)
+{
+    user_filename_encoding = encoding;
+}
+
+int erts_get_user_requested_filename_encoding(void)
+{
+    return user_filename_encoding;
+}
+
+void erts_init_sys_common_misc(void)
+{
+#if defined(__WIN32__)
+    /* win_efile will totally fail if this is not set. */
+    filename_encoding = ERL_FILENAME_WIN_WCHAR;
+#else
+    if (user_filename_encoding != ERL_FILENAME_UNKNOWN) {
+	filename_encoding = user_filename_encoding;
+    } else {
+	char *l;
+	filename_encoding = ERL_FILENAME_LATIN1;
+#  ifdef PRIMITIVE_UTF8_CHECK
+	setlocale(LC_CTYPE, "");  /* Set international environment, 
+				     ignore result */
+	if (((l = getenv("LC_ALL"))   && *l) ||
+	    ((l = getenv("LC_CTYPE")) && *l) ||
+	    ((l = getenv("LANG"))     && *l)) {
+	    if (strstr(l, "UTF-8")) {
+		filename_encoding = ERL_FILENAME_UTF8;
+	    } 
+	}
+	
+#  else
+	l = setlocale(LC_CTYPE, "");  /* Set international environment */
+	if (l != NULL) {
+	    if (strcmp(nl_langinfo(CODESET), "UTF-8") == 0) {
+		filename_encoding = ERL_FILENAME_UTF8;
+	    }
+	}
+#  endif
+    }
+#  if defined(__DARWIN__)
+    if (filename_encoding == ERL_FILENAME_UTF8) {
+	filename_encoding = ERL_FILENAME_UTF8_MAC;
+    }
+#  endif
+#endif
+}
+
+int erts_get_native_filename_encoding(void)
+{
+    return filename_encoding;
+}
diff --git a/erts/emulator/sys/unix/erl9_start.c b/erts/emulator/sys/unix/erl9_start.c
deleted file mode 100644
index 578062d7e2..0000000000
--- a/erts/emulator/sys/unix/erl9_start.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-#include "sys.h"
-#include "erl_vm.h"
-#include "global.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/*
- * XXX This is a temporary dummy to make sys.c happy until we'll rewrite it.
- */
-unsigned preloaded_size_ring0 = 1;
-unsigned char preloaded_ring0[1] = {0};
-
-Preload pre_loaded[] = {
-    {"ring0", 1, preloaded_ring0},
-    {0, 0, 0}
-};
-
-int
-main(int argc, char** argv)
-{
-    char sbuf[1024];
-    struct {
-	void* p;
-	int sz;
-    } bins[2];
-    int bin_num = 0;
-    FILE* fp;
-    char* progname = argv[0];
-    char* eq;
-
-    argv++, argc--;
-
-    if (argc > 0 && argv[0][0] == '-') {
-	argv++, argc--;
-    }
-    if (argc < 1) {
-	abort();
-    }
-    if ((fp = fopen(argv[0], "r")) == NULL) {
-	abort();
-    }
-    
-    /* Needs to be called before any memory allocation */
-    erts_short_init();
-
-    while (fgets(sbuf, sizeof sbuf, fp)) {
-	if (sbuf[0] == '#') {
-	    continue;		/* Comment */
-	} else if (sbuf[0] == 'e' && strncmp("exec", sbuf, 4) == 0) {
-	    continue;		/* Comment ;-) */
-	} else if ((eq = strchr(sbuf, '=')) != NULL) {
-	    char* val;
-	    char* p = strchr(sbuf, '\n');
-	    if (p) {
-		*p = '\0';
-	    }
-	    *eq = '\0';
-	    val = erts_read_env(sbuf);
-	    if (val == NULL) {
-		*eq = '=';
-		erts_sys_putenv(sbuf, eq - &sbuf[0]);
-	    }
-	    erts_free_read_env(val);
-	} else if (sbuf[0] == ':' && '0' <= sbuf[1] && sbuf[1] <= '9') {
-	    int load_size = atoi(sbuf+1);
-	    void* bin;
-	    
-	    bin = malloc(load_size);
-	    if (fread(bin, 1, load_size, fp) != load_size) {
-		abort();
-	    }
-	    bins[bin_num].p = bin;
-	    bins[bin_num].sz = load_size;
-	    bin_num++;
-	} else if (strcmp(sbuf, "--end--\n") == 0) {
-	    int rval;
-	    Eterm mod = NIL;
-	    char *val;
-
-	    fclose(fp);
-
-	    if (bin_num != 2) {
-		abort();
-	    }
-
-	    val = erts_read_env("ERLBREAKHANDLER");
- 	    if (val) {
-		init_break_handler();
-	    }
-	    erts_free_read_env(val);
-
-	    if ((rval = erts_load_module(NULL, 0, NIL, &mod, bins[0].p, bins[0].sz)) < 0) {
-		fprintf(stderr, "%s: Load of initial module failed: %d\n",
-			progname, rval);
-		abort();
-	    }
-	    erts_first_process(mod, bins[1].p, bins[1].sz, argc, argv);
-	    free(bins[0].p);
-	    free(bins[1].p);
-	    process_main();
-	    abort();
-	} else {
-	    fprintf(stderr, "%s: bad line: %s\n", progname, sbuf);
-	    abort();
-	}
-    }
-    abort();
-}
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 2d5ef882f6..d8d51b192c 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -329,11 +329,4 @@ extern int exit_async(void);
 
 #define ERTS_EXIT_AFTER_DUMP _exit
 
-#ifdef ERTS_TIMER_THREAD
-struct erts_iwait; /* opaque for clients */
-extern struct erts_iwait *erts_iwait_init(void);
-extern void erts_iwait_wait(struct erts_iwait *iwait, struct timeval *delay);
-extern void erts_iwait_interrupt(struct erts_iwait *iwait);
-#endif /* ERTS_TIMER_THREAD */
-
 #endif /* #ifndef _ERL_UNIX_SYS_H */
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 31ab5d03de..bafbbb0f6c 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -53,6 +53,11 @@
 #define WANT_NONBLOCKING    /* must define this to pull in defs from sys.h */
 #include "sys.h"
 
+#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
+#define __DARWIN__ 1
+#endif
+
+
 #ifdef USE_THREADS
 #include "erl_threads.h"
 #endif
@@ -75,6 +80,7 @@ static erts_smp_rwmtx_t environ_rwmtx;
 
 #include "erl_sys_driver.h"
 #include "erl_check_io.h"
+#include "erl_cpu_topology.h"
 
 #ifndef DISABLE_VFORK
 #define DISABLE_VFORK 0
@@ -123,8 +129,6 @@ static ErtsSysReportExit *report_exit_transit_list;
 
 extern int  check_async_ready(void);
 extern int  driver_interrupt(int, int);
-/*EXTERN_FUNCTION(void, increment_time, (int));*/
-/*EXTERN_FUNCTION(int, next_time, (_VOID_));*/
 extern void do_break(void);
 
 extern void erl_sys_args(int*, char**);
@@ -161,14 +165,14 @@ static int debug_log = 0;
 #endif
 
 #ifdef ERTS_SMP
-erts_smp_atomic_t erts_got_sigusr1;
+erts_smp_atomic32_t erts_got_sigusr1;
 #define ERTS_SET_GOT_SIGUSR1 \
-  erts_smp_atomic_set(&erts_got_sigusr1, 1)
+  erts_smp_atomic32_set(&erts_got_sigusr1, 1)
 #define ERTS_UNSET_GOT_SIGUSR1 \
-  erts_smp_atomic_set(&erts_got_sigusr1, 0)
-static erts_smp_atomic_t have_prepared_crash_dump;
+  erts_smp_atomic32_set(&erts_got_sigusr1, 0)
+static erts_smp_atomic32_t have_prepared_crash_dump;
 #define ERTS_PREPARED_CRASH_DUMP \
-  ((int) erts_smp_atomic_xchg(&have_prepared_crash_dump, 1))
+  ((int) erts_smp_atomic32_xchg(&have_prepared_crash_dump, 1))
 #else
 volatile int erts_got_sigusr1;
 #define ERTS_SET_GOT_SIGUSR1 (erts_got_sigusr1 = 1)
@@ -221,10 +225,10 @@ static struct fd_data {
 } *fd_data;			/* indexed by fd */
 
 /* static FUNCTION(int, write_fill, (int, char*, int)); unused? */
-static FUNCTION(void, note_child_death, (int, int));
+static void note_child_death(int, int);
 
 #if CHLDWTHR
-static FUNCTION(void *, child_waiter, (void *));
+static void* child_waiter(void *);
 #endif
 
 /********************* General functions ****************************/
@@ -236,11 +240,11 @@ static int max_files = -1;
  * a few variables used by the break handler 
  */
 #ifdef ERTS_SMP
-erts_smp_atomic_t erts_break_requested;
+erts_smp_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 1)
+  erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 0)
+  erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0)
 #else
 volatile int erts_break_requested = 0;
 #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
@@ -367,7 +371,7 @@ erts_sys_misc_mem_sz(void)
 /*
  * reset the terminal to the original settings on exit
  */
-void sys_tty_reset(void)
+void sys_tty_reset(int exit_code)
 {
   if (using_oldshell && !replace_intr) {
     SET_BLOCKING(0);
@@ -384,18 +388,6 @@ MALLOC_USE_HASH(1);
 #endif
 
 #ifdef USE_THREADS
-static void *ethr_internal_alloc(size_t size)
-{
-    return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size);
-}
-static void *ethr_internal_realloc(void *ptr, size_t size)
-{
-    return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size);
-}
-static void ethr_internal_free(void *ptr)
-{
-    erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr);
-}
 
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
 /*
@@ -413,7 +405,7 @@ typedef struct {
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
     sigset_t saved_sigmask;
 #endif
-    int unbind_child;
+    int sched_bind_data;
 } erts_thr_create_data_t;
 
 /*
@@ -424,15 +416,13 @@ static void *
 thr_create_prepare(void)
 {
     erts_thr_create_data_t *tcdp;
-    ErtsSchedulerData *esdp;
 
     tcdp = erts_alloc(ERTS_ALC_T_TMP, sizeof(erts_thr_create_data_t));
 
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
     erts_thr_sigmask(SIG_BLOCK, &thr_create_sigmask, &tcdp->saved_sigmask);
 #endif
-    esdp = erts_get_scheduler_data();
-    tcdp->unbind_child = esdp && erts_is_scheduler_bound(esdp);
+    tcdp->sched_bind_data = erts_sched_bind_atthrcreate_prepare();
 
     return (void *) tcdp;
 }
@@ -444,6 +434,8 @@ thr_create_cleanup(void *vtcdp)
 {
     erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp;
 
+    erts_sched_bind_atthrcreate_parent(tcdp->sched_bind_data);
+
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
     /* Restore signalmask... */
     erts_thr_sigmask(SIG_SETMASK, &tcdp->saved_sigmask, NULL);
@@ -457,6 +449,10 @@ thr_create_prepare_child(void *vtcdp)
 {
     erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp;
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_thread_setup();
+#endif
+
 #ifndef NO_FPE_SIGNALS
     /*
      * We do not want fp exeptions in other threads than the
@@ -466,12 +462,7 @@ thr_create_prepare_child(void *vtcdp)
     erts_thread_disable_fpe();
 #endif
 
-    if (tcdp->unbind_child) {
-	erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
-	erts_unbind_from_cpu(erts_cpuinfo);
-	erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
-    }
-    
+    erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data);
 }
 
 #endif /* #ifdef USE_THREADS */
@@ -484,9 +475,6 @@ erts_sys_pre_init(void)
 #ifdef USE_THREADS
     {
     erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER;
-    eid.alloc = ethr_internal_alloc;
-    eid.realloc = ethr_internal_realloc;
-    eid.free = ethr_internal_free;
 
     eid.thread_create_child_func = thr_create_prepare_child;
     /* Before creation in parent */
@@ -521,9 +509,9 @@ erts_sys_pre_init(void)
 #endif
     }
 #ifdef ERTS_SMP
-    erts_smp_atomic_init(&erts_break_requested, 0);
-    erts_smp_atomic_init(&erts_got_sigusr1, 0);
-    erts_smp_atomic_init(&have_prepared_crash_dump, 0);
+    erts_smp_atomic32_init(&erts_break_requested, 0);
+    erts_smp_atomic32_init(&erts_got_sigusr1, 0);
+    erts_smp_atomic32_init(&have_prepared_crash_dump, 0);
 #else
     erts_break_requested = 0;
     erts_got_sigusr1 = 0;
@@ -534,13 +522,14 @@ erts_sys_pre_init(void)
 #endif
 #endif /* USE_THREADS */
     erts_smp_atomic_init(&sys_misc_mem_sz, 0);
-    erts_smp_rwmtx_init(&environ_rwmtx, "environ");
 }
 
 void
 erl_sys_init(void)
 {
+    erts_smp_rwmtx_init(&environ_rwmtx, "environ");
 #if !DISABLE_VFORK
+ {
     int res;
     char bindir[MAXPATHLEN];
     size_t bindirsz = sizeof(bindir);
@@ -570,6 +559,7 @@ erl_sys_init(void)
             bindir,
             DIR_SEPARATOR_CHAR,
             CHILD_SETUP_PROG_NAME);
+ }
 #endif
 
 #ifdef USE_SETLINEBUF
@@ -1324,9 +1314,18 @@ static char **build_unix_environment(char *block)
 	}
     }
 
-    for (j = 0; j < i; j++) {
-	if (cpp[j][strlen(cpp[j])-1] == '=') {
+    for (j = 0; j < i; ) {
+        size_t last = strlen(cpp[j])-1;
+	if (cpp[j][last] == '=' && strchr(cpp[j], '=') == cpp[j]+last) {
 	    cpp[j] = cpp[--len];
+	    if (len < i) {
+		i--;
+	    } else {
+		j++;
+	    }
+	}
+	else {
+	    j++;
 	}
     }
 
@@ -1463,9 +1462,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op
 
     CHLD_STAT_LOCK;
 
-    unbind = erts_is_scheduler_bound(NULL);
-    if (unbind)
-	erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
+    unbind = erts_sched_bind_atfork_prepare();
 
 #if !DISABLE_VFORK
     /* See fork/vfork discussion before this function. */
@@ -1478,7 +1475,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op
 	if (pid == 0) {
 	    /* The child! Setup child... */
 
-	    if (unbind && erts_unbind_from_cpu(erts_cpuinfo) != 0)
+	    if (erts_sched_bind_atfork_child(unbind) != 0)
 		goto child_error;
 
 	    /* OBSERVE!
@@ -1579,8 +1576,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op
 
 	cs_argv[CS_ARGV_PROGNAME_IX] = child_setup_prog;
 	cs_argv[CS_ARGV_WD_IX] = opts->wd ? opts->wd : ".";
-	cs_argv[CS_ARGV_UNBIND_IX]
-	    = (unbind ? erts_get_unbind_from_cpu_str(erts_cpuinfo) : "false");
+	cs_argv[CS_ARGV_UNBIND_IX] = erts_sched_bind_atvfork_child(unbind);
 	cs_argv[CS_ARGV_FD_CR_IX] = fd_close_range;
 	for (i = 0; i < CS_ARGV_NO_OF_DUP2_OPS; i++)
 	    cs_argv[CS_ARGV_DUP2_OP_IX(i)] = &dup2_op[i][0];
@@ -1629,8 +1625,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op
     }
 #endif
 
-    if (unbind)
-	erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
+    erts_sched_bind_atfork_parent(unbind);
 
     if (pid == -1) {
         saved_errno = errno;
@@ -2562,7 +2557,6 @@ extern Preload pre_loaded[];
 
 void erts_sys_alloc_init(void)
 {
-    elib_ensure_initialized();
 }
 
 void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz)
@@ -3000,11 +2994,27 @@ init_smp_sig_notify(void)
 			NULL,
 			&thr_opts);
 }
+#ifdef __DARWIN__
 
+int erts_darwin_main_thread_pipe[2];
+int erts_darwin_main_thread_result_pipe[2];
+
+static void initialize_darwin_main_thread_pipes(void) 
+{
+    if (pipe(erts_darwin_main_thread_pipe) < 0 || 
+	pipe(erts_darwin_main_thread_result_pipe) < 0) {
+	erl_exit(1,"Fatal error initializing Darwin main thread stealing");
+    }
+}
+
+#endif
 void
 erts_sys_main_thread(void)
 {
     erts_thread_disable_fpe();
+#ifdef __DARWIN__
+    initialize_darwin_main_thread_pipes();
+#endif
     /* Become signal receiver thread... */
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_set_thread_name("signal_receiver");
@@ -3013,6 +3023,27 @@ erts_sys_main_thread(void)
     smp_sig_notify(0); /* Notify initialized */
     while (1) {
 	/* Wait for a signal to arrive... */
+#ifdef __DARWIN__
+	/*
+	 * The wx driver needs to be able to steal the main thread for Cocoa to
+	 * work properly.
+	 */
+	fd_set readfds;
+	int res;
+
+	FD_ZERO(&readfds);
+	FD_SET(erts_darwin_main_thread_pipe[0], &readfds);
+	res = select(erts_darwin_main_thread_pipe[0] + 1, &readfds, NULL, NULL, NULL);
+	if (res > 0 && FD_ISSET(erts_darwin_main_thread_pipe[0],&readfds)) {
+	    void* (*func)(void*);
+	    void* arg;
+	    void *resp;
+	    read(erts_darwin_main_thread_pipe[0],&func,sizeof(void* (*)(void*)));
+	    read(erts_darwin_main_thread_pipe[0],&arg, sizeof(void*));
+	    resp = (*func)(arg);
+	    write(erts_darwin_main_thread_result_pipe[1],&resp,sizeof(void *));
+	}
+#else
 #ifdef DEBUG
 	int res =
 #else
@@ -3021,6 +3052,7 @@ erts_sys_main_thread(void)
 	    select(0, NULL, NULL, NULL, NULL);
 	ASSERT(res < 0);
 	ASSERT(errno == EINTR);
+#endif
     }
 }
 
@@ -3120,226 +3152,3 @@ erl_sys_args(int* argc, char** argv)
     }
     *argc = j;
 }
-
-#ifdef ERTS_TIMER_THREAD
-
-/*
- * Interruptible-wait facility: low-level synchronisation state
- * and methods that are implementation dependent.
- *
- * Constraint: Every implementation must define 'struct erts_iwait'
- * with a field 'erts_smp_atomic_t state;'.
- */
-
-/* values for struct erts_iwait's state field */
-#define IWAIT_WAITING	0
-#define IWAIT_AWAKE	1
-#define IWAIT_INTERRUPT	2
-
-#if 0	/* XXX: needs feature test in erts/configure.in */
-
-/*
- * This is an implementation of the interruptible wait facility on
- * top of Linux-specific futexes.
- */
-#include <asm/unistd.h>
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
-static int sys_futex(void *futex, int op, int val, const struct timespec *timeout)
-{
-    return syscall(__NR_futex, futex, op, val, timeout);
-}
-
-struct erts_iwait {
-    erts_smp_atomic_t state; /* &state.counter is our futex */
-};
-
-static void iwait_lowlevel_init(struct erts_iwait *iwait) { /* empty */ }
-
-static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay)
-{
-    struct timespec timeout;
-    int res;
-
-    timeout.tv_sec = delay->tv_sec;
-    timeout.tv_nsec = delay->tv_usec * 1000;
-    res = sys_futex((void*)&iwait->state.counter, FUTEX_WAIT, IWAIT_WAITING, &timeout);
-    if (res < 0 && errno != ETIMEDOUT && errno != EWOULDBLOCK && errno != EINTR)
-	perror("FUTEX_WAIT");
-}
-
-static void iwait_lowlevel_interrupt(struct erts_iwait *iwait)
-{
-    int res = sys_futex((void*)&iwait->state.counter, FUTEX_WAKE, 1, NULL);
-    if (res < 0)
-	perror("FUTEX_WAKE");
-}
-
-#else	/* using poll() or select() */
-
-/*
- * This is an implementation of the interruptible wait facility on
- * top of pipe(), poll() or select(), read(), and write().
- */
-struct erts_iwait {
-    erts_smp_atomic_t state;
-    int read_fd;	/* wait polls and reads this fd */
-    int write_fd;	/* interrupt writes this fd */
-};
-
-static void iwait_lowlevel_init(struct erts_iwait *iwait)
-{
-    int fds[2];
-
-    if (pipe(fds) < 0) {
-	perror("pipe()");
-	exit(1);
-    }
-    iwait->read_fd = fds[0];
-    iwait->write_fd = fds[1];
-}
-
-#if defined(ERTS_USE_POLL)
-
-#include <sys/poll.h>
-#define PERROR_POLL "poll()"
-
-static int iwait_lowlevel_poll(int read_fd, struct timeval *delay)
-{
-    struct pollfd pollfd;
-    int timeout;
-
-    pollfd.fd = read_fd;
-    pollfd.events = POLLIN;
-    pollfd.revents = 0;
-    timeout = delay->tv_sec * 1000 + delay->tv_usec / 1000;
-    return poll(&pollfd, 1, timeout);
-}
-
-#else	/* !ERTS_USE_POLL */
-
-#include <sys/select.h>
-#define PERROR_POLL "select()"
-
-static int iwait_lowlevel_poll(int read_fd, struct timeval *delay)
-{
-    fd_set readfds;
-
-    FD_ZERO(&readfds);
-    FD_SET(read_fd, &readfds);
-    return select(read_fd + 1, &readfds, NULL, NULL, delay);
-}
-
-#endif	/* !ERTS_USE_POLL */
-
-static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay)
-{
-    int res;
-    char buf[64];
-
-    res = iwait_lowlevel_poll(iwait->read_fd, delay);
-    if (res > 0)
-	(void)read(iwait->read_fd, buf, sizeof buf);
-    else if (res < 0 && errno != EINTR)
-	perror(PERROR_POLL);
-}
-
-static void iwait_lowlevel_interrupt(struct erts_iwait *iwait)
-{
-    int res = write(iwait->write_fd, "!", 1);
-    if (res < 0)
-	perror("write()");
-}
-
-#endif	/* using poll() or select() */
-
-#if 0	/* not using poll() or select() */
-/*
- * This is an implementation of the interruptible wait facility on
- * top of pthread_cond_timedwait(). This has two problems:
- * 1. pthread_cond_timedwait() requires an absolute time point,
- *    so the relative delay must be converted to absolute time.
- *    Worse, this breaks if the machine's time is adjusted while
- *    we're preparing to wait.
- * 2. Each cond operation requires additional mutex lock/unlock operations.
- *
- * Problem 2 is probably not too bad on Linux (they'll just become
- * relatively cheap futex operations), but problem 1 is the real killer.
- * Only use this implementation if no better alternatives are available!
- */
-struct erts_iwait {
-    erts_smp_atomic_t state;
-    pthread_cond_t cond;
-    pthread_mutex_t mutex;
-};
-
-static void iwait_lowlevel_init(struct erts_iwait *iwait)
-{
-    iwait->cond = (pthread_cond_t) PTHREAD_COND_INITIALIZER;
-    iwait->mutex = (pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER;
-}
-
-static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay)
-{
-    struct timeval tmp;
-    struct timespec timeout;
-
-    /* Due to pthread_cond_timedwait()'s use of absolute
-       time, this must be the real gettimeofday(), _not_
-       the "smoothed" one beam/erl_time_sup.c implements. */
-    gettimeofday(&tmp, NULL);
-
-    tmp.tv_sec += delay->tv_sec;
-    tmp.tv_usec += delay->tv_usec;
-    if (tmp.tv_usec >= 1000*1000) {
-	tmp.tv_usec -= 1000*1000;
-	tmp.tv_sec += 1;
-    }
-    timeout.tv_sec = tmp.tv_sec;
-    timeout.tv_nsec = tmp.tv_usec * 1000;
-    pthread_mutex_lock(&iwait->mutex);
-    pthread_cond_timedwait(&iwait->cond, &iwait->mutex, &timeout);
-    pthread_mutex_unlock(&iwait->mutex);
-}
-
-static void iwait_lowlevel_interrupt(struct erts_iwait *iwait)
-{
-    pthread_mutex_lock(&iwait->mutex);
-    pthread_cond_signal(&iwait->cond);
-    pthread_mutex_unlock(&iwait->mutex);
-}
-
-#endif /* not using POLL */
-
-/*
- * Interruptible-wait facility. This is just a wrapper around the
- * low-level synchronisation code, where we maintain our logical
- * state in order to suppress some state transitions.
- */
-
-struct erts_iwait *erts_iwait_init(void)
-{
-    struct erts_iwait *iwait = malloc(sizeof *iwait);
-    if (!iwait) {
-	perror("malloc");
-	exit(1);
-    }
-    iwait_lowlevel_init(iwait);
-    erts_smp_atomic_init(&iwait->state, IWAIT_AWAKE);
-    return iwait;
-}
-
-void erts_iwait_wait(struct erts_iwait *iwait, struct timeval *delay)
-{
-    if (erts_smp_atomic_xchg(&iwait->state, IWAIT_WAITING) != IWAIT_INTERRUPT)
-	iwait_lowlevel_wait(iwait, delay);
-    erts_smp_atomic_set(&iwait->state, IWAIT_AWAKE);
-}
-
-void erts_iwait_interrupt(struct erts_iwait *iwait)
-{
-    if (erts_smp_atomic_xchg(&iwait->state, IWAIT_INTERRUPT) == IWAIT_WAITING)
-	iwait_lowlevel_interrupt(iwait);
-}
-
-#endif /* ERTS_TIMER_THREAD */
diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c
index c59c99f65e..8ec7b31ce0 100644
--- a/erts/emulator/sys/unix/sys_float.c
+++ b/erts/emulator/sys/unix/sys_float.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -36,11 +36,6 @@ erts_sys_init_float(void)
 # endif
 }
 
-static ERTS_INLINE void set_current_fp_exception(unsigned long pc)
-{
-    /* nothing to do */
-}
-
 #else  /* !NO_FPE_SIGNALS */
 
 #ifdef ERTS_SMP
@@ -476,7 +471,7 @@ static int mask_fpe(void)
 
 #endif
 
-#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__))) || (defined(__OpenBSD__) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__))
+#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__))) || ((defined(__NetBSD__) || defined(__OpenBSD__)) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__))
 
 #if defined(__linux__) && defined(__i386__)
 #if !defined(X86_FXSR_MAGIC)
@@ -519,6 +514,10 @@ static int mask_fpe(void)
 #define mc_pc(mc)	((mc)->mc_rip)
 #elif defined(__FreeBSD__) && defined(__i386__)
 #define mc_pc(mc)	((mc)->mc_eip)
+#elif defined(__NetBSD__) && defined(__x86_64__)
+#define mc_pc(mc)	((mc)->__gregs[_REG_RIP])
+#elif defined(__NetBSD__) && defined(__i386__)
+#define mc_pc(mc)	((mc)->__gregs[_REG_EIP])
 #elif defined(__OpenBSD__) && defined(__x86_64__)
 #define mc_pc(mc)	((mc)->sc_rip)
 #elif defined(__sun__) && defined(__x86_64__)
@@ -610,6 +609,23 @@ static void fpe_sig_action(int sig, siginfo_t *si, void *puc)
 	struct env87 *env87 = &savefpu->sv_87.sv_env;
 	env87->en_sw &= ~0xFF;
     }
+#elif defined(__NetBSD__) && defined(__x86_64__)
+    mcontext_t *mc = &uc->uc_mcontext;
+    struct fxsave64 *fxsave = (struct fxsave64 *)&mc->__fpregs;
+    pc = mc_pc(mc);
+    fxsave->fx_mxcsr = 0x1F80;
+    fxsave->fx_fsw &= ~0xFF;
+#elif defined(__NetBSD__) && defined(__i386__)
+    mcontext_t *mc = &uc->uc_mcontext;
+    pc = mc_pc(mc);
+    if (uc->uc_flags & _UC_FXSAVE) {
+	struct envxmm *envxmm = (struct envxmm *)&mc->__fpregs;
+	envxmm->en_mxcsr = 0x1F80;
+	envxmm->en_sw &= ~0xFF;
+    } else {
+	struct env87 *env87 = (struct env87 *)&mc->__fpregs;
+	env87->en_sw &= ~0xFF;
+    }
 #elif defined(__OpenBSD__) && defined(__x86_64__)
     struct fxsave64 *fxsave = uc->sc_fpstate;
     pc = mc_pc(uc);
@@ -799,8 +815,17 @@ sys_chars_to_double(char* buf, double* fp)
     }
 
 #ifdef NO_FPE_SIGNALS
-    if (errno == ERANGE && (*fp == 0.0 || *fp == HUGE_VAL || *fp == -HUGE_VAL)) {
-	return -1;
+    if (errno == ERANGE) {
+	if (*fp == HUGE_VAL || *fp == -HUGE_VAL) {
+	    /* overflow, should give error */
+	    return -1;
+	} else if (t == s && *fp == 0.0) {
+	    /* This should give 0.0 - OTP-7178 */
+	    errno = 0;
+
+	} else if (*fp == 0.0) {
+	    return -1;
+	}
     }
 #endif
     return 0;
@@ -810,7 +835,9 @@ int
 matherr(struct exception *exc)
 {
 #if !defined(NO_FPE_SIGNALS)
-    set_current_fp_exception((unsigned long)__builtin_return_address(0));
+    volatile unsigned long *fpexnp = erts_get_current_fp_exception();
+    if (fpexnp != NULL)
+	*fpexnp = (unsigned long)__builtin_return_address(0);
 #endif
     return 1;
 }
diff --git a/erts/emulator/sys/vxworks/sys.c b/erts/emulator/sys/vxworks/sys.c
index abddc7e107..c6e7b65f32 100644
--- a/erts/emulator/sys/vxworks/sys.c
+++ b/erts/emulator/sys/vxworks/sys.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1997-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -85,7 +85,7 @@ EXTERN_FUNCTION(void, erl_exit, (int n, char*, _DOTS_));
 EXTERN_FUNCTION(void, erl_error, (char*, va_list));
 EXTERN_FUNCTION(int, driver_interrupt, (int, int));
 EXTERN_FUNCTION(void, increment_time, (int));
-EXTERN_FUNCTION(int, next_time, (_VOID_));
+EXTERN_FUNCTION(int, erts_next_time, (_VOID_));
 EXTERN_FUNCTION(void, set_reclaim_free_function, (FreeFunction));
 EXTERN_FUNCTION(int, erl_mem_info_get, (MEM_PART_STATS *));
 EXTERN_FUNCTION(void, erl_crash_dump, (char* file, int line, char* fmt, ...));
@@ -143,6 +143,14 @@ volatile int erts_break_requested;
 
 /********************* General functions ****************************/
 
+/*
+ * Reset the terminal to the original settings on exit
+ * (nothing to do for WxWorks).
+ */
+void sys_tty_reset(int exit_code)
+{
+}
+
 Uint
 erts_sys_misc_mem_sz(void)
 {
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index d816cc2c07..074e2e247f 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 #ifdef HAVE_CONFIG_H
@@ -274,7 +274,6 @@ struct ErtsPollSet_ {
     Waiter** waiter;
     int allocated_waiters;  /* Size ow waiter array */ 
     int num_waiters;	    /* Number of waiter threads. */
-    erts_atomic_t sys_io_ready; /* Tells us there is I/O ready (already). */
     int restore_events;        /* Tells us to restore waiters events 
 				  next time around */
     HANDLE event_io_ready;     /* To be used when waiting for io */
@@ -282,12 +281,11 @@ struct ErtsPollSet_ {
     volatile int standby_wait_counter; /* Number of threads to wait for */
     CRITICAL_SECTION standby_crit;     /* CS to guard the counter */
     HANDLE standby_wait_event;         /* Event signalled when counte == 0 */
+    erts_atomic32_t wakeup_state;
 #ifdef ERTS_SMP
-    erts_smp_atomic_t woken;
     erts_smp_mtx_t mtx;
-    erts_smp_atomic_t interrupt;
 #endif
-    erts_smp_atomic_t timeout;
+    erts_smp_atomic32_t timeout;
 };
 
 #ifdef ERTS_SMP
@@ -296,99 +294,24 @@ struct ErtsPollSet_ {
   erts_smp_mtx_lock(&(PS)->mtx)
 #define ERTS_POLLSET_UNLOCK(PS) \
   erts_smp_mtx_unlock(&(PS)->mtx)
-#define ERTS_POLLSET_SET_POLLED_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->polled, (long) 1))
-#define ERTS_POLLSET_SET_POLLED(PS) \
-  erts_smp_atomic_set(&(PS)->polled, (long) 1)
-#define ERTS_POLLSET_UNSET_POLLED(PS) \
-  erts_smp_atomic_set(&(PS)->polled, (long) 0)
-#define ERTS_POLLSET_IS_POLLED(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->polled))
-#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->woken, (long) 1))
-#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \
-  erts_smp_atomic_set(&(PS)->woken, (long) 1)
-#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \
-  erts_smp_atomic_set(&(PS)->woken, (long) 0)
-#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->woken))
-
-#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->interrupt, (long) 0))
-#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \
-  erts_smp_atomic_set(&(PS)->interrupt, (long) 0)
-#define ERTS_POLLSET_SET_INTERRUPTED(PS) \
-  erts_smp_atomic_set(&(PS)->interrupt, (long) 1)
-#define ERTS_POLLSET_IS_INTERRUPTED(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->interrupt))
 
 #else
 
 #define ERTS_POLLSET_LOCK(PS)
 #define ERTS_POLLSET_UNLOCK(PS)
-#define ERTS_POLLSET_SET_POLLED_CHK(PS) 0
-#define ERTS_POLLSET_UNSET_POLLED(PS)
-#define ERTS_POLLSET_IS_POLLED(PS) 0
-#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) 1
-#define ERTS_POLLSET_SET_POLLER_WOKEN(PS)
-#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS)
-#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) 1
-
 
 #endif
 
 /*
- * While atomics are not yet implemented for windows in the common library...
- *
- * MSDN doc states that SMP machines and old compilers require
- * InterLockedExchange to properly read and write interlocked
- * variables, otherwise the processors might reschedule
- * the access and order of atomics access is destroyed... 
- * While they only mention it in white-papers, the problem
- * in VS2003 is due to the IA64 arch, so we can still count
- * on the CPU not rescheduling the access to volatile in X86 arch using 
- * even the slightly older compiler...
- *
- * So here's (hopefully) a subset of the generally working atomic 
- * variable access...
- */
-
-#if defined(__GNUC__)
-#  if defined(__i386__) || defined(__x86_64__)
-#    define VOLATILE_IN_SEQUENCE 1
-#  else
-#    define VOLATILE_IN_SEQUENCE 0
-#  endif 
-#elif defined(_MSC_VER)
-#  if _MSC_VER < 1300
-#    define VOLATILE_IN_SEQUENCE 0 /* Dont trust really old compilers */
-#  else
-#    if defined(_M_IX86)
-#      define VOLATILE_IN_SEQUENCE 1
-#    else /* I.e. IA64 */
-#      if _MSC_VER >= 1400 
-#        define VOLATILE_IN_SEQUENCE 1
-#      else
-#        define VOLATILE_IN_SEQUENCE 0
-#      endif
-#    endif
-#  endif
-#else
-# define VOLATILE_IN_SEQUENCE 0
-#endif
- 
-
-
-/*
  * Communication with sys_interrupt
  */
 
 #ifdef ERTS_SMP
-extern erts_smp_atomic_t erts_break_requested;
+extern erts_smp_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 1)
+  erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 0)
+  erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0)
 #else
 extern volatile int erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
@@ -397,7 +320,7 @@ extern volatile int erts_break_requested;
 
 static erts_mtx_t break_waiter_lock;
 static HANDLE break_happened_event;
-static erts_atomic_t break_waiter_state;
+static erts_atomic32_t break_waiter_state;
 #define BREAK_WAITER_GOT_BREAK 1
 #define BREAK_WAITER_GOT_HALT 2
 
@@ -440,29 +363,172 @@ do { \
     wait_standby(PS); \
  } while(0)
 
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
+#define ERTS_POLL_NOT_WOKEN		((erts_aint32_t) 0)
+#define ERTS_POLL_WOKEN_IO_READY	((erts_aint32_t) 1)
+#define ERTS_POLL_WOKEN_INTR		((erts_aint32_t) 2)
+#define ERTS_POLL_WOKEN_TIMEDOUT	((erts_aint32_t) 3)
 
 static ERTS_INLINE int
-unset_interrupted_chk(ErtsPollSet ps)
+is_io_ready(ErtsPollSet ps)
 {
-    /* This operation isn't atomic, but we have no need at all for an
-       atomic operation here... */
-    int res = ps->interrupt;
-    ps->interrupt = 0;
-    return res;
+    return erts_atomic32_read(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY;
 }
 
+static ERTS_INLINE void
+woke_up(ErtsPollSet ps)
+{
+    if (erts_atomic32_read(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN)
+	erts_atomic32_cmpxchg(&ps->wakeup_state,
+			      ERTS_POLL_WOKEN_TIMEDOUT,
+			      ERTS_POLL_NOT_WOKEN);
+#ifdef DEBUG
+    {
+	erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+	switch (wakeup_state) {
+	case ERTS_POLL_WOKEN_IO_READY:
+	case ERTS_POLL_WOKEN_INTR:
+	case ERTS_POLL_WOKEN_TIMEDOUT:
+	    break;
+	default:
+	    ASSERT(0);
+	    break;
+	}
+    }
 #endif
+}
+
+static ERTS_INLINE int
+wakeup_cause(ErtsPollSet ps)
+{
+    int res;
+    erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+    switch (wakeup_state) {
+    case ERTS_POLL_WOKEN_IO_READY:
+	res = 0;
+	break;
+    case ERTS_POLL_WOKEN_INTR:
+	res = EINTR;
+	break;
+    case ERTS_POLL_WOKEN_TIMEDOUT:
+	res = ETIMEDOUT;
+	break;
+    default:
+	res = 0;
+	erl_exit(ERTS_ABORT_EXIT,
+		 "%s:%d: Internal error: Invalid wakeup_state=%d\n",
+		 __FILE__, __LINE__, (int) wakeup_state);
+    }
+    return res;
+}
+
+static ERTS_INLINE DWORD
+poll_wait_timeout(ErtsPollSet ps, SysTimeval *tvp)
+{
+    time_t timeout = tvp->tv_sec * 1000 + tvp->tv_usec / 1000;
+
+    if (timeout <= 0) {
+	woke_up(ps);
+	return (DWORD) 0;
+    }
+
+    ResetEvent(ps->event_io_ready);
+    /*
+     * Since we don't know the internals of ResetEvent() we issue
+     * a memory barrier as a safety precaution ensuring that
+     * the load of wakeup_state wont be reordered with stores made
+     * by ResetEvent().
+     */
+    ERTS_THR_MEMORY_BARRIER;
+    if (erts_atomic32_read(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN)
+	return (DWORD) 0;
+
+    if (timeout > ERTS_AINT32_T_MAX) /* Also prevents DWORD overflow */
+	timeout = ERTS_AINT32_T_MAX;
+
+    erts_smp_atomic32_set_relb(&ps->timeout, (erts_aint32_t) timeout);
+    return (DWORD) timeout;
+}
 
-#ifdef ERTS_SMP
 static ERTS_INLINE void
-wake_poller(ErtsPollSet ps)
+wake_poller(ErtsPollSet ps, int io_ready)
 {
-    if (!ERTS_POLLSET_SET_POLLER_WOKEN_CHK(ps)) {
+    erts_aint32_t wakeup_state;
+    if (io_ready) {
+	/* We may set the event multiple times. This is, however, harmless. */
+	wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+	erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY);
+    }
+    else {
+	ERTS_THR_MEMORY_BARRIER;
+	wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+	while (wakeup_state != ERTS_POLL_WOKEN_IO_READY
+	       && wakeup_state != ERTS_POLL_WOKEN_INTR) {
+	    erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state,
+						      ERTS_POLL_WOKEN_INTR,
+						      wakeup_state);
+	    if (act == wakeup_state) {
+		wakeup_state = act;
+		break;
+	    }
+	    wakeup_state = act;
+	}
+    }
+    if (wakeup_state == ERTS_POLL_NOT_WOKEN) {
+	/*
+	 * Since we don't know the internals of SetEvent() we issue
+	 * a memory barrier as a safety precaution ensuring that
+	 * the store we just made to wakeup_state wont be reordered
+	 * with loads in SetEvent().
+	 */
+	ERTS_THR_MEMORY_BARRIER;
 	SetEvent(ps->event_io_ready);
     }
 }
-#endif
+
+static ERTS_INLINE void
+reset_io_ready(ErtsPollSet ps)
+{
+    erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
+}
+
+static ERTS_INLINE void
+restore_io_ready(ErtsPollSet ps)
+{
+    erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY);
+}
+
+/*
+ * notify_io_ready() is used by threads waiting for events, when
+ * notifying a poller thread about I/O ready.
+ */
+static ERTS_INLINE void
+notify_io_ready(ErtsPollSet ps)
+{
+    wake_poller(ps, 1);
+}
+
+static ERTS_INLINE void
+reset_interrupt(ErtsPollSet ps)
+{
+    /* We need to keep io-ready if set */
+    erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+    while (wakeup_state != ERTS_POLL_WOKEN_IO_READY
+	   && wakeup_state != ERTS_POLL_NOT_WOKEN) {
+	erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state,
+						  ERTS_POLL_NOT_WOKEN,
+						  wakeup_state);
+	if (wakeup_state == act)
+	    break;
+	wakeup_state = act;
+    }
+    ERTS_THR_MEMORY_BARRIER;
+}
+
+static ERTS_INLINE void
+set_interrupt(ErtsPollSet ps)
+{
+    wake_poller(ps, 0);
+}
 
 static void setup_standby_wait(ErtsPollSet ps, int num_threads)
 {
@@ -626,14 +692,16 @@ static void *break_waiter(void *param)
 	case WAIT_OBJECT_0:
 	    ResetEvent(harr[0]);
 	    erts_mtx_lock(&break_waiter_lock);
-	    erts_atomic_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK);
+	    erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK);
+	    ERTS_THR_MEMORY_BARRIER;
 	    SetEvent(break_happened_event);
 	    erts_mtx_unlock(&break_waiter_lock);
 	    break;
 	case (WAIT_OBJECT_0+1):
 	    ResetEvent(harr[1]);
 	    erts_mtx_lock(&break_waiter_lock);
-	    erts_atomic_set(&break_waiter_state,BREAK_WAITER_GOT_HALT);
+	    erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_HALT);
+	    ERTS_THR_MEMORY_BARRIER;
 	    SetEvent(break_happened_event);
 	    erts_mtx_unlock(&break_waiter_lock);
 	    break;
@@ -740,12 +808,7 @@ event_happened:
 	    consistency_check(w);
 #endif
 	    ASSERT(WAIT_OBJECT_0 < i && i < WAIT_OBJECT_0+w->active_events);
-	    if (!erts_atomic_xchg(&ps->sys_io_ready,1)) { 
-		HARDDEBUGF(("SET EventIoReady (%d)",erts_atomic_read(&ps->sys_io_ready)));
-		SetEvent(ps->event_io_ready);
-	    } else {
-		HARDDEBUGF(("DONT SET EventIoReady"));
-	    }
+	    notify_io_ready(ps);
 
 	    /* 
 	     * The main thread wont start working on our arrays untill we're
@@ -940,15 +1003,10 @@ static int cancel_driver_select(ErtsPollSet ps, HANDLE event)
 void  erts_poll_interrupt(ErtsPollSet ps, int set /* bool */)
 {
     HARDTRACEF(("In erts_poll_interrupt(%d)",set));
-#ifdef ERTS_SMP
-    if (set) {
-	ERTS_POLLSET_SET_INTERRUPTED(ps);
-	wake_poller(ps);
-    }
-    else {
-	ERTS_POLLSET_UNSET_INTERRUPTED(ps);
-    }
-#endif
+    if (!set)
+	reset_interrupt(ps);
+    else
+	set_interrupt(ps);
     HARDTRACEF(("Out erts_poll_interrupt(%d)",set));
 }
 
@@ -957,17 +1015,10 @@ void erts_poll_interrupt_timed(ErtsPollSet ps,
 			       long msec)
 {
     HARDTRACEF(("In erts_poll_interrupt_timed(%d,%ld)",set,msec));
-#ifdef ERTS_SMP
-    if (set) {
-	if (erts_smp_atomic_read(&ps->timeout) > msec) {
-	    ERTS_POLLSET_SET_INTERRUPTED(ps);
-	    wake_poller(ps);
-	}
-    }
-    else {
-	ERTS_POLLSET_UNSET_INTERRUPTED(ps);
-    }
-#endif
+    if (!set)
+	reset_interrupt(ps);
+    else if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec)
+	set_interrupt(ps);
     HARDTRACEF(("Out erts_poll_interrupt_timed"));
 }
 
@@ -1041,10 +1092,8 @@ void erts_poll_controlv(ErtsPollSet ps,
 int erts_poll_wait(ErtsPollSet ps,
 		   ErtsPollResFd pr[],
 		   int *len,
-		   SysTimeval *utvp)
+		   SysTimeval *tvp)
 {
-    SysTimeval *tvp = utvp;
-    SysTimeval itv;
     int no_fds;
     DWORD timeout;
     EventData* ev;
@@ -1057,7 +1106,7 @@ int erts_poll_wait(ErtsPollSet ps,
     HARDTRACEF(("In erts_poll_wait"));
     ERTS_POLLSET_LOCK(ps);
 
-    if (!erts_atomic_read(&ps->sys_io_ready) && ps->restore_events) {
+    if (!is_io_ready(ps) && ps->restore_events) {
 	HARDDEBUGF(("Restore events: %d",ps->num_waiters));
 	ps->restore_events = 0;
 	for (i = 0; i < ps->num_waiters; ++i) {
@@ -1075,7 +1124,7 @@ int erts_poll_wait(ErtsPollSet ps,
 	       if (w->highwater != w->active_events) {
 		   HARDDEBUGF(("Oups!"));
 		   /* Oups, got signalled before we took the lock, can't reset */
-		   if(erts_atomic_read(&ps->sys_io_ready) == 0) {
+		   if(!is_io_ready(ps)) {
 		       erl_exit(1,"Internal error: "
 				"Inconsistent io structures in erl_poll.\n");
 		   }
@@ -1100,39 +1149,27 @@ int erts_poll_wait(ErtsPollSet ps,
 	no_fds = ERTS_POLL_MAX_RES;
 #endif
 
+    timeout = poll_wait_timeout(ps, tvp);
 
-    ResetEvent(ps->event_io_ready);
-    ERTS_POLLSET_UNSET_POLLER_WOKEN(ps);
-
-#ifdef ERTS_SMP
-    if (ERTS_POLLSET_IS_INTERRUPTED(ps)) {
-	/* Interrupt use zero timeout */
-	itv.tv_sec = 0;
-	itv.tv_usec = 0;
-	tvp = &itv;
-    }
-#endif
-
-    timeout = tvp->tv_sec * 1000 + tvp->tv_usec / 1000;
     /*HARDDEBUGF(("timeout = %ld",(long) timeout));*/
-    erts_smp_atomic_set(&ps->timeout, timeout);
 
-    if (timeout > 0 && ! erts_atomic_read(&ps->sys_io_ready) && ! erts_atomic_read(&break_waiter_state)) {
+    if (timeout > 0 && !erts_atomic32_read(&break_waiter_state)) {
 	HANDLE harr[2] = {ps->event_io_ready, break_happened_event};
 	int num_h = 2;
 
-	HARDDEBUGF(("Start waiting %d [%d]",num_h, (long) timeout));
+	HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout));
 	ERTS_POLLSET_UNLOCK(ps);
 	WaitForMultipleObjects(num_h, harr, FALSE, timeout);
 	ERTS_POLLSET_LOCK(ps);
-	HARDDEBUGF(("Stop waiting %d [%d]",num_h, (long) timeout));
+	HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout));
+	woke_up(ps);
     }
 
     ERTS_UNSET_BREAK_REQUESTED;
-    if(erts_atomic_read(&break_waiter_state)) {
+    if(erts_atomic32_read(&break_waiter_state)) {
 	erts_mtx_lock(&break_waiter_lock);
-	break_state = erts_atomic_read(&break_waiter_state);
-	erts_atomic_set(&break_waiter_state,0);
+	break_state = erts_atomic32_read(&break_waiter_state);
+	erts_atomic32_set(&break_waiter_state,0);
 	ResetEvent(break_happened_event);
 	erts_mtx_unlock(&break_waiter_lock);
 	switch (break_state) {
@@ -1147,15 +1184,13 @@ int erts_poll_wait(ErtsPollSet ps,
 	}
     }
 
-    ERTS_POLLSET_SET_POLLER_WOKEN(ps);
-
-    if (!erts_atomic_read(&ps->sys_io_ready)) {
-	res = EINTR;
-	HARDDEBUGF(("EINTR!"));
-	goto done; 
+    res = wakeup_cause(ps);
+    if (res != 0) {
+	HARDDEBUGF(("%s!", res == EINTR ? "EINTR" : "ETIMEDOUT"));
+	goto done;
     }
 
-    erts_atomic_set(&ps->sys_io_ready,0);
+    reset_io_ready(ps);
 
     n = ps->num_waiters;	
 
@@ -1177,9 +1212,9 @@ int erts_poll_wait(ErtsPollSet ps,
 	    if (num >= no_fds) {
 		w->highwater=j+1;
 		erts_mtx_unlock(&w->mtx);
-		/* This might mean we still have data to report, set
-		   back the global flag! */
-		erts_atomic_set(&ps->sys_io_ready,1);
+		/* This might mean we still have data to report,
+		   restore flag indicating I/O ready! */
+		restore_io_ready(ps);
 		HARDDEBUGF(("To many FD's to report!"));
 		goto done;
 	    }
@@ -1201,7 +1236,7 @@ int erts_poll_wait(ErtsPollSet ps,
 	erts_mtx_unlock(&w->mtx);
     }
  done:
-    erts_smp_atomic_set(&ps->timeout, LONG_MAX);
+    erts_smp_atomic32_set(&ps->timeout, ERTS_AINT32_T_MAX);
     *len = num;
     ERTS_POLLSET_UNLOCK(ps);
     HARDTRACEF(("Out erts_poll_wait"));
@@ -1279,15 +1314,13 @@ ErtsPollSet erts_poll_create_pollset(void)
     ps->standby_wait_counter = 0;
     ps->event_io_ready = CreateManualEvent(FALSE);
     ps->standby_wait_event = CreateManualEvent(FALSE); 
-    erts_atomic_init(&ps->sys_io_ready,0);
     ps->restore_events = 0;
 
+    erts_atomic32_init(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
 #ifdef ERTS_SMP
-    erts_smp_atomic_init(&ps->woken, 0);
     erts_smp_mtx_init(&ps->mtx, "pollset");
-    erts_smp_atomic_init(&ps->interrupt, 0);
 #endif
-    erts_smp_atomic_init(&ps->timeout, LONG_MAX);
+    erts_smp_atomic32_init(&ps->timeout, ERTS_AINT32_T_MAX);
 
     HARDTRACEF(("Out erts_poll_create_pollset"));
     return ps;
@@ -1339,7 +1372,7 @@ void  erts_poll_init(void)
 
     erts_mtx_init(&break_waiter_lock,"break_waiter_lock");
     break_happened_event = CreateManualEvent(FALSE);
-    erts_atomic_init(&break_waiter_state, 0); 
+    erts_atomic32_init(&break_waiter_state, 0); 
 
     erts_thr_create(&thread, &break_waiter, NULL, NULL);
     ERTS_UNSET_BREAK_REQUESTED;
diff --git a/erts/emulator/sys/win32/erl_win_dyn_driver.h b/erts/emulator/sys/win32/erl_win_dyn_driver.h
index 4949998abc..ecb06868d5 100644
--- a/erts/emulator/sys/win32/erl_win_dyn_driver.h
+++ b/erts/emulator/sys/win32/erl_win_dyn_driver.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -87,15 +87,15 @@ WDD_TYPEDEF(unsigned long, erts_alc_test, (unsigned long,
 					   unsigned long,
 					   unsigned long,
 					   unsigned long));
-WDD_TYPEDEF(long, driver_binary_get_refc, (ErlDrvBinary *dbp));
-WDD_TYPEDEF(long, driver_binary_inc_refc, (ErlDrvBinary *dbp));
-WDD_TYPEDEF(long, driver_binary_dec_refc, (ErlDrvBinary *dbp));
+WDD_TYPEDEF(ErlDrvSInt, driver_binary_get_refc, (ErlDrvBinary *dbp));
+WDD_TYPEDEF(ErlDrvSInt, driver_binary_inc_refc, (ErlDrvBinary *dbp));
+WDD_TYPEDEF(ErlDrvSInt, driver_binary_dec_refc, (ErlDrvBinary *dbp));
 WDD_TYPEDEF(ErlDrvPDL, driver_pdl_create, (ErlDrvPort));
 WDD_TYPEDEF(void, driver_pdl_lock, (ErlDrvPDL));
 WDD_TYPEDEF(void, driver_pdl_unlock, (ErlDrvPDL));
-WDD_TYPEDEF(long, driver_pdl_get_refc, (ErlDrvPDL));
-WDD_TYPEDEF(long, driver_pdl_inc_refc, (ErlDrvPDL));
-WDD_TYPEDEF(long, driver_pdl_dec_refc, (ErlDrvPDL));
+WDD_TYPEDEF(ErlDrvSInt, driver_pdl_get_refc, (ErlDrvPDL));
+WDD_TYPEDEF(ErlDrvSInt, driver_pdl_inc_refc, (ErlDrvPDL));
+WDD_TYPEDEF(ErlDrvSInt, driver_pdl_dec_refc, (ErlDrvPDL));
 WDD_TYPEDEF(void, driver_system_info, (ErlDrvSysInfo *, size_t));
 WDD_TYPEDEF(int, driver_get_now, (ErlDrvNowData *));
 WDD_TYPEDEF(int, driver_monitor_process, (ErlDrvPort port, 
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 3194493ac8..a2159d063c 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -31,12 +31,13 @@
 #include "global.h"
 #include "erl_threads.h"
 #include "../../drivers/win32/win_con.h"
+#include "erl_cpu_topology.h"
 
 
 void erts_sys_init_float(void);
 
 void erl_start(int, char**);
-void erl_exit(int n, char*, _DOTS_);
+void erl_exit(int n, char*, ...);
 void erl_error(char*, va_list);
 void erl_crash_dump(char*, int, char*, ...);
 
@@ -66,14 +67,17 @@ static void async_read_file(struct async_io* aio, LPVOID buf, DWORD numToRead);
 static int async_write_file(struct async_io* aio, LPVOID buf, DWORD numToWrite);
 static int get_overlapped_result(struct async_io* aio,
 				 LPDWORD pBytesRead, BOOL wait);
-static FUNCTION(BOOL, CreateChildProcess, (char *, HANDLE, HANDLE,
-					   HANDLE, LPHANDLE, BOOL,
-					   LPVOID, LPTSTR, unsigned, 
-					   char **, int *));
+static BOOL create_child_process(char *, HANDLE, HANDLE,
+			       HANDLE, LPHANDLE, BOOL,
+			       LPVOID, LPTSTR, unsigned,
+			       char **, int *);
 static int create_pipe(LPHANDLE, LPHANDLE, BOOL, BOOL);
-static int ApplicationType(const char* originalName, char fullPath[MAX_PATH],
+static int application_type(const char* originalName, char fullPath[MAX_PATH],
 			   BOOL search_in_path, BOOL handle_quotes,
 			   int *error_return);
+static int application_type_w(const WCHAR *originalName, WCHAR fullPath[MAX_PATH],
+			      BOOL search_in_path, BOOL handle_quotes,
+			      int *error_return);
 
 HANDLE erts_service_event;
 
@@ -86,15 +90,19 @@ static erts_smp_atomic_t pipe_creation_counter;
 static erts_smp_mtx_t sys_driver_data_lock;
 
 
-/* Results from ApplicationType is one of */
+/* Results from application_type(_w) is one of */
 #define APPL_NONE 0
 #define APPL_DOS  1
 #define APPL_WIN3X 2
 #define APPL_WIN32 3
 
-static FUNCTION(int, driver_write, (long, HANDLE, byte*, int));
+static int driver_write(long, HANDLE, byte*, int);
 static void common_stop(int);
 static int create_file_thread(struct async_io* aio, int mode);
+#ifdef ERTS_SMP
+static void close_active_handle(ErlDrvPort, HANDLE handle);
+static DWORD WINAPI threaded_handle_closer(LPVOID param);
+#endif
 static DWORD WINAPI threaded_reader(LPVOID param);
 static DWORD WINAPI threaded_writer(LPVOID param);
 static DWORD WINAPI threaded_exiter(LPVOID param);
@@ -132,6 +140,13 @@ static BOOL win_console = FALSE;
 
 static OSVERSIONINFO int_os_version;	/* Version information for Win32. */
 
+/*#define USE_CANCELIOEX
+    Disabled the use of CancelIoEx as its been seen to cause problem with some
+    drivers. Not sure what to blame; faulty drivers or some form of invalid use.
+*/
+#if defined(ERTS_SMP) && defined(USE_CANCELIOEX)
+static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED);
+#endif
 
 /* This is the system's main function (which may or may not be called "main")
    - do general system-dependent initialization
@@ -187,6 +202,17 @@ erts_sys_misc_mem_sz(void)
     return res;
 }
 
+/*
+ * Reset the terminal to the original settings on exit
+ */
+void sys_tty_reset(int exit_code)
+{
+    if (exit_code > 0)
+	ConWaitForExit();
+    else
+	ConNormalExit();
+}
+
 void erl_sys_args(int* argc, char** argv)
 {
     char *event_name;
@@ -234,7 +260,7 @@ erts_sys_prepare_crash_dump(void)
 }
 
 static void
-init_console()
+init_console(void)
 {
     char* mode = erts_read_env("ERL_CONSOLE_MODE");
 
@@ -254,7 +280,7 @@ init_console()
     erts_free_read_env(mode);
 }
 
-int sys_max_files() 
+int sys_max_files(void) 
 {
     return max_files;
 }
@@ -270,10 +296,7 @@ int sys_max_files()
  */
 
 static int
-get_and_remove_option(argc, argv, option)
-    int* argc;			/* Number of arguments. */
-    char* argv[];		/* The argument vector. */
-    const char* option;		/* Option to search for and remove. */
+get_and_remove_option(int* argc, char* argv[], const char *option)
 {
     int i;
 
@@ -323,9 +346,7 @@ static char *get_and_remove_option2(int *argc, char **argv,
 char os_type[] = "win32";
 
 void
-os_flavor(namebuf, size)
-char* namebuf;			/* Where to return the name. */
-unsigned size;			/* Size of name buffer. */
+os_flavor(char *namebuf, unsigned size)
 {
     switch (int_os_version.dwPlatformId) {
     case VER_PLATFORM_WIN32_WINDOWS:
@@ -598,12 +619,7 @@ struct erl_drv_entry async_driver_entry = {
  */
 
 static DriverData*
-new_driver_data(port_num, packet_bytes, wait_objs_required, use_threads)
-    int port_num;		/* The port number. */
-    int packet_bytes;		/* Number of bytes in header. */
-    int wait_objs_required;	/* The number objects this port is going
-				/* wait for (typically 1 or 2). */
-    int use_threads;		/* TRUE if threads are intended to be used. */
+new_driver_data(int port_num, int packet_bytes, int wait_objs_required, int use_threads)
 {
     DriverData* dp;
     
@@ -665,25 +681,50 @@ release_driver_data(DriverData* dp)
     erts_smp_mtx_lock(&sys_driver_data_lock);
 
 #ifdef ERTS_SMP
-    /* This is a workaround for the fact that CancelIo cant cancel
-       requests issued by another thread and that we still cant use
-       CancelIoEx as that's only availabele in Vista etc. */
-    if(dp->in.async_io_active && dp->in.fd != INVALID_HANDLE_VALUE) {
-	CloseHandle(dp->in.fd);
-	dp->in.fd = INVALID_HANDLE_VALUE;
-	DEBUGF(("Waiting for the in event thingie"));
-	WaitForSingleObject(dp->in.ov.hEvent,INFINITE);
-	DEBUGF(("...done\n"));
-    }
-    if(dp->out.async_io_active && dp->out.fd != INVALID_HANDLE_VALUE) {
-	CloseHandle(dp->out.fd);
-	dp->out.fd = INVALID_HANDLE_VALUE;
-	DEBUGF(("Waiting for the out event thingie"));
-	WaitForSingleObject(dp->out.ov.hEvent,INFINITE);
-	DEBUGF(("...done\n"));
+#ifdef USE_CANCELIOEX
+    if (fpCancelIoEx != NULL) {
+	if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) {
+	    (*fpCancelIoEx)(dp->in.fd, NULL);
+	}
+	if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) {
+	    (*fpCancelIoEx)(dp->out.fd, NULL);
+	}
+    }
+    else
+#endif
+    {
+	/* This is a workaround for the fact that CancelIo cant cancel
+	   requests issued by another thread and that we cant use
+	   CancelIoEx as that's only available in Vista etc.
+	   R14: Avoid scheduler deadlock by only wait for 10ms, and then spawn
+	    a thread that will keep waiting in in order to close handles. */
+	HANDLE handles[2];
+	int i = 0;
+	int timeout = 10;
+	if(dp->in.async_io_active && dp->in.fd != INVALID_HANDLE_VALUE) {
+	    CloseHandle(dp->in.fd);
+	    dp->in.fd = INVALID_HANDLE_VALUE;
+	    DEBUGF(("Waiting for the in event thingie"));
+	    if (WaitForSingleObject(dp->in.ov.hEvent,timeout) == WAIT_TIMEOUT) {
+		close_active_handle(dp->port_num, dp->in.ov.hEvent);
+		dp->in.ov.hEvent = NULL;
+		timeout = 0;
+	    }
+	    DEBUGF(("...done\n"));
+	}
+	if(dp->out.async_io_active && dp->out.fd != INVALID_HANDLE_VALUE) {
+	    CloseHandle(dp->out.fd);
+	    dp->out.fd = INVALID_HANDLE_VALUE;
+	    DEBUGF(("Waiting for the out event thingie"));
+	    if (WaitForSingleObject(dp->out.ov.hEvent,timeout) == WAIT_TIMEOUT) {
+		close_active_handle(dp->port_num, dp->out.ov.hEvent);
+		dp->out.ov.hEvent = NULL;
+	    }
+	    DEBUGF(("...done\n"));
+	}
     }
 #else
-    	if (dp->out.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) {
+	if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) {
 	     CancelIo(dp->in.fd); 
 	}
 	if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) {
@@ -726,6 +767,88 @@ release_driver_data(DriverData* dp)
     erts_smp_mtx_unlock(&sys_driver_data_lock);
 }
 
+#ifdef ERTS_SMP
+
+struct handles_to_be_closed {
+    HANDLE handles[MAXIMUM_WAIT_OBJECTS];
+    unsigned cnt;
+};
+static struct handles_to_be_closed* htbc_curr = NULL;
+CRITICAL_SECTION htbc_lock;
+
+static void close_active_handle(ErlDrvPort port_num, HANDLE handle)
+{
+    struct handles_to_be_closed* htbc;
+    int i;
+    EnterCriticalSection(&htbc_lock);
+    htbc = htbc_curr;
+    if (htbc == NULL || htbc->cnt >= MAXIMUM_WAIT_OBJECTS) {
+	DWORD tid;
+	HANDLE thread;
+
+	htbc = (struct handles_to_be_closed*) erts_alloc(ERTS_ALC_T_DRV_TAB,
+							 sizeof(*htbc));
+	htbc->handles[0] = CreateAutoEvent(FALSE);
+	htbc->cnt = 1;
+	thread = (HANDLE *) _beginthreadex(NULL, 0, threaded_handle_closer, htbc, 0, &tid);
+	CloseHandle(thread);
+    }
+    htbc->handles[htbc->cnt++] = handle;
+    driver_select(port_num, (ErlDrvEvent)handle, ERL_DRV_USE_NO_CALLBACK, 0);
+    SetEvent(htbc->handles[0]);
+    htbc_curr = htbc;
+    LeaveCriticalSection(&htbc_lock);
+}
+
+static DWORD WINAPI
+threaded_handle_closer(LPVOID param)
+{
+    struct handles_to_be_closed* htbc = (struct handles_to_be_closed*) param;
+    unsigned ix;
+    DWORD res;
+    DEBUGF(("threaded_handle_closer %p started\r\n", htbc));
+    EnterCriticalSection(&htbc_lock);
+    for (;;) {
+	{
+	    HANDLE* handles = htbc->handles;
+	    unsigned cnt = htbc->cnt;
+	    DWORD timeout = (htbc == htbc_curr) ? INFINITE : 10*1000;
+
+	    LeaveCriticalSection(&htbc_lock);
+	    DEBUGF(("threaded_handle_closer %p waiting for %d handles\r\n", htbc, cnt));
+	    res = WaitForMultipleObjects(cnt, handles, FALSE, timeout);
+	}
+	EnterCriticalSection(&htbc_lock);
+	switch (res) {
+	case WAIT_OBJECT_0:
+	case WAIT_TIMEOUT:
+	    break; /* got some more handles to wait for maybe */
+	default:
+	    ix = res - WAIT_OBJECT_0;
+	    if (ix > 0 && ix < htbc->cnt) {
+		CloseHandle(htbc->handles[ix]);
+		htbc->handles[ix] = htbc->handles[--htbc->cnt];
+	    }
+	}
+	if (htbc != htbc_curr) {
+	    if (htbc->cnt == 1) { /* no real handles left */
+		break;
+	    }
+	    /* The thread with most free slots will be "current" */
+	    if (htbc->cnt < htbc_curr->cnt) {
+		htbc_curr = htbc;
+		DEBUGF(("threaded_handle_closer %p made current\r\n", htbc));
+	    }
+	}
+    }
+    LeaveCriticalSection(&htbc_lock);
+    CloseHandle(htbc->handles[0]);
+    erts_free(ERTS_ALC_T_DRV_TAB, htbc);
+    DEBUGF(("threaded_handle_closer %p terminating\r\n", htbc));
+    return 0;
+}
+#endif /* ERTS_SMP */
+
 /*
  * Stores input and output file descriptors in the DriverData structure,
  * and calls driver_select().
@@ -734,12 +857,7 @@ release_driver_data(DriverData* dp)
  */
 
 static ErlDrvData
-set_driver_data(dp, ifd, ofd, read_write, report_exit)
-    DriverData* dp;
-    HANDLE ifd;
-    HANDLE ofd;
-    int read_write;
-    int report_exit;
+set_driver_data(DriverData* dp, HANDLE ifd, HANDLE ofd, int read_write, int report_exit)
 {
     int index = dp - driver_data;
     int result;
@@ -763,6 +881,31 @@ set_driver_data(dp, ifd, ofd, read_write, report_exit)
     return (ErlDrvData)index;
 }
 
+static ErlDrvData
+reuse_driver_data(DriverData *dp, HANDLE ifd, HANDLE ofd, int read_write, ErlDrvPort port_num)
+{
+    int index = dp - driver_data;
+    int result;
+
+    dp->port_num = port_num;
+    dp->in.fd = ifd;
+    dp->out.fd = ofd;
+    dp->report_exit = 0;
+
+    if (read_write & DO_READ) {
+	result = driver_select(dp->port_num, (ErlDrvEvent)dp->in.ov.hEvent,
+			       ERL_DRV_READ|ERL_DRV_USE, 1);
+	ASSERT(result != -1);
+    }
+
+    if (read_write & DO_WRITE) {
+	result = driver_select(dp->port_num, (ErlDrvEvent)dp->out.ov.hEvent,
+			       ERL_DRV_WRITE|ERL_DRV_USE, 1);
+	ASSERT(result != -1);
+    }
+    return (ErlDrvData)index;
+}
+
 /*
  * Initialises an AsyncIo structure.
  */
@@ -836,10 +979,7 @@ release_async_io(AsyncIo* aio, ErlDrvPort port_num)
  */
 
 static void
-async_read_file(aio, buf, numToRead)
-    AsyncIo* aio;		/* Pointer to driver data. */
-    LPVOID buf;			/* Pointer to buffer to receive data. */
-    DWORD numToRead;		/* Number of bytes to read. */
+async_read_file(AsyncIo* aio, LPVOID buf, DWORD numToRead)
 {
     aio->pendingError = NO_ERROR;
 #ifdef HARD_POLL_DEBUG
@@ -890,10 +1030,9 @@ async_read_file(aio, buf, numToRead)
  * ----------------------------------------------------------------------
  */
 static int
-async_write_file(aio, buf, numToWrite)
-    AsyncIo* aio;		/* Pointer to async control block. */
-    LPVOID buf;			/* Pointer to buffer with data to write. */
-    DWORD numToWrite;		/* Number of bytes to write. */
+async_write_file(AsyncIo* aio,		/* Pointer to async control block. */
+		 LPVOID buf,		/* Pointer to buffer with data to write. */
+		 DWORD numToWrite)	/* Number of bytes to write. */
 {
     aio->pendingError = NO_ERROR;
     if (aio->thread != (HANDLE) -1) {
@@ -937,12 +1076,12 @@ async_write_file(aio, buf, numToWrite)
  * ----------------------------------------------------------------------
  */
 static int
-get_overlapped_result(aio, pBytesRead, wait)
-    AsyncIo* aio;		/* Pointer to async control block. */
-    LPDWORD pBytesRead;		/* Where to place the number of bytes
-				 * transferred.
-				 */
-    BOOL wait;			/* If true, wait until result is ready. */
+get_overlapped_result(AsyncIo* aio,		/* Pointer to async control block. */
+		      LPDWORD pBytesRead,	/* Where to place the number of bytes
+						 * transferred.
+						 */
+		      BOOL wait			/* If true, wait until result is ready. */
+		      )
 {
     DWORD error = NO_ERROR;	/* Error status from last function. */
 
@@ -1012,15 +1151,21 @@ fd_init(void)
     return 0;
 }
 static int
-spawn_init()
+spawn_init(void)
 {
     int i;
-  
+#if defined(ERTS_SMP) && defined(USE_CANCELIOEX)
+    HMODULE module = GetModuleHandle("kernel32");
+    fpCancelIoEx = (BOOL (WINAPI *)(HANDLE,LPOVERLAPPED))
+	((module != NULL) ? GetProcAddress(module,"CancelIoEx") : NULL);
+    DEBUGF(("fpCancelIoEx = %p\r\n", fpCancelIoEx));
+#endif
     driver_data = (struct driver_data *)
 	erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data));
     erts_smp_atomic_add(&sys_misc_mem_sz, max_files*sizeof(struct driver_data));
     for (i = 0; i < max_files; i++)
 	driver_data[i].port_num = PORT_FREE;
+
     return 0;
 }
 
@@ -1099,8 +1244,10 @@ spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts)
      */
 
     DEBUGF(("Spawning \"%s\"\n", name));
-    envir = win_build_environment(envir);
-    ok = CreateChildProcess(name, 
+    envir = win_build_environment(envir); /* Still an ansi environment, could be
+					     converted to unicode for spawn_executable, but
+					     that is not done (yet) */ 
+    ok = create_child_process(name, 
 			    hChildStdin, 
 			    hChildStdout,
 			    hChildStderr,
@@ -1179,7 +1326,7 @@ create_file_thread(AsyncIo* aio, int mode)
 }
 
 /* 
- *  A helper function used by CreateChildProcess().
+ *  A helper function used by create_child_process().
  *  Parses a command line with arguments and returns the length of the
  *  first part containing the program name.
  *  Example: input = "\"Program Files\"\\erl arg1 arg2"
@@ -1220,24 +1367,25 @@ int parse_command(char* cmd){
     return i;
 }
 
-BOOL need_quotes(char *str)
+static BOOL need_quotes(WCHAR *str) 
 {
     int in_quote = 0;
     int backslashed = 0;
     int naked_space = 0;
-    while (*str != '\0') {
+
+    while (*str != L'\0') {
 	switch (*str) {
-	case '\\' :
+	case L'\\' :
 	    backslashed = !backslashed;
 	    break;
-	case '"':
+	case L'"':
 	    if (backslashed) {
 		backslashed=0;
 	    } else {
 		in_quote = !in_quote;
 	    }
 	    break;
-	case ' ':
+	case L' ':
 	    backslashed = 0;
 	    if (!(backslashed || in_quote)) {
 		naked_space++;
@@ -1256,7 +1404,7 @@ BOOL need_quotes(char *str)
 /*
  *----------------------------------------------------------------------
  *
- * CreateChildProcess --
+ * create_child_process --
  *
  *	Create a child process that has pipes as its 
  *	standard input, output, and error.  The child process runs
@@ -1281,7 +1429,7 @@ BOOL need_quotes(char *str)
  */
 
 static BOOL
-CreateChildProcess
+create_child_process
 (
  char *origcmd,  /* Command line for child process (including
 		  * name of executable). Or whole executable if st is
@@ -1300,14 +1448,12 @@ CreateChildProcess
  )
 { 
     PROCESS_INFORMATION piProcInfo = {0};
-    STARTUPINFO siStartInfo = {0};
     BOOL ok = FALSE;
     int applType;
     /* Not to be changed for different types of executables */
     int staticCreateFlags = GetPriorityClass(GetCurrentProcess()); 
     int createFlags = DETACHED_PROCESS;
     char *newcmdline = NULL;
-    char execPath[MAX_PATH];
     int cmdlength;
     char* thecommand;
     LPTSTR appname = NULL;
@@ -1315,14 +1461,17 @@ CreateChildProcess
     
     *errno_return = -1;
 
-    siStartInfo.cb = sizeof(STARTUPINFO); 
-    siStartInfo.dwFlags = STARTF_USESTDHANDLES;
-    siStartInfo.hStdInput = hStdin;
-    siStartInfo.hStdOutput = hStdout;
-    siStartInfo.hStdError = hStderr;
-
 
     if (st != ERTS_SPAWN_EXECUTABLE) {
+	STARTUPINFO siStartInfo = {0};
+	char execPath[MAX_PATH];
+
+	siStartInfo.cb = sizeof(STARTUPINFO); 
+	siStartInfo.dwFlags = STARTF_USESTDHANDLES;
+	siStartInfo.hStdInput = hStdin;
+	siStartInfo.hStdOutput = hStdout;
+	siStartInfo.hStdError = hStderr;
+
 	/*
 	 * Parse out the program name from the command line (it can be quoted and
 	 * contain spaces).
@@ -1334,9 +1483,9 @@ CreateChildProcess
 	thecommand[cmdlength] = '\0';
 	DEBUGF(("spawn command: %s\n", thecommand));
     
-	applType = ApplicationType(thecommand, execPath, TRUE, 
+	applType = application_type(thecommand, execPath, TRUE, 
 				   TRUE, errno_return);
-	DEBUGF(("ApplicationType returned for (%s) is %d\n", thecommand, applType));
+	DEBUGF(("application_type returned for (%s) is %d\n", thecommand, applType));
 	erts_free(ERTS_ALC_T_TMP, (void *) thecommand);
 	if (applType == APPL_NONE) {
 	    erts_free(ERTS_ALC_T_TMP,newcmdline);
@@ -1365,126 +1514,147 @@ CreateChildProcess
 
 	strcat(newcmdline, execPath);
 	strcat(newcmdline, origcmd+cmdlength);
-    } else { /* ERTS_SPAWN_EXECUTABLE */
+	DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags));
+	ok = CreateProcessA(appname, 
+			    newcmdline, 
+			    NULL, 
+			    NULL, 
+			    TRUE, 
+			    createFlags | staticCreateFlags, 
+			    env, 
+			    wd, 
+			    &siStartInfo, 
+			    &piProcInfo);
+
+    } else { /* ERTS_SPAWN_EXECUTABLE, filename and args are in unicode ({utf16,little}) */
 	int run_cmd = 0;
-	applType = ApplicationType(origcmd, execPath, FALSE, FALSE, 
-				   errno_return);
+	STARTUPINFOW siStartInfo = {0};
+	WCHAR execPath[MAX_PATH];
+
+
+	siStartInfo.cb = sizeof(STARTUPINFOW); 
+	siStartInfo.dwFlags = STARTF_USESTDHANDLES;
+	siStartInfo.hStdInput = hStdin;
+	siStartInfo.hStdOutput = hStdout;
+	siStartInfo.hStdError = hStderr;
+
+	applType = application_type_w((WCHAR *) origcmd, execPath, FALSE, FALSE, 
+				      errno_return);
 	if (applType == APPL_NONE) {
 	    return FALSE;
 	} 
 	if (applType == APPL_DOS) {
-		/*
-		 * See comment above
-		 */
+	    /*
+	     * See comment above
+	     */
 		
-		siStartInfo.wShowWindow = SW_HIDE;
-		siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
-		createFlags = CREATE_NEW_CONSOLE;
-		run_cmd = 1;
+	    siStartInfo.wShowWindow = SW_HIDE;
+	    siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
+	    createFlags = CREATE_NEW_CONSOLE;
+	    run_cmd = 1;
 	} else if (hide) {
-		DEBUGF(("hiding window\n"));
-		siStartInfo.wShowWindow = SW_HIDE;
-		siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
-		createFlags = 0;
+	    DEBUGF(("hiding window\n"));
+	    siStartInfo.wShowWindow = SW_HIDE;
+	    siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
+	    createFlags = 0;
 	}
 	if (run_cmd) {
-	    char cmdPath[MAX_PATH];
+	    WCHAR cmdPath[MAX_PATH];
 	    int cmdType;
-	    cmdType = ApplicationType("cmd.exe", cmdPath, TRUE, FALSE, errno_return);
+	    cmdType = application_type_w(L"cmd.exe", cmdPath, TRUE, FALSE, errno_return);
 	    if (cmdType == APPL_NONE || cmdType == APPL_DOS) {
 		return FALSE;
 	    }
-	    appname = (char *) erts_alloc(ERTS_ALC_T_TMP, strlen(cmdPath)+1);
-	    strcpy(appname,cmdPath);
+	    appname = (char *) erts_alloc(ERTS_ALC_T_TMP, (wcslen(cmdPath)+1)*sizeof(WCHAR));
+	    wcscpy((WCHAR *) appname,cmdPath);
 	} else {
-	    appname = (char *) erts_alloc(ERTS_ALC_T_TMP, strlen(execPath)+1);
-	    strcpy(appname,execPath);
+	    appname = (char *) erts_alloc(ERTS_ALC_T_TMP, (wcslen(execPath)+1)*sizeof(WCHAR));
+	    wcscpy((WCHAR *) appname, execPath);
 	}
-	if (argv == NULL) {
+	if (argv == NULL) { 
 	    BOOL orig_need_q = need_quotes(execPath);
-	    char *ptr;
-	    int ocl = strlen(execPath);
+	    WCHAR *ptr;
+	    int ocl = wcslen(execPath);
 	    if (run_cmd) {
 		newcmdline = (char *) erts_alloc(ERTS_ALC_T_TMP, 
-						 ocl + ((orig_need_q) ? 3 : 1)
-						 + 11);
-		memcpy(newcmdline,"cmd.exe /c ",11);
-		ptr = newcmdline + 11;
+						 (ocl + ((orig_need_q) ? 3 : 1)
+						  + 11)*sizeof(WCHAR));
+		memcpy(newcmdline,L"cmd.exe /c ",11*sizeof(WCHAR));
+		ptr = (WCHAR *) (newcmdline + (11*sizeof(WCHAR)));
 	    } else {
 		newcmdline = (char *) erts_alloc(ERTS_ALC_T_TMP, 
-						 ocl + ((orig_need_q) ? 3 : 1));
-		ptr = newcmdline;
+						 (ocl + ((orig_need_q) ? 3 : 1))*sizeof(WCHAR));
+		ptr = (WCHAR *) newcmdline;
 	    }
 	    if (orig_need_q) {
-		*ptr++ = '"';
+		*ptr++ = L'"';
 	    }
-	    memcpy(ptr,execPath,ocl);
+	    memcpy(ptr,execPath,ocl*sizeof(WCHAR));
 	    ptr += ocl;
 	    if (orig_need_q) {
-		*ptr++ = '"';
+		*ptr++ = L'"';
 	    }
-	    *ptr = '\0';
+	    *ptr = L'\0';
 	} else {
 	    int sum = 1; /* '\0' */
-	    char **ar = argv;
-	    char *n;
+	    WCHAR **ar = (WCHAR **) argv;
+	    WCHAR *n;
 	    char *save_arg0 = NULL;
 	    if (argv[0] == erts_default_arg0 || run_cmd) {
 		save_arg0 = argv[0];
-		argv[0] = execPath;
+		argv[0] = (char *) execPath;
 	    }
 	    if (run_cmd) {
 		sum += 11; /* cmd.exe /c */
 	    }
 	    while (*ar != NULL) {
-		sum += strlen(*ar);
+		sum += wcslen(*ar);
 		if (need_quotes(*ar)) {
 		    sum += 2; /* quotes */
 		}
 		sum++; /* space */
 		++ar;
 	    }
-	    ar = argv;
-	    newcmdline = erts_alloc(ERTS_ALC_T_TMP, sum);
-	    n = newcmdline;
+	    ar = (WCHAR **) argv;
+	    newcmdline = erts_alloc(ERTS_ALC_T_TMP, sum*sizeof(WCHAR));
+	    n = (WCHAR *) newcmdline;
 	    if (run_cmd) {
-		memcpy(n,"cmd.exe /c ",11);
+		memcpy(n,L"cmd.exe /c ",11*sizeof(WCHAR));
 		n += 11;
 	    }
 	    while (*ar != NULL) {
 		int q = need_quotes(*ar);
-		sum = strlen(*ar);
+		sum = wcslen(*ar);
 		if (q) {
-		    *n++ = '"';
+		    *n++ = L'"';
 		}
-		memcpy(n,*ar,sum);
+		memcpy(n,*ar,sum*sizeof(WCHAR));
 		n += sum;
 		if (q) {
-		    *n++ = '"';
+		    *n++ = L'"';
 		}
-		*n++ = ' ';
+		*n++ = L' ';
 		++ar;
 	    }
-	    ASSERT(n > newcmdline);
-	    *(n-1) = '\0';
+	    *(n-1) = L'\0';
 	    if (save_arg0 != NULL) {
 		argv[0] = save_arg0;
 	    }
 	}	    
 	    
-    }
-    DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags));
-    ok = CreateProcess(appname, 
-		       newcmdline, 
-		       NULL, 
-		       NULL, 
-		       TRUE, 
-		       createFlags | staticCreateFlags, 
-		       env, 
-		       wd, 
-		       &siStartInfo, 
-		       &piProcInfo);
-
+	DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags));
+	ok = CreateProcessW((WCHAR *) appname, 
+			    (WCHAR *) newcmdline, 
+			    NULL, 
+			    NULL, 
+			    TRUE, 
+			    createFlags | staticCreateFlags, 
+			    env, 
+			    (WCHAR *) wd, 
+			    &siStartInfo, 
+			    &piProcInfo);
+
+    } /* end SPAWN_EXECUTABLE */
     if (newcmdline != NULL) {
 	    erts_free(ERTS_ALC_T_TMP,newcmdline);
     }	
@@ -1603,7 +1773,7 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o
 
 
 
-static int ApplicationType
+static int application_type
 (
  const char *originalName, /* Name of the application to find. */ 
  char fullPath[MAX_PATH],  /* Filled with complete path to 
@@ -1757,6 +1927,145 @@ static int ApplicationType
     return applType;
 }
 
+static int application_type_w (const WCHAR *originalName, /* Name of the application to find. */ 
+			       WCHAR wfullpath[MAX_PATH],/* Filled with complete path to 
+							  * application. */
+			       BOOL search_in_path,      /* If we should search the system wide path */
+			       BOOL handle_quotes,       /* If we should handle quotes around executable */
+			       int *error_return)         /* A place to put an error code */
+{
+    int applType, i;
+    HANDLE hFile;
+    WCHAR *ext, *rest;
+    char buf[2];
+    DWORD read;
+    IMAGE_DOS_HEADER header;
+    static WCHAR extensions[][5] = {L"", L".com", L".exe", L".bat"};
+    int is_quoted;
+    int len;
+    WCHAR xfullpath[MAX_PATH];
+
+    len = wcslen(originalName);
+    is_quoted = handle_quotes && len > 0 && originalName[0] == L'"' && 
+	originalName[len-1] == L'"';
+
+    applType = APPL_NONE;
+    *error_return = ENOENT;
+    for (i = 0; i < (int) (sizeof(extensions) / sizeof(extensions[0])); i++) {
+	if(is_quoted) {
+	   lstrcpynW(xfullpath, originalName+1, MAX_PATH - 7); /* Cannot start using StringCchCopy yet, we support
+							   older platforms */
+	   len = wcslen(xfullpath);
+	   if(len > 0) {
+	       xfullpath[len-1] = L'\0';
+	   }
+	} else {
+	    lstrcpynW(xfullpath, originalName, MAX_PATH - 5);
+	}
+	wcscat(xfullpath, extensions[i]);
+	/* It seems that the Unicode version does not allow in and out parameter to overlap. */
+	SearchPathW((search_in_path) ? NULL : L".", xfullpath, NULL, MAX_PATH, wfullpath, &rest);
+
+	/*
+	 * Ignore matches on directories or data files, return if identified
+	 * a known type.
+	 */
+
+	if (GetFileAttributesW(wfullpath) & FILE_ATTRIBUTE_DIRECTORY) {
+	    continue;
+	}
+
+	ext = wcsrchr(wfullpath, L'.');
+	if ((ext != NULL) && (_wcsicmp(ext, L".bat") == 0)) {
+	    *error_return = EACCES;
+	    applType = APPL_DOS;
+	    break;
+	}
+
+	hFile = CreateFileW(wfullpath, GENERIC_READ, FILE_SHARE_READ, NULL, 
+		OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+	if (hFile == INVALID_HANDLE_VALUE) {
+	    continue;
+	}
+
+	*error_return = EACCES; /* If considered an error, 
+				    it's an access error */
+	header.e_magic = 0;
+	ReadFile(hFile, (void *) &header, sizeof(header), &read, NULL);
+	if (header.e_magic != IMAGE_DOS_SIGNATURE) {
+	    /* 
+	     * Doesn't have the magic number for relocatable executables.  If 
+	     * filename ends with .com, assume it's a DOS application anyhow.
+	     * Note that we didn't make this assumption at first, because some
+	     * supposed .com files are really 32-bit executables with all the
+	     * magic numbers and everything.  
+	     */
+
+	    CloseHandle(hFile);
+	    if ((ext != NULL) && (_wcsicmp(ext, L".com") == 0)) {
+		applType = APPL_DOS;
+		break;
+	    }
+	    continue;
+	}
+	if (header.e_lfarlc != sizeof(header)) {
+	    /* 
+	     * All Windows 3.X and Win32 and some DOS programs have this value
+	     * set here.  If it doesn't, assume that since it already had the 
+	     * other magic number it was a DOS application.
+	     */
+
+	    CloseHandle(hFile);
+	    applType = APPL_DOS;
+	    break;
+	}
+
+	/* 
+	 * The DWORD at header.e_lfanew points to yet another magic number.
+	 */
+
+	buf[0] = '\0';
+	SetFilePointer(hFile, header.e_lfanew, NULL, FILE_BEGIN);
+	ReadFile(hFile, (void *) buf, 2, &read, NULL);
+	CloseHandle(hFile);
+
+	if ((buf[0] == 'L') && (buf[1] == 'E')) {
+	    applType = APPL_DOS;
+	} else if ((buf[0] == 'N') && (buf[1] == 'E')) {
+	    applType = APPL_WIN3X;
+	} else if ((buf[0] == 'P') && (buf[1] == 'E')) {
+	    applType = APPL_WIN32;
+	} else {
+	    continue;
+	}
+	break;
+    }
+
+    if (applType == APPL_NONE) {
+	return APPL_NONE;
+    }
+
+    if ((applType == APPL_DOS) || (applType == APPL_WIN3X)) {
+	/* 
+	 * Replace long path name of executable with short path name for 
+	 * 16-bit applications.  Otherwise the application may not be able
+	 * to correctly parse its own command line to separate off the 
+	 * application name from the arguments.
+	 */
+
+	GetShortPathNameW(wfullpath, wfullpath, MAX_PATH);
+    }
+    if (is_quoted) {
+	/* restore quotes on quoted program name */
+	len = wcslen(wfullpath);
+	memmove(wfullpath+1,wfullpath,len*sizeof(WCHAR));
+	wfullpath[0]=L'"';
+	wfullpath[len+1]=L'"';
+	wfullpath[len+2]=L'\0';
+    }
+    return applType;
+}
+
 /*
  * Thread function used to emulate overlapped reading.
  */
@@ -1776,9 +2085,10 @@ threaded_reader(LPVOID param)
 	buf = OV_BUFFER_PTR(aio);
 	numToRead = OV_NUM_TO_READ(aio);
 	aio->pendingError = 0;
-	if (!ReadFile(aio->fd, buf, numToRead, &aio->bytesTransferred, NULL))
-	    aio->pendingError = GetLastError();
-	else if (aio->flags & DF_XLAT_CR) {
+	if (!ReadFile(aio->fd, buf, numToRead, &aio->bytesTransferred, NULL)) {
+	    int error = GetLastError();
+	    aio->pendingError = error;
+	} else if (aio->flags & DF_XLAT_CR) {
 	    char *s;
 	    int n;
 	    
@@ -1905,56 +2215,79 @@ translate_fd(int fd)
     return handle;
 }
 
+/* Driver level locking, start function is serialized */
+static DriverData *save_01_port = NULL;
+static DriverData *save_22_port = NULL;
+
 static ErlDrvData
 fd_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts)
 {
     DriverData* dp;
     int is_std_error = (opts->ofd == 2);
-    
-    opts->ifd = (int) translate_fd(opts->ifd);
-    opts->ofd = (int) translate_fd(opts->ofd);
-    if ((dp = new_driver_data(port_num, opts->packet_bytes, 2, TRUE)) == NULL)
-	return ERL_DRV_ERROR_GENERAL;
-    
-    if (!create_file_thread(&dp->in, DO_READ)) {
-	dp->port_num = PORT_FREE;
-	return ERL_DRV_ERROR_GENERAL;
-    }
-
-    if (!create_file_thread(&dp->out, DO_WRITE)) {
-	dp->port_num = PORT_FREE;
-	return ERL_DRV_ERROR_GENERAL;
-    }
-    
-    fd_driver_input = &(dp->in);
-    dp->in.flags = DF_XLAT_CR;
-    if (is_std_error) {
-	dp->out.flags |= DF_DROP_IF_INVH; /* Just drop messages if stderror
-					     is an invalid handle */
+    int in = opts->ifd, out = opts->ofd;
+
+    opts->ifd = (Uint) translate_fd(in);
+    opts->ofd = (Uint) translate_fd(out);
+    if ( in == 0 && out == 1 && save_01_port != NULL) {
+	dp = save_01_port;
+	return reuse_driver_data(dp, (HANDLE) opts->ifd, (HANDLE) opts->ofd, opts->read_write, port_num);
+    } else if (in == 2 && out == 2 && save_22_port != NULL) {
+	dp = save_22_port;
+	return reuse_driver_data(dp, (HANDLE) opts->ifd, (HANDLE) opts->ofd, opts->read_write, port_num);
+    } else {
+	if ((dp = new_driver_data(port_num, opts->packet_bytes, 2, TRUE)) == NULL)
+	    return ERL_DRV_ERROR_GENERAL;
+	
+	if (!create_file_thread(&dp->in, DO_READ)) {
+	    dp->port_num = PORT_FREE;
+	    return ERL_DRV_ERROR_GENERAL;
+	}
+	
+	if (!create_file_thread(&dp->out, DO_WRITE)) {
+	    dp->port_num = PORT_FREE;
+	    return ERL_DRV_ERROR_GENERAL;
+	}
+	
+	fd_driver_input = &(dp->in);
+	dp->in.flags = DF_XLAT_CR;
+	if (is_std_error) {
+	    dp->out.flags |= DF_DROP_IF_INVH; /* Just drop messages if stderror
+						 is an invalid handle */
+	}
+	
+	if ( in == 0 && out == 1) {
+	    save_01_port = dp;
+	} else if (in == 2 && out == 2) {
+	    save_22_port = dp;
+	}
+	return set_driver_data(dp, (HANDLE) opts->ifd, (HANDLE) opts->ofd, opts->read_write, 0);
     }
-    return set_driver_data(dp, opts->ifd, opts->ofd, opts->read_write, 0);
 }
 
 static void fd_stop(ErlDrvData d)
 {
   int fd = (int)d;
+  DriverData* dp = driver_data+fd;
   /*
-   * I don't know a clean way to terminate the threads
-   * (TerminateThread() doesn't release the stack),
-   * so will we'll let the threads live.  Normally, the fd
-   * driver is only used to support the -oldshell option,
-   * so this shouldn't be a problem in practice.
-   *
-   * Since we will not attempt to terminate the threads,
-   * better not close the input or output files either.
+   * There's no way we can terminate an fd port in a consistent way.
+   * Instead we let it live until it's opened again (which it is,
+   * as the only FD-drivers are for 0,1 and 2 adn the only time they
+   * get closed is by init:reboot).
+   * So - just deselect them and let everything be as is. 
+   * They get woken up in fd_start again, where the DriverData is
+   * remembered. /PaN
    */
+  if (dp->in.ov.hEvent != NULL) {
+      (void) driver_select(dp->port_num,
+			   (ErlDrvEvent)dp->in.ov.hEvent,
+			   ERL_DRV_READ, 0);
+  }
+  if (dp->out.ov.hEvent != NULL) {
+      (void) driver_select(dp->port_num,
+			   (ErlDrvEvent)dp->out.ov.hEvent,
+			   ERL_DRV_WRITE, 0);
+  }    
 
-  driver_data[fd].in.thread = (HANDLE) -1;
-  driver_data[fd].out.thread = (HANDLE) -1;
-  driver_data[fd].in.fd = INVALID_HANDLE_VALUE;
-  driver_data[fd].out.fd = INVALID_HANDLE_VALUE;
-
-  /*return */ common_stop(fd);
 }
 
 static ErlDrvData
@@ -2046,7 +2379,6 @@ threaded_exiter(LPVOID param)
      * because it is an auto reset event.  Therefore, always set the
      * exit flag and signal the event.
      */
-
     i = 0;
     if (dp->out.thread != (HANDLE) -1) {
 	dp->out.flags = DF_EXIT_THREAD;
@@ -2414,6 +2746,7 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event)
 	    driver_failure_eof(dp->port_num);
 	} else {			/* Report real errors. */
 	    int error = GetLastError();
+
 	    (void) driver_select(dp->port_num, ready_event, ERL_DRV_READ, 0);
 	    _dosmaperr(error);
 	    driver_failure_posix(dp->port_num, errno);
@@ -2534,7 +2867,6 @@ erts_sys_main_thread(void)
 
 void erts_sys_alloc_init(void)
 {
-    elib_ensure_initialized();
 }
 
 void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz)
@@ -2881,19 +3213,51 @@ check_supported_os_version(void)
 }
 
 #ifdef USE_THREADS
-static void *ethr_internal_alloc(size_t size)
+
+typedef struct {
+    int sched_bind_data;
+} erts_thr_create_data_t;
+
+/*
+ * thr_create_prepare() is called in parent thread before thread creation.
+ * Returned value is passed as argument to thr_create_cleanup().
+ */
+static void *
+thr_create_prepare(void)
 {
-    return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size);
+    erts_thr_create_data_t *tcdp;
+
+    tcdp = erts_alloc(ERTS_ALC_T_TMP, sizeof(erts_thr_create_data_t));
+    tcdp->sched_bind_data = erts_sched_bind_atthrcreate_prepare();
+
+    return (void *) tcdp;
 }
-static void *ethr_internal_realloc(void *ptr, size_t size)
+
+
+/* thr_create_cleanup() is called in parent thread after thread creation. */
+static void
+thr_create_cleanup(void *vtcdp)
 {
-    return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size);
+    erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp;
+
+    erts_sched_bind_atthrcreate_parent(tcdp->sched_bind_data);
+
+    erts_free(ERTS_ALC_T_TMP, tcdp);
 }
-static void ethr_internal_free(void *ptr)
+
+static void
+thr_create_prepare_child(void *vtcdp)
 {
-    erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr);
+    erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp;
+
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_thread_setup();
+#endif /* ERTS_ENABLE_LOCK_COUNT */
+
+    erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data);
 }
-#endif
+
+#endif /* USE_THREADS */
 
 void
 erts_sys_pre_init(void)
@@ -2904,9 +3268,13 @@ erts_sys_pre_init(void)
 #ifdef USE_THREADS
     {
 	erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER;
-	eid.alloc = ethr_internal_alloc;
-	eid.realloc = ethr_internal_realloc;
-	eid.free = ethr_internal_free;
+
+	eid.thread_create_child_func = thr_create_prepare_child;
+	/* Before creation in parent */
+	eid.thread_create_prepare_func = thr_create_prepare;
+	/* After creation in parent */
+	eid.thread_create_parent_func = thr_create_cleanup,
+
 	erts_thr_init(&eid);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 	erts_lcnt_init();
@@ -2914,15 +3282,8 @@ erts_sys_pre_init(void)
     }
 #endif
     erts_smp_atomic_init(&sys_misc_mem_sz, 0);
-    erts_sys_env_init();
 }
 
-/*
- * the last two only used for standalone erlang
- * they should are used by sae_main in beam dll to
- * enable standalone execution via erl_api-routines
- */
-
 void noinherit_std_handle(DWORD type)
 {
     HANDLE h = GetStdHandle(type);
@@ -2936,6 +3297,8 @@ void erl_sys_init(void)
 {
     HANDLE handle;
 
+    erts_sys_env_init();
+
     noinherit_std_handle(STD_OUTPUT_HANDLE);
     noinherit_std_handle(STD_INPUT_HANDLE);
     noinherit_std_handle(STD_ERROR_HANDLE);
@@ -2945,6 +3308,7 @@ void erl_sys_init(void)
 
 #ifdef ERTS_SMP
     erts_smp_tsd_key_create(&win32_errstr_key);
+    InitializeCriticalSection(&htbc_lock);
 #endif
     erts_smp_atomic_init(&pipe_creation_counter,0);
     /*
diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c
index ac4be3f316..02c8433a10 100644
--- a/erts/emulator/sys/win32/sys_env.c
+++ b/erts/emulator/sys/win32/sys_env.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -145,15 +145,17 @@ merge_environment(char *old, char *add)
     for(j = 0; a_arg[j] != NULL; ++j){
 	char **tmp;
 	char *current = a_arg[j];
+	char *eq_p = strchr(current,'=');
+	int unset = (eq_p!=NULL && eq_p[1]=='\0');
 
 	if ((tmp = find_arg(c_arg, current)) != NULL) {
-	    if (current[strlen(current)-1] != '=') {
+	    if (!unset) {
 		*tmp = current;
 	    } else {
 		*tmp = c_arg[--i];
 		c_arg[i] = NULL;
 	    }
-	} else if (current[strlen(current)-1] != '=') {
+	} else if (!unset) {
 	    c_arg[i++] = current;
 	    c_arg[i] = NULL;
 	}
diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c
index d2449a1bdb..943c338794 100644
--- a/erts/emulator/sys/win32/sys_interrupt.c
+++ b/erts/emulator/sys/win32/sys_interrupt.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -31,11 +31,11 @@
 #endif
 
 #ifdef ERTS_SMP
-erts_smp_atomic_t erts_break_requested;
+erts_smp_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 1)
+  erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 0)
+  erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0)
 #else
 volatile int erts_break_requested = 0;
 #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index b1374950b2..4d0c87bf12 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -1,19 +1,19 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 
@@ -61,7 +61,7 @@ MODULES= \
 	exception_SUITE \
 	float_SUITE \
 	fun_SUITE \
-	fun_r11_SUITE \
+	fun_r12_SUITE \
 	gc_SUITE \
 	guard_SUITE \
 	hash_SUITE \
@@ -75,14 +75,15 @@ MODULES= \
 	node_container_SUITE \
 	nofrag_SUITE \
 	num_bif_SUITE \
-	obsolete_SUITE \
 	op_SUITE \
 	port_SUITE \
 	port_bif_SUITE \
 	process_SUITE \
 	pseudoknot_SUITE \
+	receive_SUITE \
 	ref_SUITE \
 	register_SUITE \
+	mtx_SUITE \
 	save_calls_SUITE \
 	send_term_SUITE \
 	sensitive_SUITE \
@@ -100,6 +101,7 @@ MODULES= \
 	trace_local_SUITE \
 	trace_meta_SUITE \
 	trace_call_count_SUITE \
+	trace_call_time_SUITE \
 	scheduler_SUITE \
 	old_scheduler_SUITE \
 	z_SUITE \
@@ -117,12 +119,17 @@ NO_OPT= bs_bincomp \
 	bs_match_int \
 	bs_match_tail \
 	bs_match_misc \
-	bs_utf
+	bs_utf \
+	guard
 
+NATIVE= hibernate
 
 NO_OPT_MODULES= $(NO_OPT:%=%_no_opt_SUITE)
 NO_OPT_ERL_FILES= $(NO_OPT_MODULES:%=%.erl)
 
+NATIVE_MODULES= $(NATIVE:%=%_native_SUITE)
+NATIVE_ERL_FILES= $(NATIVE_MODULES:%=%.erl)
+
 ERL_FILES= $(MODULES:%=%.erl)
 
 TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR))
@@ -148,7 +155,7 @@ ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/test_server/include
 # Targets
 # ----------------------------------------------------
 
-make_emakefile: $(NO_OPT_ERL_FILES)
+make_emakefile: $(NO_OPT_ERL_FILES) $(NATIVE_ERL_FILES)
 	# This special rule can be removed when communication with R7B nodes 
 	# is no longer supported.
 	$(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) +compressed -o$(EBIN) \
@@ -157,6 +164,8 @@ make_emakefile: $(NO_OPT_ERL_FILES)
 	$(MODULES) >> $(EMAKEFILE)
 	$(ERL_TOP)/make/make_emakefile +no_copt +no_postopt $(ERL_COMPILE_FLAGS) \
 	-o$(EBIN) $(NO_OPT_MODULES) >> $(EMAKEFILE)
+	$(ERL_TOP)/make/make_emakefile +native $(ERL_COMPILE_FLAGS) \
+	-o$(EBIN) $(NATIVE_MODULES) >> $(EMAKEFILE)
 
 tests debug opt: make_emakefile
 	erl $(ERL_MAKE_FLAGS) -make
@@ -175,6 +184,9 @@ docs:
 %_no_opt_SUITE.erl: %_SUITE.erl
 	sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@
 
+%_native_SUITE.erl: %_SUITE.erl
+	sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@
+
 # ----------------------------------------------------
 # Release Target
 # ---------------------------------------------------- 
@@ -187,7 +199,8 @@ release_tests_spec: make_emakefile
 	$(INSTALL_DATA) $(EMAKEFILE) $(TEST_SPEC_FILES) \
 		$(ERL_FILES) $(RELSYSDIR)
 	$(INSTALL_DATA) $(NO_OPT_ERL_FILES) $(RELSYSDIR)
-	chmod -f -R u+w $(RELSYSDIR)
+	$(INSTALL_DATA) $(NATIVE_ERL_FILES) $(RELSYSDIR)
+	chmod -R u+w $(RELSYSDIR)
 	tar cf - *_SUITE_data | (cd $(RELSYSDIR); tar xf -)
 
 release_docs_spec:
diff --git a/erts/emulator/test/a_SUITE.erl b/erts/emulator/test/a_SUITE.erl
index e9d653a7c4..b541be3df6 100644
--- a/erts/emulator/test/a_SUITE.erl
+++ b/erts/emulator/test/a_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -26,15 +26,32 @@
 %%%-------------------------------------------------------------------
 -module(a_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1, long_timers/1, pollset_size/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, long_timers/1, pollset_size/1]).
 
-all(doc) ->
-    [];
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [long_timers, pollset_size].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 long_timers(doc) ->
     [];
 long_timers(suite) ->
diff --git a/erts/emulator/test/after_SUITE.erl b/erts/emulator/test/after_SUITE.erl
index 3e1a871408..7cc329cc69 100644
--- a/erts/emulator/test/after_SUITE.erl
+++ b/erts/emulator/test/after_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -21,27 +21,48 @@
 
 %% Tests receive after.
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1, t_after/1, receive_after/1, receive_after_big/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 t_after/1, receive_after/1, receive_after_big/1,
 	 receive_after_errors/1, receive_var_zero/1, receive_zero/1,
 	 multi_timeout/1, receive_after_32bit/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 %% Internal exports.
 
 -export([timeout_g/0]).
 
-all(suite) ->
-    [t_after, receive_after, receive_after_big, receive_after_errors,
-     receive_var_zero, receive_zero, multi_timeout, receive_after_32bit].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [t_after, receive_after, receive_after_big,
+     receive_after_errors, receive_var_zero, receive_zero,
+     multi_timeout, receive_after_32bit].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(3)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl
index 94766dc6e9..22b5d93983 100644
--- a/erts/emulator/test/alloc_SUITE.erl
+++ b/erts/emulator/test/alloc_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2003-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2003-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,7 +18,8 @@
 
 -module(alloc_SUITE).
 -author('[email protected]').
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 -export([basic/1,
 	 coalesce/1,
@@ -29,28 +30,40 @@
 	 rbtree/1,
 	 mseg_clear_cache/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(DEFAULT_TIMETRAP_SECS, 240).
 
-all(doc) -> [];
-all(suite) -> [basic,
-	       coalesce,
-	       threads,
-	       realloc_copy,
-	       bucket_index,
-	       bucket_mask,
-	       rbtree,
-	       mseg_clear_cache].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [basic, coalesce, threads, realloc_copy, bucket_index,
+     bucket_mask, rbtree, mseg_clear_cache].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 
 init_per_testcase(Case, Config) when is_list(Config) ->
     Dog = ?t:timetrap(?t:seconds(?DEFAULT_TIMETRAP_SECS)),
     [{watchdog, Dog},{testcase, Case}|Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/test/beam_SUITE.erl b/erts/emulator/test/beam_SUITE.erl
index cc1626630b..02c6e19686 100644
--- a/erts/emulator/test/beam_SUITE.erl
+++ b/erts/emulator/test/beam_SUITE.erl
@@ -1,34 +1,55 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 1998-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
 -module(beam_SUITE).
 
--export([all/1, packed_registers/1, apply_last/1, apply_last_bif/1,
-	 buildo_mucho/1, heap_sizes/1, big_lists/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 packed_registers/1, apply_last/1, apply_last_bif/1,
+	 buildo_mucho/1, heap_sizes/1, big_lists/1, fconv/1,
+	 select_val/1]).
 
 -export([applied/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [packed_registers, apply_last, apply_last_bif,
+     buildo_mucho, heap_sizes, big_lists, select_val].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [packed_registers, apply_last, apply_last_bif, buildo_mucho,
-     heap_sizes, big_lists].
 
 
 %% Verify that apply(M, F, A) is really tail recursive.
@@ -279,3 +300,42 @@ b() ->
          _} ->
 	    ok
     end.
+
+fconv(Config) when is_list(Config) ->
+    ?line do_fconv(atom),
+    ?line do_fconv(nil),
+    ?line do_fconv(tuple_literal),
+    ?line 3.0 = do_fconv(1.0, 2.0),
+    ok.
+
+do_fconv(Type) ->
+    try
+	do_fconv(Type, 1.0),
+	test_server:fail()
+    catch
+	error:badarith ->
+	    ok
+    end.
+
+do_fconv(atom, Float) when is_float(Float) ->
+    Float + a;
+do_fconv(nil, Float) when is_float(Float) ->
+    Float + [];
+do_fconv(tuple_literal, Float) when is_float(Float) ->
+    Float + {a,b}.
+
+select_val(Config) when is_list(Config) ->
+    ?line zero = do_select_val(0),
+    ?line big = do_select_val(1 bsl 64),
+    ?line integer = do_select_val(42),
+    ok.
+
+do_select_val(X) ->
+    case X of
+	0 ->
+	    zero;
+	1 bsl 64 ->
+	    big;
+	Int when is_integer(Int) ->
+	    integer
+    end.
diff --git a/erts/emulator/test/beam_literals_SUITE.erl b/erts/emulator/test/beam_literals_SUITE.erl
index 75841adbfc..85236e4203 100644
--- a/erts/emulator/test/beam_literals_SUITE.erl
+++ b/erts/emulator/test/beam_literals_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,21 +18,41 @@
 %%
 
 -module(beam_literals_SUITE).
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 -export([putting/1, matching_smalls/1, matching_smalls_jt/1,
 	 matching_bigs/1, matching_more_bigs/1,
 	 matching_bigs_and_smalls/1, badmatch/1, case_clause/1,
 	 receiving/1, literal_type_tests/1,
-	 put_list/1, fconv/1, literal_case_expression/1]).
+	 put_list/1, fconv/1, literal_case_expression/1,
+	 increment/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [putting, matching_smalls, matching_smalls_jt,
      matching_bigs, matching_more_bigs,
      matching_bigs_and_smalls, badmatch, case_clause,
-     receiving, literal_type_tests,
-     put_list, fconv, literal_case_expression].
+     receiving, literal_type_tests, put_list, fconv,
+     literal_case_expression, increment].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 putting(doc) -> "Test creating lists and tuples containing big number literals.";
 putting(Config) when is_list(Config) ->
@@ -48,6 +68,7 @@ matching_bigs(doc) -> "Test matching of a few big number literals (in Beam,"
 matching_bigs(Config) when is_list(Config) ->
     a = matching1(3972907842873739),
     b = matching1(-389789298378939783333333333333333333784),
+    other = matching1(3141699999999999999999999999999999999),
     other = matching1(42).
 
 matching_smalls(doc) -> "Test matching small numbers (both positive and negative).";
@@ -236,14 +257,14 @@ make_test([{T,L}|Ts]) ->
 make_test([]) -> [].
 
 test(T, L) ->
-    S = lists:flatten(io_lib:format("begin io:format(\"~~p~~n\", [{~p,~p}]), if ~w(~w) -> true; true -> false end end. ", [T, L, T, L])),
+    S = lists:flatten(io_lib:format("begin io:format(\"~~p~n\", [{~p,~p}]), if ~w(~w) -> true; true -> false end end. ", [T, L, T, L])),
     {ok,Toks,_Line} = erl_scan:string(S),
     {ok,E} = erl_parse:parse_exprs(Toks),
     {value,Val,_Bs} = erl_eval:exprs(E, []),
     {match,0,{atom,0,Val},hd(E)}.
 
 test(T, A, L) ->
-    S = lists:flatten(io_lib:format("begin io:format(\"~~p~~n\", [{~p,~p,~p}]), if ~w(~w, ~w) -> true; true -> false end end. ",
+    S = lists:flatten(io_lib:format("begin io:format(\"~~p~n\", [{~p,~p,~p}]), if ~w(~w, ~w) -> true; true -> false end end. ",
 				    [T,L,A,T,L,A])),
     {ok,Toks,_Line} = erl_scan:string(S),
     {ok,E} = erl_parse:parse_exprs(Toks),
@@ -405,14 +426,51 @@ fconv_2(F) when is_float(F) ->
 literal_case_expression(Config) when is_list(Config) ->
     ?line DataDir = ?config(data_dir, Config),
     ?line Src = filename:join(DataDir, "literal_case_expression"),
-    ?line {ok,literal_case_expression=Mod,Code} = compile:file(Src, [from_asm,binary]),
+    ?line {ok,literal_case_expression=Mod,Code} =
+	compile:file(Src, [from_asm,binary]),
     ?line {module,Mod} = code:load_binary(Mod, Src, Code),
     ?line ok = Mod:x(),
     ?line ok = Mod:y(),
+    ?line ok = Mod:zi1(),
+    ?line ok = Mod:zi2(),
+    ?line ok = Mod:za1(),
+    ?line ok = Mod:za2(),
     ?line true = code:delete(Mod),
     ?line code:purge(Mod),
     ok.
 
+%% Test the i_increment instruction.
+increment(Config) when is_list(Config) ->
+    %% In the 32-bit emulator, Neg32 can be represented as a small,
+    %% but -Neg32 cannot. Therefore the i_increment instruction must
+    %% not be used in the subtraction that follows (since i_increment
+    %% cannot handle a bignum literal).
+    Neg32 = -(1 bsl 27),
+    Big32 = id(1 bsl 32),
+    Result32 = (1 bsl 32) + (1 bsl 27),
+    ?line Result32 = Big32 + (1 bsl 27),
+    ?line Result32 = Big32 - Neg32,
+
+    %% Same thing, but for the 64-bit emulator.
+    Neg64 = -(1 bsl 59),
+    Big64 = id(1 bsl 64),
+    Result64 = (1 bsl 64) + (1 bsl 59),
+    ?line Result64 = Big64 + (1 bsl 59),
+    ?line Result64 = Big64 - Neg64,
+
+    %% Test error handling for the i_increment instruction.
+    Bad = id(bad),
+    ?line {'EXIT',{badarith,_}} = (catch Bad + 42),
+
+    %% Small operands, but a big result.
+    Res32 = 1 bsl 27,
+    Small32 = id(Res32-1),
+    ?line Res32 = Small32 + 1,
+    Res64 = 1 bsl 59,
+    Small64 = id(Res64-1),
+    ?line Res64 = Small64 + 1,
+    ok.
+
 %% Help functions.
 
 chksum(Term) ->
diff --git a/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S b/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S
index c0ffe9ab53..bfdfc079dc 100644
--- a/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S
+++ b/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S
@@ -1,10 +1,11 @@
 {module, literal_case_expression}.  %% version = 0
 
-{exports, [{module_info,0},{module_info,1},{x,0},{y,0}]}.
+{exports, [{module_info,0},{module_info,1},{x,0},{y,0},
+	   {zi1,0},{zi2,0},{za1,0},{za2,0}]}.
 
 {attributes, []}.
 
-{labels, 15}.
+{labels, 32}.
 
 
 {function, x, 0, 2}.
@@ -52,6 +53,81 @@
   {label,10}.
     {case_end,{float,34.0000}}.
 
+{function, zi1, 0, 16}.
+  {label,15}.
+    {func_info,{atom,literal_case_expression},{atom,zi1},0}.
+  {label,16}.
+    {test,is_integer,{f,19},[{integer,42}]}.
+    {select_val,{integer,42},
+                {f,18},
+                {list,[{integer,42},
+                       {f,17},
+                       {integer,1000},
+                       {f,18}]}}.
+  {label,17}.
+    {move,{atom,ok},{x,0}}.
+    return.
+  {label,18}.
+    {move,{atom,error},{x,0}}.
+    return.
+  {label,19}.
+    {case_end,{integer,42}}.
+
+{function, zi2, 0, 16}.
+  {label,20}.
+    {func_info,{atom,literal_case_expression},{atom,zi2},0}.
+  {label,21}.
+    {test,is_integer,{f,23},[{integer,42}]}.
+    {select_val,{integer,42},
+                {f,23},
+                {list,[{integer,42},
+                       {f,22},
+                       {integer,1000},
+                       {f,23}]}}.
+  {label,22}.
+    {move,{atom,ok},{x,0}}.
+    return.
+  {label,23}.
+    {move,{atom,error},{x,0}}.
+    return.
+
+{function, za1, 0, 25}.
+  {label,24}.
+    {func_info,{atom,literal_case_expression},{atom,za1},0}.
+  {label,25}.
+    {test,is_atom,{f,28},[{atom,x}]}.
+    {select_val,{atom,x},
+                {f,27},
+                {list,[{atom,a},
+                       {f,27},
+                       {atom,x},
+                       {f,26}]}}.
+  {label,26}.
+    {move,{atom,ok},{x,0}}.
+    return.
+  {label,27}.
+    {move,{atom,error},{x,0}}.
+    return.
+  {label,28}.
+    {case_end,{atom,x}}.
+
+{function, za2, 0, 30}.
+  {label,29}.
+    {func_info,{atom,literal_case_expression},{atom,za2},0}.
+  {label,30}.
+    {test,is_atom,{f,32},[{atom,x}]}.
+    {select_val,{atom,x},
+                {f,32},
+                {list,[{atom,a},
+                       {f,32},
+                       {atom,x},
+                       {f,31}]}}.
+  {label,31}.
+    {move,{atom,ok},{x,0}}.
+    return.
+  {label,32}.
+    {move,{atom,error},{x,0}}.
+    return.
 
 {function, module_info, 0, 12}.
   {label,11}.
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index cfbc5dfe81..c7617d3b90 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -1,44 +1,159 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2005-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
 -module(bif_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 display/1, display_huge/0,
+	 types/1,
 	 t_list_to_existing_atom/1,os_env/1,otp_7526/1,
 	 binary_to_atom/1,binary_to_existing_atom/1,
 	 atom_to_binary/1,min_max/1]).
 
-all(suite) ->
-    [t_list_to_existing_atom,os_env,otp_7526,
-     atom_to_binary,binary_to_atom,binary_to_existing_atom,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [types, t_list_to_existing_atom, os_env, otp_7526,
+     display,
+     atom_to_binary, binary_to_atom, binary_to_existing_atom,
      min_max].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(1)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
+
+display(suite) ->
+    [];
+display(doc) ->
+    ["Uses erlang:display to test that erts_printf does not do deep recursion"];
+display(Config) when is_list(Config) ->
+    Pa = filename:dirname(code:which(?MODULE)),
+    {ok, Node} = test_server:start_node(display_huge_term,peer,
+					[{args, "-pa "++Pa}]),
+    true = rpc:call(Node,?MODULE,display_huge,[]),
+    test_server:stop_node(Node),
+    ok.
+
+display_huge() ->
+    erlang:display(deeep(100000)).
+
+deeep(0,Acc) ->
+    Acc;
+deeep(N,Acc) ->
+    deeep(N-1,[Acc|[]]).
+
+deeep(N) ->
+    deeep(N,[hello]).
+
+
+types(Config) when is_list(Config) ->
+    c:l(erl_bif_types),
+    case erlang:function_exported(erl_bif_types, module_info, 0) of
+	false ->
+	    %% Fail cleanly.
+	    ?line ?t:fail("erl_bif_types not compiled");
+	true ->
+	    types_1()
+    end.
+
+types_1() ->
+    ?line List0 = erlang:system_info(snifs),
+
+    %% Ignore missing type information for hipe BIFs.
+    ?line List = [MFA || {M,_,_}=MFA <- List0, M =/= hipe_bifs],
+
+    case [MFA || MFA <- List, not known_types(MFA)] of
+	[] ->
+	    types_2(List);
+	BadTypes ->
+	    io:put_chars("No type information:\n"),
+	    io:format("~p\n", [lists:sort(BadTypes)]),
+	    ?line ?t:fail({length(BadTypes),bifs_without_types})
+    end.
+
+types_2(List) ->
+    BadArity = [MFA || {M,F,A}=MFA <- List,
+		       begin
+			   Types = erl_bif_types:arg_types(M, F, A),
+			   length(Types) =/= A
+		       end],
+    case BadArity of
+	[] ->
+	    types_3(List);
+	[_|_] ->
+	    io:put_chars("Bifs with bad arity\n"),
+	    io:format("~p\n", [BadArity]),
+	    ?line ?t:fail({length(BadArity),bad_arity})
+    end.
+
+types_3(List) ->
+    BadSmokeTest = [MFA || {M,F,A}=MFA <- List,
+			   begin
+			       try erl_bif_types:type(M, F, A) of
+				   Type ->
+				       %% Test that type is returned.
+				       not erl_types:is_erl_type(Type)
+			       catch
+				   Class:Error ->
+				       io:format("~p: ~p ~p\n",
+						 [MFA,Class,Error]),
+				       true
+			       end
+			   end],
+    case BadSmokeTest of
+	[] ->
+	    ok;
+	[_|_] ->
+	    io:put_chars("Bifs with failing calls to erlang_bif_types:type/3 "
+			 "(or with bogus return values):\n"),
+	    io:format("~p\n", [BadSmokeTest]),
+	    ?line ?t:fail({length(BadSmokeTest),bad_smoke_test})
+    end.
+
+known_types({M,F,A}) ->
+    erl_bif_types:is_known(M, F, A).
+
 t_list_to_existing_atom(Config) when is_list(Config) ->
     ?line all = list_to_existing_atom("all"),
     ?line ?MODULE = list_to_existing_atom(?MODULE_STRING),
@@ -308,6 +423,18 @@ min_max(Config) when is_list(Config) ->
 
     ?line 42.0 = erlang:min(42.0, 42),
     ?line 42.0 = erlang:max(42.0, 42),
+    %% And now (R14) they are also autoimported!
+    ?line a = min(id(a), a),
+    ?line a = min(id(a), b),
+    ?line a = min(id(b), a),
+    ?line b = min(id(b), b),
+    ?line a = max(id(a), a),
+    ?line b = max(id(a), b),
+    ?line b = max(id(b), a),
+    ?line b = max(id(b), b),
+
+    ?line 42.0 = min(42.0, 42),
+    ?line 42.0 = max(42.0, 42),
 
     ok.
 
diff --git a/erts/emulator/test/big_SUITE.erl b/erts/emulator/test/big_SUITE.erl
index 6cedd39009..3487917677 100644
--- a/erts/emulator/test/big_SUITE.erl
+++ b/erts/emulator/test/big_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,10 @@
 -module(big_SUITE).
 
 
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 -export([t_div/1, eq_28/1, eq_32/1, eq_big/1, eq_math/1, big_literals/1,
-	 borders/1, negative/1, big_float/1, big_float_1/1, big_float_2/1,
+	 borders/1, negative/1, big_float_1/1, big_float_2/1,
 	 shift_limit_1/1, powmod/1, system_limit/1, otp_6692/1]).
 
 %% Internal exports.
@@ -30,19 +31,38 @@
 
 -export([fac/1, fib/1, pow/2, gcd/2, lcm/2]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [t_div, eq_28, eq_32, eq_big, eq_math, big_literals,
+     borders, negative, {group, big_float}, shift_limit_1,
+     powmod, system_limit, otp_6692].
+
+groups() -> 
+    [{big_float, [], [big_float_1, big_float_2]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [t_div, eq_28, eq_32, eq_big, eq_math, big_literals, borders,
-     negative, big_float, shift_limit_1, powmod, system_limit, otp_6692].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(3)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -260,10 +280,6 @@ big_literals(Config) when is_list(Config) ->
     ?line ok = Mod:t(),
     ok.
 
-big_float(doc) ->
-    ["Test cases for mixing bignums and floats"];
-big_float(suite) ->
-    [big_float_1, big_float_2].
 
 big_float_1(doc) ->
     ["OTP-2436, part 1"];
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index db2b3e10db..4e82381fba 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -23,12 +23,12 @@
 %% Tests binaries and the BIFs:
 %%	list_to_binary/1
 %%      iolist_to_binary/1
-%%      bitstr_to_list/1
+%%      list_to_bitstring/1
 %%	binary_to_list/1
 %%	binary_to_list/3
 %%	binary_to_term/1
 %%  	binary_to_term/2
-%%      bitstr_to_list/1
+%%      bitstring_to_list/1
 %%	term_to_binary/1
 %%      erlang:external_size/1
 %%	size(Binary)
@@ -40,9 +40,11 @@
 %%      phash2(Binary, N)
 %%
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2,
 	 copy_terms/1, conversions/1, deep_lists/1, deep_bitstr_lists/1,
 	 bad_list_to_binary/1, bad_binary_to_list/1,
 	 t_split_binary/1, bad_split/1, t_concat_binary/1,
@@ -55,31 +57,48 @@
 	 otp_5484/1,otp_5933/1,
 	 ordering/1,unaligned_order/1,gc_test/1,
 	 bit_sized_binary_sizes/1,
-	 bitlevel_roundtrip/1,
 	 otp_6817/1,deep/1,obsolete_funs/1,robustness/1,otp_8117/1,
 	 otp_8180/1]).
 
 %% Internal exports.
 -export([sleeper/0]).
 
-all(suite) ->
-    [copy_terms,conversions,deep_lists,deep_bitstr_lists,
+suite() -> [{ct_hooks,[ts_install_cth]},
+	    {timetrap,{minutes,2}}].
+
+all() -> 
+    [copy_terms, conversions, deep_lists, deep_bitstr_lists,
      t_split_binary, bad_split, t_concat_binary,
-     bad_list_to_binary, bad_binary_to_list, terms, terms_float,
-     external_size, t_iolist_size,
-     bad_binary_to_term_2,safe_binary_to_term2,
-     bad_binary_to_term, bad_terms, t_hash, bad_size, bad_term_to_binary,
-     more_bad_terms, otp_5484, otp_5933, ordering, unaligned_order,
-     gc_test, bit_sized_binary_sizes, bitlevel_roundtrip, otp_6817, otp_8117,
-     deep,obsolete_funs,robustness,otp_8180].
+     bad_list_to_binary, bad_binary_to_list, terms,
+     terms_float, external_size, t_iolist_size,
+     bad_binary_to_term_2, safe_binary_to_term2,
+     bad_binary_to_term, bad_terms, t_hash, bad_size,
+     bad_term_to_binary, more_bad_terms, otp_5484, otp_5933,
+     ordering, unaligned_order, gc_test,
+     bit_sized_binary_sizes, otp_6817, otp_8117, deep,
+     obsolete_funs, robustness, otp_8180].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
-    Dog=?t:timetrap(?t:minutes(2)),
-    [{watchdog, Dog}|Config].
+    Config.
 
-fin_per_testcase(_Func, Config) ->
-    Dog=?config(watchdog, Config),
-    ?t:timetrap_cancel(Dog).
+end_per_testcase(_Func, _Config) ->
+    ok.
 
 -define(heap_binary_size, 64).
 
@@ -256,12 +275,33 @@ bad_list_to_binary(Config) when is_list(Config) ->
     ?line test_bad_bin(fun(X, Y) -> X*Y end),
     ?line test_bad_bin([1,fun(X) -> X + 1 end,2|fun() -> 0 end]),
     ?line test_bad_bin([fun(X) -> X + 1 end]),
+
+    %% Test iolists that do not fit in the address space.
+    %% Unfortunately, it would be too slow to test in a 64-bit emulator.
+    case erlang:system_info(wordsize) of
+	4 -> huge_iolists();
+	_ -> ok
+    end.
+
+huge_iolists() ->
+    FourGigs = 1 bsl 32,
+    ?line Sizes = [FourGigs+N || N <- lists:seq(0, 64)] ++
+	[1 bsl N || N <- lists:seq(33, 37)],
+    ?line Base = <<0:(1 bsl 20)/unit:8>>,
+    [begin
+	 L = build_iolist(Sz, Base),
+	 ?line {'EXIT',{system_limit,_}} = (catch list_to_binary([L])),
+	 ?line {'EXIT',{system_limit,_}} = (catch list_to_bitstring([L])),
+	 ?line {'EXIT',{system_limit,_}} = (catch binary:list_to_bin([L])),
+	 ?line {'EXIT',{system_limit,_}} = (catch iolist_to_binary(L))
+	 end || Sz <- Sizes],
     ok.
 
 test_bad_bin(List) ->
     {'EXIT',{badarg,_}} = (catch list_to_binary(List)),
     {'EXIT',{badarg,_}} = (catch iolist_to_binary(List)),
-    {'EXIT',{badarg,_}} = (catch list_to_bitstring(List)).
+    {'EXIT',{badarg,_}} = (catch list_to_bitstring(List)),
+    {'EXIT',{badarg,_}} = (catch iolist_size(List)).
 
 bad_binary_to_list(doc) -> "Tries binary_to_list/1,3 with bad arguments.";
 bad_binary_to_list(Config) when is_list(Config) ->
@@ -439,11 +479,11 @@ terms(Config) when is_list(Config) ->
 			      ok
 		      end,
 		      Term = binary_to_term(Bin),
-		      Term = erlang:binary_to_term(Bin, [safe]),
+		      Term = binary_to_term(Bin, [safe]),
 		      Unaligned = make_unaligned_sub_binary(Bin),
 		      Term = binary_to_term(Unaligned),
-		      Term = erlang:binary_to_term(Unaligned, []),
-		      Term = erlang:binary_to_term(Bin, [safe]),
+		      Term = binary_to_term(Unaligned, []),
+		      Term = binary_to_term(Bin, [safe]),
 		      BinC = erlang:term_to_binary(Term, [compressed]),
 		      Term = binary_to_term(BinC),
 		      true = size(BinC) =< size(Bin),
@@ -497,18 +537,65 @@ external_size_1(Term, Size0, Limit) when Size0 < Limit ->
 external_size_1(_, _, _) -> ok.
 
 t_iolist_size(Config) when is_list(Config) ->
-    %% Build a term whose external size only fits in a big num (on 32-bit CPU).
-    Bin = iolist_to_binary(lists:seq(0, 254)),
-    ?line ok = t_iolist_size_1(Bin, 0, 16#7FFFFFFF),
-    ?line ok = t_iolist_size_1(make_unaligned_sub_binary(Bin), 0, 16#7FFFFFFF).
+    ?line Seed = now(),
+    ?line io:format("Seed: ~p", [Seed]),
+    ?line random:seed(Seed),
+    ?line Base = <<0:(1 bsl 20)/unit:8>>,
+    ?line Powers = [1 bsl N || N <- lists:seq(2, 37)],
+    ?line Sizes0 = [[N - random:uniform(N div 2),
+		     lists:seq(N-2, N+2),
+		     N+N div 2,
+		     N + random:uniform(N div 2)] ||
+		       N <- Powers],
+    %% Test sizes around 1^32 more thoroughly.
+    FourGigs = 1 bsl 32,
+    ?line Sizes1 = [FourGigs+N || N <- lists:seq(-8, 40)] ++ Sizes0,
+    ?line Sizes2 = lists:flatten(Sizes1),
+    ?line Sizes = lists:usort(Sizes2),
+    io:format("~p sizes:", [length(Sizes)]),
+    io:format("~p\n", [Sizes]),
+    ?line [Sz = iolist_size(build_iolist(Sz, Base)) || Sz <- Sizes],
+    ok.
 
-t_iolist_size_1(IOList, Size0, Limit) when Size0 < Limit ->
-    case iolist_size(IOList) of
-	Size when is_integer(Size), Size0 < Size ->
-	    io:format("~p", [Size]),
-	    t_iolist_size_1([IOList|IOList], Size, Limit)
+build_iolist(N, Base) when N < 16 ->
+    case random:uniform(3) of
+	1 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    Bin;
+	_ ->
+	    lists:seq(1, N)
     end;
-t_iolist_size_1(_, _, _) -> ok.
+build_iolist(N, Base) when N =< byte_size(Base) ->
+    case random:uniform(3) of
+	1 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    Bin;
+	2 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    [Bin];
+	3 ->
+	    case N rem 2 of
+		0 ->
+		    L = build_iolist(N div 2, Base),
+		    [L,L];
+		1 ->
+		    L = build_iolist(N div 2, Base),
+		    [L,L,45]
+	    end
+    end;
+build_iolist(N0, Base) ->
+    Small = random:uniform(15),
+    Seq = lists:seq(1, Small),
+    N = N0 - Small,
+    case N rem 2 of
+	0 ->
+	    L = build_iolist(N div 2, Base),
+	    [L,L|Seq];
+	1 ->
+	    L = build_iolist(N div 2, Base),
+	    [47,L,L|Seq]
+    end.
+
 
 bad_binary_to_term_2(doc) -> "OTP-4053.";
 bad_binary_to_term_2(suite) -> [];
@@ -543,7 +630,7 @@ bad_bin_to_term(BadBin) ->
     {'EXIT',{badarg,_}} = (catch binary_to_term(BadBin)).
 
 bad_bin_to_term(BadBin,Opts) ->
-    {'EXIT',{badarg,_}} = (catch erlang:binary_to_term(BadBin,Opts)).
+    {'EXIT',{badarg,_}} = (catch binary_to_term(BadBin,Opts)).
 
 safe_binary_to_term2(doc) -> "Test safety options for binary_to_term/2";
 safe_binary_to_term2(Config) when is_list(Config) ->
@@ -554,7 +641,7 @@ safe_binary_to_term2(Config) when is_list(Config) ->
     BadRef = <<131,114,0,3,BadHostAtom/binary,0,<<0,0,0,255>>/binary,
 	      Empty/binary,Empty/binary>>,
     ?line bad_bin_to_term(BadRef, [safe]), % good ref, with a bad atom
-    ?line fullsweep_after = erlang:binary_to_term(<<131,100,0,15,"fullsweep_after">>, [safe]), % should be a good atom
+    ?line fullsweep_after = binary_to_term(<<131,100,0,15,"fullsweep_after">>, [safe]), % should be a good atom
     BadExtFun = <<131,113,100,0,4,98,108,117,101,100,0,4,109,111,111,110,97,3>>,
     ?line bad_bin_to_term(BadExtFun, [safe]),
     ok.
@@ -1042,7 +1129,7 @@ test_terms(Test_Func) ->
     ?line Test_Func(F = fun(A) -> 42*A end),
     ?line Test_Func(lists:duplicate(32, F)),
 
-    ?line Test_Func(FF = fun binary_SUITE:all/1),
+    ?line Test_Func(FF = fun binary_SUITE:all/0),
     ?line Test_Func(lists:duplicate(32, FF)),
 
     ok.
@@ -1150,35 +1237,6 @@ bsbs_1(A) ->
     Bin = binary_to_term(<<131,$M,5:32,A,0,0,0,0,0>>),
     BinSize = bit_size(Bin).
 
-bitlevel_roundtrip(Config) when is_list(Config) ->
-    case ?t:is_release_available("r11b") of
-	true -> bitlevel_roundtrip_1();
-	false -> {skip,"No R11B found"}
-    end.
-
-bitlevel_roundtrip_1() ->
-    Name = bitlevelroundtrip,
-    ?line N = list_to_atom(atom_to_list(Name) ++ "@" ++ hostname()),
-    ?line ?t:start_node(Name, slave, [{erl,[{release,"r11b"}]}]),
-
-    ?line {<<128>>,1} = roundtrip(N, <<1:1>>),
-    ?line {<<64>>,2} = roundtrip(N, <<1:2>>),
-    ?line {<<16#E0>>,3} = roundtrip(N, <<7:3>>),
-    ?line {<<16#70>>,4} = roundtrip(N, <<7:4>>),
-    ?line {<<16#10>>,5} = roundtrip(N, <<2:5>>),
-    ?line {<<16#8>>,6} = roundtrip(N, <<2:6>>),
-    ?line {<<16#2>>,7} = roundtrip(N, <<1:7>>),
-    ?line {<<8,128>>,1} = roundtrip(N, <<8,1:1>>),
-    ?line {<<42,248>>,5} = roundtrip(N, <<42,31:5>>),
-
-    ?line ?t:stop_node(N),
-    ok.
-
-roundtrip(Node, Term) ->
-    {badrpc,{'EXIT',Res}} = rpc:call(Node, erlang, exit, [Term]),
-    io:format("<<~p bits>> => ~w", [bit_size(Term),Res]),
-    Res.
-
 deep(Config) when is_list(Config) ->
     ?line deep_roundtrip(lists:foldl(fun(E, A) ->
 					     [E,A]
@@ -1193,34 +1251,7 @@ deep(Config) when is_list(Config) ->
 
 deep_roundtrip(T) ->
     B = term_to_binary(T),
-    true = deep_eq(T, binary_to_term(B)).
-
-%%
-%% FIXME: =:= runs out of stack.
-%%
-deep_eq([H1|T1], [H2|T2]) ->
-    deep_eq(H1, H2) andalso deep_eq(T1, T2);
-deep_eq(T1, T2) when tuple_size(T1) =:= tuple_size(T2) ->
-    deep_eq_tup(T1, T2, tuple_size(T1));
-deep_eq(T1, T2) when is_function(T1), is_function(T2) ->
-    {uniq,U1} = erlang:fun_info(T1, uniq),
-    {index,I1} = erlang:fun_info(T1, index),
-    {arity,A1} = erlang:fun_info(T1, arity),
-    {env,E1} = erlang:fun_info(T1, env),
-    {uniq,U2} = erlang:fun_info(T2, uniq),
-    {index,I2} = erlang:fun_info(T2, index),
-    {arity,A2} = erlang:fun_info(T2, arity),
-    {env,E2} = erlang:fun_info(T2, env),
-    U1 =:= U2 andalso I1 =:= I2 andalso A1 =:= A2 andalso
-	deep_eq(E1, E2);
-deep_eq(T1, T2) ->
-    T1 =:= T2.
-
-deep_eq_tup(_T1, _T2, 0) ->
-    true;
-deep_eq_tup(T1, T2, N) ->
-    deep_eq(element(N, T1), element(N, T2)) andalso
-	deep_eq_tup(T1, T2, N-1).
+    T = binary_to_term(B).
 
 obsolete_funs(Config) when is_list(Config) ->
     erts_debug:set_internal_state(available_internal_state, true),
@@ -1331,11 +1362,4 @@ unaligned_sub_bin(Bin0, Offs) ->
     <<_:Offs,Bin:Sz/binary,_:Roffs>> = id(Bin1),
     Bin.
 
-hostname() ->
-    from($@, atom_to_list(node())).
-
-from(H, [H | T]) -> T;
-from(H, [_ | T]) -> from(H, T);
-from(_, []) -> [].
-
 id(I) -> I.
diff --git a/erts/emulator/test/bs_bincomp_SUITE.erl b/erts/emulator/test/bs_bincomp_SUITE.erl
index 4e83d97689..f1c2dff560 100644
--- a/erts/emulator/test/bs_bincomp_SUITE.erl
+++ b/erts/emulator/test/bs_bincomp_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,15 +22,34 @@
 
 -module(bs_bincomp_SUITE).
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 byte_aligned/1,bit_aligned/1,extended_byte_aligned/1,
 	 extended_bit_aligned/1,mixed/1,tracing/1]).
 
--include("test_server.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [byte_aligned, bit_aligned, extended_byte_aligned,
+     extended_bit_aligned, mixed, tracing].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [byte_aligned,bit_aligned,extended_byte_aligned,
-     extended_bit_aligned,mixed,tracing].
 
 
 byte_aligned(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/bs_bit_binaries_SUITE.erl b/erts/emulator/test/bs_bit_binaries_SUITE.erl
index 52bb925385..ff1088118d 100644
--- a/erts/emulator/test/bs_bit_binaries_SUITE.erl
+++ b/erts/emulator/test/bs_bit_binaries_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,18 +22,38 @@
 
 -module(bs_bit_binaries_SUITE).
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 misc/1,horrid_match/1,test_bitstr/1,test_bit_size/1,asymmetric_tests/1,
 	 big_asymmetric_tests/1,binary_to_and_from_list/1,
 	 big_binary_to_and_from_list/1,send_and_receive/1,
 	 send_and_receive_alot/1,append/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [misc, horrid_match, test_bitstr, test_bit_size,
+     asymmetric_tests, big_asymmetric_tests,
+     binary_to_and_from_list, big_binary_to_and_from_list,
+     send_and_receive, send_and_receive_alot, append].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [misc,horrid_match,test_bitstr,test_bit_size,asymmetric_tests,
-     big_asymmetric_tests,binary_to_and_from_list,big_binary_to_and_from_list,
-     send_and_receive,send_and_receive_alot,append].
 
 misc(Config) when is_list(Config) ->
     ?line <<1:100>> = id(<<1:100>>),
diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl
index 3d9b51d278..1959803385 100644
--- a/erts/emulator/test/bs_construct_SUITE.erl
+++ b/erts/emulator/test/bs_construct_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -21,22 +21,39 @@
 
 -module(bs_construct_SUITE).
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 test1/1, test2/1, test3/1, test4/1, test5/1, testf/1,
 	 not_used/1, in_guard/1,
 	 mem_leak/1, coerce_to_float/1, bjorn/1,
 	 huge_float_field/1, huge_binary/1, system_limit/1, badarg/1,
 	 copy_writable_binary/1, kostis/1, dynamic/1, bs_add/1,
-	 otp_7422/1]).
+	 otp_7422/1, zero_width/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
-all(suite) ->
-    [test1, test2, test3, test4, test5, testf,
-     not_used, in_guard, mem_leak, coerce_to_float, bjorn,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [test1, test2, test3, test4, test5, testf, not_used,
+     in_guard, mem_leak, coerce_to_float, bjorn,
      huge_float_field, huge_binary, system_limit, badarg,
-     copy_writable_binary, kostis, dynamic, bs_add,
-     otp_7422].
+     copy_writable_binary, kostis, dynamic, bs_add, otp_7422, zero_width].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 big(1) ->
     57285702734876389752897683.
@@ -786,5 +803,20 @@ otp_7422_bin(N) when N < 512 ->
 	       end),
     otp_7422_bin(N+1);
 otp_7422_bin(_) -> ok.
+
+zero_width(Config) when is_list(Config) ->
+    ?line Z = id(0),
+    Small = id(42),
+    Big = id(1 bsl 128),
+    ?line <<>> = <<Small:Z>>,
+    ?line <<>> = <<Small:0>>,
+    ?line <<>> = <<Big:Z>>,
+    ?line <<>> = <<Big:0>>,
+    
+    ?line {'EXIT',{badarg,_}} = (catch <<not_a_number:0>>),
+    ?line {'EXIT',{badarg,_}} = (catch <<(id(not_a_number)):Z>>),
+    ?line {'EXIT',{badarg,_}} = (catch <<(id(not_a_number)):0>>),
+
+    ok.
     
 id(I) -> I.
diff --git a/erts/emulator/test/bs_match_bin_SUITE.erl b/erts/emulator/test/bs_match_bin_SUITE.erl
index 3d054a279f..96e69dbc0b 100644
--- a/erts/emulator/test/bs_match_bin_SUITE.erl
+++ b/erts/emulator/test/bs_match_bin_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,12 +19,32 @@
 
 -module(bs_match_bin_SUITE).
 
--export([all/1,byte_split_binary/1,bit_split_binary/1,match_huge_bin/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 byte_split_binary/1,bit_split_binary/1,match_huge_bin/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [byte_split_binary, bit_split_binary, match_huge_bin].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [byte_split_binary,bit_split_binary,match_huge_bin].
 
 byte_split_binary(doc) -> "Tries to split a binary at all byte-aligned positions.";
 byte_split_binary(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/bs_match_int_SUITE.erl b/erts/emulator/test/bs_match_int_SUITE.erl
index 99dee7c7bc..ce03ecb548 100644
--- a/erts/emulator/test/bs_match_int_SUITE.erl
+++ b/erts/emulator/test/bs_match_int_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,16 +18,36 @@
 
 -module(bs_match_int_SUITE).
 
--export([all/1,integer/1,signed_integer/1,dynamic/1,more_dynamic/1,mml/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 integer/1,signed_integer/1,dynamic/1,more_dynamic/1,mml/1,
 	 match_huge_int/1,bignum/1,unaligned_32_bit/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -import(lists, [seq/2]).
 
-all(suite) ->
-    [integer,signed_integer,dynamic,more_dynamic,mml,match_huge_int,bignum,
-     unaligned_32_bit].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [integer, signed_integer, dynamic, more_dynamic, mml,
+     match_huge_int, bignum, unaligned_32_bit].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 integer(Config) when is_list(Config) ->
     ?line 0 = get_int(mkbin([])),
diff --git a/erts/emulator/test/bs_match_misc_SUITE.erl b/erts/emulator/test/bs_match_misc_SUITE.erl
index 6de2ef67e5..b022f96740 100644
--- a/erts/emulator/test/bs_match_misc_SUITE.erl
+++ b/erts/emulator/test/bs_match_misc_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2000-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,18 +18,38 @@
 %%
 -module(bs_match_misc_SUITE).
 
--export([all/1,bound_var/1,bound_tail/1,t_float/1,little_float/1,sean/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 bound_var/1,bound_tail/1,t_float/1,little_float/1,sean/1,
 	 kenneth/1,encode_binary/1,native/1,happi/1,
 	 size_var/1,wiger/1,x0_context/1,huge_float_field/1,
 	 writable_binary_matched/1,otp_7198/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [bound_var, bound_tail, t_float, little_float, sean,
+     kenneth, encode_binary, native, happi, size_var, wiger,
+     x0_context, huge_float_field, writable_binary_matched,
+     otp_7198].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [bound_var,bound_tail,t_float,little_float,sean,
-     kenneth,encode_binary,native,happi,
-     size_var,wiger,x0_context,huge_float_field,
-     writable_binary_matched,otp_7198].
 
 bound_var(doc) -> "Test matching of bound variables.";
 bound_var(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/bs_match_tail_SUITE.erl b/erts/emulator/test/bs_match_tail_SUITE.erl
index b0b0779b65..1397f2069c 100644
--- a/erts/emulator/test/bs_match_tail_SUITE.erl
+++ b/erts/emulator/test/bs_match_tail_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,11 +20,31 @@
 -module(bs_match_tail_SUITE).
 
 -author('[email protected]').
--export([all/1,aligned/1,unaligned/1,zero_tail/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,aligned/1,unaligned/1,zero_tail/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [aligned, unaligned, zero_tail].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) -> [aligned,unaligned,zero_tail].
 
 aligned(doc) -> "Test aligned tails.";
 aligned(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/bs_utf_SUITE.erl b/erts/emulator/test/bs_utf_SUITE.erl
index 87adc5197b..72c656c400 100644
--- a/erts/emulator/test/bs_utf_SUITE.erl
+++ b/erts/emulator/test/bs_utf_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2008-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,13 +19,15 @@
 
 -module(bs_utf_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 utf8_roundtrip/1,utf16_roundtrip/1,utf32_roundtrip/1,
 	 utf8_illegal_sequences/1,utf16_illegal_sequences/1,
 	 utf32_illegal_sequences/1,
 	 bad_construction/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(FAIL(Expr), ?line fail_check(catch Expr, ??Expr, [])).
 
@@ -33,14 +35,32 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(6)),
     [{watchdog,Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-all(suite) ->
-    [utf8_roundtrip,utf16_roundtrip,utf32_roundtrip,
-     utf8_illegal_sequences,utf16_illegal_sequences,
-     utf32_illegal_sequences,bad_construction].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [utf8_roundtrip, utf16_roundtrip, utf32_roundtrip,
+     utf8_illegal_sequences, utf16_illegal_sequences,
+     utf32_illegal_sequences, bad_construction].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 utf8_roundtrip(Config) when is_list(Config) ->
     ?line utf8_roundtrip(0, 16#D7FF),
diff --git a/erts/emulator/test/busy_port_SUITE.erl b/erts/emulator/test/busy_port_SUITE.erl
index 9b16170293..8365e1c540 100644
--- a/erts/emulator/test/busy_port_SUITE.erl
+++ b/erts/emulator/test/busy_port_SUITE.erl
@@ -1,39 +1,58 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
 -module(busy_port_SUITE).
 
--export([all/1, io_to_busy/1, message_order/1, send_3/1, 
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 io_to_busy/1, message_order/1, send_3/1, 
 	 system_monitor/1, no_trap_exit/1,
 	 no_trap_exit_unlinked/1, trap_exit/1, multiple_writers/1,
 	 hard_busy_driver/1, soft_busy_driver/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %% Internal exports.
 -export([init/2]).
 
-all(suite) -> {req, [dynamic_loading],
-	       [io_to_busy, message_order, send_3, 
-		system_monitor, no_trap_exit,
-		no_trap_exit_unlinked, trap_exit, multiple_writers,
-		hard_busy_driver, soft_busy_driver]}.
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [io_to_busy, message_order, send_3, system_monitor,
+     no_trap_exit, no_trap_exit_unlinked, trap_exit,
+     multiple_writers, hard_busy_driver, soft_busy_driver].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 %% Tests I/O operations to a busy port, to make sure a suspended send
 %% operation is correctly restarted.  This used to crash Beam.
@@ -182,7 +201,7 @@ system_monitor(Config) when is_list(Config) ->
     ?line Master ! {Owner, {command, "u"}},
     ?line {Busy,beta} = rec(Void),
     ?line Void = rec(Void),
-    ?line NewMonitor = erlang:system_monitor(OldMonitor),
+    ?line _NewMonitor = erlang:system_monitor(OldMonitor),
     ?line OldMonitor = erlang:system_monitor(),
     ?line OldMonitor = erlang:system_monitor(OldMonitor),
     %%
@@ -361,7 +380,6 @@ soft_busy_driver(Config) when is_list(Config) ->
     hs_test(Config, false).
 
 hs_test(Config, HardBusy) when is_list(Config) ->
-    ?line Me = self(),
     ?line DrvName = case HardBusy of
 			true -> 'hard_busy_drv';
 			false -> 'soft_busy_drv'
@@ -479,7 +497,7 @@ hs_busy_pcmd(Prt, Opts, StartFun, EndFun) ->
 			   Tester ! {self(), doing_port_command},
 			   Start = os:timestamp(),
 			   Res = try {return,
-				      erlang:port_command(Prt, [], Opts)}
+				      port_command(Prt, [], Opts)}
 				 catch Exception:Error -> {Exception, Error}
 				 end,
 			   End = os:timestamp(),
diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl
index e0528955b0..93fdc157f7 100644
--- a/erts/emulator/test/call_trace_SUITE.erl
+++ b/erts/emulator/test/call_trace_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,7 +20,9 @@
 
 -module(call_trace_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 hipe/1,process_specs/1,basic/1,flags/1,errors/1,pam/1,change_pam/1,
 	 return_trace/1,exception_trace/1,on_load/1,deep_exception/1,
 	 exception_nocatch/1,bit_syntax/1]).
@@ -35,25 +37,44 @@
 -export([abbr/1,abbr/2]).
 
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(P, 20).
 
-all(suite) ->
-    Common = [errors,on_load],
-    NotHipe = [process_specs,basic,flags,pam,change_pam,return_trace,
-	       exception_trace,deep_exception,exception_nocatch,bit_syntax],
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    Common = [errors, on_load],
+    NotHipe = [process_specs, basic, flags, pam, change_pam,
+	       return_trace, exception_trace, deep_exception,
+	       exception_nocatch, bit_syntax],
     Hipe = [hipe],
-    case test_server:is_native(?MODULE) of
+    case test_server:is_native(call_trace_SUITE) of
 	true -> Hipe ++ Common;
 	false -> NotHipe ++ Common
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:seconds(30)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 33351a3cc9..a062cea117 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,18 +18,38 @@
 %%
 
 -module(code_SUITE).
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 new_binary_types/1,t_check_process_code/1,t_check_process_code_ets/1,
 	 external_fun/1,get_chunk/1,module_md5/1,make_stub/1,
 	 make_stub_many_funs/1,constant_pools/1,
 	 false_dependency/1,coverage/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [new_binary_types, t_check_process_code,
+     t_check_process_code_ets, external_fun, get_chunk,
+     module_md5, make_stub, make_stub_many_funs,
+     constant_pools, false_dependency, coverage].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [new_binary_types,t_check_process_code,t_check_process_code_ets,
-     external_fun,get_chunk,module_md5,make_stub,make_stub_many_funs,
-     constant_pools,false_dependency,coverage].
 
 new_binary_types(Config) when is_list(Config) ->
     ?line Data = ?config(data_dir, Config),
@@ -320,6 +340,9 @@ make_stub(Config) when is_list(Config) ->
 	(catch code:make_stub_module(my_code_test,
 				     bit_sized_binary(Code),
 				     {[],[]})),
+    ?line {'EXIT',{badarg,_}} =
+	(catch code:make_stub_module(my_code_test_with_wrong_name,
+				     Code, {[],[]})),
     ok.
 
 make_stub_many_funs(Config) when is_list(Config) ->
@@ -460,7 +483,7 @@ do_false_dependency(Init, Code) ->
     %% Spawn process. Make sure it has the appropriate init function
     %% and returned. CP should not contain garbage after the return.
     Parent = self(),
-    ?line Pid = spawn_link(fun() -> false_dependency_loop(Parent, Init) end),
+    ?line Pid = spawn_link(fun() -> false_dependency_loop(Parent, Init, true) end),
     ?line receive initialized -> ok end,
 
     %% Reload the module. Make sure the process is still alive.
@@ -475,14 +498,23 @@ do_false_dependency(Init, Code) ->
     ?line unlink(Pid), exit(Pid, kill),
     ?line true = erlang:purge_module(cpbugx),
     ?line true = erlang:delete_module(cpbugx),
+    ?line code:is_module_native(cpbugx),  % test is_module_native on deleted code
     ?line true = erlang:purge_module(cpbugx),
+    ?line code:is_module_native(cpbugx),  % test is_module_native on purged code
     ok.
     
-false_dependency_loop(Parent, Init) ->
+false_dependency_loop(Parent, Init, SendInitAck) ->
     Init(),
-    Parent ! initialized,
+    case SendInitAck of
+	true -> Parent ! initialized;
+	false -> void
+		 %% Just send one init-ack. I guess the point of this test
+		 %% wasn't to fill parents msg-queue (?). Seen to cause
+		 %% out-of-mem (on halfword-vm for some reason) by
+		 %% 91 million msg in queue. /sverker
+    end,
     receive
-	_ -> false_dependency_loop(Parent, Init)
+	_ -> false_dependency_loop(Parent, Init, false)
     end.
 
 coverage(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/crypto_SUITE.erl b/erts/emulator/test/crypto_SUITE.erl
index e3d34b923d..a82bd4fe38 100644
--- a/erts/emulator/test/crypto_SUITE.erl
+++ b/erts/emulator/test/crypto_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,14 +19,34 @@
 
 -module(crypto_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 t_md5/1,t_md5_update/1,error/1,unaligned_context/1,random_lists/1,
 	 misc_errors/1]).
 
-all(suite) ->
-    [t_md5,t_md5_update,error,unaligned_context,random_lists,misc_errors].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [t_md5, t_md5_update, error, unaligned_context,
+     random_lists, misc_errors].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 
 misc_errors(doc) ->
diff --git a/erts/emulator/test/crypto_reference.erl b/erts/emulator/test/crypto_reference.erl
index 99107e3b57..b91535a50e 100644
--- a/erts/emulator/test/crypto_reference.erl
+++ b/erts/emulator/test/crypto_reference.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2008-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/ddll_SUITE.erl b/erts/emulator/test/ddll_SUITE.erl
index 79047d7de5..6e15c228cd 100644
--- a/erts/emulator/test/ddll_SUITE.erl
+++ b/erts/emulator/test/ddll_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -30,7 +30,8 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 
--export([all/1, ddll_test/1, errors/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, ddll_test/1, errors/1,
 	 reference_count/1,
 	 kill_port/1, dont_kill_port/1]).
 -export([unload_on_process_exit/1, delayed_unload_with_ports/1, 
@@ -50,35 +51,39 @@
 
 -import(ordsets, [subtract/2]).
 
--include("test_server.hrl").
-
-all(suite) ->
-    [ddll_test, errors,
-     reference_count,
-     kill_port,
-     dont_kill_port,
-     properties,
-     load_and_unload,
-     unload_on_process_exit, 
-     delayed_unload_with_ports, 
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [ddll_test, errors, reference_count, kill_port,
+     dont_kill_port, properties, load_and_unload,
+     unload_on_process_exit, delayed_unload_with_ports,
      unload_due_to_process_exit,
-     no_unload_due_to_process_exit, 
-     no_unload_due_to_process_exit_2,
-     unload_reload_thingie, 
-     unload_reload_thingie_2,
-     unload_reload_thingie_3,
-     reload_pending,
-     load_fail_init,
-     reload_pending_fail_init,
-     reload_pending_kill,
-     more_error_codes, 
-     forced_port_killing, 
-     no_trap_exit_and_kill_ports,
-     monitor_demonitor, 
-     monitor_demonitor_load,
-     new_interface,
-     lock_driver
-    ].
+     no_unload_due_to_process_exit,
+     no_unload_due_to_process_exit_2, unload_reload_thingie,
+     unload_reload_thingie_2, unload_reload_thingie_3,
+     reload_pending, load_fail_init,
+     reload_pending_fail_init, reload_pending_kill,
+     more_error_codes, forced_port_killing,
+     no_trap_exit_and_kill_ports, monitor_demonitor,
+     monitor_demonitor_load, new_interface, lock_driver].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 unload_on_process_exit(suite) ->
     [];
diff --git a/erts/emulator/test/decode_packet_SUITE.erl b/erts/emulator/test/decode_packet_SUITE.erl
index 13f17e972c..c0499554eb 100644
--- a/erts/emulator/test/decode_packet_SUITE.erl
+++ b/erts/emulator/test/decode_packet_SUITE.erl
@@ -1,19 +1,19 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2008-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 2008-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
@@ -21,13 +21,33 @@
 
 -module(decode_packet_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 basic/1, packet_size/1, neg/1, http/1, line/1, ssl/1, otp_8536/1]).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
-	 basic/1, packet_size/1, neg/1, http/1, line/1, ssl/1]).
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [basic, packet_size, neg, http, line, ssl, otp_8536].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [basic, packet_size, neg, http, line, ssl].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Seed = {S1,S2,S3} = now(),
@@ -36,7 +56,7 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(1)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -304,6 +324,10 @@ http(Config) when is_list(Config) ->
 		   {ok, {http_request, 'GET', ResB, {1,1}}, Rest} = decode_pkt(http_bin,Bin) 
 	   end,
     lists:foreach(UriF, http_uri_variants()),
+
+    %% Response with empty phrase
+    ?line {ok,{http_response,{1,1},200,[]},<<>>} = decode_pkt(http, <<"HTTP/1.1 200\r\n">>, []),
+    ?line {ok,{http_response,{1,1},200,<<>>},<<>>} = decode_pkt(http_bin, <<"HTTP/1.1 200\r\n">>, []),
     ok.
 	     
 http_with_bin(http) ->
@@ -504,6 +528,27 @@ ssl(Config) when is_list(Config) ->
     F(v2hello),
     ok.
 
+otp_8536(doc) -> ["Corrupt sub-binary-strings from httph_bin"];
+otp_8536(Config) when is_list(Config) ->
+    lists:foreach(fun otp_8536_do/1, lists:seq(1,50)),
+    ok.
+
+otp_8536_do(N) ->
+    Data = <<"some data 123">>,
+    Letters = <<"bcdefghijklmnopqrstuvwxyzyxwvutsrqponmlkjihgfedcba">>,
+    <<HdrTail:N/binary,_/binary>> = Letters,
+    Hdr = <<$A, HdrTail/binary>>,
+    Bin = <<Hdr/binary, ": ", Data/binary, "\r\n\r\n">>,
+
+    io:format("Bin='~p'\n",[Bin]),
+    ?line {ok,{http_header,0,Hdr2,undefined,Data2},<<"\r\n">>} = decode_pkt(httph_bin, Bin,  []),
+
+    %% Do something to trash the C-stack, how about another decode_packet:
+    decode_pkt(httph_bin,<<Letters/binary, ": ", Data/binary, "\r\n\r\n">>, []),
+
+    %% Now check that we got the expected binaries
+    {Hdr, Data} = {Hdr2, Data2}.
+
 decode_pkt(Type,Bin) ->
     decode_pkt(Type,Bin,[]).		       
 decode_pkt(Type,Bin,Opts) ->
diff --git a/erts/emulator/test/dgawd_handler.erl b/erts/emulator/test/dgawd_handler.erl
index 881354b9da..27085b7b7e 100644
--- a/erts/emulator/test/dgawd_handler.erl
+++ b/erts/emulator/test/dgawd_handler.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 8f48d8a992..4bebae51cc 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -1,65 +1,92 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
 -module(distribution_SUITE).
+-compile(r12).
 
 %% Tests distribution and the tcp driver.
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,
-	 ping/1, bulk_send/1, bulk_send_small/1,
-	 bulk_send_big/1,
-	 local_send/1, local_send_small/1, local_send_big/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 ping/1, bulk_send_small/1,
+	 bulk_send_big/1, bulk_send_bigbig/1,
+	 local_send_small/1, local_send_big/1,
 	 local_send_legal/1, link_to_busy/1, exit_to_busy/1,
 	 lost_exit/1, link_to_dead/1, link_to_dead_new_node/1,
 	 applied_monitor_node/1, ref_port_roundtrip/1, nil_roundtrip/1,
-	 trap_bif/1, trap_bif_1/1, trap_bif_2/1, trap_bif_3/1,
-	 stop_dist/1, dist_auto_connect/1, 
+	 trap_bif_1/1, trap_bif_2/1, trap_bif_3/1,
+	 stop_dist/1, 
 	 dist_auto_connect_never/1, dist_auto_connect_once/1,
 	 dist_parallel_send/1,
 	 atom_roundtrip/1,
 	 atom_roundtrip_r12b/1,
 	 contended_atom_cache_entry/1,
-	 bad_dist_ext/1,
+	 bad_dist_structure/1,
 	 bad_dist_ext_receive/1,
 	 bad_dist_ext_process_info/1,
 	 bad_dist_ext_control/1,
 	 bad_dist_ext_connection_id/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 %% Internal exports.
 -export([sender/3, receiver2/2, dummy_waiter/0, dead_process/0,
 	 roundtrip/1, bounce/1, do_dist_auto_connect/1, inet_rpc_server/1,
 	 dist_parallel_sender/3, dist_parallel_receiver/0,
-	 dist_evil_parallel_receiver/0]).
-
-all(suite) -> [
-	       ping, bulk_send, local_send, link_to_busy, exit_to_busy,
-	       lost_exit, link_to_dead, link_to_dead_new_node,
-	       applied_monitor_node, ref_port_roundtrip, nil_roundtrip,
-	       stop_dist, trap_bif, dist_auto_connect, dist_parallel_send,
-	       atom_roundtrip, atom_roundtrip_r12b,
-	       contended_atom_cache_entry,
-	       bad_dist_ext
-	      ].
+	 dist_evil_parallel_receiver/0,
+         sendersender/4, sendersender2/4]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [ping, {group, bulk_send}, {group, local_send},
+     link_to_busy, exit_to_busy, lost_exit, link_to_dead,
+     link_to_dead_new_node, applied_monitor_node,
+     ref_port_roundtrip, nil_roundtrip, stop_dist,
+     {group, trap_bif}, {group, dist_auto_connect},
+     dist_parallel_send, atom_roundtrip, atom_roundtrip_r12b,
+     contended_atom_cache_entry, bad_dist_structure, {group, bad_dist_ext}].
+
+groups() -> 
+    [{bulk_send, [], [bulk_send_small, bulk_send_big, bulk_send_bigbig]},
+     {local_send, [],
+      [local_send_small, local_send_big, local_send_legal]},
+     {trap_bif, [], [trap_bif_1, trap_bif_2, trap_bif_3]},
+     {dist_auto_connect, [],
+      [dist_auto_connect_never, dist_auto_connect_once]},
+     {bad_dist_ext, [],
+      [bad_dist_ext_receive, bad_dist_ext_process_info,
+       bad_dist_ext_control, bad_dist_ext_connection_id]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 -define(DEFAULT_TIMETRAP, 4*60*1000).
 
@@ -67,7 +94,7 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?DEFAULT_TIMETRAP),
     [{watchdog, Dog},{testcase, Func}|Config].
 
-fin_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
+end_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -115,19 +142,15 @@ ping(Config) when is_list(Config) ->
 
     ok.
 
-bulk_send(doc) ->
- ["Tests sending large amount of data to another node and measure",
-  "the time. This tests that a process that is suspended on a ",
-  "busy port will eventually be resumed."];
-bulk_send(suite) ->
-    [bulk_send_small, bulk_send_big].
-
 bulk_send_small(Config) when is_list(Config) ->
     ?line bulk_send(64, 32).
 
 bulk_send_big(Config) when is_list(Config) ->
     ?line bulk_send(32, 64).
 
+bulk_send_bigbig(Config) when is_list(Config) ->
+    ?line bulk_sendsend(32*5, 4).
+
 bulk_send(Terms, BinSize) ->
     ?line Dog = test_server:timetrap(test_server:seconds(30)),
 
@@ -144,6 +167,53 @@ bulk_send(Terms, BinSize) ->
     ?line test_server:timetrap_cancel(Dog),
     {comment, integer_to_list(trunc(Size/1024/Elapsed+0.5)) ++ " K/s"}.
 
+bulk_sendsend(Terms, BinSize) ->
+    {Rate1, MonitorCount1} = bulk_sendsend2(Terms, BinSize,   5),
+    {Rate2, MonitorCount2} = bulk_sendsend2(Terms, BinSize, 995),
+    Ratio = if MonitorCount2 == 0 -> MonitorCount1 / 1.0;
+               true               -> MonitorCount1 / MonitorCount2
+            end,
+    %% A somewhat arbitrary ratio, but hopefully one that will accomodate
+    %% a wide range of CPU speeds.
+    true = (Ratio > 8.0),
+    {comment,
+     integer_to_list(Rate1) ++ " K/s, " ++
+     integer_to_list(Rate2) ++ " K/s, " ++
+     integer_to_list(MonitorCount1) ++ " monitor msgs, " ++
+     integer_to_list(MonitorCount2) ++ " monitor msgs, " ++
+     float_to_list(Ratio) ++ " monitor ratio"}.    
+
+bulk_sendsend2(Terms, BinSize, BusyBufSize) ->
+    ?line Dog = test_server:timetrap(test_server:seconds(30)),
+
+    ?line io:format("Sending ~w binaries, each of size ~w K",
+		    [Terms, BinSize]),
+    ?line {ok, NodeRecv} = start_node(bulk_receiver),
+    ?line Recv = spawn(NodeRecv, erlang, apply, [fun receiver/2, [0, 0]]),
+    ?line Bin = list_to_binary(lists:duplicate(BinSize*1024, 253)),
+    %%?line Size = Terms*size(Bin),
+
+    %% SLF LEFT OFF HERE.
+    %% When the caller uses small hunks, like 4k via
+    %% bulk_sendsend(32*5, 4), then (on my laptop at least), we get
+    %% zero monitor messages.  But if we use "+zdbbl 5", then we
+    %% get a lot of monitor messages.  So, if we can count up the
+    %% total number of monitor messages that we get when running both
+    %% default busy size and "+zdbbl 5", and if the 5 case gets
+    %% "many many more" monitor messages, then we know we're working.
+
+    ?line {ok, NodeSend} = start_node(bulk_sender, "+zdbbl " ++ integer_to_list(BusyBufSize)),
+    ?line _Send = spawn(NodeSend, erlang, apply, [fun sendersender/4, [self(), Recv, Bin, Terms]]),
+    ?line {Elapsed, {_TermsN, SizeN}, MonitorCount} =
+        receive {sendersender, BigRes} ->
+                BigRes
+        end,
+    ?line stop_node(NodeRecv),
+    ?line stop_node(NodeSend),
+
+    ?line test_server:timetrap_cancel(Dog),
+    {trunc(SizeN/1024/Elapsed+0.5), MonitorCount}.
+
 sender(To, _Bin, 0) ->
     To ! {done, self()},
     receive
@@ -154,6 +224,43 @@ sender(To, Bin, Left) ->
     To ! {term, Bin},
     sender(To, Bin, Left-1).
 
+%% Sender process to be run on a slave node
+
+sendersender(Parent, To, Bin, Left) ->
+    erlang:system_monitor(self(), [busy_dist_port]),
+    [spawn(fun() -> sendersender2(To, Bin, Left, false) end) ||
+        _ <- lists:seq(1,1)],
+    {USec, {Res, MonitorCount}} =
+        timer:tc(?MODULE, sendersender2, [To, Bin, Left, true]),
+    Parent ! {sendersender, {USec/1000000, Res, MonitorCount}}.
+
+sendersender2(To, Bin, Left, SendDone) ->
+    sendersender3(To, Bin, Left, SendDone, 0).
+
+sendersender3(To, _Bin, 0, SendDone, MonitorCount) ->
+    if SendDone ->
+            To ! {done, self()};
+       true ->
+            ok
+    end,
+    receive
+        {monitor, _Pid, _Type, _Info} ->
+            sendersender3(To, _Bin, 0, SendDone, MonitorCount + 1)
+    after 0 ->
+            if SendDone ->
+                    receive
+                        Any when is_tuple(Any), size(Any) == 2 ->
+                            {Any, MonitorCount}
+                    end;
+               true ->
+                    exit(normal)
+            end
+    end;
+sendersender3(To, Bin, Left, SendDone, MonitorCount) ->
+    To ! {term, Bin},
+    %%timer:sleep(50),
+    sendersender3(To, Bin, Left-1, SendDone, MonitorCount).
+
 %% Receiver process to be run on a slave node.
 
 receiver(Terms, Size) ->
@@ -165,17 +272,14 @@ receiver(Terms, Size) ->
     end.
 
 
-local_send(suite) ->
-    [local_send_small, local_send_big, local_send_legal];
-local_send(doc) ->
-    ["Tests sending small and big messages to a non-existing ",
-     "local registered process."].
 
 local_send_big(doc) ->
     ["Sends several big message to an non-registered process on ",
      "the local node."];
 local_send_big(Config) when is_list(Config) ->
-    Data0=local_send_big(doc)++local_send(doc),
+    Data0=local_send_big(doc)++
+	["Tests sending small and big messages to a non-existing ",
+        "local registered process."],
     Data1=[Data0,[Data0, Data0, [Data0], Data0],Data0],
     Data2=Data0++lists:flatten(Data1)++
 	list_to_binary(lists:flatten(Data1)),
@@ -432,7 +536,7 @@ sink1() ->
 
 lost_exit(doc) ->
     "Test that EXIT and DOWN messages send to another node are not lost if "
-	"if the distribution port is busy.";
+	"the distribution port is busy.";
 lost_exit(Config) when is_list(Config) ->
     ?line {ok, Node} = start_node(lost_exit),
 
@@ -661,9 +765,6 @@ stop_dist(Config) when is_list(Config) ->
 
     ok.
 
-trap_bif(doc) ->
-    ["Verifies that BIFs which are traps to Erlang work (OTP-2680)."];
-trap_bif(suite) -> [trap_bif_1, trap_bif_2, trap_bif_3].
 
 trap_bif_1(doc) ->
     [""];
@@ -700,10 +801,6 @@ tr3() ->
 
 
 
-dist_auto_connect(doc) ->
-    ["Tests the kernel parameter 'dist_auto_connect'."];
-dist_auto_connect(suite) ->
-    [dist_auto_connect_never, dist_auto_connect_once].
 
 % This has to be done by nodes with differrent cookies, otherwise global
 % will connect nodes, which is correct, but makes it hard to test.
@@ -1053,8 +1150,7 @@ contended_atom_cache_entry(Config) when is_list(Config) ->
     ?line {ok, SNode} = start_node(Config),
     ?line {ok, RNode} = start_node(Config),
     ?line Success = make_ref(),
-    ?line Mstr
-	= spawn_link(
+    ?line spawn_link(
 	    SNode,
 	    fun () ->
 		    erts_debug:set_internal_state(available_internal_state,
@@ -1111,13 +1207,13 @@ contended_atom_cache_entry(Config) when is_list(Config) ->
     ?line stop_node(RNode),
     ?line ok.
 
-send_ref_atom(To, Ref, Atom, 0) ->
+send_ref_atom(_To, _Ref, _Atom, 0) ->
     ok;
 send_ref_atom(To, Ref, Atom, N) ->
     To ! {Ref, Atom},
     send_ref_atom(To, Ref, Atom, N-1).
 
-receive_ref_atom(Ref, Atom, 0) ->
+receive_ref_atom(_Ref, _Atom, 0) ->
     ok;
 receive_ref_atom(Ref, Atom, N) ->
     receive
@@ -1152,7 +1248,7 @@ unwanted_cixs() ->
 	      nodes()).
     
     
-get_conflicting_atoms(CIX, 0) ->
+get_conflicting_atoms(_CIX, 0) ->
     [];
 get_conflicting_atoms(CIX, N) ->
     {A, B, C} = now(),
@@ -1166,13 +1262,187 @@ get_conflicting_atoms(CIX, N) ->
 	    get_conflicting_atoms(CIX, N)
     end.
 
+-define(COOKIE, '').
+-define(DOP_LINK,		1).
+-define(DOP_SEND,		2).
+-define(DOP_EXIT,		3).
+-define(DOP_UNLINK,		4).
+-define(DOP_REG_SEND,		6).
+-define(DOP_GROUP_LEADER,	7).
+-define(DOP_EXIT2,		8).
+
+-define(DOP_SEND_TT,		12).
+-define(DOP_EXIT_TT,		13).
+-define(DOP_REG_SEND_TT,	16).
+-define(DOP_EXIT2_TT,		18).
+
+-define(DOP_MONITOR_P,		19).
+-define(DOP_DEMONITOR_P,	20).
+-define(DOP_MONITOR_P_EXIT,	21).
+
+start_monitor(Offender,P) ->
+    ?line Parent = self(),
+    ?line Q = spawn(Offender,
+		    fun () -> 
+			    Ref = erlang:monitor(process,P),
+			    Parent ! {self(),ref,Ref},
+			    receive
+				just_stay_alive -> ok
+			    end
+		    end),
+    ?line Ref = receive
+		    {Q,ref,R} ->
+			R
+		after  5000 ->
+			error
+		end,
+    io:format("Ref is ~p~n",[Ref]),
+    ok.
+start_link(Offender,P) ->
+    ?line Parent = self(),
+    ?line Q = spawn(Offender,
+		    fun () ->
+			    process_flag(trap_exit,true),
+			    link(P),
+			    Parent ! {self(),ref,P},
+			    receive
+				just_stay_alive -> ok
+			    end
+		    end),
+    ?line Ref = receive
+		    {Q,ref,R} ->
+			R
+		after  5000 ->
+			error
+		end,
+    io:format("Ref is ~p~n",[Ref]),
+    ok.
+
+bad_dist_structure(suite) ->
+    [];
+bad_dist_structure(doc) ->
+    ["Test dist messages with valid structure (binary to term ok) but malformed"
+     "control content"];
+bad_dist_structure(Config) when is_list(Config) ->
+    %process_flag(trap_exit,true),
+    ODog = ?config(watchdog, Config),
+    ?t:timetrap_cancel(ODog),
+    Dog = ?t:timetrap(?t:seconds(15)),
+
+    ?line {ok, Offender} = start_node(bad_dist_structure_offender),
+    ?line {ok, Victim} = start_node(bad_dist_structure_victim),
+    ?line start_node_monitors([Offender,Victim]),
+    ?line Parent = self(),
+    ?line P = spawn(Victim,
+		    fun () ->
+			    process_flag(trap_exit,true),
+			    Parent ! {self(), started},
+			    receive check_msgs -> ok end,
+			    bad_dist_struct_check_msgs([one,
+							two]),
+			    Parent ! {self(), messages_checked},
+			    receive done -> ok end
+		    end),
+    ?line receive {P, started} -> ok end,
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line verify_up(Offender, Victim),
+    ?line true = lists:member(Offender, rpc:call(Victim, erlang, nodes, [])),
+    ?line start_monitor(Offender,P),
+    ?line P ! one,
+    ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_monitor(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal,normal},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_link(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_LINK},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_link(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_UNLINK,'replace'},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_link(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_UNLINK,'replace',make_ref()},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_link(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_UNLINK,make_ref(),P},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_link(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_UNLINK,normal,normal},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_monitor(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_monitor(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P,normal},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_monitor(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_monitor(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P,normal},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT,'replace',P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT,make_ref(),normal,normal},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT_TT,'replace',token,P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT_TT,make_ref(),token,normal,normal},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT2,'replace',P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT2,make_ref(),normal,normal},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT2_TT,'replace',token,P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT2_TT,make_ref(),token,normal,normal},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace'},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace','atomic'},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace',P},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name},2,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name,token},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace',''},2,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',P},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name,{token}},2,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_SEND_TT,'',P},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_SEND_TT,'',name,token},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_SEND,''},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_SEND,'',name},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_SEND,'',P,{token}},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line P ! two,
+    ?line P ! check_msgs,
+    ?line receive 
+	      {P, messages_checked} -> ok 
+	  after 5000 ->
+		  exit(victim_is_dead)
+	  end,
+
+    ?line {message_queue_len, 0}
+	= rpc:call(Victim, erlang, process_info, [P, message_queue_len]),
+    
+    ?line unlink(P),
+    ?line P ! done,
+    ?line stop_node(Offender),
+    ?line stop_node(Victim),
+    ?t:timetrap_cancel(Dog),
+    ok.
 
-bad_dist_ext(doc) -> [];
-bad_dist_ext(suite) ->
-    [bad_dist_ext_receive,
-     bad_dist_ext_process_info,
-     bad_dist_ext_control,
-     bad_dist_ext_connection_id].
 
 
 bad_dist_ext_receive(Config) when is_list(Config) ->
@@ -1393,6 +1663,22 @@ bad_dist_ext_connection_id(Config) when is_list(Config) ->
     ?line stop_node(Victim).
 
 
+bad_dist_struct_check_msgs([]) ->
+    receive
+	Msg ->
+	    exit({unexpected_message, Msg})
+    after 0 ->
+	    ok
+    end;
+bad_dist_struct_check_msgs([M|Ms]) ->
+    receive
+	{'EXIT',_,_} = EM ->
+	    io:format("Ignoring exit message: ~p~n",[EM]),
+	    bad_dist_struct_check_msgs([M|Ms]);
+	Msg ->
+	    M = Msg,
+	    bad_dist_struct_check_msgs(Ms)
+    end.
 bad_dist_ext_check_msgs([]) ->
     receive
 	Msg ->
@@ -1407,24 +1693,6 @@ bad_dist_ext_check_msgs([M|Ms]) ->
 	    bad_dist_ext_check_msgs(Ms)
     end.
     
--define(COOKIE, '').
--define(DOP_LINK,		1).
--define(DOP_SEND,		2).
--define(DOP_EXIT,		3).
--define(DOP_UNLINK,		4).
--define(DOP_NODE_LINK,		5).
--define(DOP_REG_SEND,		6).
--define(DOP_GROUP_LEADER,	7).
--define(DOP_EXIT2,		8).
-
--define(DOP_SEND_TT,		12).
--define(DOP_EXIT_TT,		13).
--define(DOP_REG_SEND_TT,	16).
--define(DOP_EXIT2_TT,		18).
-
--define(DOP_MONITOR_P,		19).
--define(DOP_DEMONITOR_P,	20).
--define(DOP_MONITOR_P_EXIT,	21).
 
 dport_reg_send(Node, Name, Msg) ->
     DPrt = case dport(Node) of
@@ -1456,6 +1724,39 @@ dport_send(To, Msg) ->
 				  ?COOKIE,
 				  To}),
 			dmsg_ext(Msg)]).
+send_bad_structure(Offender,Victim,Bad,WhereToPutSelf) ->
+    send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,[]).
+send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) ->
+    Parent = self(),
+    Done = make_ref(),
+    spawn(Offender,
+	  fun () ->
+		  Node = node(Victim),
+		  pong = net_adm:ping(Node),
+		  DPrt = dport(Node),
+		  Bad1 = case WhereToPutSelf of
+			     0 -> 
+				 Bad;
+			     N when N > 0 ->
+				 setelement(N,Bad,self())
+			 end,
+		  DData = [dmsg_hdr(),
+			   dmsg_ext(Bad1)] ++
+		      case PayLoad of
+			  [] -> [];
+			  _Other -> [dmsg_ext(PayLoad)]
+		      end,
+		  port_command(DPrt, DData),
+		  Parent ! {DData,Done}
+	  end),
+    receive 
+	{WhatSent,Done} -> 
+	    io:format("Offender sent ~p~n",[WhatSent]),
+	    ok
+    after 5000 ->
+	    exit(unable_to_send)
+    end.
+   
 
 %% send_bad_msgs():
 %% Send a valid distribution header and control message
@@ -1539,10 +1840,10 @@ dmsg_bad_hdr() ->
      255].  % 255 atom references
     
 
-dmsg_fake_hdr1() ->
-    A = <<"fake header atom 1">>,
-    [131, % Version Magic
-     $D, 1, 16#8, 0, size(A), A]. % Fake header
+%% dmsg_fake_hdr1() ->
+%%     A = <<"fake header atom 1">>,
+%%     [131, % Version Magic
+%%      $D, 1, 16#8, 0, size(A), A]. % Fake header
 
 dmsg_fake_hdr2() ->
     A1 = <<"fake header atom 1">>,
@@ -1727,7 +2028,7 @@ flush_node_changes() ->
 
 node_monitor_loop(Master) ->
     receive
-	{nodeup, Node, InfoList} = Msg ->
+	{nodeup, Node, _InfoList} = Msg ->
 	    Master ! {nodeup, node(), Node},
 	    ?t:format("~p ~p: ~p~n", [node(), erlang:now(), Msg]),
 	    node_monitor_loop(Master);
@@ -1764,9 +2065,9 @@ verify_no_down(A, B) ->
 	    ok
     end.
 
-verify_down(A, B) ->
-    receive {nodedown, A, B, _} -> ok end,
-    receive {nodedown, B, A, _} -> ok end.
+%% verify_down(A, B) ->
+%%     receive {nodedown, A, B, _} -> ok end,
+%%     receive {nodedown, B, A, _} -> ok end.
 
 verify_down(A, ReasonA, B, ReasonB) ->
     receive
@@ -1786,11 +2087,11 @@ from(H, [H | T]) -> T;
 from(H, [_ | T]) -> from(H, T);
 from(_, []) -> [].
 
-fun_spawn(Fun) ->
-    fun_spawn(Fun, []).
+%% fun_spawn(Fun) ->
+%%     fun_spawn(Fun, []).
 
-fun_spawn(Fun, Args) ->
-    spawn_link(erlang, apply, [Fun, Args]).
+%% fun_spawn(Fun, Args) ->
+%%     spawn_link(erlang, apply, [Fun, Args]).
 
 
 long_or_short() -> 
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 39b2ed395f..f6cf01ce16 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -27,18 +27,18 @@
 %%% - queueing
 
 -module(driver_SUITE).
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, 
+	 end_per_suite/1, init_per_group/2,end_per_group/2,
 	 init_per_testcase/2,
-	 fin_per_testcase/2,
-	 end_per_suite/1,
+	 end_per_testcase/2,
 	 outputv_echo/1,
-	 timer/1,
+	
 	 timer_measure/1,
 	 timer_cancel/1,
 	 timer_change/1,
 	 timer_delay/1,
 	 queue_echo/1,
-	 fun_to_port/1,
+	 outputv_errors/1,
 	 driver_unloaded/1,
 	 io_ready_exit/1,
 	 use_fallback_pollset/1,
@@ -51,7 +51,7 @@
 	 'driver_system_info_ver1.1'/1,
 	 driver_system_info_current_ver/1,
 	 driver_monitor/1,
-	 ioq_exit/1,
+	
 	 ioq_exit_ready_input/1,
 	 ioq_exit_ready_output/1,
 	 ioq_exit_timeout/1,
@@ -74,11 +74,12 @@
 	 missing_callbacks/1,
 	 smp_select/1,
 	 driver_select_use/1,
-	 thread_mseg_alloc_cache_clean/1]).
+	 thread_mseg_alloc_cache_clean/1,
+	 otp_9302/1]).
 
 -export([bin_prefix/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 
 % First byte in communication with the timer driver
@@ -120,80 +121,135 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) ->
     ?line 0 = erts_debug:get_internal_state(check_io_debug),
     [{watchdog, Dog},{testcase, Case}|Config].
 
-fin_per_testcase(Case, Config) ->
+end_per_testcase(Case, Config) ->
     Dog = ?config(watchdog, Config),
-    erlang:display({fin_per_testcase, Case}),
+    erlang:display({end_per_testcase, Case}),
     ?line 0 = erts_debug:get_internal_state(check_io_debug),
     ?t:timetrap_cancel(Dog).
 
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [outputv_errors, outputv_echo, queue_echo, {group, timer},
+     driver_unloaded, io_ready_exit, use_fallback_pollset,
+     bad_fd_in_pollset, driver_event, fd_change,
+     steal_control, otp_6602, 'driver_system_info_ver1.0',
+     'driver_system_info_ver1.1',
+     driver_system_info_current_ver, driver_monitor,
+     {group, ioq_exit}, zero_extended_marker_garb_drv,
+     invalid_extended_marker_drv, larger_major_vsn_drv,
+     larger_minor_vsn_drv, smaller_major_vsn_drv,
+     smaller_minor_vsn_drv, peek_non_existing_queue,
+     otp_6879, caller, many_events, missing_callbacks,
+     smp_select, driver_select_use,
+     thread_mseg_alloc_cache_clean,
+     otp_9302].
+
+groups() -> 
+    [{timer, [],
+      [timer_measure, timer_cancel, timer_delay,
+       timer_change]},
+     {ioq_exit, [],
+      [ioq_exit_ready_input, ioq_exit_ready_output,
+       ioq_exit_timeout, ioq_exit_ready_async, ioq_exit_event,
+       ioq_exit_ready_input_async, ioq_exit_ready_output_async,
+       ioq_exit_timeout_async, ioq_exit_event_async]}].
+
+init_per_suite(Config) ->
+    Config.
+
 end_per_suite(_Config) ->
     catch erts_debug:set_internal_state(available_internal_state, false).
 
-all(suite) ->
-    [
-     fun_to_port,
-     outputv_echo,
-     queue_echo,
-     timer,
-     driver_unloaded,
-     io_ready_exit,
-     use_fallback_pollset,
-     bad_fd_in_pollset,
-     driver_event,
-     fd_change,
-     steal_control,
-     otp_6602,
-     'driver_system_info_ver1.0',
-     'driver_system_info_ver1.1',
-     driver_system_info_current_ver,
-     driver_monitor,
-     ioq_exit,
-     zero_extended_marker_garb_drv,
-     invalid_extended_marker_drv,
-     larger_major_vsn_drv,
-     larger_minor_vsn_drv,
-     smaller_major_vsn_drv,
-     smaller_minor_vsn_drv,
-     peek_non_existing_queue,
-     otp_6879,
-     caller,
-     many_events,
-     missing_callbacks,
-     smp_select,
-     driver_select_use,
-     thread_mseg_alloc_cache_clean
-    ].
-
-fun_to_port(doc) -> "Test sending a fun to port with an outputv-capable driver.";
-fun_to_port(Config) when is_list(Config) ->
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+outputv_errors(doc) -> "Test sending bad types to port with an outputv-capable driver.";
+outputv_errors(Config) when is_list(Config) ->
     ?line Path = ?config(data_dir, Config),
     ?line erl_ddll:start(),
     ?line ok = load_driver(Path, outputv_drv),
 
-    ?line fun_to_port_1(fun() -> 33 end),
-    ?line fun_to_port_1([fun() -> 42 end]),
-    ?line fun_to_port_1([1|fun() -> 42 end]),
-    L = build_io_list(65536),
-    ?line fun_to_port_1([L,fun() -> 42 end]),
-    ?line fun_to_port_1([L|fun() -> 42 end]),
+    outputv_bad_types(fun(T) ->
+			      ?line outputv_errors_1(T),
+			      ?line outputv_errors_1([1|T]),
+			      ?line L = [1,2,3],
+			      ?line outputv_errors_1([L,T]),
+			      ?line outputv_errors_1([L|T])
+		      end),
+    outputv_errors_1(42),
+
+    %% Test iolists that do not fit in the address space.
+    %% Unfortunately, it would be too slow to test in a 64-bit emulator.
+    case erlang:system_info(wordsize) of
+	4 -> outputv_huge_iolists();
+	_ -> ok
+    end.
+
+outputv_bad_types(Test) ->
+    Types = [-1,256,atom,42.0,{a,b,c},make_ref(),fun() -> 42 end,
+	     [1|2],<<1:1>>,<<1:9>>,<<1:15>>],
+    _ = [Test(Type) || Type <- Types],
+    ok.
+
+outputv_huge_iolists() ->
+    FourGigs = 1 bsl 32,
+    ?line Sizes = [FourGigs+N || N <- lists:seq(0, 64)] ++
+	[1 bsl N || N <- lists:seq(33, 37)],
+    ?line Base = <<0:(1 bsl 20)/unit:8>>,
+    [begin
+	 ?line L = build_iolist(Sz, Base),
+	 ?line outputv_errors_1(L)
+     end || Sz <- Sizes],
     ok.
 
-fun_to_port_1(Term) ->
-    Port = open_port({spawn,outputv_drv}, []),
+outputv_errors_1(Term) ->
+    Port = open_port({spawn_driver,outputv_drv}, []),
     {'EXIT',{badarg,_}} = (catch port_command(Port, Term)),
     port_close(Port).
 
-build_io_list(0) -> [];
-build_io_list(1) -> [7];
-build_io_list(N) ->
-    L = build_io_list(N div 2),
+build_iolist(N, Base) when N < 16 ->
+    case random:uniform(3) of
+	1 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    Bin;
+	_ ->
+	    lists:seq(1, N)
+    end;
+build_iolist(N, Base) when N =< byte_size(Base) ->
+    case random:uniform(3) of
+	1 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    Bin;
+	2 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    [Bin];
+	3 ->
+	    case N rem 2 of
+		0 ->
+		    L = build_iolist(N div 2, Base),
+		    [L,L];
+		1 ->
+		    L = build_iolist(N div 2, Base),
+		    [L,L,45]
+	    end
+    end;
+build_iolist(N0, Base) ->
+    Small = random:uniform(15),
+    Seq = lists:seq(1, Small),
+    N = N0 - Small,
     case N rem 2 of
-	0 -> [L|L];
-	1 -> [7,L|L]
+	0 ->
+	    L = build_iolist(N div 2, Base),
+	    [L,L|Seq];
+	1 ->
+	    L = build_iolist(N div 2, Base),
+	    [47,L,L|Seq]
     end.
 
-
-
 outputv_echo(doc) -> ["Test echoing data with a driver that supports outputv."];
 outputv_echo(Config) when is_list(Config) ->
     ?line Dog = test_server:timetrap(test_server:minutes(10)),
@@ -308,7 +364,6 @@ compare(Got, Expected) ->
 %% 		Driver timer test suites
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-timer(suite) -> [timer_measure,timer_cancel,timer_delay,timer_change].
 
 timer_measure(doc) -> ["Check that timers time out in good time."];
 timer_measure(Config) when is_list(Config) ->
@@ -1299,17 +1354,6 @@ driver_monitor(Config) when is_list(Config) ->
     ?line stop_driver(Port, Name),
     ?line ok.
 
-ioq_exit(doc) -> [];
-ioq_exit(suite) ->
-    [ioq_exit_ready_input,
-     ioq_exit_ready_output,
-     ioq_exit_timeout,
-     ioq_exit_ready_async,
-     ioq_exit_event,
-     ioq_exit_ready_input_async,
-     ioq_exit_ready_output_async,
-     ioq_exit_timeout_async,
-     ioq_exit_event_async].
 
 -define(IOQ_EXIT_READY_INPUT, 1).
 -define(IOQ_EXIT_READY_OUTPUT, 2).
@@ -1682,7 +1726,7 @@ smp_select0(Config) ->
     ProcFun = fun()-> io:format("Worker ~p starting\n",[self()]),	
 		      ?line Port = open_port({spawn, DrvName}, []),
 		      smp_select_loop(Port, 100000),
-		      sleep(500), % wait for driver to handle pending events
+		      sleep(1000), % wait for driver to handle pending events
 		      ?line true = erlang:port_close(Port),
 		      Master ! {ok,self()},
 		      io:format("Worker ~p finished\n",[self()])
@@ -1790,8 +1834,8 @@ mseg_alloc_ccc() ->
     mseg_alloc_ccc(erlang:system_info({allocator,mseg_alloc})).
 
 mseg_alloc_ccc(MsegAllocInfo) ->
-    ?line {value,{calls, CL}}
-	= lists:keysearch(calls, 1, MsegAllocInfo),
+    ?line {value,{memkind, MKL}} = lists:keysearch(memkind,1,MsegAllocInfo),
+    ?line {value,{calls, CL}} = lists:keysearch(calls, 1, MKL),
     ?line {value,{mseg_check_cache, GigaCCC, CCC}}
 	= lists:keysearch(mseg_check_cache, 1, CL),
     ?line GigaCCC*1000000000 + CCC.
@@ -1800,12 +1844,28 @@ mseg_alloc_cached_segments() ->
     mseg_alloc_cached_segments(erlang:system_info({allocator,mseg_alloc})).
 
 mseg_alloc_cached_segments(MsegAllocInfo) ->
+    MemName = case is_halfword_vm() of
+	true -> "high memory";
+	false -> "all memory"
+    end,
+    ?line [{memkind,DrvMem}]
+	= lists:filter(fun(E) -> case E of
+				    {memkind, [{name, MemName} | _]} -> true;
+				    _ -> false
+		       end end, MsegAllocInfo),
     ?line {value,{status, SL}}
-	= lists:keysearch(status, 1, MsegAllocInfo),
+	= lists:keysearch(status, 1, DrvMem),
     ?line {value,{cached_segments, CS}}
 	= lists:keysearch(cached_segments, 1, SL),
     ?line CS.
 
+is_halfword_vm() ->
+    case {erlang:system_info({wordsize, internal}),
+	  erlang:system_info({wordsize, external})} of
+	{4, 8} -> true;
+	{WS, WS} -> false
+    end.
+
 driver_alloc_sbct() ->
     {_, _, _, As} = erlang:system_info(allocator),
     case lists:keysearch(driver_alloc, 1, As) of
@@ -1832,13 +1892,39 @@ thread_mseg_alloc_cache_clean_test(Port, N, CCI, Size) ->
     ?line ?t:format("CCC = ~p~n", [CCC]),
     ?line true = CCC > OCCC,
     ?line thread_mseg_alloc_cache_clean_test(Port, N-1, CCI, Size).
-    
-    
+
+otp_9302(Config) when is_list(Config) ->
+    ?line Path = ?config(data_dir, Config),
+    ?line erl_ddll:start(),
+    ?line ok = load_driver(Path, otp_9302_drv),
+    ?line Port = open_port({spawn, otp_9302_drv}, []),
+    ?line true = is_port(Port),
+    ?line port_command(Port, ""),
+    ?line {msg, block} = get_port_msg(Port, infinity),
+    ?line {msg, job} = get_port_msg(Port, infinity),
+    ?line case erlang:system_info(thread_pool_size) of
+	      0 ->
+		  {msg, cancel} = get_port_msg(Port, infinity);
+	      _ ->
+		  ok
+	  end,
+    ?line {msg, job} = get_port_msg(Port, infinity),
+    ?line {msg, end_of_jobs} = get_port_msg(Port, infinity),
+    ?line no_msg = get_port_msg(Port, 2000),
+    ?line port_close(Port),
+    ?line ok.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% 		Utilities
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    
+
+get_port_msg(Port, Timeout) ->
+    receive
+	{Port, What} ->
+	    {msg, What}
+    after Timeout ->
+	    no_msg
+    end.
 
 wait_until(Fun) ->
     case Fun() of
diff --git a/erts/emulator/test/driver_SUITE_data/Makefile.src b/erts/emulator/test/driver_SUITE_data/Makefile.src
index 4ac7987d2f..5b3ba1557e 100644
--- a/erts/emulator/test/driver_SUITE_data/Makefile.src
+++ b/erts/emulator/test/driver_SUITE_data/Makefile.src
@@ -11,7 +11,8 @@ MISC_DRVS =		outputv_drv@dll@ \
 			caller_drv@dll@ \
 			many_events_drv@dll@ \
 			missing_callback_drv@dll@ \
-			thr_alloc_drv@dll@
+			thr_alloc_drv@dll@ \
+			otp_9302_drv@dll@
 
 SYS_INFO_DRVS = 	sys_info_1_0_drv@dll@ \
 			sys_info_1_1_drv@dll@ \
diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
index 9e1e5e72c2..bbdb09cfcb 100644
--- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
@@ -17,7 +17,7 @@
  */
 
 #ifndef UNIX
-#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(__WIN32__) && !defined(VXWORKS)
 #define UNIX 1
 #endif
 #endif
@@ -102,6 +102,7 @@ typedef struct chkio_smp_select {
     int write_fd;
     int next_read;
     int next_write;
+    int first_write;
     enum {Closed, Opened, Selected, Waiting} state;
     int wasSelected;
     unsigned rand_state;
@@ -577,9 +578,16 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event)
 	inPipe = (pip->next_write - pip->next_read);
 	if (inPipe == 0) {
 	    bytes = read(pip->read_fd, &word, sizeof(word));
-	    printf("Unexpected empty pipe, expected %u -> %u, bytes=%d, word=%d\n",
-		   pip->next_read, pip->next_write-1, bytes, word);
-	    abort();
+	    printf("Unexpected empty pipe, expected %u -> %u, bytes=%d, word=%d, written=%d\n",
+		   pip->next_read, pip->next_write-1, bytes, word,
+		   (pip->next_write - pip->first_write));
+	    /*abort();
+    	      Allow unexpected events as it's been seen to be triggered by epoll
+	      on Linux. Most of the time the unwanted events are filtered by
+	      the erl_check_io layer. But when fd's are reused the events may
+	      slip up to the driver.
+	    */
+	    break; 
 	}
 
 	n = rand_r(&pip->rand_state) % (inPipe*4);
@@ -1252,6 +1260,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		pip->state = Opened;
 		pip->wasSelected = 0;
 		pip->next_write = pip->next_read = rand_r(&pip->rand_state) % 1024;
+		pip->first_write = pip->next_write;
 		if (op & 1) break;
 		op >>= 1;
 	    }/*fall through*/
diff --git a/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c b/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c
index 25d4b17001..6afa46b3a2 100644
--- a/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c
@@ -17,7 +17,7 @@
  */
 
 #ifndef UNIX
-#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(__WIN32__) && !defined(VXWORKS)
 #define UNIX 1
 #endif
 #endif
diff --git a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
index c7a42aa687..e49de388b4 100644
--- a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
@@ -29,7 +29,7 @@
  */
 
 #ifndef UNIX
-#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(__WIN32__) && !defined(VXWORKS)
 #define UNIX 1
 #endif
 #endif
diff --git a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c
index c80e492e3f..e7d9a294fa 100644
--- a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c
@@ -17,7 +17,7 @@
  */
 
 #ifndef UNIX
-#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(__WIN32__) && !defined(VXWORKS)
 #define UNIX 1
 #endif
 #endif
diff --git a/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c b/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c
new file mode 100644
index 0000000000..beee1b735f
--- /dev/null
+++ b/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c
@@ -0,0 +1,232 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+#ifdef __WIN32__
+#include <windows.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include "erl_driver.h"
+
+static void stop(ErlDrvData drv_data);
+static ErlDrvData start(ErlDrvPort port,
+			char *command);
+static void output(ErlDrvData drv_data,
+		   char *buf, int len);
+static void ready_async(ErlDrvData drv_data,
+			ErlDrvThreadData thread_data);
+
+static ErlDrvEntry otp_9302_drv_entry = { 
+    NULL /* init */,
+    start,
+    stop,
+    output,
+    NULL /* ready_input */,
+    NULL /* ready_output */,
+    "otp_9302_drv",
+    NULL /* finish */,
+    NULL /* handle */,
+    NULL /* control */,
+    NULL /* timeout */,
+    NULL /* outputv */,
+    ready_async,
+    NULL /* flush */,
+    NULL /* call */,
+    NULL /* event */,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    ERL_DRV_FLAG_USE_PORT_LOCKING,
+    NULL /* handle2 */,
+    NULL /* handle_monitor */
+};
+
+typedef struct Otp9302AsyncData_ Otp9302AsyncData;
+
+typedef struct {
+    ErlDrvMutex *mtx;
+    Otp9302AsyncData *start;
+    Otp9302AsyncData *end;
+} Otp9302MsgQ;
+
+typedef struct {
+    ErlDrvPort port;
+    int smp;
+    Otp9302MsgQ msgq;
+} Otp9302Data;
+
+struct Otp9302AsyncData_ {
+    Otp9302AsyncData *next;
+    ErlDrvPort port;
+    int smp;
+    int refc;
+    int block;
+    struct {
+	ErlDrvTermData port;
+	ErlDrvTermData receiver;
+	ErlDrvTermData msg;
+    } term_data;
+    Otp9302MsgQ *msgq;
+};
+
+
+DRIVER_INIT(otp_9302_drv)
+{
+    return &otp_9302_drv_entry;
+}
+
+static void stop(ErlDrvData drv_data)
+{
+    Otp9302Data *data = (Otp9302Data *) drv_data;
+    if (!data->smp)
+	erl_drv_mutex_destroy(data->msgq.mtx);
+    driver_free(data);
+}
+
+static ErlDrvData start(ErlDrvPort port,
+			char *command)
+{
+    Otp9302Data *data;
+    ErlDrvSysInfo sys_info;
+
+    data = driver_alloc(sizeof(Otp9302Data));
+    if (!data)
+	return ERL_DRV_ERROR_GENERAL;
+
+    data->port = port;
+
+    driver_system_info(&sys_info, sizeof(ErlDrvSysInfo));
+    data->smp = sys_info.smp_support;
+
+    if (!data->smp) {
+	data->msgq.start = NULL;
+	data->msgq.end = NULL;
+	data->msgq.mtx = erl_drv_mutex_create("");
+	if (!data->msgq.mtx) {
+	    driver_free(data);
+	    return ERL_DRV_ERROR_GENERAL;
+	}
+    }
+
+    return (ErlDrvData) data;
+}
+
+static void send_reply(Otp9302AsyncData *adata)
+{
+    ErlDrvTermData spec[] = {
+	ERL_DRV_PORT, adata->term_data.port,
+	ERL_DRV_ATOM, adata->term_data.msg,
+	ERL_DRV_TUPLE, 2
+    };
+    driver_send_term(adata->port, adata->term_data.receiver,
+		     spec, sizeof(spec)/sizeof(spec[0]));
+}
+
+static void enqueue_reply(Otp9302AsyncData *adata)
+{
+    Otp9302MsgQ *msgq = adata->msgq;
+    adata->next = NULL;
+    adata->refc++;
+    erl_drv_mutex_lock(msgq->mtx);
+    if (msgq->end)
+	msgq->end->next = adata;
+    else
+	msgq->end = msgq->start = adata;
+    msgq->end = adata;
+    erl_drv_mutex_unlock(msgq->mtx);
+}
+
+static void dequeue_replies(Otp9302AsyncData *adata)
+{
+    Otp9302MsgQ *msgq = adata->msgq;
+    erl_drv_mutex_lock(msgq->mtx);
+    if (--adata->refc == 0)
+	driver_free(adata);
+    while (msgq->start) {
+	send_reply(msgq->start);
+	adata = msgq->start;
+	msgq->start = msgq->start->next;
+	if (--adata->refc == 0)
+	    driver_free(adata);
+    }
+    msgq->start = msgq->end = NULL;
+    erl_drv_mutex_unlock(msgq->mtx);
+}
+
+static void async_invoke(void *data)
+{
+    Otp9302AsyncData *adata = (Otp9302AsyncData *) data;
+    if (adata->block) {
+#ifdef __WIN32__
+	Sleep((DWORD) 2000);
+#else
+	sleep(2);
+#endif
+    }
+    if (adata->smp)
+	send_reply(adata);
+    else
+	enqueue_reply(adata);
+}
+
+static void ready_async(ErlDrvData drv_data,
+			ErlDrvThreadData thread_data)
+{
+    Otp9302AsyncData *adata = (Otp9302AsyncData *) thread_data;
+    if (adata->smp)
+	driver_free(adata);
+    else
+	dequeue_replies(adata);
+}
+
+static void output(ErlDrvData drv_data,
+		   char *buf, int len)
+{
+    Otp9302Data *data = (Otp9302Data *) drv_data;
+    ErlDrvTermData td_port = driver_mk_port(data->port);
+    ErlDrvTermData td_receiver = driver_caller(data->port);
+    ErlDrvTermData td_job = driver_mk_atom("job");
+    unsigned int key = (unsigned int) data->port;
+    long id[5];
+    Otp9302AsyncData *ad[5];
+    int i;
+
+    for (i = 0; i < sizeof(ad)/sizeof(ad[0]); i++) {
+	ad[i] = driver_alloc(sizeof(Otp9302AsyncData));
+	if (!ad[i])
+	    abort();
+
+	ad[i]->smp = data->smp;
+	ad[i]->port = data->port;
+	ad[i]->block = 0;
+	ad[i]->refc = 1;
+	ad[i]->term_data.port = td_port;
+	ad[i]->term_data.receiver = td_receiver;
+	ad[i]->term_data.msg = td_job;
+	ad[i]->msgq = &data->msgq;
+    }
+    ad[0]->block = 1;
+    ad[0]->term_data.msg = driver_mk_atom("block");
+    ad[2]->term_data.msg = driver_mk_atom("cancel");
+    ad[4]->term_data.msg = driver_mk_atom("end_of_jobs");
+    for (i = 0; i < sizeof(id)/sizeof(id[0]); i++)
+	id[i] = driver_async(data->port, &key, async_invoke, ad[i], driver_free);
+    if (id[2] > 0)
+	driver_async_cancel(id[2]);
+}
diff --git a/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c b/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c
index f429a5b51e..3a5b5af13a 100644
--- a/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c
@@ -28,7 +28,7 @@
  */
 
 #ifndef UNIX
-#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(__WIN32__) && !defined(VXWORKS)
 #define UNIX 1
 #endif
 #endif
diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl
index 1d66b6ef70..9ac004200e 100644
--- a/erts/emulator/test/efile_SUITE.erl
+++ b/erts/emulator/test/efile_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -17,12 +17,32 @@
 %% %CopyrightEnd%
 
 -module(efile_SUITE).
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 -export([iter_max_files/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [iter_max_files].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) -> [iter_max_files].
 
 %%
 %% Open as many files as possible. Do this several times and check 
diff --git a/erts/emulator/test/emulator.spec b/erts/emulator/test/emulator.spec
index ed5bd48e84..1ea751cc3b 100644
--- a/erts/emulator/test/emulator.spec
+++ b/erts/emulator/test/emulator.spec
@@ -1 +1 @@
-{topcase, {dir, "../emulator_test"}}.
+{suites,"../emulator_test",all}.
diff --git a/erts/emulator/test/erl_drv_thread_SUITE.erl b/erts/emulator/test/erl_drv_thread_SUITE.erl
index ea618e9feb..84a82cced0 100644
--- a/erts/emulator/test/erl_drv_thread_SUITE.erl
+++ b/erts/emulator/test/erl_drv_thread_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,18 +19,36 @@
 
 -module(erl_drv_thread_SUITE).
 -author('[email protected]').
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 -export([basic/1, rwlock/1, tsd/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(DEFAULT_TIMETRAP_SECS, 240).
 
-all(doc) -> [];
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [basic, rwlock, tsd].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%                                                                        %%
 %% Testcases                                                              %%
diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl
index 542c8dffbe..435c0872e6 100644
--- a/erts/emulator/test/erl_link_SUITE.erl
+++ b/erts/emulator/test/erl_link_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2001-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2001-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -28,9 +28,10 @@
 -author('[email protected]').
 
 %-define(line_trace, 1).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 % Test cases
 -export([links/1,
@@ -46,7 +47,7 @@
 	 otp_5772_dist_monitor/1,
 	 otp_7946/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 % Internal exports
 -export([test_proc/0]).
@@ -77,11 +78,29 @@
 
 
 
-all(suite) -> [links, dist_links, monitor_nodes, process_monitors,
-	       dist_process_monitors, busy_dist_port_monitor,
-	       busy_dist_port_link, otp_5772_link, otp_5772_dist_link,
-	       otp_5772_monitor, otp_5772_dist_monitor,
-	       otp_7946].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [links, dist_links, monitor_nodes, process_monitors,
+     dist_process_monitors, busy_dist_port_monitor,
+     busy_dist_port_link, otp_5772_link, otp_5772_dist_link,
+     otp_5772_monitor, otp_5772_dist_monitor, otp_7946].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    catch erts_debug:set_internal_state(available_internal_state, false).
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 links(doc) -> ["Tests node local links"];
 links(suite) -> [];
@@ -678,13 +697,10 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     end,
     ?line [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     ?line Dog = ?config(watchdog, Config),
     ?line ?t:timetrap_cancel(Dog).
 
-end_per_suite(_Config) ->
-    catch erts_debug:set_internal_state(available_internal_state, false).
-
 tp_call(Tp, Fun) ->
     ?line R = make_ref(),
     ?line Tp ! {call, self(), R, Fun},
@@ -1050,7 +1066,6 @@ stop_node(Node) ->
 -define(DOP_SEND,		2).
 -define(DOP_EXIT,		3).
 -define(DOP_UNLINK,		4).
--define(DOP_NODE_LINK,		5).
 -define(DOP_REG_SEND,		6).
 -define(DOP_GROUP_LEADER,	7).
 -define(DOP_EXIT2,		8).
diff --git a/erts/emulator/test/erts_debug_SUITE.erl b/erts/emulator/test/erts_debug_SUITE.erl
index e60a999df1..4dc2fbaae2 100644
--- a/erts/emulator/test/erts_debug_SUITE.erl
+++ b/erts/emulator/test/erts_debug_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2005-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,19 +18,40 @@
 %%
 
 -module(erts_debug_SUITE).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
-	 flat_size/1,flat_size_big/1,df/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 flat_size/1,flat_size_big/1,df/1,
+	 instructions/1]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [flat_size, flat_size_big, df, instructions].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [flat_size,flat_size_big,df].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(2)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -70,3 +91,8 @@ df(Config) when is_list(Config) ->
 
 pps() ->
     {erlang:ports()}.
+
+instructions(Config) when is_list(Config) ->
+    ?line Is = erts_debug:instructions(),
+    ?line _ = [list_to_atom(I) || I <- Is],
+    ok.
diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl
index 7fb92faf0d..2417d4bcfe 100644
--- a/erts/emulator/test/estone_SUITE.erl
+++ b/erts/emulator/test/estone_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,8 +18,9 @@
 
 -module(estone_SUITE).
 %% Test functions
--export([all/1,estone/1]).
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,estone/1]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 %% Internal exports for EStone tests
 -export([lists/1,
@@ -30,7 +31,7 @@
 	 trav/1,
 	 port_io/1,
 	 large_dataset_work/1,
-	 large_local_dataset_work/1,mk_big_procs/1,big_proc/0,
+	 large_local_dataset_work/1,mk_big_procs/1,big_proc/0, very_big/1,
 	 alloc/1,
 	 bif_dispatch/1,
 	 binary_h/1,echo/1,
@@ -44,7 +45,7 @@
 	 run_micro/3,p1/1,ppp/3,macro/2,micros/0]).
 
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %% Test suite defines
 -define(default_timeout, ?t:minutes(10)).
@@ -68,12 +69,31 @@
 init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(?default_timeout),
     [{watchdog, Dog}|Config].
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
 
-all(suite) -> [estone].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [estone].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 estone(suite) ->
     [];
diff --git a/erts/emulator/test/evil_SUITE.erl b/erts/emulator/test/evil_SUITE.erl
index a8288584f4..f982b9d4ff 100644
--- a/erts/emulator/test/evil_SUITE.erl
+++ b/erts/emulator/test/evil_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,7 +18,9 @@
 
 -module(evil_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 heap_frag/1,
 	 encode_decode_ext/1,
 	 decode_integer_ext/1,
@@ -30,26 +32,37 @@
 	 decode_pos_neg_zero/1
 	]).
 
--include("test_server.hrl").
-
-all(suite) ->
-    [
-     heap_frag,
-     encode_decode_ext,
-     decode_integer_ext,
-     decode_small_big_ext,
-     decode_large_big_ext,
-     decode_small_big_ext_neg,
-     decode_large_big_ext_neg,
-     decode_too_small,
-     decode_pos_neg_zero
-    ].
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [heap_frag, encode_decode_ext, decode_integer_ext,
+     decode_small_big_ext, decode_large_big_ext,
+     decode_small_big_ext_neg, decode_large_big_ext_neg,
+     decode_too_small, decode_pos_neg_zero].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(_Case, Config) ->
     ?line Dog = test_server:timetrap(?t:minutes(0.5)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index f1e6e004ad..9d6fc9521d 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,20 +19,40 @@
 
 -module(exception_SUITE).
 
--export([all/1, badmatch/1, pending_errors/1, nil_arith/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 badmatch/1, pending_errors/1, nil_arith/1,
          stacktrace/1, nested_stacktrace/1, raise/1, gunilla/1, per/1,
 	 exception_with_heap_frag/1]).
 
 -export([bad_guy/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -import(lists, [foreach/2]).
 
-all(suite) ->
-    [badmatch, pending_errors, nil_arith, 
-     stacktrace, nested_stacktrace, raise, gunilla, per,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [badmatch, pending_errors, nil_arith, stacktrace,
+     nested_stacktrace, raise, gunilla, per,
      exception_with_heap_frag].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 -define(try_match(E),
 	catch ?MODULE:bar(),
 	{'EXIT', {{badmatch, nomatch}, _}} = (catch E = id(nomatch))).
@@ -255,7 +275,16 @@ stacktrace(Conf) when is_list(Conf) ->
     ?line [{?MODULE,stacktrace_1,3}|_] = erase(stacktrace1),
     ?line St4 = erase(stacktrace2),
     ?line St4 = erlang:get_stacktrace(),
-    ok.
+
+    try
+	?line stacktrace_2()
+    catch
+	error:{badmatch,_} ->
+	    [{?MODULE,stacktrace_2,0},
+	     {?MODULE,stacktrace,1}|_] =
+		erlang:get_stacktrace(),
+	    ok
+    end.
 
 stacktrace_1(X, C1, Y) ->
     erase(stacktrace1),
@@ -275,6 +304,9 @@ stacktrace_1(X, C1, Y) ->
         put(stacktrace2, erlang:get_stacktrace())
     end.
 
+stacktrace_2() ->
+    ok = erlang:process_info(self(), current_function),
+    ok.
 
 
 nested_stacktrace(Conf) when is_list(Conf) ->
diff --git a/erts/emulator/test/float_SUITE.erl b/erts/emulator/test/float_SUITE.erl
index 102e472ea6..736510339f 100644
--- a/erts/emulator/test/float_SUITE.erl
+++ b/erts/emulator/test/float_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,21 +19,61 @@
 
 -module(float_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 fpe/1,fp_drv/1,fp_drv_thread/1,denormalized/1,match/1,
+	 bad_float_unpack/1]).
+-export([otp_7178/1]).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
-	 fpe/1,fp_drv/1,fp_drv_thread/1,denormalized/1,match/1,bad_float_unpack/1]).
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(3)),
     [{watchdog, Dog},{testcase,Func}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-all(suite) ->
-    [fpe,fp_drv,fp_drv_thread,denormalized,match,bad_float_unpack].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [fpe, fp_drv, fp_drv_thread, otp_7178, denormalized,
+     match, bad_float_unpack].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+%%
+%% OTP-7178, list_to_float on very small numbers should give 0.0
+%% instead of exception, i.e. ignore underflow.
+%%
+otp_7178(suite) ->
+    [];
+otp_7178(doc) ->
+    ["test that list_to_float on very small numbers give 0.0"];
+otp_7178(Config) when is_list(Config) ->
+    ?line X = list_to_float("1.0e-325"),
+    ?line true = (X < 0.00000001) and (X > -0.00000001),
+    ?line Y = list_to_float("1.0e-325325325"),
+    ?line true = (Y < 0.00000001) and (Y > -0.00000001),
+    ?line {'EXIT', {badarg,_}} = (catch list_to_float("1.0e83291083210")),
+    ok.
 
 %% Forces floating point exceptions and tests that subsequent, legal,
 %% operations are calculated correctly.  Original version by Sebastian
diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl
index a7889dfe90..7795efe57e 100644
--- a/erts/emulator/test/fun_SUITE.erl
+++ b/erts/emulator/test/fun_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1999-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,7 +22,9 @@
 
 -define(default_timeout, ?t:minutes(1)).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 bad_apply/1,bad_fun_call/1,badarity/1,ext_badarity/1,
 	 equality/1,ordering/1,
 	 fun_to_port/1,t_hash/1,t_phash/1,t_phash2/1,md5/1,
@@ -32,19 +34,37 @@
 
 -export([nothing/0]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [bad_apply, bad_fun_call, badarity, ext_badarity,
+     equality, ordering, fun_to_port, t_hash, t_phash,
+     t_phash2, md5, refc, refc_ets, refc_dist,
+     const_propagation, t_arity, t_is_function2, t_fun_info].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [bad_apply,bad_fun_call,badarity,ext_badarity,equality,ordering,
-     fun_to_port,t_hash,t_phash,t_phash2,md5,
-     refc,refc_ets,refc_dist,const_propagation,
-     t_arity,t_is_function2,t_fun_info].
 
 init_per_testcase(_Case, Config) ->
     ?line Dog = test_server:timetrap(?default_timeout),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/test/fun_r11_SUITE.erl b/erts/emulator/test/fun_r12_SUITE.erl
index 61ba816cc8..3b1dfc9825 100644
--- a/erts/emulator/test/fun_r11_SUITE.erl
+++ b/erts/emulator/test/fun_r12_SUITE.erl
@@ -1,72 +1,93 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
--module(fun_r11_SUITE).
--compile(r11).
+-module(fun_r12_SUITE).
+-compile(r12).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,dist_old_release/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,dist_old_release/1]).
 
 -define(default_timeout, ?t:minutes(1)).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [dist_old_release].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) -> [dist_old_release].
 
 init_per_testcase(_Case, Config) ->
     ?line Dog = test_server:timetrap(?default_timeout),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
 
 dist_old_release(Config) when is_list(Config) ->
-    case ?t:is_release_available("r11b") of
+    case ?t:is_release_available("r12b") of
 	true -> do_dist_old(Config);
-	false -> {skip,"No R11B found"}
+	false -> {skip,"No R12B found"}
     end.
 
 do_dist_old(Config) when is_list(Config) ->
     ?line Pa = filename:dirname(code:which(?MODULE)),
-    Name = fun_dist_r11,
+    Name = fun_dist_r12,
     ?line {ok,Node} = ?t:start_node(Name, peer,
 				    [{args,"-pa "++Pa},
-				     {erl,[{release,"r11b"}]}]),
+				     {erl,[{release,"r12b"}]}]),
 
     ?line Pid = spawn_link(Node,
 			   fun() ->
 				   receive
 				       Fun when is_function(Fun) ->
-					   R11BFun = fun(H) -> cons(H, [b,c]) end,
-					   Fun(Fun, R11BFun)
+					   R12BFun = fun(H) -> cons(H, [b,c]) end,
+					   Fun(Fun, R12BFun)
 				   end
 			   end),
     Self = self(),
-    Fun = fun(F, R11BFun) ->
+    Fun = fun(F, R12BFun) ->
 		  {pid,Self} = erlang:fun_info(F, pid),
 		  {module,?MODULE} = erlang:fun_info(F, module),
-		  Self ! {ok,F,R11BFun}
+		  Self ! {ok,F,R12BFun}
 	  end,
     ?line Pid ! Fun,
     ?line receive
-	      {ok,Fun,R11BFun} ->
-		  ?line [a,b,c] = R11BFun(a);
+	      {ok,Fun,R12BFun} ->
+		  ?line [a,b,c] = R12BFun(a);
 	      Other ->
 		  ?line ?t:fail({bad_message,Other})
 	  end,
diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl
index 066aa215b2..771d2c9a7a 100644
--- a/erts/emulator/test/gc_SUITE.erl
+++ b/erts/emulator/test/gc_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -21,15 +21,34 @@
 
 -module(gc_SUITE).
 
--include("test_server.hrl").
--export([all/1]).
+-include_lib("test_server/include/test_server.hrl").
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 -define(default_timeout, ?t:minutes(10)).
 
 -export([grow_heap/1, grow_stack/1, grow_stack_heap/1]).
 
-all(suite) ->
-    [grow_heap,grow_stack, grow_stack_heap].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [grow_heap, grow_stack, grow_stack_heap].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 grow_heap(doc) -> ["Produce a growing list of elements, ",
 		   "for X calls, then drop one item per call",
diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl
index 23482a20d7..f41324c2cc 100644
--- a/erts/emulator/test/guard_SUITE.erl
+++ b/erts/emulator/test/guard_SUITE.erl
@@ -1,33 +1,55 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
 -module(guard_SUITE).
 
--export([all/1, bad_arith/1, bad_tuple/1, test_heap_guards/1, guard_bifs/1,
-	 type_tests/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, bad_arith/1, bad_tuple/1, 
+	 test_heap_guards/1, guard_bifs/1,
+	 type_tests/1,guard_bif_binary_part/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -export([init/3]).
 -import(lists, [member/2]).
 
-all(suite) -> [bad_arith, bad_tuple, test_heap_guards, guard_bifs, type_tests].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [bad_arith, bad_tuple, test_heap_guards, guard_bifs,
+     type_tests, guard_bif_binary_part].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 bad_arith(doc) -> "Test that a bad arithmetic operation in a guard works correctly.";
 bad_arith(Config) when is_list(Config) ->
@@ -136,6 +158,170 @@ init(Fun, Args, Filler) ->
 dummy(_) ->
     ok.
 
+-define(MASK_ERROR(EXPR),mask_error((catch (EXPR)))).
+mask_error({'EXIT',{Err,_}}) ->
+    Err;
+mask_error(Else) ->
+    Else.
+
+guard_bif_binary_part(doc) ->
+    ["Test the binary_part/2,3 guard BIF's extensively"];
+guard_bif_binary_part(Config) when is_list(Config) ->
+    %% Overflow tests that need to be unoptimized
+    ?line badarg =
+	?MASK_ERROR(
+	   binary_part(<<1,2,3>>,{16#FFFFFFFFFFFFFFFF,
+				 -16#7FFFFFFFFFFFFFFF-1})),
+    ?line badarg =
+	?MASK_ERROR(
+	   binary_part(<<1,2,3>>,{16#FFFFFFFFFFFFFFFF,
+				 16#7FFFFFFFFFFFFFFF})),
+    F = fun(X) ->
+		Master = self(),
+		{Pid,Ref} = spawn_monitor( fun() ->
+					     A = lists:duplicate(X,a),
+					     B = [do_binary_part_guard() | A],
+					     Master ! {self(),hd(B)},
+					     ok
+				     end),
+		receive
+		    {Pid,ok} ->
+			erlang:demonitor(Ref,[flush]),
+			ok;
+		    Error ->
+			Error
+		end
+	end,
+    [ ok = F(N) || N <- lists:seq(1,10000) ],
+    ok.
+
+
+do_binary_part_guard() ->
+    ?line 1 = bptest(<<1,2,3>>),
+    ?line 2 = bptest(<<2,1,3>>),
+    ?line error = bptest(<<1>>),
+    ?line error = bptest(<<>>),
+    ?line error = bptest(apa),
+    ?line 3 = bptest(<<2,3,3>>),
+    % With one variable (pos)
+    ?line 1 = bptest(<<1,2,3>>,1),
+    ?line 2 = bptest(<<2,1,3>>,1),
+    ?line error = bptest(<<1>>,1),
+    ?line error = bptest(<<>>,1),
+    ?line error = bptest(apa,1),
+    ?line 3 = bptest(<<2,3,3>>,1),
+    % With one variable (length)
+    ?line 1 = bptesty(<<1,2,3>>,1),
+    ?line 2 = bptesty(<<2,1,3>>,1),
+    ?line error = bptesty(<<1>>,1),
+    ?line error = bptesty(<<>>,1),
+    ?line error = bptesty(apa,1),
+    ?line 3 = bptesty(<<2,3,3>>,2),
+    % With one variable (whole tuple)
+    ?line 1 = bptestx(<<1,2,3>>,{1,1}),
+    ?line 2 = bptestx(<<2,1,3>>,{1,1}),
+    ?line error = bptestx(<<1>>,{1,1}),
+    ?line error = bptestx(<<>>,{1,1}),
+    ?line error = bptestx(apa,{1,1}),
+    ?line 3 = bptestx(<<2,3,3>>,{1,2}),
+    % With two variables
+    ?line 1 = bptest(<<1,2,3>>,1,1),
+    ?line 2 = bptest(<<2,1,3>>,1,1),
+    ?line error = bptest(<<1>>,1,1),
+    ?line error = bptest(<<>>,1,1),
+    ?line error = bptest(apa,1,1),
+    ?line 3 = bptest(<<2,3,3>>,1,2),
+    % Direct (autoimported) call, these will be evaluated by the compiler...
+    ?line <<2>> = binary_part(<<1,2,3>>,1,1),
+    ?line <<1>> = binary_part(<<2,1,3>>,1,1),
+    % Compiler warnings due to constant evaluation expected (3)
+    ?line badarg = ?MASK_ERROR(binary_part(<<1>>,1,1)),
+    ?line badarg = ?MASK_ERROR(binary_part(<<>>,1,1)),
+    ?line badarg = ?MASK_ERROR(binary_part(apa,1,1)),
+    ?line <<3,3>> = binary_part(<<2,3,3>>,1,2),
+    % Direct call through apply
+    ?line <<2>> = apply(erlang,binary_part,[<<1,2,3>>,1,1]),
+    ?line <<1>> = apply(erlang,binary_part,[<<2,1,3>>,1,1]),
+    % Compiler warnings due to constant evaluation expected (3)
+    ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[<<1>>,1,1])),
+    ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[<<>>,1,1])),
+    ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[apa,1,1])),
+    ?line <<3,3>> = apply(erlang,binary_part,[<<2,3,3>>,1,2]),
+    % Constant propagation
+    ?line  Bin = <<1,2,3>>,
+    ?line  ok = if
+		    binary_part(Bin,1,1) =:= <<2>> ->
+			ok;
+		    %% Compiler warning, clause cannot match (expected)
+		    true ->
+			error
+		end,
+    ?line  ok = if
+		    binary_part(Bin,{1,1}) =:= <<2>> ->
+			ok;
+		    %% Compiler warning, clause cannot match (expected)
+		    true ->
+			error
+		end,
+    ok.
+
+
+bptest(B) when length(B) =:= 1337 ->
+    1;
+bptest(B) when binary_part(B,{1,1}) =:= <<2>> ->
+    1;
+bptest(B) when erlang:binary_part(B,1,1) =:= <<1>> ->
+    2;
+bptest(B)  when erlang:binary_part(B,{1,2}) =:= <<3,3>> ->
+    3;
+bptest(_) ->
+    error.
+
+bptest(B,A) when length(B) =:= A ->
+    1;
+bptest(B,A) when binary_part(B,{A,1}) =:= <<2>> ->
+    1;
+bptest(B,A) when erlang:binary_part(B,A,1) =:= <<1>> ->
+    2;
+bptest(B,A)  when erlang:binary_part(B,{A,2}) =:= <<3,3>> ->
+    3;
+bptest(_,_) ->
+    error.
+
+bptestx(B,A) when length(B) =:= A ->
+    1;
+bptestx(B,A) when binary_part(B,A) =:= <<2>> ->
+    1;
+bptestx(B,A) when erlang:binary_part(B,A) =:= <<1>> ->
+    2;
+bptestx(B,A)  when erlang:binary_part(B,A) =:= <<3,3>> ->
+    3;
+bptestx(_,_) ->
+    error.
+
+bptesty(B,A) when length(B) =:= A ->
+    1;
+bptesty(B,A) when binary_part(B,{1,A}) =:= <<2>> ->
+    1;
+bptesty(B,A) when erlang:binary_part(B,1,A) =:= <<1>> ->
+    2;
+bptesty(B,A)  when erlang:binary_part(B,{1,A}) =:= <<3,3>> ->
+    3;
+bptesty(_,_) ->
+    error.
+
+bptest(B,A,_C) when length(B) =:= A ->
+    1;
+bptest(B,A,C) when binary_part(B,{A,C}) =:= <<2>> ->
+    1;
+bptest(B,A,C) when erlang:binary_part(B,A,C) =:= <<1>> ->
+    2;
+bptest(B,A,C)  when erlang:binary_part(B,{A,C}) =:= <<3,3>> ->
+    3;
+bptest(_,_,_) ->
+    error.
+
+
 guard_bifs(doc) -> "Test all guard bifs with nasty (but legal arguments).";
 guard_bifs(Config) when is_list(Config) ->
     ?line Big = -237849247829874297658726487367328971246284736473821617265433,
diff --git a/erts/emulator/test/hash_SUITE.erl b/erts/emulator/test/hash_SUITE.erl
index 85bdb8bff8..830ed91da9 100644
--- a/erts/emulator/test/hash_SUITE.erl
+++ b/erts/emulator/test/hash_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2000-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -49,7 +49,7 @@
 -define(config(A,B),config(A,B)).
 -export([config/2]).
 -else.
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -endif.
 
 -ifdef(debug).
@@ -69,22 +69,40 @@ config(priv_dir,_) ->
     ".".
 -else.
 %% When run in test server.
--export([all/1,test_basic/1,test_cmp/1,test_range/1,test_spread/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 test_basic/1,test_cmp/1,test_range/1,test_spread/1,
 	 test_phash2/1,otp_5292/1,bit_level_binaries/1,otp_7127/1,
-	 fin_per_testcase/2,init_per_testcase/2]).
+	 end_per_testcase/2,init_per_testcase/2]).
 init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(test_server:minutes(10)),
     [{watchdog, Dog}|Config].
  
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
-all(doc) ->
-    ["Test erlang:phash"];
-all(suite) ->
-    [test_basic, test_cmp, test_range, test_spread, test_phash2, otp_5292,
-     bit_level_binaries, otp_7127].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [test_basic, test_cmp, test_range, test_spread,
+     test_phash2, otp_5292, bit_level_binaries, otp_7127].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 test_basic(suite) ->
     [];
@@ -480,14 +498,14 @@ otp_5292_test() ->
     S2 = md5([md5(hash_int(S, E, PH)) || {Start, N, Sz} <- d(), 
                                          {S, E} <- int(Start, N, Sz)]),
     ?line Comment = case S1 of 
-                        <<43,186,76,102,87,4,110,245,203,177,206,6,130,69,43,99>> ->
+			<<4,248,208,156,200,131,7,1,173,13,239,173,112,81,16,174>> ->
 			    ?line big = erlang:system_info(endian),
                             "Big endian machine";
-                        <<21,206,139,15,149,28,167,81,98,225,132,254,49,125,174,195>> ->
+                        <<180,28,33,231,239,184,71,125,76,47,227,241,78,184,176,233>> ->
 			    ?line little = erlang:system_info(endian),
                             "Little endian machine"
                     end,
-    ?line <<140,37,79,80,26,242,130,22,20,229,123,240,223,244,43,99>> = S2,
+    ?line <<124,81,198,121,174,233,19,137,10,83,33,80,226,111,238,99>> = S2,
     ?line 2 = erlang:hash(1, (1 bsl 27) -1),
     ?line {'EXIT', _} = (catch erlang:hash(1, (1 bsl 27))),
     {comment, Comment}.
@@ -507,7 +525,7 @@ hash_int(Start, End, F) ->
     {Start, End, md5(HL)}.
 
 md5(T) ->
-    erlang:md5(term_to_binary(T)).
+    erlang:md5(term_to_binary(T)).   
 
 bit_level_binaries() ->
     ?line [3511317,7022633,14044578,28087749,56173436,112344123,90467083|_] =
diff --git a/erts/emulator/test/hibernate_SUITE.erl b/erts/emulator/test/hibernate_SUITE.erl
index 4d36076d12..203fa6b48e 100644
--- a/erts/emulator/test/hibernate_SUITE.erl
+++ b/erts/emulator/test/hibernate_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2003-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2003-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,23 +19,44 @@
 
 -module(hibernate_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
-	 basic/1,min_heap_size/1,bad_args/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 basic/1,dynamic_call/1,min_heap_size/1,bad_args/1,
 	 messages_in_queue/1,undefined_mfa/1, no_heap/1]).
 
 %% Used by test cases.
--export([basic_hibernator/1,messages_in_queue_restart/2, no_heap_loop/0]).
+-export([basic_hibernator/1,dynamic_call_hibernator/2,messages_in_queue_restart/2, no_heap_loop/0]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [basic, dynamic_call, min_heap_size, bad_args, messages_in_queue,
+     undefined_mfa, no_heap].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [basic,min_heap_size,bad_args,messages_in_queue,undefined_mfa,no_heap].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(3)),
     [{watchdog,Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -138,10 +159,47 @@ whats_up_calc(A1, A2, A3, A4, A5, A6, A7, A8, A9, Acc) ->
     whats_up_calc(A1-1, A2+1, A3+2, A4+3, A5+4, A6+5, A7+6, A8+7, A9+8, [A1,A2|Acc]).
 
 %%%
+%%% Testing a call to erlang:hibernate/3 that the compiler and loader do not
+%%% translate to an instruction.
+%%%
+
+dynamic_call(Config) when is_list(Config) ->
+    Ref = make_ref(),
+    Info = {self(),Ref},
+    ExpectedHeapSz = case erlang:system_info(heap_type) of
+			 private -> erts_debug:size([Info]);
+			 hybrid -> erts_debug:size([a|b])
+		     end,
+    ?line Child = spawn_link(fun() -> ?MODULE:dynamic_call_hibernator(Info, hibernate) end),
+    ?line hibernate_wake_up(100, ExpectedHeapSz, Child),
+    ?line Child ! please_quit_now,
+    ok.
+
+dynamic_call_hibernator(Info, Function) ->
+    {catchlevel,0} = process_info(self(), catchlevel),
+    receive
+	Any ->
+	    dynamic_call_hibernator_msg(Any, Function, Info),
+	    dynamic_call_hibernator(Info, Function)
+    end.
+
+dynamic_call_hibernator_msg({hibernate,_}, Function, Info) ->
+    catch apply(erlang, Function, [?MODULE, basic_hibernator, [Info]]),
+    exit(hibernate_returned);
+dynamic_call_hibernator_msg(Msg, _Function, Info) ->
+    basic_hibernator_msg(Msg, Info).
+
+%%%
 %%% Testing setting the minimum heap size.
 %%%
 
 min_heap_size(Config) when is_list(Config) ->
+    case test_server:is_native(?MODULE) of
+	true -> {skip, "Test case relies on trace which is not available in HiPE"};
+	false -> min_heap_size_1(Config)
+    end.
+
+min_heap_size_1(Config) when is_list(Config) ->
     ?line erlang:trace(new, true, [call]),
     MFA = {?MODULE,min_hibernator,1},
     ?line 1 = erlang:trace_pattern(MFA, true, [local]),
diff --git a/erts/emulator/test/ignore_cores.erl b/erts/emulator/test/ignore_cores.erl
index 1d738cbafd..8b1ac0fe6c 120000..100644
--- a/erts/emulator/test/ignore_cores.erl
+++ b/erts/emulator/test/ignore_cores.erl
@@ -1 +1,158 @@
-../../test/ignore_cores.erl
\ No newline at end of file
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File    : ignore_cores.erl
+%%% Author  : Rickard Green <[email protected]>
+%%% Description : 
+%%%
+%%% Created : 11 Feb 2008 by Rickard Green <[email protected]>
+%%%-------------------------------------------------------------------
+
+-module(ignore_cores).
+
+-include_lib("test_server/include/test_server.hrl").
+
+-export([init/1, fini/1, setup/3, setup/4, restore/1, dir/1]).
+
+-record(ignore_cores, {org_cwd,
+		       org_path,
+		       org_pwd_env,
+		       ign_dir = false,
+		       cores_dir = false}).
+
+%%
+%% Takes a testcase config
+%%
+
+init(Config) ->
+    {ok, OrgCWD} = file:get_cwd(),
+    [{ignore_cores,
+      #ignore_cores{org_cwd = OrgCWD,
+		    org_path = code:get_path(),
+		    org_pwd_env = os:getenv("PWD")}}
+     | lists:keydelete(ignore_cores, 1, Config)].
+    
+fini(Config) ->
+    #ignore_cores{org_cwd = OrgCWD,
+		  org_path = OrgPath,
+		  org_pwd_env = OrgPWD} = ?config(ignore_cores, Config),
+    ok = file:set_cwd(OrgCWD),
+    true = code:set_path(OrgPath),
+    case OrgPWD of
+	false -> ok;
+	_ -> true = os:putenv("PWD", OrgPWD)
+    end,
+    lists:keydelete(ignore_cores, 1, Config).
+
+setup(Suite, Testcase, Config) ->
+    setup(Suite, Testcase, Config, false).
+
+setup(Suite, Testcase, Config, SetCwd) when is_atom(Suite), 
+                                            is_atom(Testcase),
+                                            is_list(Config) ->
+    #ignore_cores{org_cwd = OrgCWD,
+		  org_path = OrgPath,
+		  org_pwd_env = OrgPWD} = ?config(ignore_cores, Config),
+    Path = lists:map(fun (".") -> OrgCWD; (Dir) -> Dir end, OrgPath),
+    true = code:set_path(Path),
+    PrivDir = ?config(priv_dir, Config),
+    IgnDir = filename:join([PrivDir,
+			 atom_to_list(Suite)
+			 ++ "_"
+			 ++ atom_to_list(Testcase)
+			 ++ "_wd"]),
+    ok = file:make_dir(IgnDir),
+    case SetCwd of
+	false ->
+	    ok;
+	_ ->
+	    ok = file:set_cwd(IgnDir),
+	    OrgPWD = case os:getenv("PWD") of
+			 false -> false;
+			 PWD ->
+			     os:putenv("PWD", IgnDir),
+			     PWD
+		     end
+    end,
+    ok = file:write_file(filename:join([IgnDir, "ignore_core_files"]), <<>>),
+    %% cores are dumped in /cores on MacOS X
+    CoresDir = case {?t:os_type(), filelib:is_dir("/cores")} of
+		   {{unix,darwin}, true} ->
+		       filelib:fold_files("/cores",
+					  "^core.*$",
+					  false,
+					  fun (C,Cs) -> [C|Cs] end,
+					  []);
+		   _ ->
+		       false
+	       end,
+    lists:keyreplace(ignore_cores,
+		     1,
+		     Config,
+		     {ignore_cores,
+		      #ignore_cores{org_cwd = OrgCWD,
+				    org_path = OrgPath,
+				    org_pwd_env = OrgPWD,
+				    ign_dir = IgnDir,
+				    cores_dir = CoresDir}}).
+
+restore(Config) ->
+    #ignore_cores{org_cwd = OrgCWD,
+		  org_path = OrgPath,
+		  org_pwd_env = OrgPWD,
+		  ign_dir = IgnDir,
+		  cores_dir = CoresDir} = ?config(ignore_cores, Config),
+    try
+	case CoresDir of
+	    false ->
+		ok;
+	    _ ->
+		%% Move cores dumped by these testcases in /cores
+		%% to cwd.
+		lists:foreach(fun (C) ->
+				      case lists:member(C, CoresDir) of
+					  true -> ok;
+					  _ ->
+					      Dst = filename:join(
+						      [IgnDir,
+						       filename:basename(C)]),
+					      {ok, _} = file:copy(C, Dst),
+					      file:delete(C)
+				      end
+			      end,
+			      filelib:fold_files("/cores",
+						 "^core.*$",
+						 false,
+						 fun (C,Cs) -> [C|Cs] end,
+						 []))
+	end
+    after
+	catch file:set_cwd(OrgCWD),
+	catch code:set_path(OrgPath),
+	case OrgPWD of
+	    false -> ok;
+	    _ -> catch os:putenv("PWD", OrgPWD)
+	end
+    end.
+
+
+dir(Config) ->
+    #ignore_cores{ign_dir = Dir} = ?config(ignore_cores, Config),
+    Dir.
diff --git a/erts/emulator/test/list_bif_SUITE.erl b/erts/emulator/test/list_bif_SUITE.erl
index 65ea88eb2f..45a44d8b43 100644
--- a/erts/emulator/test/list_bif_SUITE.erl
+++ b/erts/emulator/test/list_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,21 +18,42 @@
 %%
 
 -module(list_bif_SUITE).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2]).
 -export([hd_test/1,tl_test/1,t_length/1,t_list_to_pid/1,
 	 t_list_to_float/1,t_list_to_integer/1]).
 
 
-all(suite) ->
-    [hd_test,tl_test,t_length,t_list_to_pid,t_list_to_float,t_list_to_integer].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [hd_test, tl_test, t_length, t_list_to_pid,
+     t_list_to_float, t_list_to_integer].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(_Case, Config) ->
     ?line Dog = test_server:timetrap(test_server:seconds(60)),
     [{watchdog,Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/test/long_timers_test.erl b/erts/emulator/test/long_timers_test.erl
index 28626d26fb..28a4fba9f6 100644
--- a/erts/emulator/test/long_timers_test.erl
+++ b/erts/emulator/test/long_timers_test.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index 69c89f5d2d..2b21fa58f4 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,8 @@
 
 -module(match_spec_SUITE).
 
--export([all/1, not_run/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, not_run/1]).
 -export([test_1/1, test_2/1, test_3/1, bad_match_spec_bin/1,
 	 trace_control_word/1, silent/1, silent_no_ms/1, 
 	 ms_trace2/1, ms_trace3/1, boxed_and_small/1,
@@ -34,29 +35,48 @@
 % This test suite assumes that tracing in general works. What we test is
 % the match spec functionality.
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:seconds(10)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
 
-all(suite) ->
-    case test_server:is_native(?MODULE) of
-	false -> [test_1, test_2, test_3, bad_match_spec_bin,
-		  trace_control_word, silent, silent_no_ms, 
-		  ms_trace2, ms_trace3, boxed_and_small,
-		  destructive_in_test_bif, guard_exceptions,
-		  unary_plus, unary_minus, fpe, moving_labels];
-	true  -> [not_run]
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(match_spec_SUITE) of
+	false ->
+	    [test_1, test_2, test_3, bad_match_spec_bin,
+	     trace_control_word, silent, silent_no_ms, ms_trace2,
+	     ms_trace3, boxed_and_small, destructive_in_test_bif,
+	     guard_exceptions, unary_plus, unary_minus, fpe,
+	     moving_labels];
+	true -> [not_run]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+	Config.
+
+
 not_run(Config) when is_list(Config) ->
     {skipped, "Native Code"}.
 
@@ -345,15 +365,15 @@ silent_no_ms(Config) when is_list(Config) ->
 	    fun () -> 
 		    ?MODULE:f1(a),
 		    ?MODULE:f2(b, c),
-		    erlang:integer_to_list(id(1)),
+		    _ = erlang:integer_to_list(id(1)),
 		    ?MODULE:f3(d, e),
 		    ?MODULE:f1(start),
 		    ?MODULE:f2(f, g),
-		    erlang:integer_to_list(id(2)),
+		    _ = erlang:integer_to_list(id(2)),
 		    ?MODULE:f3(h, i),
 		    ?MODULE:f1(stop),
 		    ?MODULE:f2(j, k),
-		    erlang:integer_to_list(id(3)),
+		    _ = erlang:integer_to_list(id(3)),
 		    ?MODULE:f3(l, m)
 	    end,
 	    fun (Tracee) ->
@@ -393,15 +413,15 @@ silent_no_ms(Config) when is_list(Config) ->
 	    fun () -> 
 		    ?MODULE:f1(a),
 		    ?MODULE:f2(b, c),
-		    erlang:integer_to_list(id(1)),
+		    _ = erlang:integer_to_list(id(1)),
 		    ?MODULE:f3(d, e),
 		    ?MODULE:f1(start),
 		    ?MODULE:f2(f, g),
-		    erlang:integer_to_list(id(2)),
+		    _ = erlang:integer_to_list(id(2)),
 		    ?MODULE:f3(h, i),
 		    ?MODULE:f1(stop),
 		    ?MODULE:f2(j, k),
-		    erlang:integer_to_list(id(3)),
+		    _ = erlang:integer_to_list(id(3)),
 		    ?MODULE:f3(l, m)
 	    end,
 	    fun (Tracee) ->
@@ -455,18 +475,18 @@ ms_trace2(Config) when is_list(Config) ->
 	    fun () -> 
 		    ?MODULE:f1(a),
 		    ?MODULE:f2(b, c),
-		    erlang:integer_to_list(id(1)),
+		    _ = erlang:integer_to_list(id(1)),
 		    ?MODULE:f3(d, e),
 		    fn([all], [call,return_to,{tracer,Tracer}]),
 		    ?MODULE:f1(f),
 		    f2(g, h),
 		    f1(i),
-		    erlang:integer_to_list(id(2)),
+		    _ = erlang:integer_to_list(id(2)),
 		    ?MODULE:f3(j, k),
 		    fn([call,return_to], []),
 		    ?MODULE:f1(l),
 		    ?MODULE:f2(m, n),
-		    erlang:integer_to_list(id(3)),
+		    _ = erlang:integer_to_list(id(3)),
 		    ?MODULE:f3(o, p)
 	    end,
 	    fun (Tracee) ->
@@ -551,26 +571,26 @@ ms_trace3(Config) when is_list(Config) ->
 		    register(TraceeName, self()),
 		    ?MODULE:f1(a),
 		    ?MODULE:f2(b, c),
-		    erlang:integer_to_list(id(1)),
+		    _ = erlang:integer_to_list(id(1)),
 		    ?MODULE:f3(d, e),
 		    Controller ! {self(),Tag,start},
 		    receive {Controller,Tag,started} -> ok end,
 		    ?MODULE:f1(f),
 		    f2(g, h),
 		    f1(i),
-		    erlang:integer_to_list(id(2)),
+		    _ = erlang:integer_to_list(id(2)),
 		    ?MODULE:f3(j, k),
 		    Controller ! {self(),Tag,stop_1},
 		    receive {Controller,Tag,stopped_1} -> ok end,
 		    ?MODULE:f1(l),
 		    ?MODULE:f2(m, n),
-		    erlang:integer_to_list(id(3)),
+		    _ = erlang:integer_to_list(id(3)),
 		    ?MODULE:f3(o, p),
 		    Controller ! {self(),Tag,stop_2},
 		    receive {Controller,Tag,stopped_2} -> ok end,
 		    ?MODULE:f1(q),
 		    ?MODULE:f2(r, s),
-		    erlang:integer_to_list(id(4)),
+		    _ = erlang:integer_to_list(id(4)),
 		    ?MODULE:f3(t, u)
 	    end,
 	    
diff --git a/erts/emulator/test/module_info_SUITE.erl b/erts/emulator/test/module_info_SUITE.erl
index f34a2b496c..8a63d9fe3e 100644
--- a/erts/emulator/test/module_info_SUITE.erl
+++ b/erts/emulator/test/module_info_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2005-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,11 @@
 
 -module(module_info_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,end_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 exports/1,functions/1,native/1]).
 
 %%-compile(native).
@@ -29,8 +31,29 @@
 %% Helper.
 -export([native_proj/1,native_filter/1]).
 
-all(suite) ->
-    [exports,functions,native].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    modules().
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+modules() -> 
+    [exports, functions, native].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(3)),
@@ -42,14 +65,18 @@ end_per_testcase(_Func, Config) ->
 
 %% Should return all functions exported from this module. (local)
 all_exported() ->
-    All = add_arity(all(suite)),
-    lists:sort([{all,1},{init_per_testcase,2},{end_per_testcase,2},
+    All = add_arity(modules()),
+    lists:sort([{all,0},{suite,0},{groups,0},
+		{init_per_suite,1},{end_per_suite,1},
+		{init_per_group,2},{end_per_group,2},
+		{init_per_testcase,2},{end_per_testcase,2},
 		{module_info,0},{module_info,1},{native_proj,1},
 		{native_filter,1}|All]).
 
 %% Should return all functions in this module. (local)
 all_functions() ->
-    Locals = [{add_arity,1},{add_arity,2},{all_exported,0},{all_functions,0}],
+    Locals = [{add_arity,1},{add_arity,2},{all_exported,0},{all_functions,0},
+	      {modules,0}],
     lists:sort(Locals++all_exported()).
 
 %% Test that the list of exported functions from this module is correct.
diff --git a/erts/emulator/test/monitor_SUITE.erl b/erts/emulator/test/monitor_SUITE.erl
index 68e378dfec..aec59867d8 100644
--- a/erts/emulator/test/monitor_SUITE.erl
+++ b/erts/emulator/test/monitor_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,29 +19,49 @@
 
 -module(monitor_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 case_1/1, case_1a/1, case_2/1, case_2a/1, mon_e_1/1, demon_e_1/1, demon_1/1,
-	 demon_2/1, demon_3/1, demonitor_flush/1, remove_monitor/1,
+	 demon_2/1, demon_3/1, demonitor_flush/1,
 	 local_remove_monitor/1, remote_remove_monitor/1, mon_1/1, mon_2/1,
 	 large_exit/1, list_cleanup/1, mixer/1, named_down/1, otp_5827/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 -export([y2/1, g/1, g0/0, g1/0, large_exit_sub/1]).
 
-all(suite) ->
-    [case_1, case_1a, case_2, case_2a, mon_e_1, demon_e_1, demon_1, mon_1,
-     mon_2, demon_2, demon_3, demonitor_flush, remove_monitor,
-     large_exit, list_cleanup, mixer, named_down,
-     otp_5827].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [case_1, case_1a, case_2, case_2a, mon_e_1, demon_e_1,
+     demon_1, mon_1, mon_2, demon_2, demon_3,
+     demonitor_flush, {group, remove_monitor}, large_exit,
+     list_cleanup, mixer, named_down, otp_5827].
+
+groups() -> 
+    [{remove_monitor, [],
+      [local_remove_monitor, remote_remove_monitor]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(15)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -315,8 +335,6 @@ demonitor_flush_test(Node) ->
 -define(RM_MON_GROUPS, 100).
 -define(RM_MON_GPROCS, 100).
 
-remove_monitor(suite) ->
-    [local_remove_monitor, remote_remove_monitor].
 
 local_remove_monitor(Config) when is_list(Config) ->
     Gs = generate(fun () -> start_remove_monitor_group(node()) end,
diff --git a/erts/emulator/test/mtx_SUITE.erl b/erts/emulator/test/mtx_SUITE.erl
new file mode 100644
index 0000000000..e0a7878bd8
--- /dev/null
+++ b/erts/emulator/test/mtx_SUITE.erl
@@ -0,0 +1,479 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%
+%% Stress tests of rwmutex implementation.
+%%
+%% Author: Rickard Green
+%%
+-module(mtx_SUITE).
+
+%%-define(line_trace,true).
+
+-include_lib("common_test/include/ct.hrl").
+
+-export([all/0,suite/0,groups/0,
+	 init_per_group/2,end_per_group/2, init_per_suite/1, 
+	 end_per_suite/1, init_per_testcase/2, end_per_testcase/2]).
+
+-export([long_rwlock/1,
+	 hammer_ets_rwlock/1,
+	 hammer_rwlock/1,
+	 hammer_rwlock_check/1,
+	 hammer_tryrwlock/1,
+	 hammer_tryrwlock_check/1,
+	 hammer_sched_long_rwlock/1,
+	 hammer_sched_long_rwlock_check/1,
+	 hammer_sched_long_freqread_rwlock/1,
+	 hammer_sched_long_freqread_rwlock_check/1,
+	 hammer_sched_long_tryrwlock/1,
+	 hammer_sched_long_tryrwlock_check/1,
+	 hammer_sched_long_freqread_tryrwlock/1,
+	 hammer_sched_long_freqread_tryrwlock_check/1,
+	 hammer_sched_rwlock/1,
+	 hammer_sched_rwlock_check/1,
+	 hammer_sched_freqread_rwlock/1,
+	 hammer_sched_freqread_rwlock_check/1,
+	 hammer_sched_tryrwlock/1,
+	 hammer_sched_tryrwlock_check/1,
+	 hammer_sched_freqread_tryrwlock/1,
+	 hammer_sched_freqread_tryrwlock_check/1]).
+
+init_per_suite(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Lib = filename:join([DataDir, atom_to_list(?MODULE)]),
+    ok = erlang:load_nif(Lib, none),
+    Config.
+
+end_per_suite(Config) when is_list(Config) ->
+    Config.
+
+init_per_testcase(_Case, Config) ->
+    Dog = ?t:timetrap(?t:minutes(15)),
+    [{watchdog, Dog}|Config].
+
+end_per_testcase(_Func, Config) ->
+    Dog = ?config(watchdog, Config),
+    ?t:timetrap_cancel(Dog).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [long_rwlock, hammer_rwlock_check, hammer_rwlock,
+     hammer_tryrwlock_check, hammer_tryrwlock,
+     hammer_ets_rwlock, hammer_sched_long_rwlock_check,
+     hammer_sched_long_rwlock,
+     hammer_sched_long_freqread_rwlock_check,
+     hammer_sched_long_freqread_rwlock,
+     hammer_sched_long_tryrwlock_check,
+     hammer_sched_long_tryrwlock,
+     hammer_sched_long_freqread_tryrwlock_check,
+     hammer_sched_long_freqread_tryrwlock,
+     hammer_sched_rwlock_check, hammer_sched_rwlock,
+     hammer_sched_freqread_rwlock_check,
+     hammer_sched_freqread_rwlock,
+     hammer_sched_tryrwlock_check, hammer_sched_tryrwlock,
+     hammer_sched_freqread_tryrwlock_check,
+     hammer_sched_freqread_tryrwlock].
+
+groups() -> 
+    [].
+
+init_per_group(_GroupName, Config) ->
+	Config.
+
+end_per_group(_GroupName, Config) ->
+	Config.
+
+
+long_rwlock(Config) when is_list(Config) ->
+    statistics(runtime),
+    LLRes = long_rw_test(),
+    {_, RunTime} = statistics(runtime),
+    %% A very short run time is expected, since
+    %% threads in the test mostly wait
+    ?t:format("RunTime=~p~n", [RunTime]),
+    ?line true = RunTime < 100,
+    ?line RunTimeStr = "Run-time during test was "++integer_to_list(RunTime)++" ms.",
+    case LLRes of
+	ok ->
+	    {comment, RunTimeStr};
+	{comment, Comment} ->
+	    {comment, Comment ++ " " ++ RunTimeStr}
+    end.
+
+hammer_rwlock(Config) when is_list(Config) ->
+    hammer_rw_test(false).
+
+hammer_rwlock_check(Config) when is_list(Config) ->
+    hammer_rw_test(true).
+
+hammer_tryrwlock(Config) when is_list(Config) ->
+    hammer_tryrw_test(false).
+
+hammer_tryrwlock_check(Config) when is_list(Config) ->
+    hammer_tryrw_test(true).
+
+hammer_sched_rwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, false, true, 0, 0).
+
+hammer_sched_rwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, true, true, 0, 0).
+
+hammer_sched_freqread_rwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, false, true, 0, 0).
+
+hammer_sched_freqread_rwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, true, true, 0, 0).
+
+hammer_sched_tryrwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, false, false, 0, 100).
+
+hammer_sched_tryrwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, true, false, 0, 100).
+
+hammer_sched_freqread_tryrwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, false, false, 0, 100).
+
+hammer_sched_freqread_tryrwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, true, false, 0, 100).
+
+hammer_sched_long_rwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, false, true, 100, 0).
+
+hammer_sched_long_rwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, true, true, 100, 0).
+
+hammer_sched_long_freqread_rwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, false, true, 100, 0).
+
+hammer_sched_long_freqread_rwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, true, true, 100, 0).
+
+hammer_sched_long_tryrwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, false, false, 100, 100).
+
+hammer_sched_long_tryrwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, true, false, 100, 100).
+
+hammer_sched_long_freqread_tryrwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, false, false, 100, 100).
+
+hammer_sched_long_freqread_tryrwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, true, false, 100, 100).
+
+hammer_sched_rwlock_test(FreqRead, LockCheck, Blocking, WaitLocked, WaitUnlocked) ->
+    case create_rwlock(FreqRead, LockCheck) of
+	enotsup ->
+	    {skipped, "Not supported."};
+	RWLock ->
+	    Onln = erlang:system_info(schedulers_online),
+	    NWPs = case Onln div 3 of
+		       1 -> case Onln < 4 of
+				true -> 1;
+				false -> 2
+			    end;
+		       X -> X
+		   end,
+	    NRPs = Onln - NWPs,
+	    NoLockOps = ((((50000000 div Onln)
+			       div case {Blocking, WaitLocked} of
+				       {false, 0} -> 1;
+				       _ -> 10
+				   end)
+			      div (case WaitLocked == 0 of
+				       true -> 1;
+				       false -> WaitLocked*250
+				   end))
+			     div handicap()),
+	    ?t:format("NoLockOps=~p~n", [NoLockOps]),
+	    Sleep = case Blocking of
+			true -> NoLockOps;
+			false -> NoLockOps div 10
+		    end,
+	    WPs = lists:map(
+		    fun (Sched) ->
+			    spawn_opt(
+			      fun () ->
+				      io:format("Writer on scheduler ~p.~n",
+						[Sched]),
+				      Sched = erlang:system_info(scheduler_id),
+				      receive go -> gone end,
+				      hammer_sched_rwlock_proc(RWLock,
+							       Blocking,
+							       true,
+							       WaitLocked,
+							       WaitUnlocked,
+							       NoLockOps,
+							       Sleep),
+				      Sched = erlang:system_info(scheduler_id)
+			      end,
+			      [link, {scheduler, Sched}])
+		    end,
+		    lists:seq(1, NWPs)),
+	    RPs = lists:map(
+		    fun (Sched) ->
+			    spawn_opt(
+			      fun () ->
+				      io:format("Reader on scheduler ~p.~n",
+						[Sched]),
+				      Sched = erlang:system_info(scheduler_id),
+				      receive go -> gone end,
+				      hammer_sched_rwlock_proc(RWLock,
+							       Blocking,
+							       false,
+							       WaitLocked,
+							       WaitUnlocked,
+							       NoLockOps,
+							       Sleep),
+				      Sched = erlang:system_info(scheduler_id)
+			      end,
+			      [link, {scheduler, Sched}])
+		    end,
+		    lists:seq(NWPs + 1, NWPs + NRPs)),
+	    Procs = WPs ++ RPs,
+	    case {Blocking, WaitLocked} of
+		{_, 0} -> ok;
+		{false, _} -> ok;
+		_ -> statistics(runtime)
+	    end,
+	    lists:foreach(fun (P) -> P ! go end, Procs),
+	    lists:foreach(fun (P) ->
+				  M = erlang:monitor(process, P),
+				  receive
+				      {'DOWN', M, process, P, _} ->
+					  ok
+				  end
+			  end,
+			  Procs),
+	    case {Blocking, WaitLocked} of
+		{_, 0} -> ok;
+		{false, _} -> ok;
+		_ ->
+		    {_, RunTime} = statistics(runtime),
+		    ?t:format("RunTime=~p~n", [RunTime]),
+		    ?line true = RunTime < 500,
+		    {comment,
+		     "Run-time during test was "
+		     ++ integer_to_list(RunTime)
+		     ++ " ms."}
+	    end
+    end.
+
+hammer_sched_rwlock_proc(_RWLock,
+			 _Blocking,
+			 _WriteOp,
+			 _WaitLocked,
+			 _WaitUnlocked,
+			 0,
+			 _Sleep) ->
+    ok;
+hammer_sched_rwlock_proc(RWLock,
+			 Blocking,
+			 WriteOp,
+			 WaitLocked,
+			 WaitUnlocked,
+			 Times,
+			 Sleep) when Times rem Sleep == 0 ->
+    rwlock_op(RWLock, Blocking, WriteOp, WaitLocked, WaitUnlocked),
+    hammer_sched_rwlock_proc(RWLock,
+			     Blocking,
+			     WriteOp,
+			     WaitLocked,
+			     WaitUnlocked,
+			     Times - 1,
+			     Sleep);
+hammer_sched_rwlock_proc(RWLock,
+			 Blocking,
+			 WriteOp,
+			 WaitLocked,
+			 WaitUnlocked,
+			 Times,
+			 Sleep) ->
+    rwlock_op(RWLock, Blocking, WriteOp, WaitLocked, 0),
+    hammer_sched_rwlock_proc(RWLock,
+			     Blocking,
+			     WriteOp,
+			     WaitLocked,
+			     WaitUnlocked,
+			     Times - 1,
+			     Sleep).
+
+-define(HAMMER_ETS_RWLOCK_REPEAT_TIMES, 1).
+-define(HAMMER_ETS_RWLOCK_TSIZE, 500).
+
+hammer_ets_rwlock(Config) when is_list(Config) ->
+    {Ops, Procs} = case handicap() of
+		       1 -> {20000, 500};
+		       2 -> {20000, 50};
+		       3 -> {2000, 50};
+		       _ -> {200, 50}
+		   end,
+    ?t:format("Procs=~p~nOps=~p~n", [Procs, Ops]),
+    lists:foreach(fun (XOpts) ->
+			  ?t:format("Running with extra opts: ~p", [XOpts]),
+			  hammer_ets_rwlock_test(XOpts, true, 2, Ops,
+						 Procs, false)
+		  end,
+		  [[],
+		   [{read_concurrency, true}],
+		   [{write_concurrency, true}],
+		   [{read_concurrency, true},{write_concurrency, true}]]),
+    ok.
+
+%% Aux funcs
+
+long_rw_test() ->
+    exit(no_nif_implementation).
+
+hammer_rw_test(_Arg) ->
+    exit(no_nif_implementation).
+
+hammer_tryrw_test(_Arg) ->
+    exit(no_nif_implementation).
+
+create_rwlock(_FreqRead, _LockCheck) ->
+    exit(no_nif_implementation).
+
+rwlock_op(_RWLock, _Blocking, _WriteOp, _WaitLocked, _WaitUnlocked) ->
+    exit(no_nif_implementation).
+
+hammer_ets_rwlock_put_data() ->
+    put(?MODULE, {"here are some", data, "to store", make_ref()}).
+
+hammer_ets_rwlock_get_data() ->
+    get(?MODULE).
+
+hammer_ets_rwlock_ops(_T, _UW, _N, _C, _SC, 0) ->
+    ok;
+hammer_ets_rwlock_ops(T, UW, N, C, SC, Tot) when N >= ?HAMMER_ETS_RWLOCK_TSIZE ->
+    hammer_ets_rwlock_ops(T, UW, 0, C, SC, Tot);
+hammer_ets_rwlock_ops(T, UW, N, 0, SC, Tot) ->
+    case UW of
+	true ->
+	    true = ets:insert(T, {N, Tot, hammer_ets_rwlock_get_data()});
+	false ->
+	    [{N, _, _}] = ets:lookup(T, N)
+    end,
+    hammer_ets_rwlock_ops(T, UW, N+1, SC, SC, Tot-1);
+hammer_ets_rwlock_ops(T, UW, N, C, SC, Tot) ->
+    case UW of
+	false ->
+	    true = ets:insert(T, {N, Tot, hammer_ets_rwlock_get_data()});
+	true ->
+	    [{N, _, _}] = ets:lookup(T, N)
+    end,
+    hammer_ets_rwlock_ops(T, UW, N+1, C-1, SC, Tot-1).
+
+hammer_ets_rwlock_init(T, N) when N < ?HAMMER_ETS_RWLOCK_TSIZE ->
+    ets:insert(T, {N, N, N}),
+    hammer_ets_rwlock_init(T, N+1);
+hammer_ets_rwlock_init(_T, _N) ->
+    ok.
+
+hammer_ets_rwlock_test(XOpts, UW, C, N, NP, SC) ->
+    receive after 100 -> ok end,
+    {TP, TM} = spawn_monitor(
+		 fun () ->
+			 _L = repeat_list(
+				fun () ->
+					Caller = self(),
+					T = fun () ->
+						    Parent = self(),
+						    hammer_ets_rwlock_put_data(),
+						    T=ets:new(x, [public | XOpts]),
+						    hammer_ets_rwlock_init(T, 0),
+						    Ps0 = repeat_list(
+							    fun () ->
+								    spawn_link(
+								      fun () ->
+									      hammer_ets_rwlock_put_data(),
+									      receive go -> ok end,
+									      hammer_ets_rwlock_ops(T, UW, N, C, C, N),
+									      Parent ! {done, self()},
+									      receive after infinity -> ok end
+								      end)
+							    end,
+							    NP - case SC of
+								     false -> 0;
+								     _ -> 1
+								 end),
+						    Ps = case SC of
+							     false -> Ps0;
+							     _ -> [spawn_link(fun () ->
+										      hammer_ets_rwlock_put_data(),
+										      receive go -> ok end,
+										      hammer_ets_rwlock_ops(T, UW, N, SC, SC, N),
+										      Parent ! {done, self()},
+										      receive after infinity -> ok end
+									      end) | Ps0]
+							 end,
+						    Start = now(),
+						    lists:foreach(fun (P) -> P ! go end, Ps),
+						    lists:foreach(fun (P) -> receive {done, P} -> ok end end, Ps),
+						    Stop = now(),
+						    lists:foreach(fun (P) ->
+									  unlink(P),
+									  exit(P, bang),
+									  M = erlang:monitor(process, P),
+									  receive
+									      {'DOWN', M, process, P, _} -> ok
+									  end
+								  end, Ps),
+						    Res = timer:now_diff(Stop, Start)/1000000,
+						    Caller ! {?MODULE, self(), Res}
+					    end,
+					TP = spawn_link(T),
+					receive
+					    {?MODULE, TP, Res} ->
+						Res
+					end
+				end,
+				?HAMMER_ETS_RWLOCK_REPEAT_TIMES)
+		 end),
+    receive
+	{'DOWN', TM, process, TP, _} -> ok
+    end.
+
+repeat_list(Fun, N) ->
+    repeat_list(Fun, N, []).
+
+repeat_list(_Fun, 0, Acc) ->
+    Acc;
+repeat_list(Fun, N, Acc) ->
+    repeat_list(Fun, N-1, [Fun()|Acc]).
+
+
+handicap() ->
+    X0 = case catch (erlang:system_info(logical_processors_available) >=
+			 erlang:system_info(schedulers_online)) of
+	     true -> 1;
+	     _ -> 2
+	 end,
+    case erlang:system_info(build_type) of
+	opt ->
+	    X0;
+	ReallySlow when ReallySlow == debug;
+			ReallySlow == valgrind;
+			ReallySlow == purify ->
+	    X0*3;
+	_Slow ->
+	    X0*2
+    end.
+		
diff --git a/erts/emulator/test/mtx_SUITE_data/Makefile.src b/erts/emulator/test/mtx_SUITE_data/Makefile.src
new file mode 100644
index 0000000000..b6c843269c
--- /dev/null
+++ b/erts/emulator/test/mtx_SUITE_data/Makefile.src
@@ -0,0 +1,30 @@
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2010. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+
+include @erts_lib_include_internal_generated@@[email protected]
+include @erts_lib_include_internal_generated@@DS@erts_internal.mk
+
+NIF_LIBS = mtx_SUITE@dll@
+
+SHLIB_EXTRA_CFLAGS =  $(ETHR_DEFS) -I@erts_lib_include_internal@ -I@erts_lib_include_internal_generated@
+LIBS = @ERTS_LIBS@
+
+all: $(NIF_LIBS)
+
+@SHLIB_RULES@
diff --git a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c
new file mode 100644
index 0000000000..818023211c
--- /dev/null
+++ b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c
@@ -0,0 +1,692 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Stress tests of rwmutex implementation.
+ *
+ * Author: Rickard Green
+ */
+
+#include "erl_nif.h"
+
+#ifdef __WIN32__
+#  ifndef WIN32_LEAN_AND_MEAN
+#    define WIN32_LEAN_AND_MEAN
+#  endif
+#  include <windows.h>
+#else
+#  include "ethread.h"
+#  include "erl_misc_utils.h"
+#  include <unistd.h>
+#endif
+
+#include <errno.h>
+#include <stdio.h>
+
+static int
+fail(const char *file, int line, const char *function, const char *assertion);
+
+#undef ASSERT
+#define ASSERT(X) ((void) ((X) ? 1 : fail(__FILE__, __LINE__, __func__, #X)))
+
+#ifdef __WIN32__
+/*
+ * We cannot access the ethread symbols directly; test
+ * what we got in the nif api instead...
+ */
+#define HAVE_FREQREAD_SUPPORT 0
+#define RWMUTEX_T ErlNifRWLock
+#define RWMUTEX_CREATE(FR) enif_rwlock_create("dummy")
+#define RWMUTEX_DESTROY enif_rwlock_destroy
+#define RWMUTEX_WLOCK enif_rwlock_rwlock
+#define RWMUTEX_TRYWLOCK enif_rwlock_tryrwlock
+#define RWMUTEX_WUNLOCK enif_rwlock_rwunlock
+#define RWMUTEX_TRYRLOCK enif_rwlock_tryrlock
+#define RWMUTEX_RLOCK enif_rwlock_rlock
+#define RWMUTEX_RUNLOCK enif_rwlock_runlock
+#define THR_ID ErlNifTid
+#define THR_CREATE(A, B, C, D) enif_thread_create("dummy", (A), (B), (C), (D))
+#define THR_JOIN enif_thread_join
+#define ATOMIC_T volatile LONG
+#define ATOMIC_INIT(VarP, Val) (*(VarP) = (Val))
+#define ATOMIC_SET(VarP, Val) (*(VarP) = (Val))
+#define ATOMIC_READ(VarP) (*(VarP))
+#define ATOMIC_INC InterlockedIncrement
+#define ATOMIC_DEC InterlockedDecrement
+
+#else
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+#  define HAVE_FREQREAD_SUPPORT 1
+#else
+#  define HAVE_FREQREAD_SUPPORT 0
+#endif
+
+#define RWMUTEX_T ethr_rwmutex
+static ethr_rwmutex *
+RWMUTEX_CREATE(int freqread)
+{
+    ethr_rwmutex *rwmtx = enif_alloc(sizeof(ethr_rwmutex));
+    ethr_rwmutex_opt rwmtx_opt = ETHR_RWMUTEX_OPT_DEFAULT_INITER;
+    if (freqread)
+	rwmtx_opt.type = ETHR_RWMUTEX_TYPE_FREQUENT_READ;
+    ASSERT(rwmtx);
+    ASSERT(ethr_rwmutex_init_opt(rwmtx, &rwmtx_opt) == 0);
+    return rwmtx;
+}
+static void
+RWMUTEX_DESTROY(ethr_rwmutex *rwmtx)
+{
+    ASSERT(ethr_rwmutex_destroy(rwmtx) == 0);
+    enif_free(rwmtx);
+}
+#define RWMUTEX_TRYWLOCK ethr_rwmutex_tryrwlock
+#define RWMUTEX_WLOCK ethr_rwmutex_rwlock
+#define RWMUTEX_WUNLOCK ethr_rwmutex_rwunlock
+#define RWMUTEX_TRYRLOCK ethr_rwmutex_tryrlock
+#define RWMUTEX_RLOCK ethr_rwmutex_rlock
+#define RWMUTEX_RUNLOCK ethr_rwmutex_runlock
+#define THR_ID ethr_tid
+#define THR_CREATE ethr_thr_create
+#define THR_JOIN ethr_thr_join
+#define ATOMIC_T ethr_atomic_t
+#define ATOMIC_INIT ethr_atomic_init
+#define ATOMIC_SET ethr_atomic_set
+#define ATOMIC_READ ethr_atomic_read
+#define ATOMIC_INC ethr_atomic_inc
+#define ATOMIC_DEC ethr_atomic_dec
+
+#endif
+
+
+#if !defined(__func__)
+#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
+#    if !defined(__GNUC__) ||  __GNUC__ < 2
+#      define __func__ "[unknown_function]"
+#    else
+#      define __func__ __FUNCTION__
+#    endif
+#  endif
+#endif
+
+static void milli_sleep(int ms);
+static int get_bool(ErlNifEnv* env, ERL_NIF_TERM term);
+
+/*
+ * Long rwlock testcase
+ */
+
+#define LONG_RW_NO_W_THREADS 6
+#define LONG_RW_NO_THREADS 20
+#define LONG_RW_NO_WLOCK_COUNT 100
+
+typedef struct {
+    RWMUTEX_T *rwlock;
+    ATOMIC_T *is_wlocked;
+    ATOMIC_T *is_rlocked;
+    int *stop;
+    int *count;
+    int sleep;
+} long_rw_t;
+
+static void *
+long_rw_w(void *varg)
+{
+    long_rw_t *arg = varg;
+    int stop = 0;
+    do {
+	RWMUTEX_WLOCK(arg->rwlock);
+	ASSERT(!ATOMIC_READ(arg->is_wlocked));
+	ATOMIC_SET(arg->is_wlocked, 1);
+	ASSERT(!ATOMIC_READ(arg->is_rlocked));
+	milli_sleep(arg->sleep);
+	if (++(*arg->count) > LONG_RW_NO_WLOCK_COUNT)
+	    stop = *arg->stop = 1;
+	ATOMIC_SET(arg->is_wlocked, 0);
+	ASSERT(!ATOMIC_READ(arg->is_rlocked));
+	RWMUTEX_WUNLOCK(arg->rwlock);
+    } while (!stop);
+    return NULL;
+}
+
+static void *
+long_rw_r(void *varg)
+{
+    long_rw_t *arg = varg;
+    int stop;
+    do {
+	RWMUTEX_RLOCK(arg->rwlock);
+	ASSERT(!ATOMIC_READ(arg->is_wlocked));
+	ATOMIC_INC(arg->is_rlocked);
+	milli_sleep(arg->sleep);
+	stop = *arg->stop;
+	ATOMIC_DEC(arg->is_rlocked);
+	ASSERT(!ATOMIC_READ(arg->is_wlocked));
+	RWMUTEX_RUNLOCK(arg->rwlock);
+    } while (!stop);
+    return NULL;
+}
+
+
+static ERL_NIF_TERM long_rw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    int res, freqread, i, count, stop;
+    ATOMIC_T is_wlocked, is_rlocked;
+    THR_ID tid[LONG_RW_NO_THREADS];
+    long_rw_t arg;
+    long_rw_t targ[LONG_RW_NO_THREADS];
+
+    ATOMIC_INIT(&is_wlocked, 0);
+    ATOMIC_INIT(&is_rlocked, 0);
+
+    freqread = 0;
+
+    arg.is_wlocked = &is_wlocked;
+    arg.is_rlocked = &is_rlocked;
+    arg.count = &count;
+    arg.stop = &stop;
+
+ restart:
+
+    stop = 0;
+    count = 0;
+
+    arg.rwlock = RWMUTEX_CREATE(freqread);
+
+    ASSERT(arg.rwlock);
+
+    for (i = 0; i < LONG_RW_NO_W_THREADS; i++) {
+	targ[i] = arg;
+	targ[i].sleep = 100 + i*10;
+	ASSERT(THR_CREATE(&tid[i], long_rw_w, &targ[i], NULL) == 0);
+    }
+    for (; i < LONG_RW_NO_THREADS; i++) {
+	targ[i] = arg;
+	targ[i].sleep = 100;
+	ASSERT(THR_CREATE(&tid[i], long_rw_r, &targ[i], NULL) == 0);
+    }
+    for (i = 0; i < LONG_RW_NO_THREADS; i++)
+	ASSERT(THR_JOIN(tid[i], NULL) == 0);
+
+    ASSERT(!ATOMIC_READ(arg.is_wlocked));
+    ASSERT(!ATOMIC_READ(arg.is_rlocked));
+
+    RWMUTEX_DESTROY(arg.rwlock);
+
+    if (HAVE_FREQREAD_SUPPORT && !freqread) {
+	freqread = 1;
+	goto restart;
+    }
+
+    if (freqread)
+	return enif_make_atom(env, "ok");
+    else
+	return enif_make_tuple2(env,
+				enif_make_atom(env,
+					       "comment"),
+				enif_make_string(env,
+						 "No frequent read test made.",
+						 ERL_NIF_LATIN1));
+}
+
+/*
+ * Hammer rwlock testcase
+ */
+
+#define HAMMER_RW_NO_W_THREADS 6
+#define HAMMER_RW_NO_THREADS 20
+#define HAMMER_RW_NO_WLOCK_COUNT 1000000
+
+typedef struct {
+    RWMUTEX_T *rwlock;
+    ATOMIC_T is_locked;
+    int lock_check;
+    int stop;
+    int count;
+} hammer_rw_t;
+
+static void *
+hammer_rw_w(void *varg)
+{
+    hammer_rw_t *arg = varg;
+    int stop = 0;
+    do {
+	RWMUTEX_WLOCK(arg->rwlock);
+	if (arg->lock_check) {
+	    ASSERT(!ATOMIC_READ(&arg->is_locked));
+	    ATOMIC_SET(&arg->is_locked, -1);
+	}
+	if (++arg->count > HAMMER_RW_NO_WLOCK_COUNT)
+	    stop = arg->stop = 1;
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) == -1);
+	    ATOMIC_SET(&arg->is_locked, 0);
+	}
+	RWMUTEX_WUNLOCK(arg->rwlock);
+    } while (!stop);
+    return NULL;
+}
+
+static void *
+hammer_rw_r(void *varg)
+{
+    hammer_rw_t *arg = varg;
+    int stop;
+    do {
+	RWMUTEX_RLOCK(arg->rwlock);
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) >= 0);
+	    ATOMIC_INC(&arg->is_locked);
+	}
+	stop = arg->stop;
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) > 0);
+	    ATOMIC_DEC(&arg->is_locked);
+	}
+	RWMUTEX_RUNLOCK(arg->rwlock);
+    } while (!stop); 
+    return NULL;
+}
+
+
+static ERL_NIF_TERM hammer_rw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    hammer_rw_t arg;
+    char buf[10];
+    int res, freqread, i;
+    THR_ID tid[HAMMER_RW_NO_THREADS];
+
+    if (argc != 1)
+	goto badarg;
+
+    arg.lock_check = get_bool(env, argv[0]);
+    if (arg.lock_check < 0)
+	goto badarg;
+
+    ATOMIC_INIT(&arg.is_locked, 0);
+
+    freqread = 0;
+
+ restart:
+    arg.stop = 0;
+    arg.count = 0;
+
+    arg.rwlock = RWMUTEX_CREATE(freqread);
+
+    ASSERT(arg.rwlock);
+
+    for (i = 0; i < HAMMER_RW_NO_W_THREADS; i++)
+	ASSERT(THR_CREATE(&tid[i], hammer_rw_w, &arg, NULL) == 0);
+    for (; i < HAMMER_RW_NO_THREADS; i++)
+	ASSERT(THR_CREATE(&tid[i], hammer_rw_r, &arg, NULL) == 0);
+    for (i = 0; i < HAMMER_RW_NO_THREADS; i++)
+	ASSERT(THR_JOIN(tid[i], NULL) == 0);
+
+    ASSERT(!ATOMIC_READ(&arg.is_locked));
+
+    RWMUTEX_DESTROY(arg.rwlock);
+
+    if (HAVE_FREQREAD_SUPPORT && !freqread) {
+	freqread = 1;
+	goto restart;
+    }
+
+    if (freqread)
+	return enif_make_atom(env, "ok");
+    else
+	return enif_make_tuple2(env,
+				enif_make_atom(env,
+					       "comment"),
+				enif_make_string(env,
+						 "No frequent read test made.",
+						 ERL_NIF_LATIN1));
+ badarg:
+    return enif_make_badarg(env);
+}
+
+/*
+ * Hammer try rwlock testcase
+ */
+
+#define HAMMER_TRYRW_NO_W_THREADS 10
+#define HAMMER_TRYRW_NO_THREADS 20
+#define HAMMER_TRYRW_NO_WLOCK_COUNT 10000000
+#define HAMMER_TRYRW_NO_RLOCK_COUNT 10000000
+#define HAMMER_TRYRW_NO_WLOCK_WAIT_COUNT ((10*HAMMER_TRYRW_NO_WLOCK_COUNT)/8)
+#define HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT ((10*HAMMER_TRYRW_NO_RLOCK_COUNT)/8)
+
+typedef struct {
+    RWMUTEX_T *rwlock;
+    ATOMIC_T is_locked;
+    int lock_check;
+    int w_count;
+    ATOMIC_T r_count;
+} hammer_tryrw_t;
+
+static void *
+hammer_tryrw_w(void *varg)
+{
+    hammer_tryrw_t *arg = varg;
+    int stop = 0;
+    int wait = 0;
+    do {
+	while (EBUSY == RWMUTEX_TRYWLOCK(arg->rwlock));
+	if (arg->lock_check) {
+	    ASSERT(!ATOMIC_READ(&arg->is_locked));
+	    ATOMIC_SET(&arg->is_locked, -1);
+	}
+	if (++arg->w_count > HAMMER_TRYRW_NO_WLOCK_COUNT)
+	    stop = 1;
+	else if (arg->w_count > HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT)
+	    wait = 1;
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) == -1);
+	    ATOMIC_SET(&arg->is_locked, 0);
+	}
+	RWMUTEX_WUNLOCK(arg->rwlock);
+	if (wait)
+	    milli_sleep(1);
+    } while (!stop);
+    return NULL;
+}
+
+static void *
+hammer_tryrw_r(void *varg)
+{
+    hammer_tryrw_t *arg = varg;
+    long r_count;
+    int stop = 0;
+    int wait = 0;
+    do {
+	while (EBUSY == RWMUTEX_TRYRLOCK(arg->rwlock));
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) >= 0);
+	    ATOMIC_INC(&arg->is_locked);
+	}
+	ATOMIC_INC(&arg->r_count);
+	r_count = ATOMIC_READ(&arg->r_count);
+	if (r_count > HAMMER_TRYRW_NO_RLOCK_COUNT)
+	    stop = 1;
+	else if (r_count > HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT)
+	    wait = 1;
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) > 0);
+	    ATOMIC_DEC(&arg->is_locked);
+	}
+	RWMUTEX_RUNLOCK(arg->rwlock);
+	if (wait)
+	    milli_sleep(1);
+    } while (!stop);
+    return NULL;
+}
+
+
+static ERL_NIF_TERM hammer_tryrw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    hammer_tryrw_t arg;
+    char buf[10];
+    int res, freqread, i;
+    THR_ID tid[HAMMER_TRYRW_NO_THREADS];
+
+    if (argc != 1)
+	goto badarg;
+
+    arg.lock_check = get_bool(env, argv[0]);
+    if (arg.lock_check < 0)
+	goto badarg;
+
+    ATOMIC_INIT(&arg.is_locked, 0);
+    freqread = 0;
+
+ restart:
+
+    arg.w_count = 0;
+    ATOMIC_INIT(&arg.r_count, 0);
+
+    arg.rwlock = RWMUTEX_CREATE(freqread);
+
+    ASSERT(arg.rwlock);
+
+    for (i = 0; i < HAMMER_TRYRW_NO_W_THREADS; i++)
+	ASSERT(THR_CREATE(&tid[i], hammer_tryrw_w, &arg, NULL) == 0);
+    for (; i < HAMMER_TRYRW_NO_THREADS; i++)
+	ASSERT(THR_CREATE(&tid[i], hammer_tryrw_r, &arg, NULL) == 0);
+    for (i = 0; i < HAMMER_TRYRW_NO_THREADS; i++)
+	ASSERT(THR_JOIN(tid[i], NULL) == 0);
+
+    ASSERT(!ATOMIC_READ(&arg.is_locked));
+
+    RWMUTEX_DESTROY(arg.rwlock);
+
+    if (HAVE_FREQREAD_SUPPORT && !freqread) {
+	freqread = 1;
+	goto restart;
+    }
+
+    if (freqread)
+	return enif_make_atom(env, "ok");
+    else
+	return enif_make_tuple2(env,
+				enif_make_atom(env,
+					       "comment"),
+				enif_make_string(env,
+						 "No frequent read test made.",
+						 ERL_NIF_LATIN1));
+ badarg:
+    return enif_make_badarg(env);
+}
+
+typedef struct {
+    int lock_check;
+    ATOMIC_T is_locked;
+    RWMUTEX_T *rwlock;
+} rwlock_resource_t;
+
+static void
+rwlock_destructor(ErlNifEnv* env, void* obj)
+{
+    rwlock_resource_t *rwlr = obj;
+    if (rwlr->lock_check)
+	ASSERT(!ATOMIC_READ(&rwlr->is_locked));
+    RWMUTEX_DESTROY(rwlr->rwlock);
+}
+
+/*
+ * create_rwlock(FreqRead, LockCheck)
+ */
+
+static ERL_NIF_TERM
+create_rwlock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    int lock_check, freqread;
+    ERL_NIF_TERM rwlock_term;
+    rwlock_resource_t *rwlr;
+    char buf[100];
+
+    if (argc != 2)
+	goto badarg;
+
+    freqread = get_bool(env, argv[0]);
+    if (freqread < 0)
+	goto badarg;
+
+    if (!HAVE_FREQREAD_SUPPORT && freqread)
+	return enif_make_atom(env, "enotsup");
+
+    lock_check = get_bool(env, argv[1]);
+    if (lock_check < 0)
+	goto badarg;
+
+    rwlr = enif_alloc_resource(enif_priv_data(env), sizeof(rwlock_resource_t));
+    rwlr->lock_check = lock_check;
+    ATOMIC_INIT(&rwlr->is_locked, 0);
+    rwlr->rwlock = RWMUTEX_CREATE(freqread);
+    rwlock_term = enif_make_resource(env, rwlr);
+    enif_release_resource(rwlr);
+    return rwlock_term;
+
+ badarg:
+    return enif_make_badarg(env);
+}
+
+/*
+ * rwlock_op(RWLock, Blocking, WriteOp, WaitTime) 
+ */
+
+static ERL_NIF_TERM
+rwlock_op(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    rwlock_resource_t *rwlr;
+    int blocking, write, wait_locked, wait_unlocked;
+
+    if (argc != 5)
+	goto badarg;
+
+    if (!enif_get_resource(env, argv[0], enif_priv_data(env), (void **) &rwlr))
+	goto badarg;
+
+    blocking = get_bool(env, argv[1]);
+    if (blocking < 0)
+	goto badarg;
+
+    write = get_bool(env, argv[2]);
+    if (write < 0)
+	goto badarg;
+
+    if (!enif_get_int(env, argv[3], &wait_locked))
+	goto badarg;
+    if (wait_locked < 0)
+	goto badarg;
+
+    if (!enif_get_int(env, argv[4], &wait_unlocked))
+	goto badarg;
+    if (wait_unlocked < 0)
+	goto badarg;
+
+    if (write) {
+	if (blocking)
+	    RWMUTEX_WLOCK(rwlr->rwlock);
+	else
+	    while (EBUSY == RWMUTEX_TRYWLOCK(rwlr->rwlock));
+	if (rwlr->lock_check) {
+	    ASSERT(!ATOMIC_READ(&rwlr->is_locked));
+	    ATOMIC_SET(&rwlr->is_locked, -1);
+	}
+    }
+    else {
+	if (blocking)
+	    RWMUTEX_RLOCK(rwlr->rwlock);
+	else
+	    while (EBUSY == RWMUTEX_TRYRLOCK(rwlr->rwlock));
+	if (rwlr->lock_check) {
+	    ASSERT(ATOMIC_READ(&rwlr->is_locked) >= 0);
+	    ATOMIC_INC(&rwlr->is_locked);
+	}
+    }
+
+    if (wait_locked)
+	milli_sleep(wait_locked);
+
+    if (write) {
+	if (rwlr->lock_check) {
+	    ASSERT(ATOMIC_READ(&rwlr->is_locked) == -1);
+	    ATOMIC_SET(&rwlr->is_locked, 0);
+	}
+	RWMUTEX_WUNLOCK(rwlr->rwlock);
+    }
+    else {
+	if (rwlr->lock_check) {
+	    ASSERT(ATOMIC_READ(&rwlr->is_locked) > 0);
+	    ATOMIC_DEC(&rwlr->is_locked);
+	}
+	RWMUTEX_RUNLOCK(rwlr->rwlock);
+    }
+
+    if (wait_unlocked)
+	milli_sleep(wait_unlocked);
+
+    return enif_make_atom(env, "ok");
+ badarg:
+    return enif_make_badarg(env);
+}
+
+static int load_nif_lib(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    *priv_data = enif_open_resource_type(env,
+					 NULL,
+					 "rwlock_resource",
+					 rwlock_destructor,
+					 ERL_NIF_RT_CREATE,
+					 NULL);
+    if (*priv_data)
+	return 0;
+    else
+	return -1;
+}
+
+/*
+ * 0 -> false
+ * >0 -> true
+ * <0 -> error
+ */
+
+static int
+get_bool(ErlNifEnv* env, ERL_NIF_TERM term)
+{
+    int res;
+    char buf[10];
+
+    res = enif_get_atom(env, term, buf, sizeof(buf), ERL_NIF_LATIN1);
+    if (res == 0)
+	return -1;
+    if (strcmp("false", buf) == 0)
+	return 0;
+    else if (strcmp("true", buf) == 0)
+	return 1;
+    else
+	return -1;
+}
+
+static int
+fail(const char *file, int line, const char *function, const char *assertion)
+{
+    fprintf(stderr, "%s:%d: Assertion failed in %s(): %s\n",
+	    file, line, function, assertion);
+    abort();
+}
+
+static void
+milli_sleep(int ms)
+{
+#ifdef __WIN32__
+    Sleep(ms);
+#else
+    while (erts_milli_sleep(ms) != 0);
+#endif
+}
+
+static ErlNifFunc nif_funcs[] = {
+    {"long_rw_test", 0, long_rw_test},
+    {"hammer_rw_test", 1, hammer_rw_test},
+    {"hammer_tryrw_test", 1, hammer_tryrw_test},
+    {"create_rwlock", 2, create_rwlock},
+    {"rwlock_op", 5, rwlock_op}
+};
+
+ERL_NIF_INIT(mtx_SUITE, nif_funcs, load_nif_lib, NULL, NULL, NULL)
diff --git a/erts/emulator/test/nested_SUITE.erl b/erts/emulator/test/nested_SUITE.erl
index 310892424e..2cd67ebaae 100644
--- a/erts/emulator/test/nested_SUITE.erl
+++ b/erts/emulator/test/nested_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,11 +19,33 @@
 
 -module(nested_SUITE).
 
--export([all/1, case_in_case/1, case_in_after/1, catch_in_catch/1, bif_in_bif/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 case_in_case/1, case_in_after/1, catch_in_catch/1, bif_in_bif/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [case_in_case, case_in_after, catch_in_catch,
+     bif_in_bif].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) -> [case_in_case, case_in_after, catch_in_catch, bif_in_bif].
 
 case_in_case(suite) -> [];
 case_in_case(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 522caec8f1..91d695d979 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,29 +20,69 @@
 -module(nif_SUITE).
 
 %%-define(line_trace,true).
-%%-define(CHECK(Exp,Got), ?line check(Exp,Got,?LINE)).
--define(CHECK(Exp,Got), ?line Exp = Got).
+-define(CHECK(Exp,Got), check(Exp,Got,?LINE)).
+%%-define(CHECK(Exp,Got), ?line Exp = Got).
+
+-include_lib("test_server/include/test_server.hrl").
+
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, 
+	 end_per_testcase/2, basic/1, reload/1, upgrade/1, heap_frag/1,
+	 types/1, many_args/1, binaries/1, get_string/1, get_atom/1, 
+	 api_macros/1,
+	 from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, 
+	 resource_takeover/1,
+	 threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, 
+	 is_checks/1,
+	 get_length/1, make_atom/1, make_string/1]).
 
--include("test_server.hrl").
+-export([many_args_100/100]).
+
+
+%% -export([lib_version/0,call_history/0,hold_nif_mod_priv_data/1,nif_mod_call_history/0,
+%% 	 list_seq/1,type_test/0,tuple_2_list/1,is_identical/2,compare/2,
+%% 	 clone_bin/1,make_sub_bin/3,string_to_bin/2,atom_to_bin/2,macros/1,
+%% 	 tuple_2_list_and_tuple/1,iolist_2_bin/1,get_resource_type/1,alloc_resource/2,
+%% 	 make_resource/1,get_resource/2,release_resource/1,last_resource_dtor_call/0, suite/0,
+%% 	 make_new_resource/2,make_new_resource_binary/1,send_list_seq/2,send_new_blob/2,
+%% 	 alloc_msgenv/0,clear_msgenv/1,grow_blob/2,send_blob/2,send_blob_thread/3,
+%% 	 join_send_thread/1]).
 
--export([all/1, fin_per_testcase/2, basic/1, reload/1, upgrade/1, heap_frag/1,
-	 types/1, many_args/1, binaries/1, get_string/1, get_atom/1, api_macros/1,
-	 from_array/1, iolist_as_binary/1, resource/1, resource_takeover/1,
-	 threading/1, neg/1]).
 
--export([many_args_100/100]).
 -define(nif_stub,nif_stub_error(?LINE)).
 
-all(suite) ->
-    [basic, reload, upgrade, heap_frag, types, many_args, binaries, get_string,
-     get_atom, api_macros, from_array, iolist_as_binary, resource,
-     resource_takeover, threading, neg].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [basic, reload, upgrade, heap_frag, types, many_args,
+     binaries, get_string, get_atom, api_macros, from_array,
+     iolist_as_binary, resource, resource_binary,
+     resource_takeover, threading, send, send2, send3,
+     send_threaded, neg, is_checks, get_length, make_atom,
+     make_string].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
 
-%%init_per_testcase(_Case, Config) ->
-%%    ?line Dog = ?t:timetrap(?t:seconds(60*60*24)),
-%%    [{watchdog, Dog}|Config].
+init_per_group(_GroupName, Config) ->
+    Config.
 
-fin_per_testcase(_Func, _Config) ->
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+init_per_testcase(_Case, Config) ->
+%    ?line Dog = ?t:timetrap(?t:seconds(60*60*24)),
+    Config.
+
+end_per_testcase(_Func, _Config) ->
     %%Dog = ?config(watchdog, Config),
     %%?t:timetrap_cancel(Dog),
     P1 = code:purge(nif_mod),
@@ -57,7 +97,7 @@ basic(Config) when is_list(Config) ->
     ?line true = (lib_version() =/= undefined),
     ?line [{load,1,1,101},{lib_version,1,2,102}] = call_history(),
     ?line [] = call_history(),
-    ?line [?MODULE] = erlang:system_info(taints),
+    ?line true = lists:member(?MODULE, erlang:system_info(taints)),
     ok.
 
 reload(doc) -> ["Test reload callback in nif lib"];
@@ -91,7 +131,8 @@ reload(Config) when is_list(Config) ->
     ?line true = erlang:purge_module(nif_mod),
     ?line [{unload,1,3,103}] = nif_mod_call_history(),    
 
-    ?line [?MODULE, nif_mod] = erlang:system_info(taints),
+    ?line true = lists:member(?MODULE, erlang:system_info(taints)),
+    ?line true = lists:member(nif_mod, erlang:system_info(taints)),
     ?line verify_tmpmem(TmpMem),
     ok.
 
@@ -181,7 +222,8 @@ upgrade(Config) when is_list(Config) ->
     ?line true = erlang:purge_module(nif_mod),
     ?line [{unload,2,4,204}] = nif_mod_call_history(),    
 
-    ?line [?MODULE, nif_mod] = erlang:system_info(taints),    
+    ?line true = lists:member(?MODULE, erlang:system_info(taints)),
+    ?line true = lists:member(nif_mod, erlang:system_info(taints)),
     ?line verify_tmpmem(TmpMem),
     ok.
 
@@ -473,12 +515,51 @@ resource_new_do2(Type) ->
     {{PtrA,BinA}, {ResB,PtrB,BinB}}.
 
 resource_neg(TypeA) ->
+    resource_neg_do(TypeA),
+
+    catch exit(42), % dummy exception to purge saved stacktraces from earlier exception
+    erlang:garbage_collect(),
+    ?line {_,_,2} = last_resource_dtor_call(),
+    ok.
+
+resource_neg_do(TypeA) ->
     TypeB = get_resource_type(1),
-    Aptr = alloc_resource(TypeA, <<"Arnold">>),
-    Bptr = alloc_resource(TypeB, <<"Bobo">>),
-    ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeA, Bptr)),
-    ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeB, Aptr)),
+    ResA = make_new_resource(TypeA, <<"Arnold">>),
+    ResB= make_new_resource(TypeB, <<"Bobo">>),
+    ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeA, ResB)),
+    ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeB, ResA)),
     ok.
+
+resource_binary(doc) -> ["Test enif_make_resource_binary"];
+resource_binary(suite) -> [];
+resource_binary(Config) when is_list(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    ?line {Ptr,Bin} = resource_binary_do(),
+    erlang:garbage_collect(),
+    Last = last_resource_dtor_call(),
+    ?CHECK({Ptr,Bin,1}, Last),
+    ok.
+
+resource_binary_do() ->
+    Bin = <<"Hej Hopp i lingonskogen">>,
+    ?line {Ptr,ResBin1} = make_new_resource_binary(Bin),
+    ?line ResBin1 = Bin,          
+    ?line ResInfo = {Ptr,_} = get_resource(binary_resource_type,ResBin1),
+
+    Papa = self(),
+    Forwarder = spawn_link(fun() -> forwarder(Papa) end),
+    io:format("sending to forwarder pid=~p\n",[Forwarder]),  
+    Forwarder ! ResBin1,
+    ResBin2 = receive_any(),
+    ?line ResBin2 = ResBin1,
+    ?line ResInfo = get_resource(binary_resource_type,ResBin2),
+    Forwarder ! terminate,
+    ?line {Forwarder, 1} = receive_any(),
+    erlang:garbage_collect(),
+    ?line ResInfo = get_resource(binary_resource_type,ResBin1),
+    ?line ResInfo = get_resource(binary_resource_type,ResBin2),
+    ResInfo.
+
     
 -define(RT_CREATE,1).
 -define(RT_TAKEOVER,2).
@@ -672,7 +753,8 @@ resource_takeover(Config) when is_list(Config) ->
     ?line ok = forget_resource(AN4),
     ?line [] = nif_mod_call_history(),
 
-    ?line [?MODULE, nif_mod] = erlang:system_info(taints),
+    ?line true = lists:member(?MODULE, erlang:system_info(taints)),
+    ?line true = lists:member(nif_mod, erlang:system_info(taints)),
     ?line verify_tmpmem(TmpMem),    
     ok.
 
@@ -743,7 +825,282 @@ threading(Config) when is_list(Config) ->
 
     ?line ok = tester:load_nif_lib(Config, "tsd"),
     ?line ok = tester:run().
+
+send(doc) -> ["Test NIF message sending"];
+send(Config) when is_list(Config) ->    
+    ensure_lib_loaded(Config),
+
+    N = 1500,
+    List = lists:seq(1,N),
+    ?line {ok,1} = send_list_seq(N, self),
+    ?line {ok,1} = send_list_seq(N, self()),
+    ?line List = receive_any(),
+    ?line List = receive_any(),
+    Papa = self(),
+    spawn_link(fun() -> ?line {ok,1} = send_list_seq(N, Papa) end),
+    ?line List = receive_any(),
+
+    ?line {ok, 1, BlobS} = send_new_blob(self(), other_term()),
+    ?line BlobR = receive_any(),
+    io:format("Sent ~p\nGot ~p\n", [BlobS, BlobR]),
+    ?line BlobR = BlobS,
+
+    %% send to dead pid
+    {DeadPid, DeadMon} = spawn_monitor(fun() -> void end),
+    ?line {'DOWN', DeadMon, process, DeadPid, normal} = receive_any(),
+    {ok,0} = send_list_seq(7, DeadPid),
+    ok.
+
+send2(doc) -> ["More NIF message sending"];
+send2(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+
+    send2_do1(fun send_blob_dbg/2),
+    ok.
+
+send_threaded(doc) -> ["Send msg from user thread"];
+send_threaded(Config) when is_list(Config) ->
+    case erlang:system_info(smp_support) of
+	true ->
+	    send2_do1(fun(ME,To) -> send_blob_thread_dbg(ME,To,join) end),
+	    send2_do1(fun(ME,To) -> send_blob_thread_and_join(ME,To) end),
+	    ok;
+	false ->
+	    {skipped,"No threaded send on non-SMP"}
+    end.
+
+
+send2_do1(SendBlobF) ->
+    io:format("sending to self=~p\n",[self()]),
+    send2_do2(SendBlobF, self()),
+
+    Papa = self(),
+    Forwarder = spawn_link(fun() -> forwarder(Papa) end),
+    io:format("sending to forwarder pid=~p\n",[Forwarder]),
+    send2_do2(SendBlobF, Forwarder),
+    Forwarder ! terminate,
+    ?line {Forwarder, 4} = receive_any(),
+    ok.
+
+send2_do2(SendBlobF, To) ->   
+    MsgEnv = alloc_msgenv(),
+    repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []),
+    ?line {ok,1,Blob0} = SendBlobF(MsgEnv, To),
+    ?line Blob1 = receive_any(),
+    ?line Blob1 = Blob0,
     
+    clear_msgenv(MsgEnv),
+    repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []),
+    ?line {ok,1,Blob2} = SendBlobF(MsgEnv, To),
+    ?line Blob3 = receive_any(),
+    ?line Blob3 = Blob2,
+
+    clear_msgenv(MsgEnv),
+    repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []),
+
+    clear_msgenv(MsgEnv),
+    repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []),
+    ?line {ok,1,Blob4} = SendBlobF(MsgEnv, To),
+    ?line Blob5 = receive_any(),
+    ?line Blob5 = Blob4,
+
+    clear_msgenv(MsgEnv),
+    clear_msgenv(MsgEnv),
+    repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []),
+    ?line {ok,1,Blob6} = SendBlobF(MsgEnv, To),
+    ?line Blob7 = receive_any(),
+    ?line Blob7 = Blob6,
+
+    ok.
+
+
+send_blob_thread_and_join(MsgEnv, To) ->
+    ?line {ok,Blob} = send_blob_thread_dbg(MsgEnv, To, no_join),
+    ?line {ok,SendRes} = join_send_thread(MsgEnv),
+    {ok,SendRes,Blob}.
+
+send_blob_dbg(MsgEnv, To) ->
+    Ret = send_blob(MsgEnv, To),
+    %%io:format("send_blob to ~p returned ~p\n",[To,Ret]),
+    Ret.
+
+send_blob_thread_dbg(MsgEnv, To, Join) ->
+    Ret = send_blob_thread(MsgEnv, To, Join),
+    %%io:format("send_blob_thread to ~p Join=~p returned ~p\n",[To,Join,Ret]),
+    Ret.
+
+
+forwarder(To) ->
+    forwarder(To, 0).
+forwarder(To, N) ->
+    case receive_any() of
+	terminate ->
+	    To ! {self(), N};
+	Msg ->	    
+	    To ! Msg,	   
+	    forwarder(To, N+1)
+    end.
+
+other_term() ->
+    {fun(X,Y) -> X*Y end, make_ref()}.
+
+send3(doc) -> ["Message sending stress test"];
+send3(Config) when is_list(Config) ->
+    %% Let a number of processes send random message blobs between each other
+    %% using enif_send. Kill and spawn new ones randomly to keep a ~constant
+    %% number of workers running.
+    Seed = now(),
+    io:format("seed: ~p\n",[Seed]), 
+    random:seed(Seed),    
+    ets:new(nif_SUITE,[named_table,public]),
+    ?line true = ets:insert(nif_SUITE,{send3,0,0,0,0}),
+    timer:send_after(10000, timeout), % Run for 10 seconds
+    SpawnCnt = send3_controller(0, [], [], 20),
+    ?line [{_,Rcv,SndOk,SndFail,Balance}] = ets:lookup(nif_SUITE,send3),
+    io:format("spawns=~p received=~p, sent=~p send-failure=~p balance=~p\n",
+              [SpawnCnt,Rcv,SndOk,SndFail,Balance]),
+    ets:delete(nif_SUITE).
+
+send3_controller(SpawnCnt, [], _, infinity) ->
+    SpawnCnt;
+send3_controller(SpawnCnt0, Mons0, Pids0, Tick) ->
+    receive 
+        timeout ->
+            io:format("Timeout. Sending 'halt' to ~p\n",[Pids0]),            
+            lists:foreach(fun(P) -> P ! {halt,self()} end, Pids0),
+            lists:foreach(fun(P) -> receive {halted,P} -> ok end end, Pids0),
+            QTot = lists:foldl(fun(P,QSum) ->
+                                 {message_queue_len,QLen} = 
+                                    erlang:process_info(P,message_queue_len),
+                                 QSum + QLen
+                               end, 0, Pids0),
+            io:format("Total queue length ~p\n",[QTot]),            
+            lists:foreach(fun(P) -> P ! die end, Pids0),
+            send3_controller(SpawnCnt0, Mons0, [], infinity);
+        {'DOWN', MonRef, process, _Pid, _} ->
+            Mons1 = lists:delete(MonRef, Mons0),
+            %%io:format("Got DOWN from ~p. Monitors left: ~p\n",[Pid,Mons1]),            
+            send3_controller(SpawnCnt0, Mons1, Pids0, Tick)
+    after Tick -> 
+        Max = 20,
+        N = length(Pids0),
+        PidN = random:uniform(Max),
+        %%io:format("N=~p PidN=~p Pids0=~p\n", [N,PidN,Pids0]), 
+        case PidN > N of
+            true ->
+                {NewPid,Mon} = spawn_opt(fun send3_proc/0, [link,monitor]),                
+                lists:foreach(fun(P) -> P ! {is_born,NewPid} end, Pids0),
+                ?line Balance = ets:lookup_element(nif_SUITE,send3,5),
+                Inject = (Balance =< 0),
+                case Inject of
+                    true ->  ok;
+                    false -> ets:update_element(nif_SUITE,send3,{5,-1})
+                end,
+                NewPid ! {pids,Pids0,Inject},
+                send3_controller(SpawnCnt0+1, [Mon|Mons0], [NewPid|Pids0], Tick);
+            false ->
+                KillPid = lists:nth(PidN,Pids0),
+                KillPid ! die,
+                Pids1 = lists:delete(KillPid, Pids0),
+                lists:foreach(fun(P) -> P ! {is_dead,KillPid} end, Pids1),
+                send3_controller(SpawnCnt0, Mons0, Pids1, Tick)
+        end        
+   end.
+
+send3_proc() ->
+    %%io:format("Process ~p spawned\n",[self()]),
+    send3_proc([self()], {0,0,0}, {1,2,3,4,5}).
+send3_proc(Pids0, Counters={Rcv,SndOk,SndFail}, State0) ->
+    %%io:format("~p: Pids0=~p", [self(), Pids0]),
+    %%timer:sleep(10),
+    receive 
+        {pids, Pids1, Inject} ->
+            %%io:format("~p: got ~p Inject=~p\n", [self(), Pids1, Inject]), 
+            ?line Pids0 = [self()],
+            Pids2 = [self() | Pids1],
+            case Inject of
+                true -> send3_proc_send(Pids2, Counters, State0);
+                false -> send3_proc(Pids2, Counters, State0)
+            end;
+        {is_born, Pid} ->
+            %%io:format("~p: is_born ~p, got ~p\n", [self(), Pid, Pids0]), 
+            send3_proc([Pid | Pids0], Counters, State0);
+        {is_dead, Pid} ->            
+            Pids1 = lists:delete(Pid,Pids0),
+            %%io:format("~p: is_dead ~p, got ~p\n", [self(), Pid, Pids1]),
+            send3_proc(Pids1, Counters, State0);
+        {blob, Blob0} ->
+            %%io:format("~p: blob ~p\n", [self(), Blob0]),
+            State1 = send3_new_state(State0, Blob0),
+            send3_proc_send(Pids0, {Rcv+1,SndOk,SndFail}, State1);
+        die ->
+            %%io:format("Process ~p terminating, stats = ~p\n",[self(),Counters]),
+            {message_queue_len,Dropped} = erlang:process_info(self(),message_queue_len),
+            _R = ets:update_counter(nif_SUITE,send3,
+                               [{2,Rcv},{3,SndOk},{4,SndFail},{5,1-Dropped}]),
+            %%io:format("~p: dies R=~p\n", [self(), R]),
+            ok;
+        {halt,Papa} ->
+            Papa ! {halted,self()},
+            io:format("~p halted\n",[self()]),
+            receive die -> ok end,
+            io:format("~p dying\n",[self()])
+    end.
+
+send3_proc_send(Pids, {Rcv,SndOk,SndFail}, State0) ->
+    To = lists:nth(random:uniform(length(Pids)),Pids),
+    Blob = send3_make_blob(),
+    State1 = send3_new_state(State0,Blob), 
+    case send3_send(To, Blob) of
+        true ->
+            send3_proc(Pids, {Rcv,SndOk+1,SndFail}, State1);
+        false ->
+            send3_proc(Pids, {Rcv,SndOk,SndFail+1}, State1)
+    end.
+
+
+send3_make_blob() ->    
+    case random:uniform(20)-1 of
+        0 -> {term,[]};
+        N ->
+            MsgEnv = alloc_msgenv(), 
+            repeat(N bsr 1,
+                   fun(_) -> grow_blob(MsgEnv,other_term(),random:uniform(1 bsl 20))
+                   end, void),
+            case (N band 1) of
+                0 -> {term,copy_blob(MsgEnv)};
+                1 -> {msgenv,MsgEnv}
+            end
+    end.
+
+send3_send(Pid, Msg) ->
+    %% 90% enif_send and 10% normal bang
+    case random:uniform(10) of
+        1 -> send3_send_bang(Pid,Msg);
+        _ -> send3_send_nif(Pid,Msg)
+    end.
+send3_send_nif(Pid, {term,Blob}) ->
+    %%io:format("~p send term nif\n",[self()]),
+    send_term(Pid, {blob, Blob}) =:= 1;
+send3_send_nif(Pid, {msgenv,MsgEnv}) ->
+    %%io:format("~p send blob nif\n",[self()]),   
+    send3_blob(MsgEnv, Pid, blob) =:= 1.
+
+send3_send_bang(Pid, {term,Blob}) ->
+    %%io:format("~p send term bang\n",[self()]),   
+    Pid ! {blob, Blob},
+    true;
+send3_send_bang(Pid, {msgenv,MsgEnv}) ->
+    %%io:format("~p send blob bang\n",[self()]),   
+    Pid ! {blob, copy_blob(MsgEnv)},
+    true.
+
+send3_new_state(State, Blob) ->
+    case random:uniform(5+2) of
+        N when N =< 5-> setelement(N, State, Blob);
+        _ -> State  % Don't store blob
+    end.
+
 neg(doc) -> ["Negative testing of load_nif"];
 neg(Config) when is_list(Config) ->
     TmpMem = tmpmem(),
@@ -759,7 +1116,24 @@ neg(Config) when is_list(Config) ->
     ?line verify_tmpmem(TmpMem),
     ?line ok.
 
+is_checks(doc) -> ["Test all enif_is functions"];
+is_checks(Config) when is_list(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end,
+			self(), hd(erlang:ports()), [], [1,9,9,8],
+			{hejsan, "hejsan", [$h,"ejs",<<"an">>]}),
+    try
+        ?line error = check_is_exception(),
+        ?line throw(expected_badarg)
+    catch
+        error:badarg ->
+            ?line ok
+    end.
 
+get_length(doc) -> ["Test all enif_get_length functions"];
+get_length(Config) when is_list(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    ?line ok = length_test(hejsan, "hejsan", [], [], not_a_list).
 
 ensure_lib_loaded(Config) ->
     ensure_lib_loaded(Config, 1).
@@ -773,6 +1147,23 @@ ensure_lib_loaded(Config, Ver) ->
 		  ok
 	  end.
 
+make_atom(Config) when is_list(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    An0Atom = an0atom,
+    An0Atom0 = 'an\000atom\000',
+    ?line Atoms = make_atoms(),
+    ?line 7 = size(Atoms),
+    ?line Atoms = {An0Atom,An0Atom,An0Atom,An0Atom0,An0Atom,An0Atom,An0Atom0}.
+
+make_string(Config) when is_list(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    ?line Strings = make_strings(),
+    ?line 5 = size(Strings),
+    A0String = "a0string",
+    A0String0 = [$a,0,$s,$t,$r,$i,$n,$g,0],
+    AStringWithAccents = [$E,$r,$l,$a,$n,$g,$ ,16#e4,$r,$ ,$e,$t,$t,$ ,$g,$e,$n,$e,$r,$e,$l,$l,$t,$ ,$p,$r,$o,$g,$r,$a,$m,$s,$p,$r,16#e5,$k],
+    ?line Strings = {A0String,A0String,A0String,A0String0, AStringWithAccents}.
+
 tmpmem() ->
     case erlang:system_info({allocator,temp_alloc}) of
 	false -> undefined;
@@ -821,13 +1212,18 @@ call(Pid,Cmd) ->
 receive_any() ->
     receive M -> M end.	     
 
-%% check(Exp,Got,Line) ->
-%%     case Got of
-%% 	Exp -> Exp;	    
-%% 	_ ->
-%% 	    io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]),
-%% 	    Got
-%%     end.
+repeat(0, _, Arg) ->
+    Arg;
+repeat(N, Fun, Arg0) ->
+    repeat(N-1, Fun, Fun(Arg0)).
+
+check(Exp,Got,Line) ->
+    case Got of
+ 	Exp -> Exp;	    
+  	_ ->
+  	    io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]),
+ 	    Got
+    end.
 	    
 
 %% The NIFs:
@@ -855,6 +1251,24 @@ get_resource(_,_) -> ?nif_stub.
 release_resource(_) -> ?nif_stub.
 last_resource_dtor_call() -> ?nif_stub.
 make_new_resource(_,_) -> ?nif_stub.
+check_is(_,_,_,_,_,_,_,_,_,_) -> ?nif_stub.
+check_is_exception() -> ?nif_stub.
+length_test(_,_,_,_,_) -> ?nif_stub.
+make_atoms() -> ?nif_stub.
+make_strings() -> ?nif_stub.
+make_new_resource_binary(_) -> ?nif_stub.
+send_list_seq(_,_) -> ?nif_stub.     
+send_new_blob(_,_) -> ?nif_stub.     
+alloc_msgenv() -> ?nif_stub.
+clear_msgenv(_) -> ?nif_stub.
+grow_blob(_,_) -> ?nif_stub.
+grow_blob(_,_,_) -> ?nif_stub.
+send_blob(_,_) -> ?nif_stub.
+send3_blob(_,_,_) -> ?nif_stub.
+send_blob_thread(_,_,_) -> ?nif_stub.
+join_send_thread(_) -> ?nif_stub.
+copy_blob(_) -> ?nif_stub.
+send_term(_,_) -> ?nif_stub.
 
 nif_stub_error(Line) ->
     exit({nif_not_loaded,module,?MODULE,line,Line}).
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 7d05a9a880..0bb93daa33 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -1,3 +1,21 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
 #include "erl_nif.h"
 
 #include <stdio.h>
@@ -10,6 +28,12 @@
 static int static_cntA; /* zero by default */
 static int static_cntB = NIF_SUITE_LIB_VER * 100;
 
+static ERL_NIF_TERM atom_self;
+static ERL_NIF_TERM atom_ok;
+static ERL_NIF_TERM atom_join;
+static ERL_NIF_TERM atom_binary_resource_type;
+
+
 typedef struct
 {
     int ref_cnt;
@@ -20,7 +44,7 @@ typedef struct
 
 void add_call(ErlNifEnv* env, PrivData* data, const char* func_name)
 {
-    CallInfo* call = enif_alloc(env, sizeof(CallInfo)+strlen(func_name));
+    CallInfo* call = enif_alloc(sizeof(CallInfo)+strlen(func_name));
     strcpy(call->func_name, func_name);
     call->lib_ver = NIF_SUITE_LIB_VER;
     call->next = data->call_history;
@@ -31,7 +55,7 @@ void add_call(ErlNifEnv* env, PrivData* data, const char* func_name)
     call->arg_sz = 0;
 }
 
-#define ADD_CALL(FUNC_NAME) add_call(env, enif_get_data(env),FUNC_NAME)
+#define ADD_CALL(FUNC_NAME) add_call(env, enif_priv_data(env),FUNC_NAME)
 
 static void*    resource_dtor_last = NULL;
 static unsigned resource_dtor_last_sz = 0;
@@ -42,15 +66,24 @@ static void resource_dtor(ErlNifEnv* env, void* obj)
 {
     resource_dtor_last = obj;
     resource_dtor_cnt++;
-    resource_dtor_last_sz = enif_sizeof_resource(env, obj);
+    resource_dtor_last_sz = enif_sizeof_resource(obj);
     assert(resource_dtor_last_sz <= sizeof(resource_dtor_last_data));
     memcpy(resource_dtor_last_data, obj, resource_dtor_last_sz);
 }
 
+static ErlNifResourceType* msgenv_resource_type;
+static void msgenv_dtor(ErlNifEnv* env, void* obj);
+
+static ErlNifResourceType* binary_resource_type;
+static void binary_resource_dtor(ErlNifEnv* env, void* obj);
+struct binary_resource {
+    unsigned char* data;
+    unsigned size;
+};
+
 static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 {
-    /*ERL_NIF_TERM head, tail;*/
-    PrivData* data = enif_alloc(env, sizeof(PrivData));
+    PrivData* data = enif_alloc(sizeof(PrivData));
     assert(data != NULL);
     data->ref_cnt = 1;
     data->call_history = NULL;
@@ -58,41 +91,71 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 
     add_call(env, data, "load");
 
-    /*
-    head = load_info;
-    data->rt_cnt = 0;
-    for (head=load_info; enif_get_list_cell(env,load_info,&head,&tail);
-	  head=tail) {
-	char buf[20];
-	int n = enif_get_string(env,head,buf,sizeof(buf));
-	assert(n > 0);
-	assert(i < sizeof(data->rt_arr)/sizeof(*data->rt_arr));
-	data->rt_arr[data->rt_cnt++].t = enif_create_resource_type(env,buf,resource_dtor,
-								   ERL_NIF_RT_CREATE,NULL);	   
-    }
-    assert(enif_is_empty_list(env,head)); 
-    */
-    data->rt_arr[0].t = enif_open_resource_type(env,"Gold",resource_dtor,
+    data->rt_arr[0].t = enif_open_resource_type(env,NULL,"Gold",resource_dtor,
 						ERL_NIF_RT_CREATE,NULL);
-    data->rt_arr[1].t = enif_open_resource_type(env,"Silver",resource_dtor,
+    data->rt_arr[1].t = enif_open_resource_type(env,NULL,"Silver",resource_dtor,
 						ERL_NIF_RT_CREATE,NULL);
 
+    binary_resource_type =  enif_open_resource_type(env,NULL,"nif_SUITE.binary",
+						    binary_resource_dtor,
+						    ERL_NIF_RT_CREATE, NULL);
+
+    msgenv_resource_type =  enif_open_resource_type(env,NULL,"nif_SUITE.msgenv",
+						    msgenv_dtor,
+						    ERL_NIF_RT_CREATE, NULL);
+
+    atom_self = enif_make_atom(env,"self");
+    atom_ok = enif_make_atom(env,"ok");
+    atom_join = enif_make_atom(env,"join");
+    atom_binary_resource_type = enif_make_atom(env,"binary_resource_type");
+
     *priv_data = data;
     return 0;
 }
 
+static void resource_takeover(ErlNifEnv* env, PrivData* priv)
+{
+    ErlNifResourceFlags tried;
+    ErlNifResourceType* rt;
+    rt = enif_open_resource_type(env, NULL, "Gold", resource_dtor,
+				 ERL_NIF_RT_TAKEOVER, &tried);
+    assert(rt == priv->rt_arr[0].t); 
+    assert(tried == ERL_NIF_RT_TAKEOVER);
+    rt = enif_open_resource_type(env, NULL, "Silver", resource_dtor,
+				 ERL_NIF_RT_TAKEOVER, &tried);
+    assert(rt == priv->rt_arr[1].t); 
+    assert(tried == ERL_NIF_RT_TAKEOVER);
+
+    rt =  enif_open_resource_type(env, NULL, "nif_SUITE.binary", binary_resource_dtor,
+				  ERL_NIF_RT_TAKEOVER, &tried);
+    assert(rt != NULL);
+    assert(tried == ERL_NIF_RT_TAKEOVER);
+    assert(binary_resource_type==NULL || binary_resource_type == rt); 
+    binary_resource_type = rt;
+
+    rt = enif_open_resource_type(env, NULL, "nif_SUITE.msgenv", msgenv_dtor,
+				  ERL_NIF_RT_TAKEOVER, &tried);
+    assert(rt != NULL);
+    assert(tried == ERL_NIF_RT_TAKEOVER);
+    assert(msgenv_resource_type==NULL || msgenv_resource_type == rt); 
+    msgenv_resource_type = rt;
+}
+
 static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 {
-    add_call(env, *priv_data, "reload");
+    PrivData* priv = (PrivData*) *priv_data;
+    add_call(env, priv, "reload");
+    resource_takeover(env,priv);
     return 0;
 }
 
 static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
 {
-    PrivData* data = *old_priv_data;
-    add_call(env, data, "upgrade");
-    data->ref_cnt++;
-    *priv_data = *old_priv_data;    
+    PrivData* priv = (PrivData*) *old_priv_data;
+    add_call(env, priv, "upgrade");
+    priv->ref_cnt++;
+    *priv_data = *old_priv_data;
+    resource_takeover(env,priv);
     return 0;
 }
 
@@ -101,7 +164,10 @@ static void unload(ErlNifEnv* env, void* priv_data)
     PrivData* data = priv_data;
     add_call(env, data, "unload");
     if (--data->ref_cnt == 0) {
-	enif_free(env, priv_data);
+	if (data->nif_mod != NULL) {
+	    NifModPrivData_release(data->nif_mod);
+	}
+	enif_free(priv_data);
     }
 }
 
@@ -120,11 +186,10 @@ static ERL_NIF_TERM make_call_history(ErlNifEnv* env, CallInfo** headp)
 	ERL_NIF_TERM func_term = enif_make_atom(env,call->func_name);
 	ERL_NIF_TERM tpl;
 	if (call->arg != NULL) {
-	    ErlNifBinary arg_bin;
-	    enif_alloc_binary(env, call->arg_sz, &arg_bin);
-	    memcpy(arg_bin.data, call->arg, call->arg_sz);
-	    func_term = enif_make_tuple2(env, func_term,
-					 enif_make_binary(env, &arg_bin));
+	    ERL_NIF_TERM arg_bin;	    
+	    memcpy(enif_make_new_binary(env, call->arg_sz, &arg_bin),
+		   call->arg, call->arg_sz);
+	    func_term = enif_make_tuple2(env, func_term, arg_bin);
 	}
 	tpl = enif_make_tuple4(env, func_term, 					    
 			       enif_make_int(env,call->lib_ver),
@@ -132,28 +197,28 @@ static ERL_NIF_TERM make_call_history(ErlNifEnv* env, CallInfo** headp)
 			       enif_make_int(env,call->static_cntB));
 	list = enif_make_list_cell(env, tpl, list);
 	*headp = call->next;
-	enif_free(env,call);
+	enif_free(call);
     }
     return list;
 }
 
 static ERL_NIF_TERM call_history(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    PrivData* data = (PrivData*) enif_get_data(env);
+    PrivData* data = (PrivData*) enif_priv_data(env);
 
     return make_call_history(env,&data->call_history);
 }
 
 static ERL_NIF_TERM hold_nif_mod_priv_data(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    PrivData* data = (PrivData*) enif_get_data(env);
+    PrivData* data = (PrivData*) enif_priv_data(env);
     unsigned long ptr_as_ulong;
     
     if (!enif_get_ulong(env,argv[0],&ptr_as_ulong)) {
 	return enif_make_badarg(env);
     }
-    if (data->nif_mod != NULL && --(data->nif_mod->ref_cnt) == 0) {
-	enif_free(env,data->nif_mod);
+    if (data->nif_mod != NULL) {
+	NifModPrivData_release(data->nif_mod);
     }
     data->nif_mod = (NifModPrivData*) ptr_as_ulong;    
     return enif_make_int(env,++(data->nif_mod->ref_cnt)); 
@@ -161,7 +226,7 @@ static ERL_NIF_TERM hold_nif_mod_priv_data(ErlNifEnv* env, int argc, const ERL_N
 
 static ERL_NIF_TERM nif_mod_call_history(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    PrivData* data = (PrivData*) enif_get_data(env);
+    PrivData* data = (PrivData*) enif_priv_data(env);
     ERL_NIF_TERM ret;
     if (data->nif_mod == NULL) {
 	return enif_make_string(env,"nif_mod pointer is NULL", ERL_NIF_LATIN1);
@@ -231,6 +296,30 @@ static int test_ulong(ErlNifEnv* env, unsigned long i1)
     return 1;
 }
 
+static int test_int64(ErlNifEnv* env, ErlNifSInt64 i1)
+{
+    ErlNifSInt64 i2 = 0;
+    ERL_NIF_TERM int_term = enif_make_int64(env, i1);
+    if (!enif_get_int64(env,int_term, &i2) || i1 != i2) {
+	fprintf(stderr, "test_int64(%ld) ...FAILED i2=%ld\r\n",
+		(long)i1, (long)i2);
+	return 0;
+    }
+    return 1;
+}
+
+static int test_uint64(ErlNifEnv* env, ErlNifUInt64 i1)
+{
+    ErlNifUInt64 i2 = 0;
+    ERL_NIF_TERM int_term = enif_make_uint64(env, i1);
+    if (!enif_get_uint64(env,int_term, &i2) || i1 != i2) {
+	fprintf(stderr, "test_ulong(%lu) ...FAILED i2=%lu\r\n",
+		(unsigned long)i1, (unsigned long)i2);
+	return 0;
+    }
+    return 1;
+}
+
 static int test_double(ErlNifEnv* env, double d1)
 {
     double d2 = 0;
@@ -254,6 +343,8 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     unsigned uint;
     long slong;
     unsigned long ulong;
+    ErlNifSInt64 sint64;
+    ErlNifUInt64 uint64;
     double d;
     ERL_NIF_TERM atom, ref1, ref2;
 
@@ -287,11 +378,25 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
 	slong -= slong / 3 + 1;
     } while (slong >= 0);
 
+    sint64 = ((ErlNifSInt64)1 << 63); /* INT64_MIN */
+    do {
+	if (!test_int64(env,sint64)) {
+	    goto error;
+	}
+	sint64 += ~sint64 / 3 + 1;
+    } while (sint64 < 0);
+    sint64 = ((ErlNifUInt64)1 << 63) - 1; /* INT64_MAX */
+    do {
+	if (!test_int64(env,sint64)) {
+	    goto error;
+	}
+	sint64 -= sint64 / 3 + 1;
+    } while (sint64 >= 0);
 
     uint = UINT_MAX;
     for (;;) {
 	if (!test_uint(env,uint)) {
-	    
+	    goto error;
 	}
 	if (uint == 0) break;
 	uint -= uint / 3 + 1;
@@ -299,11 +404,19 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     ulong = ULONG_MAX;
     for (;;) {
 	if (!test_ulong(env,ulong)) {
-	    
+	    goto error;
 	}
 	if (ulong == 0) break;
 	ulong -= ulong / 3 + 1;
     }    
+    uint64 = (ErlNifUInt64)-1; /* UINT64_MAX */
+    for (;;) {
+	if (!test_uint64(env,uint64)) {
+	    goto error;
+	}
+	if (uint64 == 0) break;
+	uint64 -= uint64 / 3 + 1;
+    }    
 
     if (MAX_SMALL < INT_MAX) { /* 32-bit */
 	for (i=-10 ; i <= 10; i++) {
@@ -326,24 +439,31 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
 
     for (i=-10 ; i < 10; i++) {
 	if (!test_long(env,MAX_SMALL+i) || !test_ulong(env,MAX_SMALL+i) ||
-	    !test_long(env,MIN_SMALL+i)) {
+	    !test_long(env,MIN_SMALL+i) ||
+	    !test_int64(env,MAX_SMALL+i) || !test_uint64(env,MAX_SMALL+i) ||
+	    !test_int64(env,MIN_SMALL+i)) {
 	    goto error;
 	}
+	if (MAX_SMALL < INT_MAX) {
+	    if (!test_int(env,MAX_SMALL+i) || !test_uint(env,MAX_SMALL+i) ||
+		!test_int(env,MIN_SMALL+i)) {
+		goto error;
+	    }
+	}
     }
-
     for (d=3.141592e-100 ; d < 1e100 ; d *= 9.97) {
 	if (!test_double(env,d) || !test_double(env,-d)) {
 	    goto error;
 	}	
     }
 
-    if (!enif_make_existing_atom(env,"nif_SUITE", &atom)
-	|| !enif_is_identical(env,atom,enif_make_atom(env,"nif_SUITE"))) {
+    if (!enif_make_existing_atom(env,"nif_SUITE", &atom, ERL_NIF_LATIN1)
+	|| !enif_is_identical(atom,enif_make_atom(env,"nif_SUITE"))) {
 	fprintf(stderr, "nif_SUITE not an atom?\r\n");
 	goto error;
     }
     for (i=2; i; i--) {
-	if (enif_make_existing_atom(env,"nif_SUITE_pink_unicorn", &atom)) {
+	if (enif_make_existing_atom(env,"nif_SUITE_pink_unicorn", &atom, ERL_NIF_LATIN1)) {
 	    fprintf(stderr, "pink unicorn exist?\r\n");
 	    goto error;
 	}
@@ -351,7 +471,7 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     ref1 = enif_make_ref(env);
     ref2 = enif_make_ref(env);
     if (!enif_is_ref(env,ref1) || !enif_is_ref(env,ref2) 
-	|| enif_is_identical(env,ref1,ref2) || enif_compare(env,ref1,ref2)==0) {
+	|| enif_is_identical(ref1,ref2) || enif_compare(ref1,ref2)==0) {
 	fprintf(stderr, "strange refs?\r\n");
 	goto error;
     }
@@ -381,7 +501,7 @@ static ERL_NIF_TERM is_identical(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
     if (argc != 2) {
 	return enif_make_badarg(env);
     }
-    return enif_make_atom(env, (enif_is_identical(env,argv[0],argv[1]) ?
+    return enif_make_atom(env, (enif_is_identical(argv[0],argv[1]) ?
 				"true" : "false"));
 }
 
@@ -390,7 +510,7 @@ static ERL_NIF_TERM compare(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     if (argc != 2) {
 	return enif_make_badarg(env);
     }
-    return enif_make_int(env, enif_compare(env,argv[0],argv[1]));
+    return enif_make_int(env, enif_compare(argv[0],argv[1]));
 }
 
 static ERL_NIF_TERM many_args_100(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -412,11 +532,10 @@ static ERL_NIF_TERM clone_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
 {
     ErlNifBinary ibin;
     if (enif_inspect_binary(env,argv[0],&ibin)) {
-	ErlNifBinary obin;
-	enif_alloc_binary(env,ibin.size,&obin);
-	memcpy(obin.data,ibin.data,ibin.size);
-	/*enif_release_binary(env,&ibin);*/
-	return enif_make_binary(env,&obin);
+	ERL_NIF_TERM obin;	
+	memcpy(enif_make_new_binary(env, ibin.size, &obin),
+	       ibin.data, ibin.size);
+	return obin;
     }
     else {
 	return enif_make_badarg(env);
@@ -438,7 +557,7 @@ static ERL_NIF_TERM string_to_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM a
     unsigned size;
     int n;
     if (!enif_get_int(env,argv[1],(int*)&size) 
-	|| !enif_alloc_binary(env,size,&obin)) {
+	|| !enif_alloc_binary(size,&obin)) {
 	return enif_make_badarg(env);
     }
     n = enif_get_string(env, argv[0], (char*)obin.data, size, ERL_NIF_LATIN1);
@@ -452,10 +571,10 @@ static ERL_NIF_TERM atom_to_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
     unsigned size;
     int n;
     if (!enif_get_int(env,argv[1],(int*)&size) 
-	|| !enif_alloc_binary(env,size,&obin)) {
+	|| !enif_alloc_binary(size,&obin)) {
 	return enif_make_badarg(env);
     }
-    n = enif_get_atom(env, argv[0], (char*)obin.data, size);
+    n = enif_get_atom(env, argv[0], (char*)obin.data, size, ERL_NIF_LATIN1);
     return enif_make_tuple(env, 2, enif_make_int(env,n),
 			   enif_make_binary(env,&obin));
 }
@@ -515,14 +634,14 @@ static ERL_NIF_TERM iolist_2_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
 
 static ERL_NIF_TERM last_resource_dtor_call(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    ErlNifBinary bin;
     ERL_NIF_TERM ret;
     if (resource_dtor_last != NULL) {
-	enif_alloc_binary(env, resource_dtor_last_sz, &bin);
-	memcpy(bin.data, resource_dtor_last_data, resource_dtor_last_sz);
+	ERL_NIF_TERM bin;	
+	memcpy(enif_make_new_binary(env, resource_dtor_last_sz, &bin),
+	       resource_dtor_last_data, resource_dtor_last_sz);
 	ret = enif_make_tuple3(env,
 				enif_make_long(env, (long)resource_dtor_last),
-				enif_make_binary(env, &bin),  
+				bin,  
 				enif_make_int(env, resource_dtor_cnt));
     }
     else {
@@ -536,7 +655,7 @@ static ERL_NIF_TERM last_resource_dtor_call(ErlNifEnv* env, int argc, const ERL_
 
 static ERL_NIF_TERM get_resource_type(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    PrivData* data = (PrivData*) enif_get_data(env);
+    PrivData* data = (PrivData*) enif_priv_data(env);
     int ix;
     
     if (!enif_get_int(env, argv[0], &ix) || ix >= 2) {
@@ -552,7 +671,7 @@ static ERL_NIF_TERM alloc_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM
     union { void* p; long l;} data;
     if (!enif_get_long(env, argv[0], &type.l)
 	|| !enif_inspect_binary(env, argv[1], &data_bin)
-	|| (data.p = enif_alloc_resource(env, type.t, data_bin.size))==NULL) {
+	|| (data.p = enif_alloc_resource(type.t, data_bin.size))==NULL) {
 
 	return enif_make_badarg(env);
     }
@@ -577,28 +696,65 @@ static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TE
     ERL_NIF_TERM ret;
     if (!enif_get_long(env, argv[0], &type.l)
 	|| !enif_inspect_binary(env, argv[1], &data_bin)
-	|| (data = enif_alloc_resource(env, type.t, data_bin.size))==NULL) {
+	|| (data = enif_alloc_resource(type.t, data_bin.size))==NULL) {
 
 	return enif_make_badarg(env);
     }
     ret = enif_make_resource(env, data);
     memcpy(data, data_bin.data, data_bin.size);
-    enif_release_resource(env, data);
+    enif_release_resource(data);
     return ret;
 }
 
+static ERL_NIF_TERM make_new_resource_binary(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifBinary data_bin;
+    union { struct binary_resource* p; void* vp; long l;} br;
+    void* buf;
+    ERL_NIF_TERM ret;
+    if (!enif_inspect_binary(env, argv[0], &data_bin)
+	|| (br.vp = enif_alloc_resource(binary_resource_type,
+					sizeof(struct binary_resource)))==NULL
+	|| (buf = enif_alloc(data_bin.size)) == NULL) {
+	
+	return enif_make_badarg(env);
+    }    
+    memset(br.vp,0xba,sizeof(struct binary_resource)); /* avoid valgrind warning */
+    br.p->data = buf;
+    br.p->size = data_bin.size;
+    memcpy(br.p->data, data_bin.data, data_bin.size);    
+    ret = enif_make_resource_binary(env, br.vp, br.p->data, br.p->size);    
+    enif_release_resource(br.p);
+    return enif_make_tuple2(env, enif_make_long(env,br.l), ret);
+}
+
+static void binary_resource_dtor(ErlNifEnv* env, void* obj)
+{
+    struct binary_resource* br = (struct binary_resource*) obj;
+    resource_dtor(env,obj);
+    assert(br->data != NULL);
+    enif_free(br->data);
+    br->data = NULL;
+}
+
 static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     ErlNifBinary data_bin;
     union { ErlNifResourceType* t; long l; } type;
     union { void* p; long l; } data;
 
-    if (!enif_get_long(env, argv[0], &type.l)
+    type.t = NULL;
+    if (enif_is_identical(argv[0], atom_binary_resource_type)) {
+	type.t = binary_resource_type;
+    }
+    else {
+	enif_get_long(env, argv[0], &type.l);
+    }
+    if (type.t == NULL 
 	|| !enif_get_resource(env, argv[1], type.t, &data.p)) {
 	return enif_make_badarg(env);
     }
-
-    enif_alloc_binary(env, enif_sizeof_resource(env,data.p), &data_bin);    
+    enif_alloc_binary(enif_sizeof_resource(data.p), &data_bin);    
     memcpy(data_bin.data, data.p, data_bin.size);
     return enif_make_tuple2(env, enif_make_long(env,data.l),
 			    enif_make_binary(env, &data_bin));
@@ -610,10 +766,612 @@ static ERL_NIF_TERM release_resource(ErlNifEnv* env, int argc, const ERL_NIF_TER
     if (!enif_get_long(env, argv[0], &data.l)) {
 	return enif_make_badarg(env);
     }
-    enif_release_resource(env, data.p);
+    enif_release_resource(data.p);
     return enif_make_atom(env,"ok");
 }
 
+/*
+ * argv[0] an atom
+ * argv[1] a binary
+ * argv[2] a ref
+ * argv[3] 'ok'
+ * argv[4] a fun
+ * argv[5] a pid
+ * argv[6] a port
+ * argv[7] an empty list
+ * argv[8] a non-empty list
+ * argv[9] a tuple
+ */
+static ERL_NIF_TERM check_is(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ERL_NIF_TERM ok_atom = enif_make_atom(env, "ok");
+
+    if (!enif_is_atom(env, argv[0])) return enif_make_badarg(env);
+    if (!enif_is_binary(env, argv[1])) return enif_make_badarg(env);
+    if (!enif_is_ref(env, argv[2])) return enif_make_badarg(env);
+    if (!enif_is_identical(argv[3], ok_atom)) return enif_make_badarg(env);
+    if (!enif_is_fun(env, argv[4])) return enif_make_badarg(env);
+    if (!enif_is_pid(env, argv[5])) return enif_make_badarg(env);
+    if (!enif_is_port(env, argv[6])) return enif_make_badarg(env);
+    if (!enif_is_empty_list(env, argv[7])) return enif_make_badarg(env);
+    if (!enif_is_list(env, argv[7])) return enif_make_badarg(env);
+    if (!enif_is_list(env, argv[8])) return enif_make_badarg(env);
+    if (!enif_is_tuple(env, argv[9])) return enif_make_badarg(env);
+
+    return ok_atom;
+}
+
+/*
+ * no arguments
+ *
+ * This function is separate from check_is because it calls enif_make_badarg
+ * and so it must return the badarg exception as its return value. Thus, the
+ * badarg exception indicates success. Failure is indicated by returning an
+ * error atom.
+ */
+static ERL_NIF_TERM check_is_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ERL_NIF_TERM error_atom = enif_make_atom(env, "error");
+    ERL_NIF_TERM badarg = enif_make_badarg(env);
+    if (enif_is_exception(env, error_atom)) return error_atom;
+    if (!enif_is_exception(env, badarg)) return error_atom;
+    return badarg;
+}
+
+/*
+ * argv[0] atom with length of 6
+ * argv[1] list with length of 6
+ * argv[2] empty list
+ * argv[3] not an atom
+ * argv[4] not a list
+ */
+static ERL_NIF_TERM length_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    unsigned len;
+
+    if (!enif_get_atom_length(env, argv[0], &len, ERL_NIF_LATIN1) || len != 6)
+	return enif_make_badarg(env);
+
+    if (!enif_get_list_length(env, argv[1], &len) || len != 6)
+	return enif_make_badarg(env);
+
+    if (!enif_get_list_length(env, argv[2], &len) || len != 0)
+	return enif_make_badarg(env);
+
+    if (enif_get_atom_length(env, argv[3], &len, ERL_NIF_LATIN1))
+	return enif_make_badarg(env);
+
+    if (enif_get_list_length(env, argv[4], &len))
+	return enif_make_badarg(env);
+
+    return enif_make_atom(env, "ok");
+}
+
+static ERL_NIF_TERM make_atoms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ERL_NIF_TERM arr[7];
+    ERL_NIF_TERM existingatom0a, existingatom0b;
+    ERL_NIF_TERM existing0atom0;
+    const char * const an0atom = "an0atom";
+    const char an0atom0[8] = {'a','n','\0','a','t','o','m',0};
+
+    arr[0] = enif_make_atom(env, "an0atom");
+    arr[1] = enif_make_atom_len(env, "an0atom", 7);
+    arr[2] = enif_make_atom_len(env, an0atom, 7);
+    arr[3] = enif_make_atom_len(env, an0atom0, 8);
+
+    if (!enif_make_existing_atom(env, "an0atom", &existingatom0a, ERL_NIF_LATIN1))
+	return enif_make_atom(env, "error");
+    arr[4] = existingatom0a;
+
+    if (!enif_make_existing_atom_len(env, an0atom, 7, &existingatom0b, ERL_NIF_LATIN1))
+	return enif_make_atom(env, "error");
+    arr[5] = existingatom0b;
+
+    if (!enif_make_existing_atom_len(env, an0atom0, 8, &existing0atom0, ERL_NIF_LATIN1))
+	return enif_make_atom(env, "error");
+    arr[6] = existing0atom0;
+
+    return enif_make_tuple7(env,
+			    arr[0],arr[1],arr[2],arr[3],arr[4],arr[5],arr[6]);
+}
+
+static ERL_NIF_TERM make_strings(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    const char a0string[8] = {'a','0','s','t','r','i','n','g'};
+    const char a0string0[9] = {'a','\0','s','t','r','i','n','g',0};
+    const char astringwith8bits[37] = {'E','r','l','a','n','g',' ',0xE4 /* 'ä' */,'r',' ','e','t','t',' ','g','e','n','e','r','e','l','l','t',' ','p','r','o','g','r','a','m','s','p','r', 0xE5 /* 'å' */,'k',0};
+
+    return enif_make_tuple5(env,
+			    enif_make_string(env, "a0string", ERL_NIF_LATIN1),
+			    enif_make_string_len(env, "a0string", 8, ERL_NIF_LATIN1),
+			    enif_make_string_len(env, a0string, 8, ERL_NIF_LATIN1),
+			    enif_make_string_len(env, a0string0, 9, ERL_NIF_LATIN1),
+			    enif_make_string(env, astringwith8bits, ERL_NIF_LATIN1));
+}
+static ERL_NIF_TERM send_list_seq(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifPid to;
+    ERL_NIF_TERM msg;
+    ErlNifEnv* msg_env;
+    int i, res;
+    
+    if (!enif_get_int(env, argv[0], &i)) {
+	return enif_make_badarg(env);
+    }
+    if (argv[1] == atom_self) {
+	enif_self(env, &to);
+    }
+    else if (!enif_get_local_pid(env, argv[1], &to)) {
+	return enif_make_badarg(env);
+    }
+    msg_env = enif_alloc_env();
+    msg = enif_make_list(msg_env,0);
+    for ( ; i>0 ; i--) {
+	msg = enif_make_list_cell(msg_env, enif_make_int(msg_env, i), msg);
+    }
+    res = enif_send(env, &to, msg_env, msg);
+    enif_free_env(msg_env);
+    return enif_make_tuple2(env, atom_ok, enif_make_int(env,res));
+}
+
+static void fill(void* dst, unsigned bytes, int seed)
+{
+    unsigned char* ptr = dst;
+    int i;
+    for (i=bytes; i>0; i--) {
+	*ptr++ = seed;
+	seed += 7;
+    }
+}
+
+#define MAKE_TERM_REUSE_LEN 16
+struct make_term_info
+{
+    ErlNifEnv* caller_env;
+    ErlNifEnv* dst_env;
+    ERL_NIF_TERM reuse[MAKE_TERM_REUSE_LEN];
+    unsigned reuse_push;
+    unsigned reuse_pull;
+    ErlNifResourceType* resource_type;
+    ERL_NIF_TERM other_term;
+    ERL_NIF_TERM blob;
+    ErlNifPid to_pid;
+    ErlNifTid tid;
+    ErlNifCond* cond;
+    ErlNifMutex* mtx;
+    int send_it;
+    int send_res;
+    unsigned n;
+};
+
+
+static void push_term(struct make_term_info* mti, ERL_NIF_TERM term)
+{
+    unsigned ix = (mti->reuse_push++) % MAKE_TERM_REUSE_LEN; 
+    mti->reuse[ix] = term;
+    //enif_fprintf(stderr, "push at %u: %T\r\n", ix, term);
+}
+static ERL_NIF_TERM pull_term(struct make_term_info* mti)
+{
+    unsigned ix;
+    if (mti->reuse_pull >= mti->reuse_push &&
+	mti->reuse_push < MAKE_TERM_REUSE_LEN) {
+	mti->reuse_pull = 0;
+	if (mti->reuse_push == 0) {
+	    mti->reuse[0] = enif_make_list(mti->dst_env, 0);
+	}
+    }
+    ix = (mti->reuse_pull++) % MAKE_TERM_REUSE_LEN;
+    //enif_fprintf(stderr, "pull from %u: %T\r\n", ix, mti->reuse[ix]);
+    return mti->reuse[ix];
+}
+
+static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res);
+
+static ERL_NIF_TERM make_term_binary(struct make_term_info* mti, int n)
+{	
+    ErlNifBinary bin;
+    enif_alloc_binary(100, &bin);
+    fill(bin.data, bin.size, n);
+    return enif_make_binary(mti->dst_env, &bin);
+}
+
+static ERL_NIF_TERM make_term_int(struct make_term_info* mti, int n)
+{
+    int i;
+    fill(&i, sizeof(i), n);
+    return enif_make_int(mti->dst_env, i);
+}
+
+static ERL_NIF_TERM make_term_ulong(struct make_term_info* mti, int n)
+{
+    unsigned long ul;
+    fill(&ul, sizeof(ul), n);
+    return enif_make_ulong(mti->dst_env, ul);
+}
+
+static ERL_NIF_TERM make_term_double(struct make_term_info* mti, int n)
+{
+    double d = 3.141592;
+    return enif_make_double(mti->dst_env, d);    	
+}
+static ERL_NIF_TERM make_term_atom(struct make_term_info* mti, int n)
+{
+    return enif_make_atom(mti->dst_env, "make_term_n");    
+} 
+static ERL_NIF_TERM make_term_existing_atom(struct make_term_info* mti, int n)
+{	
+    ERL_NIF_TERM res;
+    int exist = enif_make_existing_atom(mti->dst_env, "nif_SUITE", &res,
+					ERL_NIF_LATIN1);
+    assert(exist);
+    return res;
+}
+static ERL_NIF_TERM make_term_string(struct make_term_info* mti, int n)
+{
+    return enif_make_string(mti->dst_env, "Hello!", ERL_NIF_LATIN1);
+}
+static ERL_NIF_TERM make_term_ref(struct make_term_info* mti, int n)
+{
+    return enif_make_ref(mti->dst_env);
+}
+static ERL_NIF_TERM make_term_sub_binary(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM orig;
+    unsigned char* ptr = enif_make_new_binary(mti->dst_env, 10, &orig);
+    fill(ptr, 10, n);
+    return enif_make_sub_binary(mti->dst_env, orig, 3, 5);	
+}
+static ERL_NIF_TERM make_term_uint(struct make_term_info* mti, int n)
+{
+    unsigned int ui;
+    fill(&ui, sizeof(ui), n);
+    return enif_make_uint(mti->dst_env, ui);	       
+}
+static ERL_NIF_TERM make_term_long(struct make_term_info* mti, int n)
+{
+    long l;
+    fill(&l, sizeof(l), n);
+    return enif_make_long(mti->dst_env, l);
+}
+static ERL_NIF_TERM make_term_tuple0(struct make_term_info* mti, int n)
+{
+    return enif_make_tuple(mti->dst_env, 0);
+}
+static ERL_NIF_TERM make_term_list0(struct make_term_info* mti, int n)
+{
+    return enif_make_list(mti->dst_env, 0);
+}
+static ERL_NIF_TERM make_term_resource(struct make_term_info* mti, int n)
+{
+    void* resource = enif_alloc_resource(mti->resource_type, 10);
+    ERL_NIF_TERM term;
+    fill(resource, 10, n); 
+    term = enif_make_resource(mti->dst_env, resource);
+    enif_release_resource(resource);
+    return term;
+}
+static ERL_NIF_TERM make_term_new_binary(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM res;
+    unsigned char* ptr = enif_make_new_binary(mti->dst_env,20,&res);
+    fill(ptr, 20, n);	
+    return res;
+}
+static ERL_NIF_TERM make_term_caller_pid(struct make_term_info* mti, int n)
+{
+    ErlNifPid pid;    
+    return enif_make_pid(mti->dst_env, enif_self(mti->caller_env, &pid));    		
+}	
+
+static ERL_NIF_TERM make_term_tuple(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM t[3];
+    t[0] = pull_term(mti);
+    t[1] = pull_term(mti);
+    t[2] = pull_term(mti);
+    return enif_make_tuple3(mti->dst_env, t[0], t[1], t[2]);	      
+}
+static ERL_NIF_TERM make_term_list(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM t[3];
+    t[0] = pull_term(mti);
+    t[1] = pull_term(mti);
+    t[2] = pull_term(mti);
+    return enif_make_list3(mti->dst_env, t[0], t[1], t[2]);    
+}
+static ERL_NIF_TERM make_term_list_cell(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM t[2];
+    t[0] = pull_term(mti);
+    t[1] = pull_term(mti);
+    return enif_make_list_cell(mti->dst_env, t[0], t[1]);
+}
+static ERL_NIF_TERM make_term_tuple_from_array(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM t[3];
+    t[0] = pull_term(mti);
+    t[1] = pull_term(mti);
+    t[2] = pull_term(mti);
+    return enif_make_tuple_from_array(mti->dst_env, t, 3);	
+}
+static ERL_NIF_TERM make_term_list_from_array(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM t[3];
+    t[0] = pull_term(mti);
+    t[1] = pull_term(mti);
+    t[2] = pull_term(mti);
+    return enif_make_list_from_array(mti->dst_env, t, 3);    
+}
+static ERL_NIF_TERM make_term_garbage(struct make_term_info* mti, int n)
+{
+    (void) enif_make_string(mti->dst_env, "garbage string", ERL_NIF_LATIN1);
+    return pull_term(mti);
+}
+static ERL_NIF_TERM make_term_copy(struct make_term_info* mti, int n)
+{
+    return enif_make_copy(mti->dst_env, mti->other_term);
+}
+
+typedef ERL_NIF_TERM Make_term_Func(struct make_term_info*, int);
+static Make_term_Func* make_funcs[] = {
+    make_term_binary,
+    make_term_int,
+    make_term_ulong,
+    make_term_double,
+    make_term_atom,
+    make_term_existing_atom,
+    make_term_string,
+    //make_term_ref,
+    make_term_sub_binary,
+    make_term_uint,
+    make_term_long,
+    make_term_tuple0,
+    make_term_list0,
+    make_term_resource,
+    make_term_new_binary,
+    make_term_caller_pid,
+    make_term_tuple,
+    make_term_list,
+    make_term_list_cell,
+    make_term_tuple_from_array,
+    make_term_list_from_array,
+    make_term_garbage,
+    make_term_copy
+};
+static unsigned num_of_make_funcs()
+{
+    return sizeof(make_funcs)/sizeof(*make_funcs);
+}
+static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res)
+{
+    if (n < num_of_make_funcs()) {
+	*res = make_funcs[n](mti, n);
+	push_term(mti, *res);
+	return 1;
+    }
+    return 0;
+}
+
+static ERL_NIF_TERM make_blob(ErlNifEnv* caller_env, ErlNifEnv* dst_env,
+			      ERL_NIF_TERM other_term)
+{
+    PrivData* priv = (PrivData*) enif_priv_data(caller_env);
+    ERL_NIF_TERM term, list;
+    int n = 0;
+    struct make_term_info mti;
+    mti.caller_env = caller_env;
+    mti.dst_env = dst_env;
+    mti.reuse_push = 0;
+    mti.reuse_pull = 0;
+    mti.resource_type = priv->rt_arr[0].t;
+    mti.other_term = other_term;
+
+    list = enif_make_list(dst_env, 0); 
+    while (make_term_n(&mti, n++, &term)) {
+	list = enif_make_list_cell(dst_env, term, list);
+    }
+    return list;
+}
+
+static ERL_NIF_TERM send_new_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifPid to;
+    ERL_NIF_TERM msg, copy;
+    ErlNifEnv* msg_env;
+    int res;
+    
+    if (!enif_get_local_pid(env, argv[0], &to)) {
+	return enif_make_badarg(env);
+    }
+    msg_env = enif_alloc_env();
+    msg = make_blob(env,msg_env, argv[1]);
+    copy = make_blob(env,env, argv[1]);
+    res = enif_send(env, &to, msg_env, msg);
+    enif_free_env(msg_env);
+    return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy);
+}
+
+static ERL_NIF_TERM alloc_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    PrivData* priv = (PrivData*) enif_priv_data(env);
+    struct make_term_info* mti;
+    ERL_NIF_TERM ret;
+
+    mti = (struct make_term_info*) enif_alloc_resource(msgenv_resource_type,
+						       sizeof(*mti));
+    mti->caller_env = NULL;
+    mti->dst_env = enif_alloc_env();
+    mti->reuse_push = 0;
+    mti->reuse_pull = 0;
+    mti->resource_type = priv->rt_arr[0].t;
+    mti->other_term = enif_make_list(mti->dst_env, 0);
+    mti->blob = enif_make_list(mti->dst_env, 0);
+    mti->mtx = enif_mutex_create("nif_SUITE:mtx");
+    mti->cond = enif_cond_create("nif_SUITE:cond");
+    mti->send_res = 0xcafebabe;
+    mti->n = 0;
+    ret = enif_make_resource(env, mti);
+    enif_release_resource(mti);
+    return ret;
+}
+
+static void msgenv_dtor(ErlNifEnv* env, void* obj)
+{
+    struct make_term_info* mti = (struct make_term_info*) obj;
+    if (mti->dst_env != NULL) {
+	enif_free_env(mti->dst_env);
+    }
+    enif_mutex_destroy(mti->mtx);
+    enif_cond_destroy(mti->cond);
+}
+
+static ERL_NIF_TERM clear_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    union {
+	void* vp;
+	struct make_term_info* p;
+    }mti;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) {
+	return enif_make_badarg(env);
+    }
+    enif_clear_env(mti.p->dst_env);
+    mti.p->reuse_pull = 0;
+    mti.p->reuse_push = 0;
+    mti.p->blob = enif_make_list(mti.p->dst_env, 0);
+    return atom_ok;
+}
+
+static ERL_NIF_TERM grow_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    union { void* vp; struct make_term_info* p; }mti;
+    ERL_NIF_TERM term;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)
+	|| (argc>2 && !enif_get_uint(env,argv[2], &mti.p->n))) {
+	return enif_make_badarg(env);
+    }
+    mti.p->caller_env = env;
+    mti.p->other_term = argv[1];
+    mti.p->n %= num_of_make_funcs();
+    make_term_n(mti.p, mti.p->n++, &term);
+    mti.p->blob = enif_make_list_cell(mti.p->dst_env, term, mti.p->blob);
+    return atom_ok;
+}
+
+static ERL_NIF_TERM send_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    union { void* vp; struct make_term_info* p; }mti;
+    ErlNifPid to;
+    ERL_NIF_TERM copy;
+    int res;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)
+	|| !enif_get_local_pid(env, argv[1], &to)) {
+	return enif_make_badarg(env);
+    }
+    copy = enif_make_copy(env, mti.p->blob);
+    res = enif_send(env, &to, mti.p->dst_env, mti.p->blob);
+    return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy);
+}
+
+static ERL_NIF_TERM send3_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    union { void* vp; struct make_term_info* p; }mti;
+    ErlNifPid to;
+    ERL_NIF_TERM copy;
+    int res;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)
+	|| !enif_get_local_pid(env, argv[1], &to)) {
+	return enif_make_badarg(env);
+    }
+    mti.p->blob = enif_make_tuple2(mti.p->dst_env, 
+				   enif_make_copy(mti.p->dst_env, argv[2]),
+				   mti.p->blob);
+    res = enif_send(env, &to, mti.p->dst_env, mti.p->blob);
+    return enif_make_int(env,res);
+}
+
+void* threaded_sender(void *arg)
+{
+
+    union { void* vp; struct make_term_info* p; }mti;
+    mti.vp = arg;
+
+    enif_mutex_lock(mti.p->mtx);
+    while (!mti.p->send_it) {
+	enif_cond_wait(mti.p->cond, mti.p->mtx);
+    }
+    mti.p->send_it = 0;
+    enif_mutex_unlock(mti.p->mtx);
+    mti.p->send_res = enif_send(NULL, &mti.p->to_pid, mti.p->dst_env, mti.p->blob);
+    return NULL;
+}
+
+static ERL_NIF_TERM send_blob_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    union { void* vp; struct make_term_info* p; }mti;
+    ERL_NIF_TERM copy;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)
+	|| !enif_get_local_pid(env,argv[1], &mti.p->to_pid)) {
+	return enif_make_badarg(env);
+    }
+    copy = enif_make_copy(env, mti.p->blob);
+
+    mti.p->send_it = enif_is_identical(argv[2],atom_join);
+    if (enif_thread_create("nif_SUITE:send_from_thread", &mti.p->tid,
+			   threaded_sender, mti.p, NULL) != 0) {
+	return enif_make_badarg(env);
+    }
+    if (enif_is_identical(argv[2],atom_join)) {
+	int err = enif_thread_join(mti.p->tid, NULL);
+	assert(err == 0);
+	return enif_make_tuple3(env, atom_ok, enif_make_int(env, mti.p->send_res), copy);
+    }
+    else {
+	enif_keep_resource(mti.vp);
+	return enif_make_tuple2(env, atom_ok, copy);
+    }
+}
+
+static ERL_NIF_TERM join_send_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    union { void* vp; struct make_term_info* p; }mti;
+    int err;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) {
+	return enif_make_badarg(env);
+    }
+    enif_mutex_lock(mti.p->mtx);
+    mti.p->send_it = 1;
+    enif_cond_signal(mti.p->cond);
+    enif_mutex_unlock(mti.p->mtx);
+    err = enif_thread_join(mti.p->tid, NULL);
+    assert(err == 0);
+    enif_release_resource(mti.vp);
+    return enif_make_tuple2(env, atom_ok, enif_make_int(env, mti.p->send_res));    
+}
+
+static ERL_NIF_TERM copy_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    union { void* vp; struct make_term_info* p; }mti;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) {
+	return enif_make_badarg(env);
+    }
+    return enif_make_copy(env, mti.p->blob);
+}
+
+static ERL_NIF_TERM send_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifEnv* menv;
+    ErlNifPid pid;
+    int ret;
+    if (!enif_get_local_pid(env, argv[0], &pid)) {
+	return enif_make_badarg(env);
+    }
+    menv = enif_alloc_env();
+    ret = enif_send(env, &pid, menv, enif_make_copy(menv, argv[1]));
+    enif_free_env(menv);
+    return enif_make_int(env, ret);
+}
 
 static ErlNifFunc nif_funcs[] =
 {
@@ -640,8 +1398,26 @@ static ErlNifFunc nif_funcs[] =
     {"get_resource", 2, get_resource},
     {"release_resource", 1, release_resource},
     {"last_resource_dtor_call", 0, last_resource_dtor_call},
-    {"make_new_resource", 2, make_new_resource}
-
+    {"make_new_resource", 2, make_new_resource},
+    {"check_is", 10, check_is},
+    {"check_is_exception", 0, check_is_exception},
+    {"length_test", 5, length_test},
+    {"make_atoms", 0, make_atoms},
+    {"make_strings", 0, make_strings},
+    {"make_new_resource", 2, make_new_resource},
+    {"make_new_resource_binary", 1, make_new_resource_binary},
+    {"send_list_seq", 2, send_list_seq},
+    {"send_new_blob", 2, send_new_blob},
+    {"alloc_msgenv", 0, alloc_msgenv},
+    {"clear_msgenv", 1, clear_msgenv},
+    {"grow_blob", 2, grow_blob},
+    {"grow_blob", 3, grow_blob},
+    {"send_blob", 2, send_blob},
+    {"send3_blob", 3, send3_blob},
+    {"send_blob_thread", 3, send_blob_thread},
+    {"join_send_thread", 1, join_send_thread},
+    {"copy_blob", 1, copy_blob},
+    {"send_term", 2, send_term}
 };
 
 ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload)
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c
index c075b74c57..e32d10057c 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c
@@ -1,3 +1,21 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
 #include "erl_nif.h"
 #include <string.h>
 #include <stdio.h>
@@ -24,6 +42,11 @@ static ERL_NIF_TERM am_resource_type;
 static ERL_NIF_TERM am_resource_dtor_A;
 static ERL_NIF_TERM am_resource_dtor_B;
 
+static NifModPrivData* priv_data(ErlNifEnv* env)
+{
+    return (NifModPrivData*) enif_priv_data(env);
+}
+
 static void init(ErlNifEnv* env)
 {
     am_true = enif_make_atom(env, "true");
@@ -36,7 +59,7 @@ static void init(ErlNifEnv* env)
 static void add_call_with_arg(ErlNifEnv* env, NifModPrivData* data, const char* func_name,
 			      const char* arg, int arg_sz)
 {
-    CallInfo* call = enif_alloc(env, sizeof(CallInfo)+strlen(func_name) + arg_sz);
+    CallInfo* call = (CallInfo*)enif_alloc(sizeof(CallInfo)+strlen(func_name) + arg_sz);
     strcpy(call->func_name, func_name);
     call->lib_ver = NIF_LIB_VER;
     call->static_cntA = ++static_cntA;
@@ -60,7 +83,7 @@ static void add_call(ErlNifEnv* env, NifModPrivData* data,const char* func_name)
     add_call_with_arg(env, data, func_name, NULL, 0);
 }
 
-#define ADD_CALL(FUNC_NAME) add_call(env, enif_priv_data(env),FUNC_NAME)
+#define ADD_CALL(FUNC_NAME) add_call(env, priv_data(env),FUNC_NAME)
 
 #define STRINGIFY_(X) #X
 #define STRINGIFY(X) STRINGIFY_(X)
@@ -69,56 +92,56 @@ static void resource_dtor_A(ErlNifEnv* env, void* a)
 {
     const char dtor_name[] = "resource_dtor_A_v"  STRINGIFY(NIF_LIB_VER); 
 
-    add_call_with_arg(env, enif_priv_data(env), dtor_name,
-		      a, enif_sizeof_resource(env, a));
+    add_call_with_arg(env, priv_data(env), dtor_name, (const char*)a,
+		      enif_sizeof_resource(a));
 }
 
 static void resource_dtor_B(ErlNifEnv* env, void* a)
 {
-    const char dtor_name[] = "resource_dtor_B_v"  STRINGIFY(NIF_LIB_VER); 
+    const char dtor_name[] = "resource_dtor_B_v"  STRINGIFY(NIF_LIB_VER);
 
-    add_call_with_arg(env, enif_priv_data(env), dtor_name,
-		      a, enif_sizeof_resource(env, a));
+    add_call_with_arg(env, priv_data(env), dtor_name, (const char*)a,
+		      enif_sizeof_resource(a));
 }
 
 /* {resource_type, Ix|null, ErlNifResourceFlags in, "TypeName", dtor(A|B|null), ErlNifResourceFlags out}*/
 static void open_resource_type(ErlNifEnv* env, ERL_NIF_TERM op_tpl)
 {
-    NifModPrivData* data = enif_priv_data(env);
+    NifModPrivData* data = priv_data(env);
     const ERL_NIF_TERM* arr;
     int arity;
     char rt_name[30];
-    union { enum ErlNifResourceFlags e; int i; } flags, exp_res, got_res;
+    union { ErlNifResourceFlags e; int i; } flags, exp_res, got_res;
     unsigned ix;
     ErlNifResourceDtor* dtor;
     ErlNifResourceType* got_ptr;
 
     CHECK(enif_get_tuple(env, op_tpl, &arity, &arr));
     CHECK(arity == 6);
-    CHECK(enif_is_identical(env, arr[0], am_resource_type));
+    CHECK(enif_is_identical(arr[0], am_resource_type));
     CHECK(enif_get_int(env, arr[2], &flags.i));
     CHECK(enif_get_string(env, arr[3], rt_name, sizeof(rt_name), ERL_NIF_LATIN1) > 0);
     CHECK(enif_get_int(env, arr[5], &exp_res.i));
 	
-    if (enif_is_identical(env, arr[4], am_null)) {
+    if (enif_is_identical(arr[4], am_null)) {
 	dtor = NULL;
     }
-    else if (enif_is_identical(env, arr[4], am_resource_dtor_A)) {
+    else if (enif_is_identical(arr[4], am_resource_dtor_A)) {
 	dtor = resource_dtor_A;
     }
     else {
-	CHECK(enif_is_identical(env, arr[4], am_resource_dtor_B));
+	CHECK(enif_is_identical(arr[4], am_resource_dtor_B));
 	dtor = resource_dtor_B;
     }
 
-    got_ptr = enif_open_resource_type(env, rt_name, dtor,
+    got_ptr = enif_open_resource_type(env, NULL, rt_name, dtor,
 				      flags.e, &got_res.e);
 
     if (enif_get_uint(env, arr[1], &ix) && ix < RT_MAX && got_ptr != NULL) {
 	data->rt_arr[ix] = got_ptr;
     }
     else {
-	CHECK(enif_is_identical(env, arr[1], am_null));
+	CHECK(enif_is_identical(arr[1], am_null));
 	CHECK(got_ptr == NULL);
     }
     CHECK(got_res.e == exp_res.e);
@@ -126,7 +149,7 @@ static void open_resource_type(ErlNifEnv* env, ERL_NIF_TERM op_tpl)
 
 static void do_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info)
 {
-    NifModPrivData* data = enif_priv_data(env);
+    NifModPrivData* data = priv_data(env);
     ERL_NIF_TERM head, tail;
     unsigned ix;
     for (ix=0; ix<RT_MAX; ix++) {
@@ -140,17 +163,18 @@ static void do_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info)
     CHECK(enif_is_empty_list(env, head));
 }
 
-static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+static int load(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info)
 {
     NifModPrivData* data;
 
     init(env);
-    data = enif_alloc(env, sizeof(NifModPrivData));
+    data = (NifModPrivData*) enif_alloc(sizeof(NifModPrivData));
     CHECK(data != NULL);
-    *priv_data = data;
+    *priv = data;
     data->mtx = enif_mutex_create("nif_mod_priv_data"); 
     data->ref_cnt = 1;
     data->call_history = NULL;
+
     add_call(env, data, "load");
 
     do_load_info(env, load_info);
@@ -158,39 +182,35 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     return 0;
 }
 
-static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+static int reload(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info)
 {
+    NifModPrivData* data = (NifModPrivData*) *priv;
     init(env);
-    add_call(env, *priv_data, "reload");
+    add_call(env, data, "reload");
     
     do_load_info(env, load_info);
     return 0;
 }
 
-static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+static int upgrade(ErlNifEnv* env, void** priv, void** old_priv_data, ERL_NIF_TERM load_info)
 {
-    NifModPrivData* data = *old_priv_data;
+    NifModPrivData* data = (NifModPrivData*) *old_priv_data;
     init(env);
     add_call(env, data, "upgrade");
     data->ref_cnt++;
 
-    *priv_data = *old_priv_data;    
+    *priv = *old_priv_data;    
     do_load_info(env, load_info);
     
     return 0;
 }
 
-static void unload(ErlNifEnv* env, void* priv_data)
+static void unload(ErlNifEnv* env, void* priv)
 {
-    NifModPrivData* data = priv_data;
+    NifModPrivData* data = (NifModPrivData*) priv;
+    int is_last;
     add_call(env, data, "unload");
-    enif_mutex_lock(data->mtx);
-    if (--data->ref_cnt == 0) {
-	enif_mutex_unlock(data->mtx);
-	enif_mutex_destroy(data->mtx);
-	enif_free(env, data);
-    }
-    enif_mutex_unlock(data->mtx);
+    NifModPrivData_release(data);
 }
 
 static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -202,12 +222,12 @@ static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
 static ERL_NIF_TERM get_priv_data_ptr(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     ADD_CALL("get_priv_data_ptr");
-    return enif_make_ulong(env, (unsigned long)enif_priv_data(env));
+    return enif_make_ulong(env, (unsigned long)priv_data(env));
 }
 
 static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    NifModPrivData* data = (NifModPrivData*) enif_priv_data(env);
+    NifModPrivData* data = priv_data(env);
     ErlNifBinary ibin;
     char* a;
     ERL_NIF_TERM ret;
@@ -216,22 +236,22 @@ static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TE
 	|| !enif_inspect_binary(env, argv[1], &ibin)) {
 	return enif_make_badarg(env);
     }
-    a = enif_alloc_resource(env, data->rt_arr[ix], ibin.size);
+    a = (char*) enif_alloc_resource(data->rt_arr[ix], ibin.size);
     memcpy(a, ibin.data, ibin.size);
     ret = enif_make_resource(env, a);
-    enif_release_resource(env, a);
+    enif_release_resource(a);
     return ret;
 }
 
 static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    NifModPrivData* data = (NifModPrivData*) enif_priv_data(env);
+    NifModPrivData* data = priv_data(env);
     ErlNifBinary obin;
     unsigned ix;
     void* a;
     if (!enif_get_uint(env, argv[0], &ix) || ix >= RT_MAX 
 	|| !enif_get_resource(env, argv[1], data->rt_arr[ix], &a)
-	|| !enif_alloc_binary(env, enif_sizeof_resource(env, a), &obin)) { 
+	|| !enif_alloc_binary(enif_sizeof_resource(a), &obin)) { 
 	return enif_make_badarg(env);
     }
     memcpy(obin.data, a, obin.size);
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.erl b/erts/emulator/test/nif_SUITE_data/nif_mod.erl
index 7888a589e7..6634624698 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.erl
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2005-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,7 @@
 
 -module(nif_mod).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -export([load_nif_lib/2, load_nif_lib/3, start/0, lib_version/0, call_history/0,
 	 get_priv_data_ptr/0, make_new_resource/2, get_resource/2]).
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.h b/erts/emulator/test/nif_SUITE_data/nif_mod.h
index 0eaf91d6e1..cd0ecf4b54 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.h
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.h
@@ -20,3 +20,15 @@ typedef struct
     ErlNifResourceType* rt_arr[RT_MAX];
 }NifModPrivData;
 
+#define NifModPrivData_release(NMPD) \
+    do { \
+	int is_last; \
+	enif_mutex_lock((NMPD)->mtx); \
+	is_last = (--(NMPD)->ref_cnt == 0); \
+        enif_mutex_unlock((NMPD)->mtx); \
+        if (is_last) { \
+	    enif_mutex_destroy((NMPD)->mtx); \
+	    enif_free((NMPD)); \
+        } \
+    }while (0)
+
diff --git a/erts/emulator/test/nif_SUITE_data/tester.erl b/erts/emulator/test/nif_SUITE_data/tester.erl
index 9df2158200..b393e29b82 100644
--- a/erts/emulator/test/nif_SUITE_data/tester.erl
+++ b/erts/emulator/test/nif_SUITE_data/tester.erl
@@ -1,6 +1,6 @@
 -module(tester).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -export([load_nif_lib/2, run/0]).
 
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index f3d9eb783b..aa83459ef8 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -29,10 +29,12 @@
 
 %-define(line_trace, 1).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %-compile(export_all).
--export([all/1, init_per_testcase/2, fin_per_testcase/2, end_per_suite/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, init_per_testcase/2, 
+	 end_per_testcase/2,
 	 node_container_refc_check/1]).
 
 -export([term_to_binary_to_term_eq/1,
@@ -55,25 +57,30 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(10)).
 
-all(doc) -> [];
-all(suite) ->
-    [term_to_binary_to_term_eq,
-     round_trip_eq,
-     cmp,
-     ref_eq,
-     node_table_gc,
-     dist_link_refc,
-     dist_monitor_refc,
-     node_controller_refc,
-     ets_refc,
-     match_spec_refc,
-     timer_refc,
-     otp_4715,
-     pid_wrap,
-     port_wrap,
-     bad_nc,
-     unique_pid,
-     iter_max_procs].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [term_to_binary_to_term_eq, round_trip_eq, cmp, ref_eq,
+     node_table_gc, dist_link_refc, dist_monitor_refc,
+     node_controller_refc, ets_refc, match_spec_refc,
+     timer_refc, otp_4715, pid_wrap, port_wrap, bad_nc,
+     unique_pid, iter_max_procs].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    available_internal_state(false).
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 available_internal_state(Bool) when Bool == true; Bool == false ->
     case {Bool,
@@ -95,14 +102,11 @@ init_per_testcase(_Case, Config) when is_list(Config) ->
     available_internal_state(true),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
 
-end_per_suite(_Config) ->
-    available_internal_state(false).
-
 %%%
 %%% The test cases -------------------------------------------------------------
 %%%
diff --git a/erts/emulator/test/nofrag_SUITE.erl b/erts/emulator/test/nofrag_SUITE.erl
index ece55f433c..6b6ac28e2e 100644
--- a/erts/emulator/test/nofrag_SUITE.erl
+++ b/erts/emulator/test/nofrag_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,11 @@
 
 -module(nofrag_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,end_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 error_handler/1,error_handler_apply/1,
 	 error_handler_fixed_apply/1,error_handler_fun/1,
 	 error_handler_tuple_fun/1,
@@ -30,9 +32,28 @@
 %% Exported functions for an error_handler module.
 -export([undefined_function/3,undefined_lambda/3,breakpoint/3]).
 
-all(suite) ->
-    [error_handler,error_handler_apply,error_handler_fixed_apply,
-     error_handler_fun,error_handler_tuple_fun,debug_breakpoint].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [error_handler, error_handler_apply,
+     error_handler_fixed_apply, error_handler_fun,
+     error_handler_tuple_fun, debug_breakpoint].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(3)),
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index d009994e2d..4459732257 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,7 @@
 
 -module(num_bif_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %% Tests the BIFs:
 %% 	abs/1
@@ -31,15 +31,36 @@
 %%	round/1
 %%	trunc/1
 
--export([all/1, t_abs/1, t_float/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, t_abs/1, t_float/1,
 	 t_float_to_list/1, t_integer_to_list/1,
 	 t_list_to_integer/1,
-	 t_list_to_float/1, t_list_to_float_safe/1, t_list_to_float_risky/1,
+	 t_list_to_float_safe/1, t_list_to_float_risky/1,
 	 t_round/1, t_trunc/1]).
 
-all(suite) -> [t_abs, t_float, t_float_to_list, t_integer_to_list,
-	       t_list_to_float, t_list_to_integer,
-	       t_round, t_trunc].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [t_abs, t_float, t_float_to_list, t_integer_to_list,
+     {group, t_list_to_float}, t_list_to_integer, t_round,
+     t_trunc].
+
+groups() -> 
+    [{t_list_to_float, [],
+      [t_list_to_float_safe, t_list_to_float_risky]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 t_abs(Config) when is_list(Config) ->
     %% Floats.
@@ -140,7 +161,6 @@ t_integer_to_list(Config) when is_list(Config) ->
 
 %% Tests list_to_float/1.
 
-t_list_to_float(suite) -> [t_list_to_float_safe, t_list_to_float_risky].
 
 t_list_to_float_safe(Config) when is_list(Config) ->
     ?line 0.0 = list_to_float(id("0.0")),
diff --git a/erts/emulator/test/obsolete_SUITE.erl b/erts/emulator/test/obsolete_SUITE.erl
deleted file mode 100644
index 33c4726699..0000000000
--- a/erts/emulator/test/obsolete_SUITE.erl
+++ /dev/null
@@ -1,123 +0,0 @@
-%%
-%% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2004-2009. All Rights Reserved.
-%% 
-%% The contents of this file are subject to the Erlang Public License,
-%% Version 1.1, (the "License"); you may not use this file except in
-%% compliance with the License. You should have received a copy of the
-%% Erlang Public License along with this software. If not, it can be
-%% retrieved online at http://www.erlang.org/.
-%% 
-%% Software distributed under the License is distributed on an "AS IS"
-%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-%% the License for the specific language governing rights and limitations
-%% under the License.
-%% 
-%% %CopyrightEnd%
-%%
-
-
--module(obsolete_SUITE).
--author('[email protected]').
--compile(nowarn_obsolete_guard).
-
--export([all/1]).
-
--export([erl_threads/1]).
-
--include("test_server.hrl").
-
--define(DEFAULT_TIMETRAP_SECS, 240).
-
-all(doc) -> [];
-all(suite) ->
-    case catch erlang:system_info(wordsize) of
-	4 -> [erl_threads];
-	_ -> {skip, "Only expected to work on 32-bit architectures"}
-    end.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%                                                                        %%
-%% Testcases                                                              %%
-%%                                                                        %%
-
-erl_threads(suite) -> [];
-erl_threads(doc) ->   [];
-erl_threads(Cfg) ->
-    ?line case erlang:system_info(threads) of
-	      true ->
-		  ?line drv_case(Cfg, erl_threads);
-	      false ->
-		  ?line {skip, "Emulator not compiled with threads support"}
-	  end.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%                                                                        %%
-%% Internal functions                                                     %%
-%%                                                                        %%
-
-drv_case(Config, CaseName) ->
-    drv_case(Config, CaseName, "").
-
-drv_case(Config, CaseName, TimeTrap) when integer(TimeTrap) ->
-    drv_case(Config, CaseName, "", TimeTrap);
-drv_case(Config, CaseName, Command) when list(Command) ->
-    drv_case(Config, CaseName, Command, ?DEFAULT_TIMETRAP_SECS).
-
-drv_case(Config, CaseName, TimeTrap, Command) when list(Command),
-						   integer(TimeTrap) ->
-    drv_case(Config, CaseName, Command, TimeTrap);
-drv_case(Config, CaseName, Command, TimeTrap) when list(Config),
-						   atom(CaseName),
-						   list(Command),
-						   integer(TimeTrap) ->
-    case ?t:os_type() of
-	{Family, _} when Family == unix; Family == win32 ->
-	    ?line run_drv_case(Config, CaseName, Command, TimeTrap);
-	SkipOs ->
-	    ?line {skipped,
-		   lists:flatten(["Not run on "
-				  | io_lib:format("~p",[SkipOs])])}
-    end.
-
-run_drv_case(Config, CaseName, Command, TimeTrap) ->
-    ?line Dog = test_server:timetrap(test_server:seconds(TimeTrap)),
-    ?line DataDir = ?config(data_dir,Config),
-    case erl_ddll:load_driver(DataDir, CaseName) of
-	ok -> ok;
-	{error, Error} ->
-	    io:format("~s\n", [erl_ddll:format_error(Error)]),
-	    ?line ?t:fail()
-    end,
-    ?line Port = open_port({spawn, atom_to_list(CaseName)}, []),
-    ?line true = is_port(Port),
-    ?line Port ! {self(), {command, Command}},
-    ?line Result = receive_drv_result(Port, CaseName),
-    ?line Port ! {self(), close},
-    ?line receive 
-	      {Port, closed} ->
-		  ok
-	  end,
-    ?line ok = erl_ddll:unload_driver(CaseName),
-    ?line test_server:timetrap_cancel(Dog),
-    ?line Result.
-
-receive_drv_result(Port, CaseName) ->
-    ?line receive
-	      {print, Port, CaseName, Str} ->
-		  ?line ?t:format("~s", [Str]),
-		  ?line receive_drv_result(Port, CaseName);
-	      {'EXIT', Port, Error} ->
-		  ?line ?t:fail(Error);
-	      {'EXIT', error, Error} ->
-		  ?line ?t:fail(Error);
-	      {failed, Port, CaseName, Comment} ->
-		  ?line ?t:fail(Comment);
-	      {skipped, Port, CaseName, Comment} ->
-		  ?line {skipped, Comment};
-	      {succeeded, Port, CaseName, ""} ->
-		  ?line succeeded;
-	      {succeeded, Port, CaseName, Comment} ->
-		  ?line {comment, Comment}
-	  end.
diff --git a/erts/emulator/test/obsolete_SUITE_data/Makefile.src b/erts/emulator/test/obsolete_SUITE_data/Makefile.src
deleted file mode 100644
index d8e2b861c0..0000000000
--- a/erts/emulator/test/obsolete_SUITE_data/Makefile.src
+++ /dev/null
@@ -1,33 +0,0 @@
-# ``The contents of this file are subject to the Erlang Public License,
-# Version 1.1, (the "License"); you may not use this file except in
-# compliance with the License. You should have received a copy of the
-# Erlang Public License along with this software. If not, it can be
-# retrieved via the world wide web at http://www.erlang.org/.
-# 
-# Software distributed under the License is distributed on an "AS IS"
-# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-# the License for the specific language governing rights and limitations
-# under the License.
-# 
-# The Initial Developer of the Original Code is Ericsson Utvecklings AB.
-# Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
-# AB. All Rights Reserved.''
-# 
-#     $Id$
-#
-
-TEST_DRVS = erl_threads@dll@
-CC = @CC@
-LD = @LD@
-CFLAGS = @SHLIB_CFLAGS@ -I@erl_include@ @DEFS@
-SHLIB_EXTRA_LDLIBS = testcase_driver@obj@
-
-all: $(TEST_DRVS)
-
-@SHLIB_RULES@
-
-testcase_driver@obj@: testcase_driver.c testcase_driver.h
-$(TEST_DRVS): testcase_driver@obj@
-
-
-
diff --git a/erts/emulator/test/obsolete_SUITE_data/erl_threads.c b/erts/emulator/test/obsolete_SUITE_data/erl_threads.c
deleted file mode 100644
index 27a5163121..0000000000
--- a/erts/emulator/test/obsolete_SUITE_data/erl_threads.c
+++ /dev/null
@@ -1,302 +0,0 @@
-/* ``The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved via the world wide web at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
- * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
- * AB. All Rights Reserved.''
- * 
- *     $Id$
- */
-
-#include "testcase_driver.h"
-
-#ifndef __WIN32__
-
-#define NO_OF_THREADS 2
-
-#include <unistd.h>
-#include <errno.h>
-
-static int die;
-static int cw_passed;
-static int res_tf0;
-static int res_tf1;
-static erl_mutex_t mtx;
-static erl_cond_t cnd;
-static erl_thread_t tid[NO_OF_THREADS];
-static int need_join[NO_OF_THREADS];
-
-typedef struct {
-    int n;
-} thr_arg_t;
-
-
-static void *tf0(void *vta)
-{
-    int r;
-
-    if (((thr_arg_t *) vta)->n != 0)
-	goto fail;
-
-    r = erts_mutex_lock(mtx);
-    if (r != 0) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    r = erts_cond_wait(cnd, mtx);
-    if (r != 0 || die) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    cw_passed++;
-
-    r = erts_cond_wait(cnd, mtx);
-    if (r != 0 || die) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    cw_passed++;
-
-    r = erts_mutex_unlock(mtx);
-    if (r != 0)
-	goto fail;
-
-    res_tf0 = 0;
-
-    return (void *) &res_tf0;
-
- fail:
-    return NULL;
-}
-
-
-static void *tf1(void *vta)
-{
-    int r;
-
-    if (((thr_arg_t *) vta)->n != 1)
-	goto fail;
-
-    r = erts_mutex_lock(mtx);
-    if (r != 0) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    r = erts_cond_wait(cnd, mtx);
-    if (r != 0 || die) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    cw_passed++;
-
-    r = erts_cond_wait(cnd, mtx);
-    if (r != 0 || die) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    cw_passed++;
-
-    r = erts_mutex_unlock(mtx);
-    if (r != 0)
-	goto fail;
-
-    res_tf1 = 1;
-
-    erts_thread_exit((void *) &res_tf1);
-
-    res_tf1 = 4711;
-
- fail:
-    return NULL;
-}
-
-#endif /* #ifndef __WIN32__ */
-
-void
-testcase_run(TestCaseState_t *tcs)
-{
-#ifdef __WIN32__
-    testcase_skipped(tcs, "Nothing to test; not supported on windows.");
-#else
-    int i, r;
-    void *tres[NO_OF_THREADS];
-    thr_arg_t ta[NO_OF_THREADS];
-    erl_thread_t t1;
-
-    die = 0;
-    cw_passed = 0;
-
-    for (i = 0; i < NO_OF_THREADS; i++)
-	need_join[i] = 0;
-
-    res_tf0 = 17;
-    res_tf1 = 17;
-
-    cnd = mtx = NULL;
-
-    /* Create mutex and cond */
-    mtx = erts_mutex_create();
-    ASSERT(tcs, mtx);
-    cnd = erts_cond_create();
-    ASSERT(tcs, cnd);
-
-    /* Create the threads */
-    ta[0].n = 0;
-    r = erts_thread_create(&tid[0], tf0, (void *) &ta[0], 0);
-    ASSERT(tcs, r == 0);
-    need_join[0] = 1;
-
-    ta[1].n = 1;
-    r = erts_thread_create(&tid[1], tf1, (void *) &ta[1], 0);
-    ASSERT(tcs, r == 0);
-    need_join[1] = 1;
-
-    /* Make sure the threads waits on cond wait */
-    sleep(1);
-
-    r = erts_mutex_lock(mtx);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    ASSERT_CLNUP(tcs, cw_passed == 0, (void) erts_mutex_unlock(mtx));
-
-
-    /* Let one thread pass one cond wait */
-    r = erts_cond_signal(cnd);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    r = erts_mutex_unlock(mtx);
-    ASSERT(tcs, r == 0);
-
-    sleep(1);
-
-    r = erts_mutex_lock(mtx);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    ASSERT_CLNUP(tcs, cw_passed == 1, (void) erts_mutex_unlock(mtx));
-
-
-    /* Let both threads pass one cond wait */
-    r = erts_cond_broadcast(cnd);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    r = erts_mutex_unlock(mtx);
-    ASSERT(tcs, r == 0);
-
-    sleep(1);
-
-    r = erts_mutex_lock(mtx);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    ASSERT_CLNUP(tcs, cw_passed == 3, (void) erts_mutex_unlock(mtx));
-
-
-    /* Let the thread that only have passed one cond wait pass the other one */
-    r = erts_cond_signal(cnd);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    r = erts_mutex_unlock(mtx);
-    ASSERT(tcs, r == 0);
-
-    sleep(1);
-
-    r = erts_mutex_lock(mtx);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    ASSERT_CLNUP(tcs, cw_passed == 4, (void) erts_mutex_unlock(mtx));
-
-    /* Both threads should have passed both cond waits and exited;
-       join them and check returned values */
-
-    r = erts_thread_join(tid[0], &tres[0]);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-    need_join[0] = 0;
-
-    ASSERT_CLNUP(tcs, tres[0] == &res_tf0, (void) erts_mutex_unlock(mtx));
-    ASSERT_CLNUP(tcs, res_tf0 == 0, (void) erts_mutex_unlock(mtx));
-
-    r = erts_thread_join(tid[1], &tres[1]);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-    need_join[1] = 0;
-
-    ASSERT_CLNUP(tcs, tres[1] == &res_tf1, (void) erts_mutex_unlock(mtx));
-    ASSERT_CLNUP(tcs, res_tf1 == 1, (void) erts_mutex_unlock(mtx));
-
-    /* Test signaling when noone waits */
-
-    r = erts_cond_signal(cnd);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    /* Test broadcasting when noone waits */
-
-    r = erts_cond_broadcast(cnd);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    /* erts_cond_timedwait() not supported anymore */
-    r = erts_cond_timedwait(cnd, mtx, 1000);
-    ASSERT_CLNUP(tcs, r != 0, (void) erts_mutex_unlock(mtx));
-    ASSERT_CLNUP(tcs,
-		 strcmp(erl_errno_id(r), "enotsup") == 0,
-		 (void) erts_mutex_unlock(mtx));
-
-    r = erts_mutex_unlock(mtx);
-    ASSERT(tcs, r == 0);
-
-    r = erts_mutex_destroy(mtx);
-    ASSERT(tcs, r == 0);
-    mtx = NULL;
-
-    r = erts_cond_destroy(cnd);
-    ASSERT(tcs, r == 0);
-    cnd = NULL;
-
-    /* ... */
-    t1 = erts_thread_self();
-
-    if (cw_passed == 4711) {
-	/* We don't want to execute this just check that the
-	   symbol/symbols is/are defined */
-	erts_thread_kill(t1);
-    }
-
-#endif /* #ifndef __WIN32__ */
-}
-
-char *
-testcase_name(void)
-{
-    return "erl_threads";
-}
-
-void
-testcase_cleanup(TestCaseState_t *tcs)
-{
-    int i;
-    for (i = 0; i < NO_OF_THREADS; i++) {
-	if (need_join[i]) {
-	    erts_mutex_lock(mtx);
-	    die = 1;
-	    erts_cond_broadcast(cnd);
-	    erts_mutex_unlock(mtx);
-	    erts_thread_join(tid[1], NULL);
-	}
-    }
-    if (mtx)
-	erts_mutex_destroy(mtx);
-    if (cnd)
-	erts_cond_destroy(cnd);
-}
-
diff --git a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c b/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c
deleted file mode 100644
index 99d5adb041..0000000000
--- a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c
+++ /dev/null
@@ -1,262 +0,0 @@
-/* ``The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved via the world wide web at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
- * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
- * AB. All Rights Reserved.''
- * 
- *     $Id$
- */
-
-#include "testcase_driver.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <setjmp.h>
-#include <string.h>
-
-#ifdef __WIN32__
-#undef HAVE_VSNPRINTF
-#define HAVE_VSNPRINTF 1
-#define vsnprintf _vsnprintf
-#endif
-
-#ifndef HAVE_VSNPRINTF
-#define HAVE_VSNPRINTF 0
-#endif
-
-#define COMMENT_BUF_SZ 4096
-
-#define TESTCASE_FAILED		0
-#define TESTCASE_SKIPPED	1
-#define TESTCASE_SUCCEEDED	2
-
-typedef struct {
-    TestCaseState_t visible;
-    int port;
-    int result;
-    jmp_buf done_jmp_buf;
-    char *comment;
-    char comment_buf[COMMENT_BUF_SZ];
-} InternalTestCaseState_t;
-
-long testcase_drv_start(int port, char *command);
-int testcase_drv_stop(long drv_data);
-int testcase_drv_run(long drv_data, char *buf, int len);
-
-static DriverEntry testcase_drv_entry = { 
-    NULL,
-    testcase_drv_start,
-    testcase_drv_stop,
-    testcase_drv_run
-};
-
-
-int DRIVER_INIT(testcase_drv)(void *arg)
-{
-    testcase_drv_entry.driver_name = testcase_name();
-    return (int) &testcase_drv_entry;
-}
-
-long
-testcase_drv_start(int port, char *command)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *)
-	driver_alloc(sizeof(InternalTestCaseState_t));
-    if (!itcs) {
-	return -1;
-    }
-
-    itcs->visible.testcase_name = testcase_name();
-    itcs->visible.extra = NULL;
-    itcs->port = port;
-    itcs->result = TESTCASE_FAILED;
-    itcs->comment = "";
-
-    return (long) itcs;
-}
-
-int
-testcase_drv_stop(long drv_data)
-{
-    testcase_cleanup((TestCaseState_t *) drv_data);
-    driver_free((void *) drv_data);
-    return 0;
-}
-
-int
-testcase_drv_run(long drv_data, char *buf, int len)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) drv_data;
-    DriverTermData result_atom;
-    DriverTermData msg[12];
-
-    itcs->visible.command = buf;
-    itcs->visible.command_len = len;
-
-    if (setjmp(itcs->done_jmp_buf) == 0) {
-	testcase_run((TestCaseState_t *) itcs);
-	itcs->result = TESTCASE_SUCCEEDED;
-    }
-
-    switch (itcs->result) {
-    case TESTCASE_SUCCEEDED:
-	result_atom = driver_mk_atom("succeeded");
-	break;
-    case TESTCASE_SKIPPED:
-	result_atom = driver_mk_atom("skipped");
-	break;
-    case TESTCASE_FAILED:
-    default:
-	result_atom = driver_mk_atom("failed");
-	break;
-    }
-
-    msg[0] = ERL_DRV_ATOM;
-    msg[1] = (DriverTermData) result_atom;
-
-    msg[2] = ERL_DRV_PORT;
-    msg[3] = driver_mk_port(itcs->port);
-
-    msg[4] = ERL_DRV_ATOM;
-    msg[5] = driver_mk_atom(itcs->visible.testcase_name);
- 
-    msg[6] = ERL_DRV_STRING;
-    msg[7] = (DriverTermData) itcs->comment;
-    msg[8] = (DriverTermData) strlen(itcs->comment);
-
-    msg[9] = ERL_DRV_TUPLE;
-    msg[10] = (DriverTermData) 4;
-
-    driver_output_term(itcs->port, msg, 11);
-    return 0;
-}
-
-int
-testcase_assertion_failed(TestCaseState_t *tcs,
-			  char *file, int line, char *assertion)
-{
-    testcase_failed(tcs, "%s:%d: Assertion failed: \"%s\"",
-		    file, line, assertion);
-    return 0;
-}
-
-void
-testcase_printf(TestCaseState_t *tcs, char *frmt, ...)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs;
-    DriverTermData msg[12];
-    va_list va;
-    va_start(va, frmt);
-#if HAVE_VSNPRINTF
-    vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va);
-#else
-    vsprintf(itcs->comment_buf, frmt, va);
-#endif
-    va_end(va);
-
-    msg[0] = ERL_DRV_ATOM;
-    msg[1] = (DriverTermData) driver_mk_atom("print");
-
-    msg[2] = ERL_DRV_PORT;
-    msg[3] = driver_mk_port(itcs->port);
-
-    msg[4] = ERL_DRV_ATOM;
-    msg[5] = driver_mk_atom(itcs->visible.testcase_name);
-
-    msg[6] = ERL_DRV_STRING;
-    msg[7] = (DriverTermData) itcs->comment_buf;
-    msg[8] = (DriverTermData) strlen(itcs->comment_buf);
-
-    msg[9] = ERL_DRV_TUPLE;
-    msg[10] = (DriverTermData) 4;
-
-    driver_output_term(itcs->port, msg, 11);
-}
-
-
-void testcase_succeeded(TestCaseState_t *tcs, char *frmt, ...)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs;
-    va_list va;
-    va_start(va, frmt);
-#if HAVE_VSNPRINTF
-    vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va);
-#else
-    vsprintf(itcs->comment_buf, frmt, va);
-#endif
-    va_end(va);
-
-    itcs->result = TESTCASE_SUCCEEDED;
-    itcs->comment = itcs->comment_buf;
-
-    longjmp(itcs->done_jmp_buf, 1);
-}
-
-void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs;
-    va_list va;
-    va_start(va, frmt);
-#if HAVE_VSNPRINTF
-    vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va);
-#else
-    vsprintf(itcs->comment_buf, frmt, va);
-#endif
-    va_end(va);
-
-    itcs->result = TESTCASE_SKIPPED;
-    itcs->comment = itcs->comment_buf;
-
-    longjmp(itcs->done_jmp_buf, 1);
-}
-
-void testcase_failed(TestCaseState_t *tcs, char *frmt, ...)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs;
-    char buf[10];
-    size_t bufsz = sizeof(buf);
-    va_list va;
-    va_start(va, frmt);
-#if HAVE_VSNPRINTF
-    vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va);
-#else
-    vsprintf(itcs->comment_buf, frmt, va);
-#endif
-    va_end(va);
-
-    itcs->result = TESTCASE_FAILED;
-    itcs->comment = itcs->comment_buf;
-
-    if (erl_drv_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0
-	&& strcmp("true", buf) == 0) {
-	fprintf(stderr, "Testcase \"%s\" failed: %s\n",
-		itcs->visible.testcase_name, itcs->comment);
-	abort();
-    }
-
-    longjmp(itcs->done_jmp_buf, 1);
-}
-
-void *testcase_alloc(size_t size)
-{
-    return driver_alloc(size);
-}
-
-void *testcase_realloc(void *ptr, size_t size)
-{
-    return driver_realloc(ptr, size);
-}
-
-void testcase_free(void *ptr)
-{
-    driver_free(ptr);
-}
diff --git a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h b/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h
deleted file mode 100644
index 3d85ca6df0..0000000000
--- a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/* ``The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved via the world wide web at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
- * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
- * AB. All Rights Reserved.''
- * 
- *     $Id$
- */
-
-#ifndef TESTCASE_DRIVER_H__
-#define TESTCASE_DRIVER_H__
-
-#include "obsolete/driver.h"
-#include <stdlib.h>
-
-typedef struct {
-    char *testcase_name;
-    char *command;
-    int command_len;
-    void *extra;
-} TestCaseState_t;
-
-#define ASSERT_CLNUP(TCS, B, CLN)					\
-do {									\
-    if (!(B)) {								\
-	CLN;								\
-	testcase_assertion_failed((TCS), __FILE__, __LINE__, #B);	\
-    }									\
-} while (0)
-
-#define ASSERT(TCS, B) ASSERT_CLNUP(TCS, B, (void) 0)
-
-void testcase_printf(TestCaseState_t *tcs, char *frmt, ...);
-void testcase_succeeded(TestCaseState_t *tcs, char *frmt, ...);
-void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...);
-void testcase_failed(TestCaseState_t *tcs, char *frmt, ...);
-int testcase_assertion_failed(TestCaseState_t *tcs, char *file, int line,
-			      char *assertion);
-void *testcase_alloc(size_t size);
-void *testcase_realloc(void *ptr, size_t size);
-void testcase_free(void *ptr);
-
-
-char *testcase_name(void);
-void testcase_run(TestCaseState_t *tcs);
-void testcase_cleanup(TestCaseState_t *tcs);
-
-#endif
diff --git a/erts/emulator/test/old_mod.erl b/erts/emulator/test/old_mod.erl
index 6c47ba6f8f..124842390a 100644
--- a/erts/emulator/test/old_mod.erl
+++ b/erts/emulator/test/old_mod.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2003-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2003-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/old_scheduler_SUITE.erl b/erts/emulator/test/old_scheduler_SUITE.erl
index 70348f64db..262536a068 100644
--- a/erts/emulator/test/old_scheduler_SUITE.erl
+++ b/erts/emulator/test/old_scheduler_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2004-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2004-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,24 +19,44 @@
 
 -module(old_scheduler_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 -export([equal/1, many_low/1, few_low/1, max/1, high/1]).
 
 -define(default_timeout, ?t:minutes(11)).
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     case catch erlang:system_info(modified_timing_level) of
 	Level when is_integer(Level) ->
 	    {skipped,
-	     "Modified timing (level " ++ integer_to_list(Level)
-	     ++ ") is enabled. Testcases gets messed up by modfied "
-	     "timing."};
-	_ ->
-	    [equal, many_low, few_low, max, high]
+	     "Modified timing (level " ++
+		 integer_to_list(Level) ++
+		 ") is enabled. Testcases gets messed "
+	     "up by modfied timing."};
+	_ -> [equal, many_low, few_low, max, high]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 %%-----------------------------------------------------------------------------------
 %% TEST SUITE DESCRIPTION
 %%
@@ -63,7 +83,7 @@ init_per_testcase(_Case, Config) ->
     ?line MS = erlang:system_flag(multi_scheduling, block),
     [{prio,Prio},{watchdog,Dog},{multi_scheduling, MS}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     erlang:system_flag(multi_scheduling, unblock),
     Dog=?config(watchdog, Config),
     Prio=?config(prio, Config),
diff --git a/erts/emulator/test/op_SUITE.erl b/erts/emulator/test/op_SUITE.erl
index 55d8d9ab0f..ef4689b850 100644
--- a/erts/emulator/test/op_SUITE.erl
+++ b/erts/emulator/test/op_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,22 +19,43 @@
 
 -module(op_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 bsl_bsr/1,logical/1,t_not/1,relop_simple/1,relop/1,complex_relop/1]).
 
 -export([]).
 -import(lists, [foldl/3,flatmap/2]).
 
-all(suite) ->
-    [bsl_bsr,logical,t_not,relop_simple,relop,complex_relop].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [bsl_bsr, logical, t_not, relop_simple, relop,
+     complex_relop].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(3)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index b9100738e4..eac56a867d 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -73,22 +73,23 @@
 %%
 
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2,
 	 init_per_suite/1, end_per_suite/1,
-	 stream/1, stream_small/1, stream_big/1,
+	 stream_small/1, stream_big/1,
 	 basic_ping/1, slow_writes/1, bad_packet/1, bad_port_messages/1,
-	 multiple_packets/1, mul_basic/1, mul_slow_writes/1,
+	 mul_basic/1, mul_slow_writes/1,
 	 dying_port/1, port_program_with_path/1,
 	 open_input_file_port/1, open_output_file_port/1,
 	 iter_max_ports/1, eof/1, input_only/1, output_only/1,
 	 name1/1,
-	 t_binary/1, options/1, parallell/1, t_exit/1,
+	 t_binary/1, parallell/1, t_exit/1,
 	 env/1, bad_env/1, cd/1, exit_status/1,
-	 tps/1, tps_16_bytes/1, tps_1K/1, line/1, stderr_to_stdout/1,
+	 tps_16_bytes/1, tps_1K/1, line/1, stderr_to_stdout/1,
 	 otp_3906/1, otp_4389/1, win_massive/1, win_massive_client/1,
 	 mix_up_ports/1, otp_5112/1, otp_5119/1, otp_6224/1,
 	 exit_status_multi_scheduling_block/1, ports/1,
-	 spawn_driver/1,spawn_executable/1,
+	 spawn_driver/1, spawn_executable/1, close_deaf_port/1,
 	 unregister_name/1]).
 
 -export([]).
@@ -98,31 +99,42 @@
 -export([otp_3906_forker/5, otp_3906_start_forker_starter/4]).
 -export([env_slave_main/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -include_lib("kernel/include/file.hrl").
 
-all(suite) ->
-    [
-     otp_6224, stream, basic_ping, slow_writes, bad_packet,
-     bad_port_messages, options, multiple_packets, parallell,
-     dying_port, port_program_with_path,
-     open_input_file_port, open_output_file_port,
-     name1,
-     env, bad_env, cd, exit_status,
-     iter_max_ports, t_exit, tps, line, stderr_to_stdout,
-     otp_3906, otp_4389, win_massive, mix_up_ports,
-     otp_5112, otp_5119,
-     exit_status_multi_scheduling_block,
-     ports, spawn_driver, spawn_executable,
-     unregister_name
-    ].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [otp_6224, {group, stream}, basic_ping, slow_writes,
+     bad_packet, bad_port_messages, {group, options},
+     {group, multiple_packets}, parallell, dying_port,
+     port_program_with_path, open_input_file_port,
+     open_output_file_port, name1, env, bad_env, cd,
+     exit_status, iter_max_ports, t_exit, {group, tps}, line,
+     stderr_to_stdout, otp_3906, otp_4389, win_massive,
+     mix_up_ports, otp_5112, otp_5119,
+     exit_status_multi_scheduling_block, ports, spawn_driver,
+     spawn_executable, close_deaf_port, unregister_name].
+
+groups() -> 
+    [{stream, [], [stream_small, stream_big]},
+     {options, [], [t_binary, eof, input_only, output_only]},
+     {multiple_packets, [], [mul_basic, mul_slow_writes]},
+     {tps, [], [tps_16_bytes, tps_1K]}].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(5)).
 
 init_per_testcase(Case, Config) ->
     [{testcase, Case} |Config].
 
-fin_per_testcase(_Case, _Config) ->
+end_per_testcase(_Case, _Config) ->
     ok.
 
 init_per_suite(Config) when is_list(Config) ->
@@ -191,7 +203,6 @@ win_massive_loop(P,N) ->
     
 
 
-stream(suite) -> [stream_small, stream_big].
 
 %% Test that we can send a stream of bytes and get it back.
 %% We will send only a small amount of data, to avoid deadlock.
@@ -304,7 +315,6 @@ bad_message(PortTest, Message) ->
 %% Tests various options (stream and {packet, Number} are implicitly
 %% tested in other test cases).
 
-options(suite) -> [t_binary, eof, input_only, output_only].
 
 %% Tests the 'binary' option for a port.
 
@@ -416,7 +426,6 @@ output_and_verify(Config, Filename, Options, Data) ->
 %% Test that receiving several packages written in the same
 %% write operation works.
 
-multiple_packets(suite) -> [mul_basic, mul_slow_writes].
 
 %% Basic test of receiving multiple packages, written in
 %% one operation by the other end.
@@ -740,7 +749,6 @@ suicide_port(Config) when is_list(Config) ->
     ?line exit(Port, die),
     ?line receive after infinity -> ok end.
 
-tps(suite) -> [tps_16_bytes, tps_1K].
 
 tps_16_bytes(doc) -> "";
 tps_16_bytes(suite) -> [];
@@ -878,12 +886,20 @@ env2(Config) ->
 			    "nisse" = os:getenv(Long)
 		    end),
 
-
+    
     ?line env_slave(Temp, [{"must_define_something","some_value"},
-			   {"certainly_not_existing",false},
+			    {"certainly_not_existing",false},
+                           {"ends_with_equal", "value="},
 			   {Long,false},
 			   {"glurf","a glorfy string"}]),
 
+    %% A lot of non existing variables (mingled with existing)
+    NotExistingList = [{lists:flatten(io_lib:format("V~p_not_existing",[X])),false} 
+                        ||  X <- lists:seq(1,150)],
+    ExistingList = [{lists:flatten(io_lib:format("V~p_existing",[X])),"a_value"} 
+                        ||  X <- lists:seq(1,150)],
+    ?line env_slave(Temp, lists:sort(ExistingList ++ NotExistingList)),
+
     ?line test_server:timetrap_cancel(Dog),
     ok.
 
@@ -1041,8 +1057,10 @@ otp_3906(Config)  when is_list(Config) ->
 -define(OTP_3906_MAX_CONC_OSP, 50).
 
 otp_3906(Config, OSName) ->
-    ?line TSDir = filename:dirname(code:which(test_server)),
-    ?line {ok, Variables} = file:consult(filename:join(TSDir, "variables")),
+    ?line DataDir = filename:dirname(proplists:get_value(data_dir,Config)),
+    ?line {ok, Variables} = file:consult(
+			      filename:join([DataDir,"..","..",
+					     "test_server","variables"])),
     case lists:keysearch('CC', 1, Variables) of
 	{value,{'CC', CC}} ->
 	    SuiteDir = filename:dirname(code:which(?MODULE)),
@@ -2292,3 +2310,37 @@ load_driver(Dir, Driver) ->
 	    io:format("~s\n", [erl_ddll:format_error(Error)]),
 	    Res
     end.
+
+
+close_deaf_port(doc) -> ["Send data to port program that does not read it, then close port."
+			 "Primary targeting Windows to test threaded_handle_closer in sys.c"];
+close_deaf_port(suite) -> [];
+close_deaf_port(Config) when is_list(Config) ->
+    ?line Dog = test_server:timetrap(test_server:seconds(100)),
+    ?line DataDir = ?config(data_dir, Config),
+    ?line DeadPort = os:find_executable("dead_port", DataDir),
+    ?line Port = open_port({spawn,DeadPort++" 60"},[]),
+    ?line erlang:port_command(Port,"Hello, can you hear me!?!?"),
+    ?line port_close(Port),
+
+    Res = close_deaf_port_1(0, DeadPort),
+    io:format("Waiting for OS procs to terminate...\n"),
+    receive after 5*1000 -> ok end,
+    ?line test_server:timetrap_cancel(Dog),
+    Res.
+
+close_deaf_port_1(1000, _) ->
+    ok;
+close_deaf_port_1(N, Cmd) ->
+    Timeout = integer_to_list(random:uniform(5*1000)),
+    ?line try open_port({spawn_executable,Cmd},[{args,[Timeout]}]) of
+    	Port ->
+	    ?line erlang:port_command(Port,"Hello, can you hear me!?!?"),
+	    ?line port_close(Port),
+	    close_deaf_port_1(N+1, Cmd)
+    catch
+    	_:eagain ->
+	    {comment, "Could not spawn more than " ++ integer_to_list(N) ++ " OS processes."}
+    end.
+    
+
diff --git a/erts/emulator/test/port_SUITE_data/Makefile.src b/erts/emulator/test/port_SUITE_data/Makefile.src
index d97b37c9ae..ff822ae720 100644
--- a/erts/emulator/test/port_SUITE_data/Makefile.src
+++ b/erts/emulator/test/port_SUITE_data/Makefile.src
@@ -3,7 +3,7 @@ LD = @LD@
 CFLAGS = @CFLAGS@ -I@erl_include@ @DEFS@
 CROSSLDFLAGS = @CROSSLDFLAGS@
 
-PROGS = port_test@exe@ echo_args@exe@
+PROGS = port_test@exe@ echo_args@exe@ dead_port@exe@
 DRIVERS = echo_drv@dll@ exit_drv@dll@ failure_drv@dll@
 
 all: $(PROGS) $(DRIVERS) port_test.@EMULATOR@
diff --git a/erts/emulator/test/port_SUITE_data/dead_port.c b/erts/emulator/test/port_SUITE_data/dead_port.c
new file mode 100644
index 0000000000..68e96fbf14
--- /dev/null
+++ b/erts/emulator/test/port_SUITE_data/dead_port.c
@@ -0,0 +1,102 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef VXWORKS
+#include <vxWorks.h>
+#include <taskVarLib.h>
+#include <taskLib.h>
+#include <sysLib.h>
+#include <string.h>
+#include <ioLib.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#ifndef __WIN32__
+#include <unistd.h>
+
+#ifdef VXWORKS
+#include "reclaim.h"
+#include <sys/times.h>
+#else
+#include <sys/time.h>
+#endif
+
+#define O_BINARY 0
+#define _setmode(fd, mode)
+#endif
+
+#ifdef __WIN32__
+#include "windows.h"
+#include "winbase.h"
+#endif
+
+
+#ifdef VXWORKS
+#define MAIN(argc, argv) port_test(argc, argv)
+#else
+#define MAIN(argc, argv) main(argc, argv)
+#endif
+
+
+extern int errno;
+
+static void delay(unsigned ms);
+
+
+MAIN(argc, argv)
+int argc;
+char *argv[];
+{
+    int x;
+    if (argc < 2) {
+	fprintf(stderr,"Usage %s <milliseconds>\n",argv[0]);
+	return 1;
+    }
+    if ((x = atoi(argv[1])) <= 0) {
+	fprintf(stderr,"Usage %s <milliseconds>\n",argv[0]);
+	return 1;
+    }
+    delay(x);
+    return 0;
+}
+
+static void
+delay(unsigned ms)
+{
+#ifdef VXWORKS
+  taskDelay((sysClkRateGet() * ms) / 1000);
+#else
+#ifdef __WIN32__
+  Sleep(ms);
+#else
+  struct timeval t;
+  t.tv_sec = ms/1000;
+  t.tv_usec = (ms % 1000) * 1000;
+
+  select(0, NULL, NULL, NULL, &t);
+#endif
+#endif
+}
diff --git a/erts/emulator/test/port_bif_SUITE.erl b/erts/emulator/test/port_bif_SUITE.erl
index f4e0bb9fa8..d9c82aba0e 100644
--- a/erts/emulator/test/port_bif_SUITE.erl
+++ b/erts/emulator/test/port_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,25 +20,47 @@
 -module(port_bif_SUITE).
 
 
--export([all/1, command/1, command_e/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, command/1,
 	 command_e_1/1, command_e_2/1, command_e_3/1, command_e_4/1,
-	 port_info/1, port_info1/1, port_info2/1,
+	 port_info1/1, port_info2/1,
 	 connect/1, control/1, echo_to_busy/1]).
 
 -export([do_command_e_1/1, do_command_e_2/1, do_command_e_4/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [command, {group, port_info}, connect, control,
+     echo_to_busy].
+
+groups() -> 
+    [{command_e, [],
+      [command_e_1, command_e_2, command_e_3, command_e_4]},
+     {port_info, [], [port_info1, port_info2]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [command, port_info, connect, control, echo_to_busy].
 
 
 init_per_testcase(_Func, Config) when is_list(Config) ->
     Dog=test_server:timetrap(test_server:minutes(10)),
     [{watchdog, Dog}|Config].
-fin_per_testcase(_Func, Config) when is_list(Config) ->
+end_per_testcase(_Func, Config) when is_list(Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog).
 
@@ -69,11 +91,6 @@ do_command(P, Data) ->
     end.
 
 
-command_e(suite) -> [command_e_1,
-		     command_e_2,
-		     command_e_3,
-		     command_e_4];
-command_e(doc) -> "Tests port_command/2 with errors".
 
 %% port_command/2: badarg 1st arg
 command_e_1(Config) when is_list(Config) ->
@@ -161,7 +178,6 @@ do_command_e_4(Program) ->
     ?line erlang:port_command(P, Data),
     exit(survived).
 
-port_info(suite) -> [port_info1, port_info2].
 
 %% Tests the port_info/1 BIF
 port_info1(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 77f850d0fb..f68e712268 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -25,12 +25,13 @@
 %%	process_info/1,2
 %%	register/2 (partially)
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(heap_binary_size, 64).
 
--export([all/1, spawn_with_binaries/1,
-	 t_exit_1/1, t_exit_2/1, t_exit_2_other/1, t_exit_2_other_normal/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, spawn_with_binaries/1,
+	 t_exit_1/1, t_exit_2_other/1, t_exit_2_other_normal/1,
 	 self_exit/1, normal_suicide_exit/1, abnormal_suicide_exit/1,
 	 t_exit_2_catch/1, trap_exit_badarg/1, trap_exit_badarg_in_bif/1,
 	 exit_and_timeout/1, exit_twice/1,
@@ -38,6 +39,7 @@
 	 process_info_other_dist_msg/1,
 	 process_info_2_list/1, process_info_lock_reschedule/1,
 	 process_info_lock_reschedule2/1,
+	 process_info_lock_reschedule3/1,
 	 bump_reductions/1, low_prio/1, binary_owner/1, yield/1, yield2/1,
 	 process_status_exiting/1,
 	 otp_4725/1, bad_register/1, garbage_collect/1, otp_6237/1,
@@ -46,39 +48,69 @@
 	 processes_large_tab/1, processes_default_tab/1, processes_small_tab/1,
 	 processes_this_tab/1, processes_apply_trap/1,
 	 processes_last_call_trap/1, processes_gc_trap/1,
-	 processes_term_proc_list/1, processes_bif/1,
-	 otp_7738/1, otp_7738_waiting/1, otp_7738_suspended/1,
-	 otp_7738_resume/1]).
+	 processes_term_proc_list/1,
+	 otp_7738_waiting/1, otp_7738_suspended/1,
+	 otp_7738_resume/1,
+	 garb_other_running/1]).
 -export([prio_server/2, prio_client/2]).
 
--export([init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 -export([hangaround/2, processes_bif_test/0, do_processes/1,
 	 processes_term_proc_list_test/1]).
 
-all(suite) ->
-    [spawn_with_binaries, t_exit_1, t_exit_2,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [spawn_with_binaries, t_exit_1, {group, t_exit_2},
      trap_exit_badarg, trap_exit_badarg_in_bif,
-     t_process_info, process_info_other_msg, process_info_other_dist_msg,
-     process_info_2_list,
-     process_info_lock_reschedule, process_info_lock_reschedule2,
-     process_status_exiting,
-     bump_reductions, low_prio, yield, yield2, otp_4725, bad_register,
-     garbage_collect, process_info_messages, process_flag_badarg, process_flag_heap_size,
-     spawn_opt_heap_size, otp_6237, processes_bif, otp_7738].
+     t_process_info, process_info_other_msg,
+     process_info_other_dist_msg, process_info_2_list,
+     process_info_lock_reschedule,
+     process_info_lock_reschedule2,
+     process_info_lock_reschedule3, process_status_exiting,
+     bump_reductions, low_prio, yield, yield2, otp_4725,
+     bad_register, garbage_collect, process_info_messages,
+     process_flag_badarg, process_flag_heap_size,
+     spawn_opt_heap_size, otp_6237, {group, processes_bif},
+     {group, otp_7738}, garb_other_running].
+
+groups() -> 
+    [{t_exit_2, [],
+      [t_exit_2_other, t_exit_2_other_normal, self_exit,
+       normal_suicide_exit, abnormal_suicide_exit,
+       t_exit_2_catch, exit_and_timeout, exit_twice]},
+     {processes_bif, [],
+      [processes_large_tab, processes_default_tab,
+       processes_small_tab, processes_this_tab,
+       processes_last_call_trap, processes_apply_trap,
+       processes_gc_trap, processes_term_proc_list]},
+     {otp_7738, [],
+      [otp_7738_waiting, otp_7738_suspended,
+       otp_7738_resume]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(Config) ->
+    catch erts_debug:set_internal_state(available_internal_state, false),
+    Config.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(10)),
     [{watchdog, Dog},{testcase, Func}|Config].
 
-fin_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
+end_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-end_per_suite(Config) ->
-    catch erts_debug:set_internal_state(available_internal_state, false),
-    Config.
-
 fun_spawn(Fun) ->
     spawn_link(erlang, apply, [Fun, []]).
 
@@ -117,10 +149,6 @@ t_exit_1() ->
 	      {'EXIT', Pid, Garbage} -> ok
 	  end.
 
-t_exit_2(suite) -> [t_exit_2_other, t_exit_2_other_normal, 
-		    self_exit, normal_suicide_exit,
-		    abnormal_suicide_exit, t_exit_2_catch,
-		    exit_and_timeout, exit_twice].
 
 %% Tests exit/2 with a lot of data in the exit message.
 t_exit_2_other(Config) when is_list(Config) ->
@@ -677,6 +705,52 @@ process_info_lock_reschedule2(Config) when is_list(Config) ->
     ?line unlink(P6), exit(P6, bang),
     ?line ok.
 
+many_args(0,_B,_C,_D,_E,_F,_G,_H,_I,_J) ->
+    ok;
+many_args(A,B,C,D,E,F,G,H,I,J) ->
+    many_args(A-1,B,C,D,E,F,G,H,I,J).
+
+do_pi_msg_len(PT, AT) ->
+    lists:map(fun (_) -> ok end, [a,b,c,d]),
+    {message_queue_len, _} = process_info(element(2,PT), element(2,AT)).
+    
+process_info_lock_reschedule3(doc) ->
+    [];
+process_info_lock_reschedule3(suite) ->
+    [];
+process_info_lock_reschedule3(Config) when is_list(Config) ->
+    %% We need a process that is running and an item that requires
+    %% process_info to take the main process lock.
+    ?line Target1 = spawn_link(fun tok_loop/0),
+    ?line Name1 = process_info_lock_reschedule_running,
+    ?line register(Name1, Target1),
+    ?line Target2 = spawn_link(fun () -> receive after infinity -> ok end end),
+    ?line Name2 = process_info_lock_reschedule_waiting,
+    ?line register(Name2, Target2),
+    ?line PI = fun(N) ->
+		       case N rem 10 of
+			   0 -> erlang:yield();
+			   _ -> ok
+		       end,
+		       ?line do_pi_msg_len({proc, Target1},
+					   {arg, message_queue_len})
+	       end,
+    ?line many_args(100000,1,2,3,4,5,6,7,8,9),
+    ?line lists:foreach(PI, lists:seq(1,1000000)),
+    %% Make sure Target1 still is willing to "tok loop"
+    ?line case process_info(Target1, status) of
+	      {status, OkStatus} when OkStatus == runnable;
+				      OkStatus == running;
+				      OkStatus == garbage_collecting ->
+		  ?line unlink(Target1),
+		  ?line unlink(Target2),
+		  ?line exit(Target1, bang),
+		  ?line exit(Target2, bang),
+		  ?line OkStatus;
+	      {status, BadStatus} ->
+		  ?line ?t:fail(BadStatus)
+	  end.
+
 process_status_exiting(Config) when is_list(Config) ->
     %% Make sure that erts_debug:get_internal_state({process_status,P})
     %% returns exiting if it is in status P_EXITING.
@@ -1227,17 +1301,6 @@ otp_6237_select_loop() ->
     otp_6237_select_loop().
 
 
-processes_bif(doc) ->
-    [];
-processes_bif(suite) ->
-    [processes_large_tab,
-     processes_default_tab,
-     processes_small_tab,
-     processes_this_tab,
-     processes_last_call_trap,
-     processes_apply_trap,
-     processes_gc_trap,
-     processes_term_proc_list].
 
 -define(NoTestProcs, 10000).
 -record(processes_bif_info, {min_start_reds,
@@ -1965,10 +2028,6 @@ processes_term_proc_list_test(MustChk) ->
     ?line erlang:system_flag(multi_scheduling, unblock),
     ?line as_expected.
 
-otp_7738(doc) ->
-    [];
-otp_7738(suite) ->
-    [otp_7738_waiting, otp_7738_suspended, otp_7738_resume].
 
 otp_7738_waiting(doc) ->
     [];
@@ -2058,6 +2117,41 @@ otp_7738_test(Type) ->
 	  end,
     ?line ok.
 
+gor(Reds, Stop) ->
+    receive
+	{From, reds} ->
+	    From ! {reds, Reds, self()},
+	    gor(Reds+1, Stop);
+	{From, Stop} ->
+	    From ! {stopped, Stop, Reds, self()}
+    after 0 ->
+	    gor(Reds+1, Stop)
+    end.
+
+garb_other_running(Config) when is_list(Config) ->
+    ?line Stop = make_ref(),
+    ?line {Pid, Mon} = spawn_monitor(fun () -> gor(0, Stop) end),
+    ?line Reds = lists:foldl(fun (_, OldReds) ->
+				     ?line erlang:garbage_collect(Pid),
+				     ?line receive after 1 -> ok end,
+				     ?line Pid ! {self(), reds},
+				     ?line receive
+					       {reds, NewReds, Pid} ->
+						   ?line true = (NewReds > OldReds),
+						   ?line NewReds
+					   end
+			     end,
+			     0,
+			     lists:seq(1, 10000)),
+    ?line receive after 1 -> ok end,
+    ?line Pid ! {self(), Stop},
+    ?line receive
+	      {stopped, Stop, StopReds, Pid} ->
+		  ?line true = (StopReds > Reds)
+	  end,
+    ?line receive {'DOWN', Mon, process, Pid, normal} -> ok end,
+    ?line ok.
+
 %% Internal functions
 
 wait_until(Fun) ->
diff --git a/erts/emulator/test/pseudoknot_SUITE.erl b/erts/emulator/test/pseudoknot_SUITE.erl
index 907204cf93..5a7cdcecd5 100644
--- a/erts/emulator/test/pseudoknot_SUITE.erl
+++ b/erts/emulator/test/pseudoknot_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2001-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2001-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,29 @@
 
 -module(pseudoknot_SUITE).
 
--export([all/1,test/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,test/1]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [test].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) -> [test].
     
 test(Config) when is_list(Config) ->
     statistics(runtime),
@@ -3274,13 +3294,13 @@ most_distant_atom(Sols) ->
     maximum(map(sol_most_distant_atom, Sols)).
 
 maximum([H|T]) ->
-    max(T,H).
+    max1(T,H).
 
-max([H|T],M) when is_float(H), is_float(M), H > M ->
-    max(T,H);
-max([_|T],M) ->
-    max(T,M);
-max([],M) -> M.
+max1([H|T],M) when is_float(H), is_float(M), H > M ->
+    max1(T,H);
+max1([_|T],M) ->
+    max1(T,M);
+max1([],M) -> M.
 
 map(_Func,[]) -> [];
 map(Func,[H|T]) -> 
diff --git a/erts/emulator/test/random_iolist.erl b/erts/emulator/test/random_iolist.erl
index 4bce347d9a..8f21b5a3b3 100644
--- a/erts/emulator/test/random_iolist.erl
+++ b/erts/emulator/test/random_iolist.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2008-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl
new file mode 100644
index 0000000000..b070e2b986
--- /dev/null
+++ b/erts/emulator/test/receive_SUITE.erl
@@ -0,0 +1,132 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(receive_SUITE).
+
+%% Tests receive after.
+
+-include_lib("test_server/include/test_server.hrl").
+
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 call_with_huge_message_queue/1,receive_in_between/1]).
+
+-export([init_per_testcase/2,end_per_testcase/2]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [call_with_huge_message_queue, receive_in_between].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
+    Dog=?t:timetrap(?t:minutes(3)),
+    [{watchdog, Dog}|Config].
+
+end_per_testcase(_Func, Config) ->
+    Dog=?config(watchdog, Config),
+    ?t:timetrap_cancel(Dog).
+
+call_with_huge_message_queue(Config) when is_list(Config) ->
+    ?line Pid = spawn_link(fun echo_loop/0),
+
+    ?line {Time,ok} = tc(fun() -> calls(10, Pid) end),
+
+    ?line [self() ! {msg,N} || N <- lists:seq(1, 500000)],
+    erlang:garbage_collect(),
+    ?line {NewTime,ok} = tc(fun() -> calls(10, Pid) end),
+    io:format("Time for empty message queue: ~p", [Time]),
+    io:format("Time for huge message queue: ~p", [NewTime]),
+
+    case (NewTime+1) / (Time+1) of
+	Q when Q < 10 ->
+	    ok;
+	Q ->
+	    io:format("Q = ~p", [Q]),
+	    ?line ?t:fail()
+    end,
+    ok.
+
+calls(0, _) -> ok;
+calls(N, Pid) ->
+    {ok,{ultimate_answer,42}} = call(Pid, {ultimate_answer,42}),
+    calls(N-1, Pid).
+
+call(Pid, Msg) ->
+    Mref = erlang:monitor(process, Pid),
+    Pid ! {Mref,{self(),Msg}},
+    receive
+	{Mref, Reply} ->
+	    erlang:demonitor(Mref, [flush]),
+	    {ok, Reply};
+	{'DOWN', Mref, _, _, Reason} ->
+	    exit(Reason)
+    end.
+
+receive_in_between(Config) when is_list(Config) ->
+    ?line Pid = spawn_link(fun echo_loop/0),
+    ?line [{ok,{a,b}} = call2(Pid, {a,b}) || _ <- lists:seq(1, 100000)],
+    ok.
+
+call2(Pid, Msg) ->
+    self() ! dummy,
+    Mref = erlang:monitor(process, Pid),
+    Pid ! {Mref,{self(),Msg}},
+    receive_one(),
+    receive
+	{Mref,Reply} ->
+	    erlang:demonitor(Mref, [flush]),
+	    {ok,Reply};
+	{'DOWN',Mref,_,_,Reason} ->
+	    exit(Reason)
+    end.
+
+receive_one() ->
+    receive
+	dummy -> ok
+    end.
+
+%%%
+%%% Common helpers.
+%%%
+
+echo_loop() ->
+    receive
+	{Ref,{Pid,Msg}} ->
+	    Pid ! {Ref,Msg},
+	    echo_loop()
+    end.
+
+tc(Fun) ->
+    timer:tc(erlang, apply, [Fun,[]]).
diff --git a/erts/emulator/test/ref_SUITE.erl b/erts/emulator/test/ref_SUITE.erl
index fa77095efd..e13dfa1575 100644
--- a/erts/emulator/test/ref_SUITE.erl
+++ b/erts/emulator/test/ref_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,23 +19,44 @@
 
 -module(ref_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2]).
 -export([wrap_1/1]).
 
 -export([loop_ref/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 init_per_testcase(_, Config) ->
     ?line Dog=test_server:timetrap(test_server:minutes(2)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_, Config) ->
+end_per_testcase(_, Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
 
-all(suite) -> [wrap_1].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [wrap_1].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 wrap_1(doc) -> "Check that refs don't wrap around easily.";
 wrap_1(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/register_SUITE.erl b/erts/emulator/test/register_SUITE.erl
index c03ee23b2e..9953df3458 100644
--- a/erts/emulator/test/register_SUITE.erl
+++ b/erts/emulator/test/register_SUITE.erl
@@ -1,7 +1,7 @@
 %% 
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2009. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,24 +22,43 @@
 
 %-define(line_trace, 1).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %-compile(export_all).
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
 -export([otp_8099/1]).
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(2)).
 
-all(doc) -> [];
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [otp_8099].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 init_per_testcase(Case, Config) when is_list(Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
     [{watchdog, Dog}, {testcase, Case} | Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/test/save_calls_SUITE.erl b/erts/emulator/test/save_calls_SUITE.erl
index b56c4ad0b0..390b49b604 100644
--- a/erts/emulator/test/save_calls_SUITE.erl
+++ b/erts/emulator/test/save_calls_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,17 +19,36 @@
 
 -module(save_calls_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 -export([save_calls_1/1,dont_break_reductions/1]).
 
 -export([do_bopp/1, do_bipp/0, do_bepp/0]).
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [save_calls_1, dont_break_reductions].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 dont_break_reductions(suite) ->
     [];
 dont_break_reductions(doc) ->
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index c9101b77c2..f16d0ea429 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -1,7 +1,7 @@
 %% 
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -30,10 +30,12 @@
 
 %-define(line_trace, 1).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %-compile(export_all).
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2, end_per_suite/1]).
 
 -export([equal/1,
 	 few_low/1,
@@ -44,30 +46,47 @@
 	 equal_with_high/1,
 	 equal_with_high_max/1,
 	 bound_process/1,
-	 scheduler_bind/1,
+	
 	 scheduler_bind_types/1,
 	 cpu_topology/1,
+	 update_cpu_info/1,
 	 sct_cmd/1,
 	 sbt_cmd/1,
-	 scheduler_suspend/1]).
+	 scheduler_suspend/1,
+	 reader_groups/1]).
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(10)).
 
 -define(MIN_SCHEDULER_TEST_TIMEOUT, ?t:minutes(1)).
 
-all(doc) -> [];
-all(suite) ->
-    [equal,
-     few_low,
-     many_low,
-     equal_with_part_time_high,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [equal, few_low, many_low, equal_with_part_time_high,
      equal_with_part_time_max,
-     equal_and_high_with_part_time_max,
-     equal_with_high,
-     equal_with_high_max,
-     bound_process,
-     scheduler_bind,
-     scheduler_suspend].
+     equal_and_high_with_part_time_max, equal_with_high,
+     equal_with_high_max, bound_process,
+     {group, scheduler_bind}, scheduler_suspend,
+     reader_groups].
+
+groups() -> 
+    [{scheduler_bind, [],
+      [scheduler_bind_types, cpu_topology, update_cpu_info,
+       sct_cmd, sbt_cmd]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(Config) ->
+    catch erts_debug:set_internal_state(available_internal_state, false),
+    Config.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Case, Config) when is_list(Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
@@ -76,7 +95,7 @@ init_per_testcase(Case, Config) when is_list(Config) ->
     OkRes = ok,
     [{watchdog, Dog}, {testcase, Case}, {ok_res, OkRes} |Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
@@ -240,11 +259,6 @@ bound_loop(NS, N, M, Sched) ->
     Sched = erlang:system_info(scheduler_id),
     bound_loop(NS, N-1, M, Sched).
 
-scheduler_bind(suite) ->
-    [scheduler_bind_types,
-     cpu_topology,
-     sct_cmd,
-     sbt_cmd].
 
 -define(TOPOLOGY_A_CMD,
 	"+sct"
@@ -766,6 +780,137 @@ cpu_topology_cmdline_test(Config, Topology, Cmd) ->
     ?line stop_node(Node),
     ?line ok.
 
+update_cpu_info(Config) when is_list(Config) ->
+    ?line OldOnline = erlang:system_info(schedulers_online),
+    ?line OldAff = get_affinity_mask(),
+    ?line ?t:format("START - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
+		    [OldAff, OldOnline, erlang:system_info(scheduler_bindings)]),
+    ?line case {erlang:system_info(logical_processors_available), OldAff} of
+	      {Avail, _} when Avail == unknown; OldAff == unknown ->
+		  %% Nothing much to test; just a smoke test
+		  case erlang:system_info(update_cpu_info) of
+		      unchanged -> ?line ok;
+		      changed -> ?line ok
+		  end;
+	      _ ->
+		  try
+		      ?line adjust_schedulers_online(),
+		      case erlang:system_info(schedulers_online) of
+			  1 ->
+			      %% Nothing much to test; just a smoke test
+			      ?line ok;
+			  Onln0 ->
+			      %% unset least significant bit
+			      ?line Aff = (OldAff band (OldAff - 1)),
+			      ?line set_affinity_mask(Aff),
+			      ?line Onln1 = Onln0 - 1,
+			      ?line case adjust_schedulers_online() of
+					{Onln0, Onln1} ->
+					    ?line Onln1 = erlang:system_info(schedulers_online),
+					    ?line receive after 500 -> ok end,
+					    ?line ?t:format("TEST - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
+							    [Aff, Onln1, erlang:system_info(scheduler_bindings)]),
+					    ?line unchanged = adjust_schedulers_online(),
+					    ?line ok;
+					Fail ->
+					    ?line ?t:fail(Fail)
+				    end
+		      end
+		  after
+		      set_affinity_mask(OldAff),
+		      adjust_schedulers_online(),
+		      erlang:system_flag(schedulers_online, OldOnline),
+		      receive after 500 -> ok end,
+		      ?t:format("END - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
+				[get_affinity_mask(),
+				 erlang:system_info(schedulers_online),
+				 erlang:system_info(scheduler_bindings)])
+		  end
+	  end.
+
+adjust_schedulers_online() ->
+    case erlang:system_info(update_cpu_info) of
+	unchanged ->
+	    unchanged;
+	changed ->
+	    Avail = erlang:system_info(logical_processors_available),
+	    {erlang:system_flag(schedulers_online, Avail), Avail}
+    end.
+
+read_affinity(Data) ->
+    Exp = "pid " ++ os:getpid() ++ "'s current affinity mask",
+    case string:tokens(Data, ":") of
+	[Exp, DirtyAffinityStr] ->
+	    AffinityStr = string:strip(string:strip(DirtyAffinityStr,
+						    both, $ ),
+				       both, $\n),
+	    case catch erlang:list_to_integer(AffinityStr, 16) of
+		Affinity when is_integer(Affinity) ->
+		    Affinity;
+		_ ->
+		    bad
+	    end;
+	_ ->
+	    bad
+    end.
+
+get_affinity_mask(Port, Status, Affinity) when Status == unknown;
+					       Affinity == unknown ->
+    receive
+	{Port,{data, Data}} ->
+	    get_affinity_mask(Port, Status, read_affinity(Data));
+	{Port,{exit_status,S}} ->
+	    get_affinity_mask(Port, S, Affinity)
+    end;
+get_affinity_mask(_Port, _Status, bad) ->
+    unknown;
+get_affinity_mask(_Port, _Status, Affinity) ->
+    Affinity.
+
+get_affinity_mask() ->
+    case ?t:os_type() of
+	{unix, linux} ->
+	    case catch open_port({spawn, "taskset -p " ++ os:getpid()},
+				 [exit_status]) of
+		Port when is_port(Port) ->
+		    get_affinity_mask(Port, unknown, unknown);
+		_ ->
+		    unknown
+	    end;
+	_ ->
+	    unknown
+    end.
+
+set_affinity_mask(Port, unknown) ->
+    receive
+	{Port,{data, _}} ->
+	    set_affinity_mask(Port, unknown);
+	{Port,{exit_status,Status}} ->
+	    set_affinity_mask(Port, Status)
+    end;
+set_affinity_mask(Port, Status) ->
+    receive
+	{Port,{data, _}} ->
+	    set_affinity_mask(Port, unknown)
+    after 0 ->
+	    Status
+    end.
+
+set_affinity_mask(Mask) ->
+    Cmd = lists:flatten(["taskset -p ",
+			 io_lib:format("~.16b", [Mask]),
+			 " ",
+			 os:getpid()]),
+    case catch open_port({spawn, Cmd}, [exit_status]) of
+	Port when is_port(Port) ->
+	    case set_affinity_mask(Port, unknown) of
+		0 -> ok;
+		_ -> exit(failed_to_set_affinity)
+	    end;
+	_ ->
+	    exit(failed_to_set_affinity)
+    end.
+
 sct_cmd(Config) when is_list(Config) ->
     ?line Topology = ?TOPOLOGY_A_TERM,
     ?line OldRelFlags = clear_erl_rel_flags(),
@@ -902,7 +1047,8 @@ scheduler_suspend_test(Config, Schedulers) ->
     ?line [SState] = mcall(Node, [fun () ->
 					  erlang:system_info(schedulers_state)
 				  end]),
-    ?line {Sched, _, _} = SState,
+    ?line ?t:format("SState=~p~n", [SState]),
+    ?line {Sched, SchedOnln, _SchedAvail} = SState,
     ?line true = is_integer(Sched),
     ?line [ok] = mcall(Node, [fun () -> sst0_loop(300) end]),
     ?line [ok] = mcall(Node, [fun () -> sst1_loop(300) end]),
@@ -914,6 +1060,14 @@ scheduler_suspend_test(Config, Schedulers) ->
 					fun () -> sst2_loop(200) end,
 					fun () -> sst3_loop(Sched, 200) end]),
     ?line [SState] = mcall(Node, [fun () ->
+					  case Sched == SchedOnln of
+					      false ->
+						  Sched = erlang:system_flag(
+							    schedulers_online,
+							    SchedOnln);
+					      true ->
+						  ok
+					  end,
 					  erlang:system_info(schedulers_state)
 				  end]),
     ?line stop_node(Node),
@@ -956,12 +1110,300 @@ sst3_loop(S, N) ->
     erlang:system_flag(schedulers_online, 1),
     erlang:system_flag(schedulers_online, S),
     sst3_loop(S, N-1).
+
+reader_groups(Config) when is_list(Config) ->
+    %% White box testing. These results are correct, but other results
+    %% could be too...
+
+    %% The actual tilepro64 topology
+    CPUT0 = [{processor,[{node,[{core,{logical,0}},
+				{core,{logical,1}},
+				{core,{logical,2}},
+				{core,{logical,8}},
+				{core,{logical,9}},
+				{core,{logical,10}},
+				{core,{logical,11}},
+				{core,{logical,16}},
+				{core,{logical,17}},
+				{core,{logical,18}},
+				{core,{logical,19}},
+				{core,{logical,24}},
+				{core,{logical,25}},
+				{core,{logical,27}},
+				{core,{logical,29}}]},
+			 {node,[{core,{logical,3}},
+				{core,{logical,4}},
+				{core,{logical,5}},
+				{core,{logical,6}},
+				{core,{logical,7}},
+				{core,{logical,12}},
+				{core,{logical,13}},
+				{core,{logical,14}},
+				{core,{logical,15}},
+				{core,{logical,20}},
+				{core,{logical,21}},
+				{core,{logical,22}},
+				{core,{logical,23}},
+				{core,{logical,28}},
+				{core,{logical,30}}]},
+			 {node,[{core,{logical,31}},
+				{core,{logical,36}},
+				{core,{logical,37}},
+				{core,{logical,38}},
+				{core,{logical,44}},
+				{core,{logical,45}},
+				{core,{logical,46}},
+				{core,{logical,47}},
+				{core,{logical,51}},
+				{core,{logical,52}},
+				{core,{logical,53}},
+				{core,{logical,54}},
+				{core,{logical,55}},
+				{core,{logical,60}},
+				{core,{logical,61}}]},
+			 {node,[{core,{logical,26}},
+				{core,{logical,32}},
+				{core,{logical,33}},
+				{core,{logical,34}},
+				{core,{logical,35}},
+				{core,{logical,39}},
+				{core,{logical,40}},
+				{core,{logical,41}},
+				{core,{logical,42}},
+				{core,{logical,43}},
+				{core,{logical,48}},
+				{core,{logical,49}},
+				{core,{logical,50}},
+				{core,{logical,58}}]}]}],
+
+    ?line [{0,1},{1,1},{2,1},{3,3},{4,3},{5,3},{6,3},{7,3},{8,1},{9,1},{10,1},
+	   {11,1},{12,3},{13,3},{14,4},{15,4},{16,2},{17,2},{18,2},{19,2},
+	   {20,4},{21,4},{22,4},{23,4},{24,2},{25,2},{26,7},{27,2},{28,4},
+	   {29,2},{30,4},{31,5},{32,7},{33,7},{34,7},{35,7},{36,5},{37,5},
+	   {38,5},{39,7},{40,7},{41,8},{42,8},{43,8},{44,5},{45,5},{46,5},
+	   {47,6},{48,8},{49,8},{50,8},{51,6},{52,6},{53,6},{54,6},{55,6},
+	   {58,8},{60,6},{61,6}]
+	= reader_groups_map(CPUT0, 8),
+
+    CPUT1 = [n([p([c([t(l(0)),t(l(1)),t(l(2)),t(l(3))]),
+		   c([t(l(4)),t(l(5)),t(l(6)),t(l(7))]),
+		   c([t(l(8)),t(l(9)),t(l(10)),t(l(11))]),
+		   c([t(l(12)),t(l(13)),t(l(14)),t(l(15))])]),
+		p([c([t(l(16)),t(l(17)),t(l(18)),t(l(19))]),
+		   c([t(l(20)),t(l(21)),t(l(22)),t(l(23))]),
+		   c([t(l(24)),t(l(25)),t(l(26)),t(l(27))]),
+		   c([t(l(28)),t(l(29)),t(l(30)),t(l(31))])])]),
+	     n([p([c([t(l(32)),t(l(33)),t(l(34)),t(l(35))]),
+		   c([t(l(36)),t(l(37)),t(l(38)),t(l(39))]),
+		   c([t(l(40)),t(l(41)),t(l(42)),t(l(43))]),
+		   c([t(l(44)),t(l(45)),t(l(46)),t(l(47))])]),
+		p([c([t(l(48)),t(l(49)),t(l(50)),t(l(51))]),
+		   c([t(l(52)),t(l(53)),t(l(54)),t(l(55))]),
+		   c([t(l(56)),t(l(57)),t(l(58)),t(l(59))]),
+		   c([t(l(60)),t(l(61)),t(l(62)),t(l(63))])])]),
+	     n([p([c([t(l(64)),t(l(65)),t(l(66)),t(l(67))]),
+		   c([t(l(68)),t(l(69)),t(l(70)),t(l(71))]),
+		   c([t(l(72)),t(l(73)),t(l(74)),t(l(75))]),
+		   c([t(l(76)),t(l(77)),t(l(78)),t(l(79))])]),
+		p([c([t(l(80)),t(l(81)),t(l(82)),t(l(83))]),
+		   c([t(l(84)),t(l(85)),t(l(86)),t(l(87))]),
+		   c([t(l(88)),t(l(89)),t(l(90)),t(l(91))]),
+		   c([t(l(92)),t(l(93)),t(l(94)),t(l(95))])])]),
+	     n([p([c([t(l(96)),t(l(97)),t(l(98)),t(l(99))]),
+		   c([t(l(100)),t(l(101)),t(l(102)),t(l(103))]),
+		   c([t(l(104)),t(l(105)),t(l(106)),t(l(107))]),
+		   c([t(l(108)),t(l(109)),t(l(110)),t(l(111))])]),
+		p([c([t(l(112)),t(l(113)),t(l(114)),t(l(115))]),
+		   c([t(l(116)),t(l(117)),t(l(118)),t(l(119))]),
+		   c([t(l(120)),t(l(121)),t(l(122)),t(l(123))]),
+		   c([t(l(124)),t(l(125)),t(l(126)),t(l(127))])])])],
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3},
+	   {10,3},{11,3},{12,4},{13,4},{14,4},{15,4},{16,5},{17,5},{18,5},
+	   {19,5},{20,6},{21,6},{22,6},{23,6},{24,7},{25,7},{26,7},{27,7},
+	   {28,8},{29,8},{30,8},{31,8},{32,9},{33,9},{34,9},{35,9},{36,10},
+	   {37,10},{38,10},{39,10},{40,11},{41,11},{42,11},{43,11},{44,12},
+	   {45,12},{46,12},{47,12},{48,13},{49,13},{50,13},{51,13},{52,14},
+	   {53,14},{54,14},{55,14},{56,15},{57,15},{58,15},{59,15},{60,16},
+	   {61,16},{62,16},{63,16},{64,17},{65,17},{66,17},{67,17},{68,18},
+	   {69,18},{70,18},{71,18},{72,19},{73,19},{74,19},{75,19},{76,20},
+	   {77,20},{78,20},{79,20},{80,21},{81,21},{82,21},{83,21},{84,22},
+	   {85,22},{86,22},{87,22},{88,23},{89,23},{90,23},{91,23},{92,24},
+	   {93,24},{94,24},{95,24},{96,25},{97,25},{98,25},{99,25},{100,26},
+	   {101,26},{102,26},{103,26},{104,27},{105,27},{106,27},{107,27},
+	   {108,28},{109,28},{110,28},{111,28},{112,29},{113,29},{114,29},
+	   {115,29},{116,30},{117,30},{118,30},{119,30},{120,31},{121,31},
+	   {122,31},{123,31},{124,32},{125,32},{126,32},{127,32}]
+	= reader_groups_map(CPUT1, 128),
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1},
+	   {11,1},{12,1},{13,1},{14,1},{15,1},{16,1},{17,1},{18,1},{19,1},
+	   {20,1},{21,1},{22,1},{23,1},{24,1},{25,1},{26,1},{27,1},{28,1},
+	   {29,1},{30,1},{31,1},{32,1},{33,1},{34,1},{35,1},{36,1},{37,1},
+	   {38,1},{39,1},{40,1},{41,1},{42,1},{43,1},{44,1},{45,1},{46,1},
+	   {47,1},{48,1},{49,1},{50,1},{51,1},{52,1},{53,1},{54,1},{55,1},
+	   {56,1},{57,1},{58,1},{59,1},{60,1},{61,1},{62,1},{63,1},{64,2},
+	   {65,2},{66,2},{67,2},{68,2},{69,2},{70,2},{71,2},{72,2},{73,2},
+	   {74,2},{75,2},{76,2},{77,2},{78,2},{79,2},{80,2},{81,2},{82,2},
+	   {83,2},{84,2},{85,2},{86,2},{87,2},{88,2},{89,2},{90,2},{91,2},
+	   {92,2},{93,2},{94,2},{95,2},{96,2},{97,2},{98,2},{99,2},{100,2},
+	   {101,2},{102,2},{103,2},{104,2},{105,2},{106,2},{107,2},{108,2},
+	   {109,2},{110,2},{111,2},{112,2},{113,2},{114,2},{115,2},{116,2},
+	   {117,2},{118,2},{119,2},{120,2},{121,2},{122,2},{123,2},{124,2},
+	   {125,2},{126,2},{127,2}]
+	= reader_groups_map(CPUT1, 2),
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3},{10,3},
+	   {11,3},{12,3},{13,3},{14,3},{15,3},{16,4},{17,4},{18,4},{19,4},
+	   {20,4},{21,4},{22,4},{23,4},{24,5},{25,5},{26,5},{27,5},{28,5},
+	   {29,5},{30,5},{31,5},{32,6},{33,6},{34,6},{35,6},{36,6},{37,6},
+	   {38,6},{39,6},{40,7},{41,7},{42,7},{43,7},{44,7},{45,7},{46,7},
+	   {47,7},{48,8},{49,8},{50,8},{51,8},{52,8},{53,8},{54,8},{55,8},
+	   {56,9},{57,9},{58,9},{59,9},{60,9},{61,9},{62,9},{63,9},{64,10},
+	   {65,10},{66,10},{67,10},{68,10},{69,10},{70,10},{71,10},{72,11},
+	   {73,11},{74,11},{75,11},{76,11},{77,11},{78,11},{79,11},{80,12},
+	   {81,12},{82,12},{83,12},{84,12},{85,12},{86,12},{87,12},{88,13},
+	   {89,13},{90,13},{91,13},{92,13},{93,13},{94,13},{95,13},{96,14},
+	   {97,14},{98,14},{99,14},{100,14},{101,14},{102,14},{103,14},
+	   {104,15},{105,15},{106,15},{107,15},{108,15},{109,15},{110,15},
+	   {111,15},{112,16},{113,16},{114,16},{115,16},{116,16},{117,16},
+	   {118,16},{119,16},{120,17},{121,17},{122,17},{123,17},{124,17},
+	   {125,17},{126,17},{127,17}]
+	= reader_groups_map(CPUT1, 17),
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1},
+	   {11,1},{12,1},{13,1},{14,1},{15,1},{16,2},{17,2},{18,2},{19,2},
+	   {20,2},{21,2},{22,2},{23,2},{24,2},{25,2},{26,2},{27,2},{28,2},
+	   {29,2},{30,2},{31,2},{32,3},{33,3},{34,3},{35,3},{36,3},{37,3},
+	   {38,3},{39,3},{40,3},{41,3},{42,3},{43,3},{44,3},{45,3},{46,3},
+	   {47,3},{48,4},{49,4},{50,4},{51,4},{52,4},{53,4},{54,4},{55,4},
+	   {56,4},{57,4},{58,4},{59,4},{60,4},{61,4},{62,4},{63,4},{64,5},
+	   {65,5},{66,5},{67,5},{68,5},{69,5},{70,5},{71,5},{72,5},{73,5},
+	   {74,5},{75,5},{76,5},{77,5},{78,5},{79,5},{80,6},{81,6},{82,6},
+	   {83,6},{84,6},{85,6},{86,6},{87,6},{88,6},{89,6},{90,6},{91,6},
+	   {92,6},{93,6},{94,6},{95,6},{96,7},{97,7},{98,7},{99,7},{100,7},
+	   {101,7},{102,7},{103,7},{104,7},{105,7},{106,7},{107,7},{108,7},
+	   {109,7},{110,7},{111,7},{112,7},{113,7},{114,7},{115,7},{116,7},
+	   {117,7},{118,7},{119,7},{120,7},{121,7},{122,7},{123,7},{124,7},
+	   {125,7},{126,7},{127,7}]
+	= reader_groups_map(CPUT1, 7),
+
+    ?line CPUT2 = [p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))]),
+		   p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]),
+		   p([t(l(10))]),
+		   p([c(l(11)),c(l(12)),c(l(13))]),
+		   p([c(l(14)),c(l(15))])],
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,1},
+	   {5,2},{6,2},{7,2},{8,2},{9,2},
+	   {10,3},
+	   {11,4},{12,4},{13,4},
+	   {14,5},{15,5}] = reader_groups_map(CPUT2, 5),
+
+
+    ?line [{0,1},{1,1},{2,2},{3,2},{4,2},
+	   {5,3},{6,3},{7,3},{8,3},{9,3},
+	   {10,4},
+	   {11,5},{12,5},{13,5},
+	   {14,6},{15,6}] = reader_groups_map(CPUT2, 6),
+
+    ?line [{0,1},{1,1},{2,2},{3,2},{4,2},
+	   {5,3},{6,3},{7,3},{8,3},{9,3},
+	   {10,4},
+	   {11,5},{12,6},{13,6},
+	   {14,7},{15,7}] = reader_groups_map(CPUT2, 7),
+
+    ?line [{0,1},{1,1},{2,2},{3,2},{4,2},
+	   {5,3},{6,3},{7,3},{8,3},{9,3},
+	   {10,4},
+	   {11,5},{12,6},{13,6},
+	   {14,7},{15,8}] = reader_groups_map(CPUT2, 8),
+
+    ?line [{0,1},{1,2},{2,2},{3,3},{4,3},
+	   {5,4},{6,4},{7,4},{8,4},{9,4},
+	   {10,5},
+	   {11,6},{12,7},{13,7},
+	   {14,8},{15,9}] = reader_groups_map(CPUT2, 9),
+
+    ?line [{0,1},{1,2},{2,2},{3,3},{4,3},
+	   {5,4},{6,4},{7,4},{8,4},{9,4},
+	   {10,5},
+	   {11,6},{12,7},{13,8},
+	   {14,9},{15,10}] = reader_groups_map(CPUT2, 10),
+
+    ?line [{0,1},{1,2},{2,3},{3,4},{4,4},
+	   {5,5},{6,5},{7,5},{8,5},{9,5},
+	   {10,6},
+	   {11,7},{12,8},{13,9},
+	   {14,10},{15,11}] = reader_groups_map(CPUT2, 11),
+
+    ?line [{0,1},{1,2},{2,3},{3,4},{4,5},
+	   {5,6},{6,6},{7,6},{8,6},{9,6},
+	   {10,7},
+	   {11,8},{12,9},{13,10},
+	   {14,11},{15,12}] = reader_groups_map(CPUT2, 100),
+
+    CPUT3 = [p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]),
+	     p([t(l(10))]),
+	     p([c(l(11)),c(l(12)),c(l(13))]),
+	     p([c(l(14)),c(l(15))]),
+	     p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))])],
+
+    ?line [{0,5},{1,5},{2,6},{3,6},{4,6},
+	   {5,1},{6,1},{7,1},{8,1},{9,1},
+	   {10,2},{11,3},{12,3},{13,3},
+	   {14,4},{15,4}] = reader_groups_map(CPUT3, 6),
+
+    CPUT4 = [p([t(l(0)),t(l(1)),t(l(2)),t(l(3)),t(l(4))]),
+	     p([t(l(5))]),
+	     p([c(l(6)),c(l(7)),c(l(8))]),
+	     p([c(l(9)),c(l(10))]),
+	     p([c(l(11)),c(l(12)),c(l(13)),c(l(14)),c(l(15))])],
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,1},
+	   {5,2},
+	   {6,3},{7,3},{8,3},
+	   {9,4},{10,4},
+	   {11,5},{12,5},{13,6},{14,6},{15,6}] = reader_groups_map(CPUT4, 6),
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,1},
+	   {5,2},
+	   {6,3},{7,4},{8,4},
+	   {9,5},{10,5},
+	   {11,6},{12,6},{13,7},{14,7},{15,7}] = reader_groups_map(CPUT4, 7),
+
+    ?line [{0,1},{65535,2}] = reader_groups_map([c(l(0)),c(l(65535))], 10),
+
+    ?line ok.
     
 
+reader_groups_map(CPUT, Groups) ->
+    Old = erlang:system_info({cpu_topology, defined}),
+    erlang:system_flag(cpu_topology, CPUT),
+    enable_internal_state(),
+    Res = erts_debug:get_internal_state({reader_groups_map, Groups}),
+    erlang:system_flag(cpu_topology, Old),
+    lists:sort(Res).
+
 %%
 %% Utils
 %%
 
+l(Id) ->
+    {logical, Id}.
+
+t(X) ->
+    {thread, X}.
+
+c(X) ->
+    {core, X}.
+
+p(X) ->
+    {processor, X}.
+
+n(X) ->
+    {node, X}.
+
 mcall(Node, Funs) ->
     Parent = self(),
     Refs = lists:map(fun (Fun) ->
diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl
index 489adbd660..6615873392 100644
--- a/erts/emulator/test/send_term_SUITE.erl
+++ b/erts/emulator/test/send_term_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2005-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,24 +19,43 @@
 
 -module(send_term_SUITE).
 
--export([all/1,basic/1]).
--export([init_per_testcase/2,fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,basic/1]).
+-export([init_per_testcase/2,end_per_testcase/2]).
 
 -export([generate_external_terms_files/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(3)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [basic].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 basic(Config) when is_list(Config) ->
     Drv = "send_term_drv",
     ?line P = start_driver(Config, Drv),
@@ -61,7 +80,7 @@ basic(Config) when is_list(Config) ->
     ?line ExpectExt2Term = term(P, 5),
 
     %% ERL_DRV_INT, ERL_DRV_UINT
-    ?line case erlang:system_info(wordsize) of
+    ?line case erlang:system_info({wordsize, external}) of
 	      4 ->
 		  ?line {-1, 4294967295} = term(P, 6);
 	      8 ->
@@ -76,40 +95,43 @@ basic(Config) when is_list(Config) ->
     ?line ExpectedBinTup = term(P, 7),
 
     %% single terms
-    ?line [] = term(P, 8), % ERL_DRV_NIL
-    ?line '' = term(P, 9), % ERL_DRV_ATOM
-    ?line an_atom = term(P, 10), % ERL_DRV_ATOM
-    ?line -4711 = term(P, 11), % ERL_DRV_INT
-    ?line 4711 = term(P, 12), % ERL_DRV_UINT
-    ?line P = term(P, 13), % ERL_DRV_PORT
-    ?line <<>> = term(P, 14), % ERL_DRV_BINARY
-    ?line <<"hejsan">> = term(P, 15), % ERL_DRV_BINARY
-    ?line <<>> = term(P, 16), % ERL_DRV_BUF2BINARY
-    ?line <<>> = term(P, 17), % ERL_DRV_BUF2BINARY
-    ?line <<"hoppsan">> = term(P, 18), % ERL_DRV_BUF2BINARY
-    ?line "" = term(P, 19), % ERL_DRV_STRING
-    ?line "" = term(P, 20), % ERL_DRV_STRING
-    ?line "hippsan" = term(P, 21), % ERL_DRV_STRING
-    ?line {} = term(P, 22), % ERL_DRV_TUPLE
-    ?line [] = term(P, 23), % ERL_DRV_LIST
-    ?line Self = term(P, 24), % ERL_DRV_PID
-    ?line [] = term(P, 25), % ERL_DRV_STRING_CONS
-    ?line AFloat = term(P, 26), % ERL_DRV_FLOAT
+    Singles = [{[], 8}, % ERL_DRV_NIL
+               {'', 9}, % ERL_DRV_ATOM
+               {an_atom, 10}, % ERL_DRV_ATOM
+               {-4711, 11}, % ERL_DRV_INT
+               {4711, 12}, % ERL_DRV_UINT
+               {P, 13}, % ERL_DRV_PORT
+               {<<>>, 14}, % ERL_DRV_BINARY
+               {<<"hejsan">>, 15}, % ERL_DRV_BINARY
+               {<<>>, 16}, % ERL_DRV_BUF2BINARY
+               {<<>>, 17}, % ERL_DRV_BUF2BINARY
+               {<<"hoppsan">>, 18}, % ERL_DRV_BUF2BINARY
+               {"", 19}, % ERL_DRV_STRING
+               {"", 20}, % ERL_DRV_STRING
+               {"hippsan", 21}, % ERL_DRV_STRING
+               {{}, 22}, % ERL_DRV_TUPLE
+               {[], 23}, % ERL_DRV_LIST
+               {Self, 24}, % ERL_DRV_PID
+               {[], 25}, % ERL_DRV_STRING_CONS
+               {[], 27}, % ERL_DRV_EXT2TERM
+               {18446744073709551615, 28}, % ERL_DRV_UINT64
+               {20233590931456, 29}, % ERL_DRV_UINT64
+               {4711, 30}, % ERL_DRV_UINT64
+               {0, 31}, % ERL_DRV_UINT64
+               {9223372036854775807, 32}, % ERL_DRV_INT64
+               {20233590931456, 33}, % ERL_DRV_INT64
+               {4711, 34}, % ERL_DRV_INT64
+               {0, 35}, % ERL_DRV_INT64
+               {-1, 36}, % ERL_DRV_INT64
+               {-4711, 37}, % ERL_DRV_INT64
+               {-20233590931456, 38}, % ERL_DRV_INT64
+               {-9223372036854775808, 39}], % ERL_DRV_INT64
+    ?line {Terms, Ops} = lists:unzip(Singles),
+    ?line Terms = term(P,Ops),
+
+    AFloat = term(P, 26), % ERL_DRV_FLOAT
     ?line true = AFloat < 0.001,
     ?line true = AFloat > -0.001,
-    ?line [] = term(P, 27), % ERL_DRV_EXT2TERM
-    ?line 18446744073709551615 = term(P, 28), % ERL_DRV_UINT64
-    ?line 20233590931456 = term(P, 29), % ERL_DRV_UINT64
-    ?line 4711 = term(P, 30), % ERL_DRV_UINT64
-    ?line 0 = term(P, 31), % ERL_DRV_UINT64
-    ?line 9223372036854775807 = term(P, 32), % ERL_DRV_INT64
-    ?line 20233590931456 = term(P, 33), % ERL_DRV_INT64
-    ?line 4711 = term(P, 34), % ERL_DRV_INT64
-    ?line 0 = term(P, 35), % ERL_DRV_INT64
-    ?line -1 = term(P, 36), % ERL_DRV_INT64
-    ?line -4711 = term(P, 37), % ERL_DRV_INT64
-    ?line -20233590931456 = term(P, 38), % ERL_DRV_INT64
-    ?line -9223372036854775808 = term(P, 39), % ERL_DRV_INT64
 
     %% Failure cases.
     ?line [] = term(P, 127),
diff --git a/erts/emulator/test/send_term_SUITE_data/send_term_drv.c b/erts/emulator/test/send_term_SUITE_data/send_term_drv.c
index 6638de0560..165cce2e9d 100644
--- a/erts/emulator/test/send_term_SUITE_data/send_term_drv.c
+++ b/erts/emulator/test/send_term_SUITE_data/send_term_drv.c
@@ -17,6 +17,7 @@
  */
 
 #include "erl_driver.h"
+#include <stdio.h>
 #include <errno.h>
 #include <string.h>
 
@@ -65,12 +66,21 @@ static void fail_term(ErlDrvTermData* msg, int len, int line);
 
 static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 {
-    ErlDrvTermData msg[1024];
-
-    switch (*buf) {
+    char buf7[1024];
+    ErlDrvTermData spec[1024];
+    ErlDrvTermData* msg = spec;
+    ErlDrvBinary* bins[15];
+    int bin_ix = 0;
+    ErlDrvSInt64 s64[15];
+    int s64_ix = 0;
+    ErlDrvUInt64 u64[15];
+    int u64_ix = 0;
+    int i = 0;
+
+    for (i=0; i<count; i++) switch (buf[i]) {
     case 0:
 	msg[0] = ERL_DRV_NIL;
-	output_term(msg, 1);
+	msg += 1;
 	break;
 
     case 1: 			/* Most term types inside a tuple. */
@@ -102,7 +112,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    msg[22] = driver_connected(erlang_port);
 	    msg[23] = ERL_DRV_TUPLE;
 	    msg[24] = (ErlDrvTermData) 7;
-	    output_term(msg, 25);
+	    msg += 25;
 	}
 	break;
 
@@ -117,7 +127,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    msg[i] = ERL_DRV_NIL;
 	    msg[i+1] = ERL_DRV_LIST;
 	    msg[i+2] = (ErlDrvTermData) 201;
-	    output_term(msg, i+3);
+	    msg += i+3;
 	}
 	break;
 
@@ -126,7 +136,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    ErlDrvBinary* bin;
 	    int i;
 
-	    bin = driver_alloc_binary(256);
+	    bin = bins[bin_ix++] = driver_alloc_binary(256);
 	    for (i = 0; i < 256; i++) {
 		bin->orig_bytes[i] = i;
 	    }
@@ -140,8 +150,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    msg[7] = (ErlDrvTermData) 23;
 	    msg[8] = ERL_DRV_TUPLE;
 	    msg[9] = (ErlDrvTermData) 2;
-	    output_term(msg, 10);
-	    driver_free_binary(bin);
+	    msg += 10;
 	}
 	break;
 
@@ -152,11 +161,11 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	msg[3] = driver_caller(erlang_port);
 	msg[4] = ERL_DRV_TUPLE;
 	msg[5] = (ErlDrvTermData) 2;
-	output_term(msg, 6);
+	msg += 6;
 	break;
 
     case 5:
-	output_term(msg, make_ext_term_list(msg, 0));
+	msg += make_ext_term_list(msg, 0);
 	break;
 
     case 6:
@@ -166,94 +175,91 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	msg[3] = ~((ErlDrvTermData) 0);
 	msg[4] = ERL_DRV_TUPLE;
 	msg[5] = (ErlDrvTermData) 2;
-	output_term(msg, 6);
+	msg += 6;
 	break;
 
     case 7: {
 	int len = 0;
-	char buf[1024];
-	memset(buf, 17, sizeof(buf));
+	memset(buf7, 17, sizeof(buf7));
 	/* empty heap binary */
 	msg[len++] = ERL_DRV_BUF2BINARY;
 	msg[len++] = (ErlDrvTermData) NULL; /* NULL is ok if size == 0 */
 	msg[len++] = (ErlDrvTermData) 0;
 	/* empty heap binary again */
 	msg[len++] = ERL_DRV_BUF2BINARY;
-	msg[len++] = (ErlDrvTermData) &buf[0]; /* ptr is ok if size == 0 */
+	msg[len++] = (ErlDrvTermData) buf7; /* ptr is ok if size == 0 */
 	msg[len++] = (ErlDrvTermData) 0;
 	/* heap binary */
 	msg[len++] = ERL_DRV_BUF2BINARY;
-	msg[len++] = (ErlDrvTermData) &buf[0];
+	msg[len++] = (ErlDrvTermData) buf7;
 	msg[len++] = (ErlDrvTermData) 17;
 	/* off heap binary */
 	msg[len++] = ERL_DRV_BUF2BINARY;
-	msg[len++] = (ErlDrvTermData) &buf[0];
-	msg[len++] = (ErlDrvTermData) sizeof(buf);
+	msg[len++] = (ErlDrvTermData) buf7;
+	msg[len++] = (ErlDrvTermData) sizeof(buf7);
 
 	msg[len++] = ERL_DRV_TUPLE;
 	msg[len++] = (ErlDrvTermData) 4;
 
-	output_term(msg, len);
+	msg += len;
 	break;
     }
 
     case 8:
 	msg[0] = ERL_DRV_NIL;
-	output_term(msg, 1);
+	msg += 1;
 	break;
 
     case 9:
 	msg[0] = ERL_DRV_ATOM;
 	msg[1] = (ErlDrvTermData) driver_mk_atom("");
-	output_term(msg, 2);
+	msg += 2;
 	break;
 
     case 10:
 	msg[0] = ERL_DRV_ATOM;
 	msg[1] = (ErlDrvTermData) driver_mk_atom("an_atom");
-	output_term(msg, 2);
+	msg += 2;
 	break;
 
     case 11:
 	msg[0] = ERL_DRV_INT;
 	msg[1] = (ErlDrvTermData) -4711;
-	output_term(msg, 2);
+	msg += 2;
 	break;
 	  
     case 12:  
 	msg[0] = ERL_DRV_UINT;
 	msg[1] = (ErlDrvTermData) 4711;
-	output_term(msg, 2);
+	msg += 2;
 	  
 	break;
     case 13:  
 	msg[0] = ERL_DRV_PORT;
 	msg[1] = driver_mk_port(erlang_port);
-	output_term(msg, 2);
+	msg += 2;
 	break;
 
     case 14: {
-	ErlDrvBinary *dbin = driver_alloc_binary(0);
+	ErlDrvBinary *dbin = bins[bin_ix++] = driver_alloc_binary(0);
 	msg[0] = ERL_DRV_BINARY;
 	msg[1] = (ErlDrvTermData) dbin;
 	msg[2] = (ErlDrvTermData) 0;
 	msg[3] = (ErlDrvTermData) 0;
-	output_term(msg, 4);
-	driver_free_binary(dbin);
+	msg += 4;
 	break;
 	}
 
     case 15: {
-	char buf[] = "hejsan";
-	ErlDrvBinary *dbin = driver_alloc_binary(sizeof(buf)-1);
+	static const char buf[] = "hejsan";
+	ErlDrvBinary *dbin = bins[bin_ix++] = driver_alloc_binary(sizeof(buf)-1);
 	if (dbin)
 	    memcpy((void *) dbin->orig_bytes, (void *) buf, sizeof(buf)-1);
 	msg[0] = ERL_DRV_BINARY;
 	msg[1] = (ErlDrvTermData) dbin;
 	msg[2] = (ErlDrvTermData) (dbin ? sizeof(buf)-1 : 0);
 	msg[3] = (ErlDrvTermData) 0;
-	output_term(msg, 4);
-	driver_free_binary(dbin);
+	msg += 4;
 	break;
 	}
 
@@ -261,24 +267,24 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	msg[0] = ERL_DRV_BUF2BINARY;
 	msg[1] = (ErlDrvTermData) NULL;
 	msg[2] = (ErlDrvTermData) 0;
-	output_term(msg, 3);
+	msg += 3;
 	break;
 	
     case 17: {
-	char buf[] = "";
+	static const char buf[] = "";
 	msg[0] = ERL_DRV_BUF2BINARY;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) sizeof(buf)-1;
-	output_term(msg, 3);
+	msg += 3;
 	break;
 	}
 
     case 18: {
-	char buf[] = "hoppsan";
+	static const char buf[] = "hoppsan";
 	msg[0] = ERL_DRV_BUF2BINARY;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) sizeof(buf)-1;
-	output_term(msg, 3);
+	msg += 3;
 	break;
     }
 
@@ -286,44 +292,44 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	msg[0] = ERL_DRV_STRING;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) 0;
-	output_term(msg, 3);
+	msg += 3;
 	break;
 
     case 20: {
-	char buf[] = "";
+	static const char buf[] = "";
 	msg[0] = ERL_DRV_STRING;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) sizeof(buf)-1;
-	output_term(msg, 3);
+	msg += 3;
 	break;
     }
 	
     case 21: {
-	char buf[] = "hippsan";
+	static const char buf[] = "hippsan";
 	msg[0] = ERL_DRV_STRING;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) sizeof(buf)-1;
-	output_term(msg, 3);
+	msg += 3;
 	break;
 	}
 
     case 22:
 	msg[0] = ERL_DRV_TUPLE;
 	msg[1] = (ErlDrvTermData) 0;
-	output_term(msg, 2);
+	msg += 2;
 	break;
 
     case 23:
 	msg[0] = ERL_DRV_NIL;
 	msg[1] = ERL_DRV_LIST;
 	msg[2] = (ErlDrvTermData) 1;
-	output_term(msg, 3);
+	msg += 3;
 	break;
 	
     case 24:
 	msg[0] = ERL_DRV_PID;
 	msg[1] = driver_connected(erlang_port);
-	output_term(msg, 2);
+	msg += 2;
 	break;
 	
     case 25:
@@ -331,132 +337,131 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	msg[1] = ERL_DRV_STRING_CONS;
 	msg[2] = (ErlDrvTermData) "";
 	msg[3] = (ErlDrvTermData) 0;
-	output_term(msg, 4);
+	msg += 4;
 	break;
 
     case 26: {
-	double my_float = 0.0;
+	static double my_float = 0.0;
 	msg[0] = ERL_DRV_FLOAT;
 	msg[1] = (ErlDrvTermData) &my_float; 
-	output_term(msg, 2);
+	msg += 2;
 	break;
     }
 
     case 27: {
-	char buf[] = {131, 106}; /* [] */
+	static char buf[] = {131, 106}; /* [] */
 	msg[0] = ERL_DRV_EXT2TERM;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) sizeof(buf);
-	output_term(msg, 3);
+	msg += 3;
 	break;
     }
 
     case 28: {
-	ErlDrvUInt64 x = ~((ErlDrvUInt64) 0);
+	ErlDrvUInt64* x = &u64[u64_ix++];
+	*x = ~((ErlDrvUInt64) 0);
 	msg[0] = ERL_DRV_UINT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 29: {
-	ErlDrvUInt64 x = ((ErlDrvUInt64) 4711) << 32;
+	ErlDrvUInt64* x = &u64[u64_ix++];
+	*x = ((ErlDrvUInt64) 4711) << 32;
 	msg[0] = ERL_DRV_UINT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 30: {
-	ErlDrvUInt64 x = 4711;
+	ErlDrvUInt64* x = &u64[u64_ix++];
+	*x = 4711;
 	msg[0] = ERL_DRV_UINT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 31: {
-	ErlDrvUInt64 x = 0;
+	ErlDrvUInt64* x = &u64[u64_ix++];
+	*x = 0;
 	msg[0] = ERL_DRV_UINT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 32: {
-	ErlDrvSInt64 x = ((((ErlDrvUInt64) 0x7fffffff) << 32)
-			  | ((ErlDrvUInt64) 0xffffffff));
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = ((((ErlDrvUInt64) 0x7fffffff) << 32) | ((ErlDrvUInt64) 0xffffffff));
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 33: {
-	ErlDrvSInt64 x = (ErlDrvSInt64) (((ErlDrvUInt64) 4711) << 32);
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = (ErlDrvSInt64) (((ErlDrvUInt64) 4711) << 32);
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 34: {
-	ErlDrvSInt64 x = 4711;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = 4711;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 35: {
-	ErlDrvSInt64 x = 0;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = 0;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 36: {
-	ErlDrvSInt64 x = -1;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = -1;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 37: {
-	ErlDrvSInt64 x = -4711;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = -4711;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 38: {
-	ErlDrvSInt64 x = ((ErlDrvSInt64) ((ErlDrvUInt64) 4711) << 32)*-1;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = ((ErlDrvSInt64) ((ErlDrvUInt64) 4711) << 32)*-1;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 39: {
-	ErlDrvSInt64 x = ((ErlDrvSInt64) 1) << 63;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = ((ErlDrvSInt64) 1) << 63;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
@@ -464,7 +469,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
     case 127:			/* Error cases */
 	{
 	    long refc;
-	    ErlDrvBinary* bin = driver_alloc_binary(256);
+	    ErlDrvBinary* bin = bins[bin_ix++] = driver_alloc_binary(256);
 
 	    FAIL_TERM(msg, 0);
 
@@ -537,7 +542,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    refc = driver_binary_get_refc(bin);
 	    if (refc > 3) {
 		char sbuf[128];
-		sprintf(sbuf, "bad_refc:%d", refc);
+		sprintf(sbuf, "bad_refc:%ld", refc);
 		driver_failure_atom(erlang_port, sbuf);
 	    }
 	    driver_free_binary(bin);
@@ -644,6 +649,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    /* Signal end of test case */
 	    msg[0] = ERL_DRV_NIL;
 	    driver_output_term(erlang_port, msg, 1);
+	    return;
 	}
 	break;
 
@@ -651,6 +657,16 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	driver_failure_atom(erlang_port, "bad_request");
 	break;
     }
+    if (count > 1) {
+	*msg++ = ERL_DRV_NIL;
+	*msg++ = ERL_DRV_LIST;
+	*msg++ = count + 1;
+    }
+    output_term(spec, msg-spec);
+    if ((bin_ix|s64_ix|u64_ix) > 15) abort();
+    while (bin_ix) {
+	driver_free_binary(bins[--bin_ix]);
+    }
 }
 
 static void output_term(ErlDrvTermData* msg, int len)
diff --git a/erts/emulator/test/sensitive_SUITE.erl b/erts/emulator/test/sensitive_SUITE.erl
index 458275af81..634df367ca 100644
--- a/erts/emulator/test/sensitive_SUITE.erl
+++ b/erts/emulator/test/sensitive_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,11 @@
 
 -module(sensitive_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 stickiness/1,send_trace/1,recv_trace/1,proc_trace/1,call_trace/1,
 	 meta_trace/1,running_trace/1,gc_trace/1,seq_trace/1,
 	 t_process_info/1,t_process_display/1,save_calls/1]).
@@ -34,14 +36,33 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(5)),
     [{watchdog,Dog}|Config].
 
-fin_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
+end_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-all(suite) ->
-    [stickiness,send_trace,recv_trace,proc_trace,call_trace,
-     meta_trace,running_trace,gc_trace,seq_trace,
-     t_process_info,t_process_display,save_calls].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [stickiness, send_trace, recv_trace, proc_trace,
+     call_trace, meta_trace, running_trace, gc_trace,
+     seq_trace, t_process_info, t_process_display,
+     save_calls].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
     
 stickiness(Config) when is_list(Config) ->
     ?line {Tracer,Mref} = spawn_monitor(fun() ->
diff --git a/erts/emulator/test/signal_SUITE.erl b/erts/emulator/test/signal_SUITE.erl
index e9103ca3c1..736dfe5b56 100644
--- a/erts/emulator/test/signal_SUITE.erl
+++ b/erts/emulator/test/signal_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -30,8 +30,9 @@
 -define(DEFAULT_TIMEOUT_SECONDS, 120).
 
 %-define(line_trace, 1).
--include("test_server.hrl").
--export([all/1]).
+-include_lib("test_server/include/test_server.hrl").
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 % Test cases
 -export([xm_sig_order/1,
@@ -49,38 +50,48 @@
 	 pending_exit_group_leader/1,
 	 exit_before_pending_exit/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     ?line Dog = ?t:timetrap(?t:seconds(?DEFAULT_TIMEOUT_SECONDS)),
     available_internal_state(true),
     ?line [{testcase, Func},{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     ?line Dog = ?config(watchdog, Config),
     ?line ?t:timetrap_cancel(Dog).
 
+init_per_suite(Config) ->
+    Config.
+
 end_per_suite(_Config) ->
     available_internal_state(true),
-    erts_debug:set_internal_state(not_running_optimization, true),
+    catch erts_debug:set_internal_state(not_running_optimization, true),
     available_internal_state(false).
 
-all(suite) ->
-    [xm_sig_order,
-     pending_exit_unlink_process,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [xm_sig_order, pending_exit_unlink_process,
      pending_exit_unlink_dist_process,
-     pending_exit_unlink_port,
-     pending_exit_trap_exit,
-     pending_exit_receive,
-     pending_exit_trap_exit,
-     pending_exit_gc,
-     pending_exit_is_process_alive,
+     pending_exit_unlink_port, pending_exit_trap_exit,
+     pending_exit_receive, pending_exit_trap_exit,
+     pending_exit_gc, pending_exit_is_process_alive,
      pending_exit_process_display,
      pending_exit_process_info_1,
-     pending_exit_process_info_2,
-     pending_exit_group_leader,
+     pending_exit_process_info_2, pending_exit_group_leader,
      exit_before_pending_exit].
 
+groups() -> 
+    [].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 xm_sig_order(doc) -> ["Test that exit signals and messages are received "
 		      "in correct order"];
 xm_sig_order(suite) -> [];
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index 898908c40f..0392312a6f 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -21,13 +21,14 @@
 
 %% Tests the statistics/1 bif.
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 init_per_testcase/2,
-	 fin_per_testcase/2,
-	 wall_clock/1, wall_clock_zero_diff/1, wall_clock_update/1,
-	 runtime/1, runtime_zero_diff/1,
+	 end_per_testcase/2,
+	 wall_clock_zero_diff/1, wall_clock_update/1,
+	 runtime_zero_diff/1,
 	 runtime_update/1, runtime_diff/1,
-	 run_queue/1, run_queue_one/1,
+	 run_queue_one/1,
 	 reductions/1, reductions_big/1, garbage_collection/1, io/1,
 	 badarg/1]).
 
@@ -35,24 +36,47 @@
 
 -export([hog/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 init_per_testcase(_, Config) ->
     ?line Dog = test_server:timetrap(test_server:seconds(300)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_, Config) ->
+end_per_testcase(_, Config) ->
     Dog = ?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
 
-all(suite) -> [wall_clock, runtime, reductions, reductions_big, run_queue,
-	       garbage_collection, io, badarg].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [{group, wall_clock}, {group, runtime}, reductions,
+     reductions_big, {group, run_queue}, garbage_collection,
+     io, badarg].
+
+groups() -> 
+    [{wall_clock, [],
+      [wall_clock_zero_diff, wall_clock_update]},
+     {runtime, [],
+      [runtime_zero_diff, runtime_update, runtime_diff]},
+     {run_queue, [], [run_queue_one]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 
 %%% Testing statistics(wall_clock).
 
-wall_clock(suite) -> [wall_clock_zero_diff, wall_clock_update].
 
 
 wall_clock_zero_diff(doc) ->
@@ -99,7 +123,6 @@ wall_clock_update1(0) ->
 
 %%% Test statistics(runtime).
 
-runtime(suite) -> [runtime_zero_diff, runtime_update, runtime_diff].
 
 runtime_zero_diff(doc) ->
     "Tests that the difference between the times returned from two consectuitive "
@@ -225,7 +248,6 @@ reductions_big_loop() ->
 
 %%% Tests of statistics(run_queue).
 
-run_queue(suite) -> [run_queue_one].
 
 run_queue_one(doc) ->
     "Tests that statistics(run_queue) returns 1 if we start a "
diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl
index e782d2f293..9b782b35a2 100644
--- a/erts/emulator/test/system_info_SUITE.erl
+++ b/erts/emulator/test/system_info_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -30,23 +30,44 @@
 
 %-define(line_trace, 1).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %-compile(export_all).
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
--export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1]).
+-export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1, wordsize/1]).
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(2)).
 
-all(doc) -> [];
-all(suite) -> [process_count, system_version, misc_smoke_tests, heap_size].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [process_count, system_version, misc_smoke_tests,
+     heap_size, wordsize].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
@@ -132,6 +153,7 @@ misc_smoke_tests(Config) when is_list(Config) ->
     ?line true = is_binary(erlang:system_info(procs)),
     ?line true = is_binary(erlang:system_info(loaded)),
     ?line true = is_binary(erlang:system_info(dist)),
+    ?line ok = try erlang:system_info({cpu_topology,erts_get_cpu_topology_error_case}), fail catch error:badarg -> ok end,
     ?line ok.
     
 
@@ -145,3 +167,23 @@ heap_size(Config) when is_list(Config) ->
    ?line Hmin  = proplists:get_value(min_heap_size, GCinf),
    ok.
 
+wordsize(suite) ->
+    [];
+wordsize(doc) ->
+    ["Tests the various wordsize variants"];
+wordsize(Config) when is_list(Config) ->
+    ?line A = erlang:system_info(wordsize),
+    ?line true = is_integer(A),
+    ?line A = erlang:system_info({wordsize,internal}),
+    ?line B = erlang:system_info({wordsize,external}),
+    ?line true = A =< B,
+    case {B,A} of
+	{4,4} ->
+	    {comment, "True 32-bit emulator"};
+	{8,8} ->
+	    {comment, "True 64-bit emulator"};
+	{8,4} ->
+	    {comment, "Halfword 64-bit emulator"};
+	Other ->
+	    exit({unexpected_wordsizes,Other})
+    end.
diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl
index 7b0d6d19fe..32089e8872 100644
--- a/erts/emulator/test/system_profile_SUITE.erl
+++ b/erts/emulator/test/system_profile_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,35 +22,52 @@
 
 -module(system_profile_SUITE).
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	system_profile_on_and_off/1,
 	runnable_procs/1,
 	runnable_ports/1,
 	scheduler/1
         ]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
--export([profiler_process/1, ring_loop/1, port_echo_start/0, list_load/0, run_load/2]).
+-export([profiler_process/1, ring_loop/1, port_echo_start/0, 
+	 list_load/0, run_load/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(default_timeout, ?t:minutes(1)).
 
 init_per_testcase(_Case, Config) ->
     ?line Dog=?t:timetrap(?default_timeout),
     [{watchdog, Dog}|Config].
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
 
-all(suite) -> 
-    %% Test specification on test suite level
-    [system_profile_on_and_off,
-    runnable_procs,
-    runnable_ports,
-    scheduler].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [system_profile_on_and_off, runnable_procs,
+     runnable_ports, scheduler].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 %% No specification clause needed for an init function in a conf case!!!
 
diff --git a/erts/emulator/test/time_SUITE.erl b/erts/emulator/test/time_SUITE.erl
index 2ad1f0d201..bd48a0a7db 100644
--- a/erts/emulator/test/time_SUITE.erl
+++ b/erts/emulator/test/time_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -29,12 +29,15 @@
 %%	now/0
 %%
 
--export([all/1, univ_to_local/1, local_to_univ/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, univ_to_local/1, local_to_univ/1,
 	 bad_univ_to_local/1, bad_local_to_univ/1,
 	 consistency/1,
-	 now/1, now_unique/1, now_update/1, timestamp/1]).
+	 now_unique/1, now_update/1, timestamp/1]).
 
--include("test_server.hrl").
+-export([local_to_univ_utc/1]).
+
+-include_lib("test_server/include/test_server.hrl").
 
 -export([linear_time/1]).
 
@@ -52,8 +55,59 @@
 
 -define(dst_timezone, 2).
 
-all(suite) -> [univ_to_local, local_to_univ,
-	       bad_univ_to_local, bad_local_to_univ, consistency, now, timestamp].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [univ_to_local, local_to_univ, local_to_univ_utc,
+     bad_univ_to_local, bad_local_to_univ, consistency,
+     {group, now}, timestamp].
+
+groups() -> 
+    [{now, [], [now_unique, now_update]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+local_to_univ_utc(suite) ->
+    [];
+local_to_univ_utc(doc) ->
+    ["Test that DST = true on timezones without DST is ignored"];
+local_to_univ_utc(Config) when is_list(Config) ->
+    case os:type() of
+	{unix,_} ->
+	    %% TZ variable has a meaning
+	    ?line {ok, Node} =
+		test_server:start_node(local_univ_utc,peer,
+				       [{args, "-env TZ UTC"}]),
+	    ?line {{2008,8,1},{0,0,0}} =
+		rpc:call(Node,
+			 erlang,localtime_to_universaltime,
+			 [{{2008, 8, 1}, {0, 0, 0}},
+			  false]),
+	    ?line {{2008,8,1},{0,0,0}} =
+		rpc:call(Node,
+			 erlang,localtime_to_universaltime,
+			 [{{2008, 8, 1}, {0, 0, 0}},
+			  true]),
+	    ?line [{{2008,8,1},{0,0,0}}] =
+		rpc:call(Node,
+			 calendar,local_time_to_universal_time_dst,
+			 [{{2008, 8, 1}, {0, 0, 0}}]),
+	    ?line test_server:stop_node(Node),
+	    ok;
+	_ ->
+	    {skip,"Only valid on Unix"}
+    end.
 
 
 %% Tests conversion from univeral to local time.
@@ -248,7 +302,6 @@ repeating_timestamp_check(N) ->
 
 %% Test now/0.
 
-now(suite) -> [now_unique, now_update].
 
 %% Tests that successive calls to now/0 returns different values.
 %% Also returns a comment string with the median difference between
diff --git a/erts/emulator/test/timer_bif_SUITE.erl b/erts/emulator/test/timer_bif_SUITE.erl
index 9ac5afcc45..7ff7449ff5 100644
--- a/erts/emulator/test/timer_bif_SUITE.erl
+++ b/erts/emulator/test/timer_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1998-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,9 @@
 
 -module(timer_bif_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,end_per_suite/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2]).
 -export([start_timer_1/1, send_after_1/1, send_after_2/1, send_after_3/1,
 	 cancel_timer_1/1,
 	 start_timer_big/1, send_after_big/1,
@@ -27,7 +29,7 @@
 	 read_timer_trivial/1, read_timer/1,
 	 cleanup/1, evil_timers/1, registered_process/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(test_server:seconds(30)),
@@ -37,19 +39,35 @@ init_per_testcase(_Case, Config) ->
     end,
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
 
+init_per_suite(Config) ->
+    Config.
+
 end_per_suite(_Config) ->
     catch erts_debug:set_internal_state(available_internal_state, false).
 
-all(suite) ->
-    [start_timer_1, send_after_1, send_after_2, cancel_timer_1,
-     start_timer_e, send_after_e, cancel_timer_e,
-     start_timer_big, send_after_big, read_timer_trivial, read_timer,
-     cleanup, evil_timers, registered_process].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [start_timer_1, send_after_1, send_after_2,
+     cancel_timer_1, start_timer_e, send_after_e,
+     cancel_timer_e, start_timer_big, send_after_big,
+     read_timer_trivial, read_timer, cleanup, evil_timers,
+     registered_process].
+
+groups() -> 
+    [].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 start_timer_1(doc) -> ["Basic start_timer/3 functionality"];
 start_timer_1(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index e9713fcf0f..221b65309a 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -23,7 +23,8 @@
 %%% Tests the trace BIF.
 %%%
 
--export([all/1, receive_trace/1, self_send/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, receive_trace/1, self_send/1,
 	 timeout_trace/1, send_trace/1,
 	 procs_trace/1, dist_procs_trace/1,
 	 suspend/1, mutual_suspend/1, suspend_exit/1, suspender_exit/1,
@@ -35,22 +36,39 @@
 	 system_monitor_large_heap_1/1, system_monitor_large_heap_2/1,
 	 bad_flag/1, trace_delivered/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %%% Internal exports
 -export([process/1]).
 
-all(suite) ->
-    [cpu_timestamp, receive_trace, self_send, timeout_trace, send_trace,
-     procs_trace, dist_procs_trace,
-     suspend, mutual_suspend, suspend_exit, suspender_exit,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [cpu_timestamp, receive_trace, self_send, timeout_trace,
+     send_trace, procs_trace, dist_procs_trace, suspend,
+     mutual_suspend, suspend_exit, suspender_exit,
      suspend_system_limit, suspend_opts, suspend_waiting,
-     new_clear, existing_clear,
-     set_on_spawn, set_on_first_spawn,
-     system_monitor_args, more_system_monitor_args,
-     system_monitor_long_gc_1, system_monitor_long_gc_2, 
-     system_monitor_large_heap_1, system_monitor_large_heap_2,
-     bad_flag, trace_delivered].
+     new_clear, existing_clear, set_on_spawn,
+     set_on_first_spawn, system_monitor_args,
+     more_system_monitor_args, system_monitor_long_gc_1,
+     system_monitor_long_gc_2, system_monitor_large_heap_1,
+     system_monitor_large_heap_2, bad_flag, trace_delivered].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 
 %% No longer testing anything, just reporting whether cpu_timestamp
diff --git a/erts/emulator/test/trace_bif_SUITE.erl b/erts/emulator/test/trace_bif_SUITE.erl
index 3f91f8dc08..2c78aa394f 100644
--- a/erts/emulator/test/trace_bif_SUITE.erl
+++ b/erts/emulator/test/trace_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1998-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,24 +19,44 @@
 
 -module(trace_bif_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1]).
--export([trace_bif/1, trace_bif_timestamp/1, trace_on_and_off/1, trace_bif_local/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
+-export([trace_bif/1, trace_bif_timestamp/1, trace_on_and_off/1, 
+	 trace_bif_local/1,
 	 trace_bif_timestamp_local/1, trace_bif_return/1, not_run/1,
 	 trace_info_old_code/1]).
 
 -export([bif_process/0]).
 
-all(suite) ->
-    case test_server:is_native(?MODULE) of
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(trace_bif_SUITE) of
 	true -> [not_run];
 	false ->
 	    [trace_bif, trace_bif_timestamp, trace_on_and_off,
-	     trace_bif_local, trace_bif_timestamp_local, 
+	     trace_bif_local, trace_bif_timestamp_local,
 	     trace_bif_return, trace_info_old_code]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 not_run(Config) when is_list(Config) -> 
     {skipped,"Native code"}.
 
diff --git a/erts/emulator/test/trace_call_count_SUITE.erl b/erts/emulator/test/trace_call_count_SUITE.erl
index 07aa7c8d8d..2ac58493ff 100644
--- a/erts/emulator/test/trace_call_count_SUITE.erl
+++ b/erts/emulator/test/trace_call_count_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -42,7 +42,7 @@
 -define(config(A,B),config(A,B)).
 -export([config/2]).
 -else.
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -endif.
  
 -ifdef(debug).
@@ -62,7 +62,9 @@ config(priv_dir,_) ->
     ".".
 -else.
 %% When run in test server.
--export([all/1, init_per_testcase/2, fin_per_testcase/2, not_run/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2, not_run/1]).
 -export([basic/1, on_and_off/1, info/1, 
 	 pause_and_restart/1, combo/1]).
 	 
@@ -70,7 +72,7 @@ init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(test_server:seconds(30)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_count]),
     erlang:trace_pattern(on_load, false, [local,meta,call_count]),
     erlang:trace(all, false, [all]),
@@ -78,15 +80,31 @@ fin_per_testcase(_Case, Config) ->
     test_server:timetrap_cancel(Dog),
     ok.
 
-all(doc) ->
-    ["Test call count tracing of local function calls."];
-all(suite) ->
-    case test_server:is_native(?MODULE) of
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(trace_call_count_SUITE) of
 	true -> [not_run];
-	false -> [basic, on_and_off, info, 
-		  pause_and_restart, combo]
+	false ->
+	    [basic, on_and_off, info, pause_and_restart, combo]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 not_run(Config) when is_list(Config) -> 
     {skipped,"Native code"}.
 
diff --git a/erts/emulator/test/trace_call_time_SUITE.erl b/erts/emulator/test/trace_call_time_SUITE.erl
new file mode 100644
index 0000000000..5dfa87bbee
--- /dev/null
+++ b/erts/emulator/test/trace_call_time_SUITE.erl
@@ -0,0 +1,634 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%
+%%% Define to run outside of test server
+%%%
+%%% -define(STANDALONE,1).
+%%%
+%%%
+%%% Define for debug output
+%%%
+%%% -define(debug,1).
+
+-module(trace_call_time_SUITE).
+
+%% Exported end user tests
+
+-export([seq/3, seq_r/3]).
+-export([loaded/1, a_function/1, a_called_function/1, dec/1, nif_dec/1]).
+
+-define(US_ERROR, 10000).
+-define(R_ERROR, 0.8).
+-define(SINGLE_CALL_US_TIME, 10).
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% Result examination macros
+
+-define(CT(P,MFA),{trace,P,call,MFA}).
+-define(CTT(P, MFA),{trace_ts,P,call,MFA,{_,_,_}}).
+-define(RF(P,MFA,V),{trace,P,return_from,MFA,V}).
+-define(RFT(P,MFA,V),{trace_ts,P,return_from,MFA,V,{_,_,_}}).
+-define(RT(P,MFA),{trace,P,return_to,MFA}).
+-define(RTT(P,MFA),{trace_ts,P,return_to,MFA,{_,_,_}}).
+
+-ifdef(debug).
+-define(dbgformat(A,B),io:format(A,B)).
+-else.
+-define(dbgformat(A,B),noop).
+-endif.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+-include_lib("test_server/include/test_server.hrl").
+
+%% When run in test server.
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2, not_run/1]).
+-export([basic/1, on_and_off/1, info/1,
+	 pause_and_restart/1, scheduling/1, called_function/1, combo/1, 
+	 bif/1, nif/1]).
+
+init_per_testcase(_Case, Config) ->
+    ?line Dog=test_server:timetrap(test_server:seconds(400)),
+    erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time,call_count]),
+    erlang:trace_pattern(on_load, false, [local,meta,call_time,call_count]),
+    timer:now_diff(now(),now()),
+    [{watchdog, Dog}|Config].
+
+end_per_testcase(_Case, Config) ->
+    erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time,call_count]),
+    erlang:trace_pattern(on_load, false, [local,meta,call_time,call_count]),
+    erlang:trace(all, false, [all]),
+    Dog=?config(watchdog, Config),
+    test_server:timetrap_cancel(Dog),
+    ok.
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(trace_call_time_SUITE) of
+	true -> [not_run];
+	false ->
+	    [basic, on_and_off, info, pause_and_restart, scheduling,
+	     combo, bif, nif, called_function]
+    end.
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+not_run(Config) when is_list(Config) ->
+    {skipped,"Native code"}.
+
+basic(suite) ->
+    [];
+basic(doc) ->
+    ["Tests basic call count trace"];
+basic(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 1000,
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,  '_'}, true, [call_time]),
+    ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_time]),
+    ?line Pid = setup(),
+    ?line {L,  T1} = execute(Pid, fun() -> seq(1, M, fun(X) -> (X+1) end) end),
+    ?line ok = check_trace_info({?MODULE, seq,   3}, [{Pid, M, 0, 0}], T1),
+    ?line ok = check_trace_info({?MODULE, seq_r, 3}, [], none),
+
+    ?line {Lr, T2} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> (X+1) end) end),
+    ?line ok = check_trace_info({?MODULE, seq,   3}, [{Pid, M, 0, 0}], T1),
+    ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Pid, 1, 0, 0}], T2/M),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Pid, M, 0, 0}], T2),
+    ?line L = lists:reverse(Lr),
+
+    %%
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line Pid ! quit,
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+
+on_and_off(suite) ->
+    [];
+on_and_off(doc) ->
+    ["Tests turning trace parameters on and off"];
+on_and_off(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 100,
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, true, [call_time]),
+    ?line Pid = setup(),
+    ?line {L, T1} = execute(Pid, {?MODULE, seq, [1, M, fun(X) -> X+1 end]}),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T1),
+
+    ?line N = erlang:trace_pattern({?MODULE,'_','_'}, true, [call_time]),
+    ?line {L, T2} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T2),
+
+    ?line P = erlang:trace_pattern({'_','_','_'}, true, [call_time]),
+    ?line {L, T3} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T3),
+
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, false, [call_time]),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, false, none),
+    ?line {L, _T4} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, false, none),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, [], none),
+    ?line {Lr, T5} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Pid,M,0,0}], T5),
+
+    ?line N = erlang:trace_pattern({?MODULE,'_','_'}, false, [call_time]),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, false, none),
+    ?line {Lr, _T6} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, false, none),
+    ?line L = lists:reverse(Lr),
+    %%
+    ?line Pid ! quit,
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+info(suite) ->
+    [];
+info(doc) ->
+    ["Tests the trace_info BIF"];
+info(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,3}, true, [call_time]),
+    ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time),
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, pause, [call_time]),
+    ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time),
+    ?line {all,[_|_]=L} = erlang:trace_info({?MODULE,seq,3}, all),
+    ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, L),
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, restart, [call_time]),
+    ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time),
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, false, [call_time]),
+    ?line {call_time,false} = erlang:trace_info({?MODULE,seq,3}, call_time),
+    ?line {all,false} = erlang:trace_info({?MODULE,seq,3}, all),
+    %%
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+pause_and_restart(suite) ->
+    [];
+pause_and_restart(doc) ->
+    ["Tests pausing and restarting call time counters"];
+pause_and_restart(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 100,
+    ?line Pid = setup(),
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, true, [call_time]),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [], none),
+    ?line {L, T1} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T1),
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, pause, [call_time]),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T1),
+    ?line {L, T2} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T2),
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, restart, [call_time]),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [], none),
+    ?line {L, T3} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T3),
+    %%
+    ?line Pid ! quit,
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+scheduling(suite) ->
+    [];
+scheduling(doc) ->
+    ["Tests in/out scheduling of call time counters"];
+scheduling(Config) when is_list(Config) ->
+    ?line P  = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M  = 1000000,
+    ?line Np = erlang:system_info(schedulers_online),
+    ?line F  = 12,
+
+    %% setup load processes
+    %% (single, no internal calls)
+
+    ?line erlang:trace_pattern({?MODULE,loaded,1}, true, [call_time]),
+
+    ?line Pids     = [setup() || _ <- lists:seq(1, F*Np)],
+    ?line {_Ls,T1} = execute(Pids, {?MODULE,loaded,[M]}),
+    ?line [Pid ! quit || Pid <- Pids],
+
+    %% logic dictates that each process will get ~ 1/F of the schedulers time
+
+    ?line {call_time, CT} = erlang:trace_info({?MODULE,loaded,1}, call_time),
+
+    ?line lists:foreach(fun (Pid) ->
+	    ?line ok = case check_process_time(lists:keysearch(Pid, 1, CT), M, F, T1) of
+		schedule_time_error ->
+		    test_server:comment("Warning: Failed time ratio"),
+		    ok;
+		Other -> Other
+	    end
+	end, Pids),
+    ?line P  = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+combo(suite) ->
+    [];
+combo(doc) ->
+    ["Tests combining local call trace and meta trace with call time trace"];
+combo(Config) when is_list(Config) ->
+    ?line Self = self(),
+    ?line Nbc = 3,
+    ?line MetaMs = [{'_',[],[{return_trace}]}],
+    ?line Flags = lists:sort([call, return_to]),
+    ?line LocalTracer = spawn_link(fun () -> relay_n(5 + Nbc + 3, Self) end),
+    ?line MetaTracer = spawn_link(fun () -> relay_n(9 + Nbc + 3, Self) end),
+    ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, [], [local]),
+    ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_time]),
+    ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, MetaMs, [{meta,MetaTracer}]),
+    ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_count]),
+
+    % bifs
+    ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, [], [local]),
+    ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_time]),
+    ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, MetaMs, [{meta,MetaTracer}]),
+    %% not implemented
+    %?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_count]),
+
+    ?line 1 = erlang:trace(Self, true, [{tracer,LocalTracer} | Flags]),
+    %%
+    ?line {traced,local} =
+	erlang:trace_info({?MODULE,seq_r,3}, traced),
+    ?line {match_spec,[]} =
+	erlang:trace_info({?MODULE,seq_r,3}, match_spec),
+    ?line {meta,MetaTracer} =
+	erlang:trace_info({?MODULE,seq_r,3}, meta),
+    ?line {meta_match_spec,MetaMs} =
+	erlang:trace_info({?MODULE,seq_r,3}, meta_match_spec),
+    ?line ok = check_trace_info({?MODULE, seq_r, 3}, [], none),
+
+    %% check empty trace_info for ?MODULE:seq_r/3
+    ?line {all,[_|_]=TraceInfo}     = erlang:trace_info({?MODULE,seq_r,3}, all),
+    ?line {value,{traced,local}}    = lists:keysearch(traced, 1, TraceInfo),
+    ?line {value,{match_spec,[]}}   = lists:keysearch(match_spec, 1, TraceInfo),
+    ?line {value,{meta,MetaTracer}} = lists:keysearch(meta, 1, TraceInfo),
+    ?line {value,{meta_match_spec,MetaMs}} = lists:keysearch(meta_match_spec, 1, TraceInfo),
+    ?line {value,{call_count,0}} = lists:keysearch(call_count, 1, TraceInfo),
+    ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, TraceInfo),
+
+    %% check empty trace_info for erlang:term_to_binary/1
+    ?line {all, [_|_] = TraceInfoBif} = erlang:trace_info({erlang, term_to_binary, 1}, all),
+    ?line {value,{traced,local}}     = lists:keysearch(traced, 1, TraceInfoBif),
+    ?line {value,{match_spec,[]}}    = lists:keysearch(match_spec, 1, TraceInfoBif),
+    ?line {value,{meta, MetaTracer}}  = lists:keysearch(meta, 1, TraceInfoBif),
+    ?line {value,{meta_match_spec,MetaMs}} = lists:keysearch(meta_match_spec, 1, TraceInfoBif),
+    %% not implemented
+    ?line {value,{call_count,false}} = lists:keysearch(call_count, 1, TraceInfoBif),
+    %?line {value,{call_count,0}} = lists:keysearch(call_count, 1, TraceInfoBif),
+    ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, TraceInfoBif),
+
+    %%
+    ?line [3,2,1] = seq_r(1, 3, fun(X) -> X+1 end),
+    ?line T0 = now(),
+    ?line with_bif(Nbc),
+    ?line T1 = now(),
+    ?line TimeB = timer:now_diff(T1,T0),
+    %%
+
+    ?line List = collect(100),
+    ?line {MetaR, LocalR} =
+	lists:foldl(
+	  fun ({P,X}, {M,L}) when P == MetaTracer ->
+		  {[X|M],L};
+	      ({P,X}, {M,L}) when P == LocalTracer ->
+		  {M,[X|L]}
+	  end,
+	  {[],[]},
+	  List),
+    ?line Meta = lists:reverse(MetaR),
+    ?line Local = lists:reverse(LocalR),
+
+    ?line [?CTT(Self,{?MODULE,seq_r,[1,3,_]}),
+	   ?CTT(Self,{?MODULE,seq_r,[1,3,_,[]]}),
+	   ?CTT(Self,{?MODULE,seq_r,[2,3,_,[1]]}),
+	   ?CTT(Self,{?MODULE,seq_r,[3,3,_,[2,1]]}),
+	   ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]),
+	   ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]),
+	   ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]),
+	   ?RFT(Self,{?MODULE,seq_r,3},[3,2,1]),
+	   ?CTT(Self,{erlang,term_to_binary,[3]}), % bif
+	   ?RFT(Self,{erlang,term_to_binary,1},<<131,97,3>>),
+	   ?CTT(Self,{erlang,term_to_binary,[2]}),
+	   ?RFT(Self,{erlang,term_to_binary,1},<<131,97,2>>)
+	] = Meta,
+
+    ?line [?CT(Self,{?MODULE,seq_r,[1,3,_]}),
+	   ?CT(Self,{?MODULE,seq_r,[1,3,_,[]]}),
+	   ?CT(Self,{?MODULE,seq_r,[2,3,_,[1]]}),
+	   ?CT(Self,{?MODULE,seq_r,[3,3,_,[2,1]]}),
+	   ?RT(Self,{?MODULE,combo,1}),
+	   ?CT(Self,{erlang,term_to_binary,[3]}), % bif
+	   ?RT(Self,{?MODULE,with_bif,1}),
+	   ?CT(Self,{erlang,term_to_binary,[2]}),
+	   ?RT(Self,{?MODULE,with_bif,1})
+	] = Local,
+
+    ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Self,1,0,0}], 1),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Self,3,0,0}], 1),
+    ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Self,1,0,0}], 1),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Self,3,0,0}], 1),
+    ?line ok = check_trace_info({erlang, term_to_binary, 1}, [{self(), Nbc - 1, 0, 0}], TimeB),
+    %%
+    ?line erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time]),
+    ?line erlang:trace_pattern(on_load, false, [local,meta,call_time]),
+    ?line erlang:trace(all, false, [all]),
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+bif(suite) ->
+    [];
+bif(doc) ->
+    ["Tests tracing of bifs"];
+bif(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 1000000,
+    %%
+    ?line 2 = erlang:trace_pattern({erlang, binary_to_term, '_'}, true, [call_time]),
+    ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_time]),
+    ?line Pid = setup(),
+    ?line {L, T1} = execute(Pid, fun() -> with_bif(M) end),
+
+    ?line ok = check_trace_info({erlang, binary_to_term, 1}, [{Pid, M - 1, 0, 0}], T1/2),
+    ?line ok = check_trace_info({erlang, term_to_binary, 1}, [{Pid, M - 1, 0, 0}], T1/2),
+
+    % disable term2binary
+
+    ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, false, [call_time]),
+
+    ?line {L, T2} = execute(Pid, fun() -> with_bif(M) end),
+
+    ?line ok = check_trace_info({erlang, binary_to_term, 1}, [{Pid, M*2 - 2, 0, 0}], T1/2 + T2),
+    ?line ok = check_trace_info({erlang, term_to_binary, 1}, false, none),
+
+    %%
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line Pid ! quit,
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+nif(suite) ->
+    [];
+nif(doc) ->
+    ["Tests tracing of nifs"];
+nif(Config) when is_list(Config) ->
+    load_nif(Config),
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 1000000,
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE, nif_dec,  '_'}, true, [call_time]),
+    ?line 1 = erlang:trace_pattern({?MODULE, with_nif, '_'}, true, [call_time]),
+    ?line Pid = setup(),
+    ?line {_, T1} = execute(Pid, fun() -> with_nif(M) end),
+
+    % the nif is called M - 1 times, the last time the function with 'with_nif'
+    % returns ok and does not call the nif.
+    ?line ok = check_trace_info({?MODULE, nif_dec,  1}, [{Pid, M-1, 0, 0}], T1/5*4),
+    ?line ok = check_trace_info({?MODULE, with_nif, 1}, [{Pid, M, 0, 0}], T1/5),
+
+    %%
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line Pid ! quit,
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+called_function(suite) ->
+    [];
+called_function(doc) ->
+    ["Tests combining nested function calls and that the time accumulates to the right function"];
+called_function(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 2100,
+    ?line Pid = setup(),
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE,a_function,'_'}, true, [call_time]),
+    ?line {L, T1} = execute(Pid, {?MODULE, a_function, [M]}),
+    ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M, 0, 0}], T1),
+
+    ?line 1 = erlang:trace_pattern({?MODULE,a_called_function,'_'}, true, [call_time]),
+    ?line {L, T2} = execute(Pid, {?MODULE, a_function, [M]}),
+    ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M+M, 0, 0}], T1 + M*?SINGLE_CALL_US_TIME),
+    ?line ok = check_trace_info({?MODULE, a_called_function, 1}, [{Pid, M, 0, 0}], T2),
+
+
+    ?line 1 = erlang:trace_pattern({?MODULE,dec,'_'}, true, [call_time]),
+    ?line {L, T3} = execute(Pid, {?MODULE, a_function, [M]}),
+    ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M+M+M, 0, 0}], T1 + (M+M)*?SINGLE_CALL_US_TIME),
+    ?line ok = check_trace_info({?MODULE, a_called_function, 1}, [{Pid, M+M, 0, 0}], T2 + M*?SINGLE_CALL_US_TIME ),
+    ?line ok = check_trace_info({?MODULE, dec, 1}, [{Pid, M, 0, 0}], T3),
+
+    ?line Pid ! quit,
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ok.
+
+%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% The Tests
+%%%
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% Local helpers
+
+
+load_nif(Config) ->
+    ?line Path = ?config(data_dir, Config),
+    ?line ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0).
+
+
+%% Stack recursive seq
+seq(Stop, Stop, Succ) when is_function(Succ) ->
+    [Stop];
+seq(Start, Stop, Succ) when is_function(Succ) ->
+    [Start | seq(Succ(Start), Stop, Succ)].
+
+
+a_function(1) -> a_called_function(1);
+a_function(N) when N > 1 -> a_function(a_called_function(N)).
+
+a_called_function(N) -> dec(N).
+
+with_bif(1) -> ok;
+with_bif(N) ->
+    with_bif(erlang:binary_to_term(erlang:term_to_binary(N)) - 1).
+
+with_nif(0) -> error;
+with_nif(1) -> ok;
+with_nif(N) ->
+    with_nif(?MODULE:nif_dec(N)).
+
+
+nif_dec(_) -> 0.
+
+dec(N) ->
+    loaded(10000),
+    N - 1.
+
+loaded(N) when N > 1 -> loaded(N - 1);
+loaded(_) -> 5.
+
+
+%% Tail recursive seq, result list is reversed
+seq_r(Start, Stop, Succ) when is_function(Succ) ->
+    seq_r(Start, Stop, Succ, []).
+
+seq_r(Stop, Stop, _, R) ->
+    [Stop | R];
+seq_r(Start, Stop, Succ, R) ->
+    seq_r(Succ(Start), Stop, Succ, [Start | R]).
+
+% Check call time tracing data and print mismatches
+check_trace_info(Mfa, [{Pid, C,_,_}] = Expect, Time) ->
+    case erlang:trace_info(Mfa, call_time) of
+	% Time tests are somewhat problematic. We want to know if Time (EXPECTED_TIME) and S*1000000 + Us (ACTUAL_TIME)
+	% is the same.
+	% If the ratio EXPECTED_TIME/ACTUAL_TIME is ~ 1 or if EXPECTED_TIME - ACTUAL_TIME is near zero, the test is ok.
+	{call_time,[{Pid,C,S,Us}]} when S >= 0, Us >= 0,  abs(1 - Time/(S*1000000 + Us)) < ?R_ERROR; abs(Time - S*1000000 - Us) < ?US_ERROR ->
+	    ok;
+	{call_time,[{Pid,C,S,Us}]} ->
+	    Sum = S*1000000 + Us,
+	    io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~w s. ~w us. = ~w us. - ~w -> delta ~w (ratio ~.2f, should be 1.0)~n",
+		[Mfa, Expect, Time, S, Us, Sum, Time, Sum - Time, Time/Sum]),
+	    time_error;
+	Other ->
+	    io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~p~n", [ Mfa, Expect, Time, Other]),
+	    time_count_error
+    end;
+check_trace_info(Mfa, Expect, _) ->
+    case erlang:trace_info(Mfa, call_time) of
+	{call_time, Expect} ->
+	    ok;
+	Other ->
+	    io:format("Expected ~p -> {call_time, ~p}~n - got ~p~n", [Mfa, Expect, Other]),
+	    result_not_expected_error
+    end.
+
+
+%check process time
+check_process_time({value,{Pid, M, S, Us}}, M, F, Time) ->
+    ?line Sum = S*1000000 + Us,
+    if
+	abs(1 - (F/(Time/Sum))) < ?R_ERROR ->
+	    ok;
+	true ->
+	    io:format("- Pid ~p, Got ratio ~.2f, expected ratio ~w~n", [Pid, Time/Sum,F]),
+	    schedule_time_error
+    end;
+check_process_time(Other, M, _, _) ->
+    io:format(" - Got ~p, expected count ~w~n", [Other, M]),
+    error.
+
+
+
+%% Message relay process
+relay_n(0, _) ->
+    ok;
+relay_n(N, Dest) ->
+    receive Msg ->
+	    Dest ! {self(), Msg},
+	    relay_n(N-1, Dest)
+    end.
+
+
+
+%% Collect received messages
+collect(Time) ->
+    Ref = erlang:start_timer(Time, self(), done),
+    L = lists:reverse(collect([], Ref)),
+    ?dbgformat("Got: ~p~n",[L]),
+    L.
+
+collect(A, 0) ->
+    receive
+	Mess ->
+	    collect([Mess | A], 0)
+    after 0 ->
+	    A
+    end;
+collect(A, Ref) ->
+    receive
+	{timeout, Ref, done} ->
+	    collect(A, 0);
+	Mess ->
+	    collect([Mess | A], Ref)
+    end.
+
+setup() ->
+    Pid = spawn_link(fun() -> loop() end),
+    ?line 1 = erlang:trace(Pid, true, [call]),
+    Pid.
+
+execute(Pids, Mfa) when is_list(Pids) ->
+    T0 = now(),
+    [P  ! {self(), execute, Mfa} || P <- Pids],
+    As = [receive {P, answer, Answer} -> Answer end || P <- Pids],
+    T1 = now(),
+    {As, timer:now_diff(T1,T0)};
+execute(P, Mfa) ->
+    T0 = now(),
+    P  ! {self(), execute, Mfa},
+    A  = receive {P, answer, Answer} -> Answer end,
+    T1 = now(),
+    {A, timer:now_diff(T1,T0)}.
+
+
+
+loop() ->
+    receive
+	quit ->
+	    ok;
+	{Pid, execute, Fun } when is_function(Fun) ->
+	    Pid ! {self(), answer, erlang:apply(Fun, [])},
+	    loop();
+	{Pid, execute, {M, F, A}} ->
+	    Pid ! {self(), answer, erlang:apply(M, F, A)},
+	    loop()
+    end.
diff --git a/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src b/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src
new file mode 100644
index 0000000000..2b2a35bd2c
--- /dev/null
+++ b/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src
@@ -0,0 +1,6 @@
+
+NIF_LIBS = trace_nif@dll@
+
+all: $(NIF_LIBS)
+
+@SHLIB_RULES@
diff --git a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c
new file mode 100644
index 0000000000..33b346aab7
--- /dev/null
+++ b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c
@@ -0,0 +1,37 @@
+#include "erl_nif.h"
+
+
+static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    return 0;
+}
+
+static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    return 0;
+}
+
+static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+{
+    return 0;
+}
+
+static void unload(ErlNifEnv* env, void* priv_data)
+{
+}
+
+static ERL_NIF_TERM nif_dec_1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    int x = 0;
+    enif_get_uint(env, argv[0], &x);
+    return enif_make_int(env, x - 1);
+}
+
+
+
+static ErlNifFunc nif_funcs[] =
+{
+    {"nif_dec", 1, nif_dec_1}
+};
+
+ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,reload,upgrade,unload)
diff --git a/erts/emulator/test/trace_local_SUITE.erl b/erts/emulator/test/trace_local_SUITE.erl
index 24005774ba..091e960610 100644
--- a/erts/emulator/test/trace_local_SUITE.erl
+++ b/erts/emulator/test/trace_local_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2000-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -45,7 +45,7 @@
 -export([config/2]).
 -define(DEFAULT_RECEIVE_TIMEOUT, 1000).
 -else.
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -define(DEFAULT_RECEIVE_TIMEOUT, infinity).
 -endif.
  
@@ -68,7 +68,8 @@ config(priv_dir,_) ->
 
 %%% When run in test server %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
--export([all/1, basic/1, bit_syntax/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, basic/1, bit_syntax/1,
 	 return/1, on_and_off/1, stack_grow/1,info/1, delete/1,
 	 exception/1, exception_apply/1,
 	 exception_function/1, exception_apply_function/1,
@@ -79,34 +80,51 @@ config(priv_dir,_) ->
 	 exception_meta_nocatch/1, exception_meta_nocatch_apply/1,
 	 exception_meta_nocatch_function/1,
 	 exception_meta_nocatch_apply_function/1,
-	 init_per_testcase/2, fin_per_testcase/2]).
+	 init_per_testcase/2, end_per_testcase/2]).
 init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(test_server:minutes(2)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     shutdown(),
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
-all(doc) ->
-    ["Test tracing of local function calls and return traces."];
-all(suite) ->
-    case test_server:is_native(?MODULE) of
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(trace_local_SUITE) of
 	true -> [not_run];
-	false -> [basic, bit_syntax, return, on_and_off, stack_grow, info, delete,
-		  exception, exception_apply,
-		  exception_function, exception_apply_function,
-		  exception_nocatch, exception_nocatch_apply,
-		  exception_nocatch_function, 
-		  exception_nocatch_apply_function,
-		  exception_meta, exception_meta_apply,
-		  exception_meta_function, exception_meta_apply_function,
-		  exception_meta_nocatch, exception_meta_nocatch_apply,
-		  exception_meta_nocatch_function,
-		  exception_meta_nocatch_apply_function]
+	false ->
+	    [basic, bit_syntax, return, on_and_off, stack_grow,
+	     info, delete, exception, exception_apply,
+	     exception_function, exception_apply_function,
+	     exception_nocatch, exception_nocatch_apply,
+	     exception_nocatch_function,
+	     exception_nocatch_apply_function, exception_meta,
+	     exception_meta_apply, exception_meta_function,
+	     exception_meta_apply_function, exception_meta_nocatch,
+	     exception_meta_nocatch_apply,
+	     exception_meta_nocatch_function,
+	     exception_meta_nocatch_apply_function]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 not_run(Config) when is_list(Config) -> 
     {skipped,"Native code"}.
 
@@ -796,9 +814,6 @@ loop(D1,D2,D3,0) ->
 loop(D1,D2,D3,N) ->
     max(N,loop(D1,D2,D3,N-1)).
 
-max(A, B) when A > B -> A;
-max(_, B) -> B.
-
 exported_wrap(Val) ->
     exported(Val).
 
diff --git a/erts/emulator/test/trace_meta_SUITE.erl b/erts/emulator/test/trace_meta_SUITE.erl
index d84cb3cdf2..45987cc319 100644
--- a/erts/emulator/test/trace_meta_SUITE.erl
+++ b/erts/emulator/test/trace_meta_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -45,7 +45,7 @@
 -define(config(A,B),config(A,B)).
 -export([config/2]).
 -else.
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -endif.
  
 -ifdef(debug).
@@ -65,7 +65,9 @@ config(priv_dir,_) ->
     ".".
 -else.
 %% When run in test server.
--export([all/1, init_per_testcase/2, fin_per_testcase/2, not_run/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2, not_run/1]).
 -export([basic/1, return/1, on_and_off/1, stack_grow/1, 
 	 info/1, tracer/1, combo/1, nosilent/1]).
 	 
@@ -73,19 +75,36 @@ init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(test_server:minutes(5)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     shutdown(),
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
-all(doc) ->
-    ["Test meta tracing of local function calls and return trace."];
-all(suite) ->
-    case test_server:is_native(?MODULE) of
-	true -> [not_run];
-	false -> [basic, return, on_and_off, stack_grow, 
-		  info, tracer, combo, nosilent]
-    end.
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+case test_server:is_native(trace_meta_SUITE) of
+  true -> [not_run];
+  false ->
+      [basic, return, on_and_off, stack_grow, info, tracer,
+       combo, nosilent]
+end.
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+	Config.
+
+end_per_group(_GroupName, Config) ->
+	Config.
+
 
 not_run(Config) when is_list(Config) -> 
     {skipped,"Native code"}.
@@ -594,11 +613,6 @@ loop(D1,D2,D3,0) ->
 loop(D1,D2,D3,N) ->
     max(N,loop(D1,D2,D3,N-1)).
 
-max(A,B) when A > B ->
-    A;
-max(_A,B) ->
-    B.
-
 id(X) ->
     X.
 
diff --git a/erts/emulator/test/trace_nif_SUITE.erl b/erts/emulator/test/trace_nif_SUITE.erl
index 587cc08979..a7484a22fd 100644
--- a/erts/emulator/test/trace_nif_SUITE.erl
+++ b/erts/emulator/test/trace_nif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2009. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,10 @@
 
 -module(trace_nif_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 -export([trace_nif/1,
 	 trace_nif_timestamp/1,
 	 trace_nif_local/1,
@@ -32,19 +33,33 @@
 
 -export([nif_process/0, nif/0, nif/1]).
 
-all(suite) ->
-    case test_server:is_native(?MODULE) of
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(trace_nif_SUITE) of
 	true -> [not_run];
 	false ->
-	    [trace_nif,
- 	     trace_nif_timestamp,
-	     trace_nif_local,
-	     trace_nif_meta,
-	     trace_nif_timestamp_local,
- 	     trace_nif_return
-	    ]
+	    [trace_nif, trace_nif_timestamp, trace_nif_local,
+	     trace_nif_meta, trace_nif_timestamp_local,
+	     trace_nif_return]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 not_run(Config) when is_list(Config) -> 
     {skipped,"Native code"}.
 
diff --git a/erts/emulator/test/trace_port_SUITE.erl b/erts/emulator/test/trace_port_SUITE.erl
index 5febe177f9..0026da4979 100644
--- a/erts/emulator/test/trace_port_SUITE.erl
+++ b/erts/emulator/test/trace_port_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,7 +20,9 @@
 
 -module(trace_port_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 call_trace/1,
 	 return_trace/1,
 	 send/1,
@@ -34,29 +36,42 @@
 	 gc/1,
 	 default_tracer/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
-test_cases() ->
-    [call_trace,
-     return_trace,
-     send,
-     receive_trace,
-     process_events,
-     schedule,
-     fake_schedule,
+test_cases() -> 
+    [call_trace, return_trace, send, receive_trace,
+     process_events, schedule, fake_schedule,
      fake_schedule_after_register,
      fake_schedule_after_getting_linked,
-     fake_schedule_after_getting_unlinked,
-     gc,
+     fake_schedule_after_getting_unlinked, gc,
      default_tracer].
 
-all(suite) -> test_cases().
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    test_cases().
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:seconds(30)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
diff --git a/erts/emulator/test/tuple_SUITE.erl b/erts/emulator/test/tuple_SUITE.erl
index c4edb16d68..bfc3910742 100644
--- a/erts/emulator/test/tuple_SUITE.erl
+++ b/erts/emulator/test/tuple_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -17,11 +17,13 @@
 %% %CopyrightEnd%
 %%
 -module(tuple_SUITE).
--export([all/1, t_size/1, t_tuple_size/1, t_element/1, t_setelement/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 t_size/1, t_tuple_size/1, t_element/1, t_setelement/1,
 	 t_list_to_tuple/1, t_tuple_to_list/1,
 	 t_make_tuple_2/1, t_make_tuple_3/1, t_append_element/1,
 	 build_and_match/1, tuple_with_case/1, tuple_in_guard/1]).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %% Tests tuples and the BIFs:
 %%
@@ -33,13 +35,30 @@
 %% make_tuple/2
 %%
 
-all(suite) ->
-    [build_and_match, t_size, t_tuple_size,
-     t_list_to_tuple, t_tuple_to_list,
-     t_element, t_setelement, t_make_tuple_2,
-     t_make_tuple_3, t_append_element,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [build_and_match, t_size, t_tuple_size, t_list_to_tuple,
+     t_tuple_to_list, t_element, t_setelement,
+     t_make_tuple_2, t_make_tuple_3, t_append_element,
      tuple_with_case, tuple_in_guard].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 build_and_match(Config) when is_list(Config) ->
     ?line {} = id({}),
     ?line {1} = id({1}),
@@ -80,7 +99,7 @@ t_tuple_size(Config) when is_list(Config) ->
 
 ludicrous_tuple_size(T)
   when tuple_size(T) =:= 16#7777777777777777777777777777777777 -> ok;
-ludicrous_tuple_size(T) -> error.
+ludicrous_tuple_size(_) -> error.
 
 %% Tests element/2.
 
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index 67d2b288a2..4b3075a164 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -29,10 +29,12 @@
 
 %-define(line_trace, 1).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %-compile(export_all).
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, init_per_testcase/2, 
+	 end_per_testcase/2]).
 
 -export([schedulers_alive/1, node_container_refc_check/1,
 	 long_timers/1, pollset_size/1,
@@ -40,19 +42,33 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(5)).
 
-all(doc) -> [];
-all(suite) ->
-    [schedulers_alive,
-     node_container_refc_check,
-     long_timers,
-     pollset_size,
-     check_io_debug].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [schedulers_alive, node_container_refc_check,
+     long_timers, pollset_size, check_io_debug].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 4a859c3094..e7c57142c0 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1,20 +1,20 @@
 #!/usr/bin/env perl
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1998-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 1998-2010. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 use strict;
@@ -27,6 +27,7 @@ my $outdir = ".";		# Directory for output files.
 my $verbose = 0;
 my $hot = 1;
 my $num_file_opcodes = 0;
+my $wordsize = 32;
 
 # This is shift counts and mask for the packer.
 my $WHOLE_WORD = '';
@@ -36,12 +37,20 @@ my @pack_mask;
 
 $pack_instr[2] = ['6', 'i'];
 $pack_instr[3] = ['0', '0', 'i'];
+$pack_instr[4] = ['6', '6', '6', 'i']; # Only for 64 bit wordsize
 
 $pack_shift[2] = ['0', 'BEAM_LOOSE_SHIFT'];
 $pack_shift[3] = ['0', 'BEAM_TIGHT_SHIFT', '(2*BEAM_TIGHT_SHIFT)'];
+$pack_shift[4] = ['0', 'BEAM_LOOSE_SHIFT', # Only for 64 bit wordsize
+		  '(2*BEAM_LOOSE_SHIFT)',
+		  '(3*BEAM_LOOSE_SHIFT)'];
 
 $pack_mask[2]  = ['BEAM_LOOSE_MASK', $WHOLE_WORD];
 $pack_mask[3]  = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK'];
+$pack_mask[4]  = ['BEAM_LOOSE_MASK', # Only for 64 bit wordsize
+		  'BEAM_LOOSE_MASK',
+		  'BEAM_LOOSE_MASK',
+		  $WHOLE_WORD];
 
 # There are two types of instructions: generic and specific.
 # The generic instructions are those generated by the Beam compiler.
@@ -80,6 +89,8 @@ my %cold_code;
 my @unnumbered_generic;
 my %unnumbered;
 
+my %is_transformed;
+
 #
 # Code transformations.
 #
@@ -118,7 +129,8 @@ my %arg_size = ('r' => 0,	# x(0) - x register zero
 		't' => 1,	# untagged integer -- can be packed
 		'b' => 1,	# pointer to bif
 		'A' => 1,	# arity value
-		'P' => 1,	# byte offset into tuple
+		'P' => 1,	# byte offset into tuple or stack
+		'Q' => 1,	# like 'P', but packable
 		'h' => 1,	# character
 		'l' => 1,	# float reg
 		'q' => 1,	# literal term
@@ -157,6 +169,7 @@ my @tag_type;
     $type_bit{'U'} = $type_bit{'u'};
     $type_bit{'e'} = $type_bit{'u'};
     $type_bit{'P'} = $type_bit{'u'};
+    $type_bit{'Q'} = $type_bit{'u'};
 }
 
 #
@@ -169,6 +182,7 @@ while (@ARGV && $ARGV[0] =~ /^-(.*)/) {
     ($target = \&emulator_output), next if /^emulator/;
     ($target = \&compiler_output), next if /^compiler/;
     ($outdir = shift), next if /^outdir/;
+    ($wordsize = shift), next if /^wordsize/;
     ($verbose = 1), next if /^v/;
     die "$0: Bad option: -$_\n";
 }
@@ -474,8 +488,9 @@ sub emulator_output {
 		$gen_transform_offset{$key} : -1;
 	    my($spec_op) = $gen_to_spec{$key};
 	    my($num_specific) = $num_specific{$key};
-	    defined $spec_op or $tr != -1 or
+	    defined $spec_op or
 		$obsolete[$gen_opnum{$name,$arity}] or
+		$is_transformed{$name,$arity} or
 		error("instruction $key has no specific instruction");
 	    $spec_op = -1 unless defined $spec_op;
 	    &init_item($name, $arity, $spec_op, $num_specific, $tr, $min_window{$key});
@@ -498,8 +513,14 @@ sub emulator_output {
     print "#define NUM_SPECIFIC_OPS ", scalar(@op_to_name), "\n";
     print "\n";
     print "#ifdef ARCH_64\n";
+    print "#  define BEAM_WIDE_MASK 0xFFFFUL\n";
     print "#  define BEAM_LOOSE_MASK 0x1FFFUL\n";
+    print "#if HALFWORD_HEAP\n";
+    print "#  define BEAM_TIGHT_MASK 0x1FFCUL\n";
+    print "#else\n";
     print "#  define BEAM_TIGHT_MASK 0x1FF8UL\n";
+    print "#endif\n";
+    print "#  define BEAM_WIDE_SHIFT 32\n";
     print "#  define BEAM_LOOSE_SHIFT 16\n";
     print "#  define BEAM_TIGHT_SHIFT 16\n";
     print "#else\n";
@@ -792,6 +813,7 @@ sub basic_generator {
 		     'I' => 1,
 		     't' => 1,
 		     'P' => 1,
+		     'Q' => 1,
 		     );
 
     # Pick up the macro to use and its flags (if any).
@@ -813,7 +835,7 @@ sub basic_generator {
     #
 
     if ($flags =~ /-pack/ && $hot) {
-	($prefix, $pack_spec, @args) = &do_pack(@args);
+        ($prefix, $pack_spec, @args) = &do_pack(@args);
     }
 
     #
@@ -907,16 +929,27 @@ sub basic_generator {
     my($code);
     if (defined $macro{$name}) {
 	my($macro_code) = "$prefix$macro(" . join(', ', @f) . ");";
-	$var_decls .= "Uint tmp_packed1;"
+	$var_decls .= "BeamInstr tmp_packed1;"
 	    if $macro_code =~ /tmp_packed1/;
-	$var_decls .= "Uint tmp_packed2;"
+	$var_decls .= "BeamInstr tmp_packed2;"
 	    if $macro_code =~ /tmp_packed2/;
 	if ($flags =~ /-nonext/) {
-	    $code = "$macro_code\n";
+	    $code = join("\n",
+			 "{ $var_decls",
+			 $macro_code,
+			 "}");
+	} elsif ($flags =~ /-goto:(\S*)/) {
+	    my $goto = $1;
+	    $code = join("\n",
+			 "{ $var_decls",
+			 $macro_code,
+			 "I += $size + 1;",
+			 "goto $goto;",
+			 "}");
 	} else {
 	    $code = join("\n",
 			 "{ $var_decls",
-			 "Eterm* next;",
+			 "BeamInstr* next;",
 			 "PreFetch($size, next);",
 			 "$macro_code",
 			 "NextPF($size, next);",
@@ -931,18 +964,31 @@ sub basic_generator {
 
 sub do_pack {
     my(@args) = @_;
-    my($i);
     my($packable_args) = 0;
+    my @is_packable;		# Packability (boolean) for each argument.
+    my $wide_packing = 0;
 
     #
     # Count the number of packable arguments.  If we encounter any 's' or 'd'
     # arguments, packing is not possible.
     #
-    for ($i = 0; $i < @args; $i++) {
-	if ($args[$i] =~ /[xyt]/) {
+    my $packable_types = "xytQ";
+    foreach my $arg (@args) {
+	if ($arg =~ /^[$packable_types]/) {
 	    $packable_args++;
-	} elsif ($args[$i] =~ /[sd]/) {
+	    push @is_packable, 1;
+	} elsif ($arg =~ /^I/ and $wordsize == 64 and $packable_args < 2) {
+	    $wide_packing = 1;
+	    push @is_packable, 1;
+	    if (++$packable_args == 2) {
+		# We can only pack two arguments. Turn off packing
+		# for the rest of the arguments.
+		$packable_types = "\xFF";
+	    }
+	} elsif ($arg =~ /^[sd]/) {
 	    return ('', '', @args);
+	} else {
+	    push @is_packable, 0;
 	}
     }
 
@@ -958,10 +1004,27 @@ sub do_pack {
 				# beginning).
     my($up) = '';		# Pack commands (storing back while
 				# moving forward).
-    my($args_per_word) = $packable_args < 4 ? $packable_args : 2;
-    my(@shift) = @{$pack_shift[$args_per_word]};
-    my(@mask) = @{$pack_mask[$args_per_word]};
-    my(@pack_instr) = @{$pack_instr[$args_per_word]};
+    my $args_per_word;
+    if ($packable_args < 4 or $wordsize == 64) {
+	$args_per_word = $packable_args;
+    } else {
+	# 4 packable argument, 32 bit wordsize. Need 2 words.
+	$args_per_word = 2;
+    }
+
+    my @shift;
+    my @mask;
+    my @instr;
+
+    if ($wide_packing) {
+	@shift = ('0', 'BEAM_WIDE_SHIFT');
+	@mask = ('BEAM_WIDE_MASK', $WHOLE_WORD);
+	@instr = ('w', 'i');
+    } else {
+	@shift = @{$pack_shift[$args_per_word]};
+	@mask = @{$pack_mask[$args_per_word]};
+	@instr = @{$pack_instr[$args_per_word]};
+    }
 
     #
     # Now generate the packing instructions.  One complication is that
@@ -975,10 +1038,10 @@ sub do_pack {
     my($ap) = 0;		# Argument number within word.
     my($tmpnum) = 1;		# Number of temporary variable.
     my($expr) = '';
-    for ($i = 0; $i < @args; $i++) {
+    for (my $i = 0; $i < @args; $i++) {
 	my($reg) = $args[$i];
 	my($this_size) = $arg_size{$reg};
-	if ($reg =~ /[xyt]/) {
+	if ($is_packable[$i]) {
 	    $this_size = 0;
 	    $did_some_packing = 1;
 
@@ -989,7 +1052,7 @@ sub do_pack {
 		$this_size = 1;
 	    }
 
-	    $down = "$pack_instr[$ap]$down";
+	    $down = "$instr[$ap]$down";
 	    my($unpack) = &make_unpack($tmpnum, $shift[$ap], $mask[$ap]);
 	    $args[$i] = "pack:$this_size:$reg" . "b($unpack)";
 
@@ -1099,6 +1162,10 @@ sub compile_transform {
     if ($obsolete[$gen_opnum{$name,$arity}]) {
 	error("obsolete function must not be used in transformations");
     }
+
+    if ($src) {
+	$is_transformed{$name,$arity} = 1;
+    }
     
     [$name,$arity,@ops];
 }
@@ -1287,13 +1354,28 @@ sub tr_gen_from {
 	    my($var, $type, $type_val, $cond, $val) = @$op;
 
 	    if ($type ne '' && $type ne '*') {
-		my($types) = '';
-		my($type_mask) = 0;
-		foreach (split('', $type)) {
-		    $types .= "$_ ";
-		    $type_mask |= $type_bit{$_};
+		#
+		# The is_bif, is_not_bif, and is_func instructions have
+		# their own built-in type test and don't need to
+		# be guarded with a type test instruction.
+		#
+		unless ($cond eq 'is_bif' or
+			$cond eq 'is_not_bif' or
+			$cond eq 'is_func') {
+		    my($types) = '';
+		    my($type_mask) = 0;
+		    foreach (split('', $type)) {
+			$types .= "$_ ";
+			$type_mask |= $type_bit{$_};
+		    }
+		    if ($cond ne 'is_eq') {
+			push(@code, &make_op($types, 'is_type', $type_mask));
+		    } else {
+			$cond = '';
+			push(@code, &make_op($types, 'is_type_eq',
+					     $type_mask, $val));
+		    }
 		}
-		push(@code, &make_op($types, 'is_type', $type_mask));
 	    }
 
 	    if ($cond eq 'is_func') {
diff --git a/erts/emulator/utils/count b/erts/emulator/utils/count
new file mode 100755
index 0000000000..617f5c25e8
--- /dev/null
+++ b/erts/emulator/utils/count
@@ -0,0 +1,127 @@
+%% -*- erlang -*-
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1998-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-mode(compile).
+
+main(_) ->
+    DisDir = "./dis",
+    ok = filelib:ensure_dir(filename:join(DisDir, "dummy")),
+    io:format("Dissambling to ~s\n", [DisDir]),
+    ok = file:set_cwd(DisDir),
+    Path = code:get_path() -- ["."],
+    Beams0 = [filelib:wildcard(filename:join(Dir, "*.beam")) ||
+		 Dir <- Path],
+    Beams = lists:append(Beams0),
+    Mods0 = [list_to_atom(filename:rootname(filename:basename(F))) ||
+	       F <- Beams],
+    Mods = lists:usort(Mods0),
+    start_sem(),
+    Ps = [begin
+	      {_,Ref} = spawn_monitor(fun() -> count(M) end),
+	      Ref
+	  end || M <- Mods],
+    [put(list_to_atom(I), 0) || I <- erts_debug:instructions()],
+    Res = wait_for_all(Ps, 1),
+    OutFile = "count",
+    {ok,Out} = file:open(OutFile, [write]),
+    [io:format(Out, "~s ~p\n", [I,C]) || {I,C} <- Res],
+    ok = file:close(Out),
+    io:format("\nResult written to ~s\n",
+	      [filename:join(DisDir, OutFile)]),
+    ok.
+
+wait_for_all([], _) ->
+    lists:reverse(lists:keysort(2, get()));
+wait_for_all([_|_]=Ps, I) ->
+    receive
+	{'DOWN',Ref,process,_,Result} ->
+	    io:format("\r~p", [I]),
+	    [increment(Key, Count) || {Key,Count} <- Result],
+	    wait_for_all(Ps -- [Ref], I+1)
+    end.
+
+count(M) ->
+    down(),
+    erts_debug:df(M),
+    {ok,Fd} = file:open(atom_to_list(M) ++ ".dis", [read,raw]),
+    count_is(Fd),
+    ok = file:close(Fd),
+    exit(get()).
+
+count_is(Fd) ->
+    case file:read_line(Fd) of
+	{ok,Line} ->
+	    count_instr(Line),
+	    count_is(Fd);
+	eof ->
+	    ok
+    end.
+
+count_instr([$\s|T]) ->
+    count_instr_1(T, []);
+count_instr([_|T]) ->
+    count_instr(T);
+count_instr([]) ->
+    %% Empty line.
+    ok.
+
+count_instr_1([$\s|_], Acc) ->
+    Instr = list_to_atom(lists:reverse(Acc)),
+    increment(Instr, 1);
+count_instr_1([H|T], Acc) ->
+    count_instr_1(T, [H|Acc]).
+
+increment(Key, Inc) -> 
+    case get(Key) of
+	undefined ->
+	    put(Key, Inc);
+	Count ->
+	    put(Key, Count+Inc)
+    end.
+
+%%%
+%%% Counting sempahore to limit the number of processes that
+%%% can run concurrently.
+%%%
+
+down() ->
+    sem ! {down,self()},
+    receive
+	sem_taken -> ok
+    end.
+
+start_sem() ->	    
+    spawn(fun() ->
+		  register(sem, self()),
+		  process_flag(trap_exit, true),
+		  do_sem(erlang:system_info(schedulers)+1) end).
+
+do_sem(0) ->
+    receive
+	{'EXIT',_,_} ->
+	    do_sem(1)
+    end;
+do_sem(C) ->
+    receive
+	{down,Pid} ->
+	    link(Pid),
+	    Pid ! sem_taken,
+	    do_sem(C-1)
+    end.
diff --git a/erts/emulator/utils/loaded b/erts/emulator/utils/loaded
new file mode 100644
index 0000000000..d124a64a78
--- /dev/null
+++ b/erts/emulator/utils/loaded
@@ -0,0 +1,44 @@
+%% -*- erlang -*-
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%% Run like:
+%%   $ERL_TOP/bin/escript erts/emulator/utils/loaded
+
+-mode(compile).
+
+main(_) ->
+    LibDir = code:lib_dir(),
+    io:format("Library root is ~s\n", [LibDir]),
+    Wc = filename:join(LibDir, "*/ebin/*.beam"),
+    Beams = filelib:wildcard(Wc),
+    BeamFileSize = lists:sum([filelib:file_size(Beam) || Beam <- Beams]),
+    io:format("~w BEAM files containing ~w bytes\n",
+	      [length(Beams),BeamFileSize]),
+    Ms = [list_to_atom(filename:rootname(filename:basename(Beam))) ||
+	     Beam <- Beams],
+    [{module,_} = code:ensure_loaded(M) || M <- Ms],
+    <<"Current code: ",T/binary>> = erlang:system_info(loaded),
+    Digits = grab_digits(T),
+    io:format("~w modules comprising ~s words when loaded\n",
+	      [length(Ms),Digits]).
+
+grab_digits(<<H,T/binary>>) when $0 =< H, H =< $9 ->
+    [H|grab_digits(T)];
+grab_digits(<<$\n,_/binary>>) -> [].
diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables
index b5391234cf..918ef62094 100755
--- a/erts/emulator/utils/make_tables
+++ b/erts/emulator/utils/make_tables
@@ -1,20 +1,20 @@
 #!/usr/bin/env perl 
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1999-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 1999-2010. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 use strict;
@@ -184,7 +184,7 @@ for ($i = 0; $i < @bif; $i++) {
     my $arity = $bif[$i]->[2];
     my $args = join(', ', 'Process*', ('Eterm') x $arity);
     print "Eterm $bif[$i]->[3]($args);\n";
-    print "Eterm wrap_$bif[$i]->[3]($args, Uint *I);\n";
+    print "Eterm wrap_$bif[$i]->[3]($args, UWord *I);\n";
 }
 print "#endif\n";
 
@@ -225,15 +225,15 @@ for ($i = 0; $i < @bif; $i++) {
     for ($arg = 1; $arg <= $arity; $arg++) {
 	print ", Eterm arg$arg";
     }
-    print ", Uint *I)\n";
+    print ", UWord *I)\n";
     print "{\n";
     print "    return erts_bif_trace($i, p";
     for ($arg = 1; $arg <= 3; $arg++) {
 	if ($arg <= $arity) {
 	    print ", arg$arg";
-	} elsif ($arg == ($arity + 1)) { 
-	    # Place I in correct position
-	    print ", (Eterm) I";
+	#} elsif ($arg == ($arity + 1)) {
+	  #  # Place I in correct position
+	   # print ", (Eterm) I";
 	} else {
 	    print ", 0";
 	}
diff --git a/erts/emulator/zlib/zutil.h b/erts/emulator/zlib/zutil.h
index d560382691..a8872e1c88 100644
--- a/erts/emulator/zlib/zutil.h
+++ b/erts/emulator/zlib/zutil.h
@@ -142,6 +142,7 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 #ifdef WIN32
 #  ifndef __CYGWIN__  /* Cygwin is Unix, not Win32 */
 #    define OS_CODE  0x0b
+#    define F_OPEN(name, mode) _wfopen((WCHAR *)(name), (WCHAR *)(mode)) /* Unicode */
 #  endif
 #endif
 
diff --git a/erts/epmd/src/Makefile.in b/erts/epmd/src/Makefile.in
index 498756b468..2f5296a5e8 100644
--- a/erts/epmd/src/Makefile.in
+++ b/erts/epmd/src/Makefile.in
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1998-2009. All Rights Reserved.
+# Copyright Ericsson AB 1998-2010. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -49,15 +49,31 @@ include ../epmd.mk
 
 BINDIR  = $(ERL_TOP)/bin/$(TARGET)
 OBJDIR  = $(ERL_TOP)/erts/obj$(TYPEMARKER)/$(TARGET)
+ERTS_INCL = -I$(ERL_TOP)/erts/include \
+            -I$(ERL_TOP)/erts/include/$(TARGET) \
+            -I$(ERL_TOP)/erts/include/internal \
+            -I$(ERL_TOP)/erts/include/internal/$(TARGET)
+
+# On windows we always need reentrant libraries.
+ifeq ($(TARGET),win32)
+ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal_r$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
+else
+ifeq ($(findstring vxworks,$(TARGET)),vxworks)
+ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
+else
+ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ -lm
+endif
+endif
 
 CC      = @CC@
 WFLAGS  = @WFLAGS@
-CFLAGS  = @CFLAGS@ @DEFS@ $(TYPE_FLAGS) $(WFLAGS)
+CFLAGS  = @CFLAGS@ @DEFS@ $(TYPE_FLAGS) $(WFLAGS) $(ERTS_INCL)
 LD      = @LD@
-LIBS    = @LIBS@
+LIBS    = @LIBS@ $(ERTS_INTERNAL_LIBS)
 LDFLAGS = @LDFLAGS@
 
 
+
 # ----------------------------------------------------
 # Release directory specification
 # ----------------------------------------------------
@@ -90,7 +106,7 @@ EPMD_OBJS = $(OBJDIR)/epmd.o       \
 #---------------------------------
 
 
-all: $(BINDIR)/$(EPMD)
+all: erts_lib $(BINDIR)/$(EPMD)
 
 docs:
 
@@ -109,9 +125,12 @@ clean:
 $(BINDIR)/$(EPMD): $(EPMD_OBJS)
 	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(EPMD_OBJS) $(LIBS)
 
-$(OBJDIR)/%.o: %.c
+$(OBJDIR)/%.o: %.c epmd.h epmd_int.h
 	$(CC) $(CFLAGS) $(EPMD_FLAGS) -o $@ -c $<
 
+erts_lib:
+	cd $(ERL_TOP)/erts/lib_src && $(MAKE) $(TYPE)
+
 include $(ERL_TOP)/make/otp_release_targets.mk
 
 release_spec: all
diff --git a/erts/epmd/src/epmd.c b/erts/epmd/src/epmd.c
index 23ac421446..08576d923f 100644
--- a/erts/epmd/src/epmd.c
+++ b/erts/epmd/src/epmd.c
@@ -1,20 +1,20 @@
 /* -*- c-indent-level: 2; c-continued-statement-offset: 2 -*- */ 
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -23,11 +23,7 @@
 #endif
 #include "epmd.h"     /* Renamed from 'epmd_r4.h' */
 #include "epmd_int.h"
-
-#ifdef _OSE_
-#  include "ose.h"
-#  include "efs.h"
-#endif
+#include "erl_printf.h"
 
 #ifdef HAVE_STDLIB_H
 #  include <stdlib.h>
@@ -37,7 +33,9 @@
 
 static void usage(EpmdVars *);
 static void run_daemon(EpmdVars*);
+static char* get_addresses(void);
 static int get_port_no(void);
+static int check_relaxed(void);
 #ifdef __WIN32__
 static int has_console(void);
 #endif
@@ -82,86 +80,7 @@ static char *mystrdup(char *s)
     return r;
 }
 
-#ifdef _OSE_
-
-struct args_sig {
-  SIGSELECT sig_no;
-  int argc ;
-  char argv[20][20];
-};
-
-union SIGNAL {
-  SIGSELECT sig_no;
-  struct args_sig args;
-};
-
-/* Start function. It may be called from the start script as well as from
-   the OSE shell directly (using late start hooks). It spawns epmd as an
-   OSE process which calls the epmd main function. */
-int start_ose_epmd(int argc, char **argv) {
-  union SIGNAL *sig;
-  PROCESS epmd_;
-  OSENTRYPOINT ose_epmd;
-  int i;
-
-  if(hunt("epmd", 0, &epmd_, NULL)) {
-    fprintf(stderr, "Warning! EPMD already exists (%u).\n", epmd_);
-    return 0;
-  }
-  else {
-    /* copy start args to signal */
-    sig = alloc(sizeof(struct args_sig), 0);
-    sig->args.argc = argc;
-    for(i=0; i<argc; i++) {
-      strcpy((sig->args.argv)[i], argv[i]);
-    }    
-    /* start epmd and send signal */
-    epmd_ = create_process(OS_BG_PROC, /* processtype */
-			   "epmd",      /* name        */
-			   ose_epmd,    /* entrypoint  */
-			   16383,	/* stacksize   */
-			   20,	        /* priority    */
-			   0,	        /* timeslice   */
-			   0,	        /* block       */
-			   NULL,0,0);   /* not used    */
-    efs_clone(epmd_);
-    start(epmd_);
-    send(&sig, epmd_);	
-#ifdef DEBUG
-    printf("EPMD ID: %li\n", epmd_);
-#endif
-  }
-  return 0;
-}
-
-OS_PROCESS(ose_epmd) {
-  union SIGNAL *sig;
-  static const SIGSELECT rec_any_sig[] = { 0 };
-  int i, argc;
-  char **argv;
-
-  sig = receive((SIGSELECT*)rec_any_sig);
-
-  argc = sig->args.argc;
-  argv = (char **)malloc((argc+1)*sizeof(char *));
-  for(i=0; i<argc; i++) {
-    argv[i] = (char *)malloc(strlen((sig->args.argv)[i])+1);
-    strcpy(argv[i], (sig->args.argv)[i]);
-  }
-  argv[argc] = NULL;
-  free_buf(&sig);
-
-  epmd(argc, argv);
-
-  for(i=0; i<argc; i++) {
-    free(argv[i]);
-  }
-  free(argv);
-}
-
-#else  /* ifdef _OSE_ */
-
-/* VxWorks start function */
+#ifdef VXWORKS
 int start_epmd(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
 char *a0, *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8, *a9;     
 {
@@ -200,10 +119,7 @@ char *a0, *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8, *a9;
 		   argc,(int) argv,1,
 		   0,0,0,0,0,0,0);
 }
-    
-#endif    /* _OSE_ */
-    
-    
+#endif    /* WxWorks */
 
 
 int epmd(int argc, char **argv)
@@ -218,6 +134,7 @@ int main(int argc, char** argv)
 {
     EpmdVars g_empd_vars;
     EpmdVars *g = &g_empd_vars;
+    int i;
 #ifdef __WIN32__
     WORD wVersionRequested;
     WSADATA wsaData;
@@ -243,21 +160,25 @@ int main(int argc, char** argv)
     g->argv = NULL;
 #endif
 
-    g->port  = get_port_no();
-    g->debug = 0;
+    g->addresses      = get_addresses();
+    g->port           = get_port_no();
+    g->debug          = 0;
 
     g->silent         = 0; 
     g->is_daemon      = 0;
+    g->brutal_kill    = check_relaxed();
     g->packet_timeout = CLOSE_TIMEOUT; /* Default timeout */
     g->delay_accept   = 0;
     g->delay_write    = 0;
     g->progname       = argv[0];
-    g->listenfd       = -1;
     g->conn           = NULL;
     g->nodes.reg = g->nodes.unreg = g->nodes.unreg_tail = NULL;
     g->nodes.unreg_count = 0;
     g->active_conn    = 0;
 
+    for (i = 0; i < MAX_LISTEN_SOCKETS; i++)
+	g->listenfd[i] = -1;
+
     argc--;
     argv++;
     while (argc > 0) {
@@ -283,12 +204,20 @@ int main(int argc, char** argv)
 	} else if (strcmp(argv[0], "-daemon") == 0) {
 	    g->is_daemon = 1;
 	    argv++; argc--;
+	} else if (strcmp(argv[0], "-relaxed_command_check") == 0) {
+	    g->brutal_kill = 1;
+	    argv++; argc--;
 	} else if (strcmp(argv[0], "-kill") == 0) {
 	    if (argc == 1)
 		kill_epmd(g);
 	    else
 		usage(g);
 	    epmd_cleanup_exit(g,0);
+	} else if (strcmp(argv[0], "-address") == 0) {
+	    if (argc == 1)
+	      usage(g);
+	    g->addresses = argv[1];
+	    argv += 2; argc -= 2;
 	} else if (strcmp(argv[0], "-port") == 0) {
 	    if ((argc == 1) ||
 		((g->port = atoi(argv[1])) == 0))
@@ -313,11 +242,17 @@ int main(int argc, char** argv)
 	    else
 		usage(g);
 	    epmd_cleanup_exit(g,0);
+	} else if (strcmp(argv[0], "-stop") == 0) {
+	    if (argc == 2)
+		stop_cli(g, argv[1]);
+	    else
+		usage(g);
+	    epmd_cleanup_exit(g,0);
 	}
 	else
 	    usage(g);
     }
-    dbg_printf(g,0,"epmd running - daemon = %d",g->is_daemon);
+    dbg_printf(g,1,"epmd running - daemon = %d",g->is_daemon);
 
 #ifndef NO_SYSCONF
     if ((g->max_conn = sysconf(_SC_OPEN_MAX)) <= 0)
@@ -327,13 +262,10 @@ int main(int argc, char** argv)
     /*
      * max_conn must not be greater than FD_SETSIZE.
      * (at least QNX crashes)
-     *
-     * More correctly, it must be FD_SETSIZE - 1, beacuse the
-     * listen FD is stored outside the connection array.
      */
   
     if (g->max_conn > FD_SETSIZE) {
-      g->max_conn = FD_SETSIZE - 1;
+      g->max_conn = FD_SETSIZE;
     }
 
     if (g->is_daemon)  {
@@ -453,7 +385,7 @@ static void run_daemon(EpmdVars *g)
 }
 #endif
 
-#if (defined(VXWORKS) || defined(_OSE_))
+#if defined(VXWORKS)
 static void run_daemon(EpmdVars *g)
 {
     run(g);
@@ -468,10 +400,14 @@ static void run_daemon(EpmdVars *g)
 
 static void usage(EpmdVars *g)
 {
-    fprintf(stderr, "usage: epmd [-d|-debug] [DbgExtra...] [-port No] [-daemon]\n");
-    fprintf(stderr, "            [-d|-debug] [-port No] [-names|-kill]\n\n");
-    fprintf(stderr, "See the Erlang epmd manual page for info about the usage.\n");
-    fprintf(stderr, "The -port and DbgExtra options are\n\n");
+    fprintf(stderr, "usage: epmd [-d|-debug] [DbgExtra...] [-address List]\n");
+    fprintf(stderr, "            [-port No] [-daemon] [-relaxed_command_check]\n");
+    fprintf(stderr, "       epmd [-d|-debug] [-port No] [-names|-kill|-stop name]\n\n");
+    fprintf(stderr, "See the Erlang epmd manual page for info about the usage.\n\n");
+    fprintf(stderr, "Regular options\n");
+    fprintf(stderr, "    -address List\n");
+    fprintf(stderr, "        Let epmd listen only on the comma-separated list of IP\n");
+    fprintf(stderr, "        addresses (and on the loopback interface).\n");
     fprintf(stderr, "    -port No\n");
     fprintf(stderr, "        Let epmd listen to another port than default %d\n",
 	    EPMD_PORT_NO);
@@ -481,8 +417,16 @@ static void usage(EpmdVars *g)
     fprintf(stderr, "        the standard error stream. It will shorten\n");
     fprintf(stderr, "        the number of saved used node names to 5.\n\n");
     fprintf(stderr, "        If you give more than one debug flag you may\n");
-    fprintf(stderr, "        get more debugging information.\n\n");
-    fprintf(stderr, "    -packet_timout Seconds\n");
+    fprintf(stderr, "        get more debugging information.\n");
+    fprintf(stderr, "    -daemon\n");
+    fprintf(stderr, "        Start epmd detached (as a daemon)\n");
+    fprintf(stderr, "    -relaxed_command_check\n");
+    fprintf(stderr, "        Allow this instance of epmd to be killed with\n");
+    fprintf(stderr, "        epmd -kill even if there "
+	    "are registered nodes.\n");
+    fprintf(stderr, "        Also allows forced unregister (epmd -stop).\n");
+    fprintf(stderr, "\nDbgExtra options\n");
+    fprintf(stderr, "    -packet_timeout Seconds\n");
     fprintf(stderr, "        Set the number of seconds a connection can be\n");
     fprintf(stderr, "        inactive before epmd times out and closes the\n");
     fprintf(stderr, "        connection (default 60).\n\n");
@@ -494,6 +438,18 @@ static void usage(EpmdVars *g)
     fprintf(stderr, "    -delay_write Seconds\n");
     fprintf(stderr, "        Also a simulation of a busy server. Inserts\n");
     fprintf(stderr, "        a delay before a reply is sent.\n");
+    fprintf(stderr, "\nInteractive options\n");
+    fprintf(stderr, "    -names\n");
+    fprintf(stderr, "        List names registered with the currently "
+	    "running epmd\n");
+    fprintf(stderr, "    -kill\n");
+    fprintf(stderr, "        Kill the currently running epmd\n");
+    fprintf(stderr, "        (only allowed if -names show empty database or\n");
+    fprintf(stderr, "        -relaxed_command_check was given when epmd was started).\n");
+    fprintf(stderr, "    -stop Name\n");
+    fprintf(stderr, "        Forcibly unregisters a name with epmd\n");
+    fprintf(stderr, "        (only allowed if -relaxed_command_check was given when \n");
+    fprintf(stderr, "        epmd was started).\n");
     epmd_cleanup_exit(g,1);
 }
 
@@ -513,20 +469,20 @@ static void usage(EpmdVars *g)
  *      args...      Arguments to print out according to the format
  *      
  */
-
+#define DEBUG_BUFFER_SIZE 2048
 static void dbg_gen_printf(int onsyslog,int perr,int from_level,
 			   EpmdVars *g,const char *format, va_list args)
 {
   time_t now;
   char *timestr;
-  char buf[2048];
+  char buf[DEBUG_BUFFER_SIZE];
 
   if (g->is_daemon)
     {
 #ifndef NO_SYSLOG
       if (onsyslog)
 	{
-	  vsprintf(buf, format, args);
+	  erts_vsnprintf(buf, DEBUG_BUFFER_SIZE, format, args);
 	  syslog(LOG_ERR,"epmd: %s",buf);
 	}
 #endif
@@ -537,11 +493,12 @@ static void dbg_gen_printf(int onsyslog,int perr,int from_level,
 
       time(&now);
       timestr = (char *)ctime(&now);
-      sprintf(buf, "epmd: %.*s: ", (int) strlen(timestr)-1, timestr);
+      erts_snprintf(buf, DEBUG_BUFFER_SIZE, "epmd: %.*s: ",
+		    (int) strlen(timestr)-1, timestr);
       len = strlen(buf);
-      vsprintf(buf + len, format, args);
-      if (perr == 1)
-	perror(buf);
+      erts_vsnprintf(buf + len, DEBUG_BUFFER_SIZE - len, format, args);
+      if (perr != 0)
+	fprintf(stderr,"%s: %s\r\n",buf,strerror(perr));
       else
 	fprintf(stderr,"%s\r\n",buf);
     }
@@ -552,7 +509,7 @@ void dbg_perror(EpmdVars *g,const char *format,...)
 {
   va_list args;
   va_start(args, format);
-  dbg_gen_printf(1,1,0,g,format,args);
+  dbg_gen_printf(1,errno,0,g,format,args);
   va_end(args);
 }
 
@@ -608,8 +565,9 @@ void epmd_cleanup_exit(EpmdVars *g, int exitval)
 	      epmd_conn_close(g,&g->conn[i]);
       free(g->conn);
   }
-  if(g->listenfd >= 0)
-      close(g->listenfd);
+  for(i=0; i < MAX_LISTEN_SOCKETS; i++)
+      if(g->listenfd[i] >= 0)
+          close(g->listenfd[i]);
   free_all_nodes(g);
   if(g->argv){
       for(i=0; g->argv[i] != NULL; ++i)
@@ -621,9 +579,18 @@ void epmd_cleanup_exit(EpmdVars *g, int exitval)
   exit(exitval);
 }
 
+static char* get_addresses(void)
+{
+    return getenv("ERL_EPMD_ADDRESS");
+}
 static int get_port_no(void)
 {
     char* port_str = getenv("ERL_EPMD_PORT");
     return (port_str != NULL) ? atoi(port_str) : EPMD_PORT_NO;
 }
+static int check_relaxed(void)
+{
+    char* port_str = getenv("ERL_EPMD_RELAXED_COMMAND_CHECK");
+    return (port_str != NULL) ? 1 : 0;
+}
 
diff --git a/erts/epmd/src/epmd.h b/erts/epmd/src/epmd.h
index 9e939ee38e..5d6e9ac165 100644
--- a/erts/epmd/src/epmd.h
+++ b/erts/epmd/src/epmd.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -17,21 +17,18 @@
  * %CopyrightEnd%
  */
 
-/* The port number is now defined in a makefile */
+/* The port number is defined in a makefile */
 
 /* Definitions of message codes */
 
-#define EPMD_ALIVE_REQ 'a'
-#define EPMD_ALIVE_OK_RESP 'Y'
-#define EPMD_PORT_REQ 'p'
+/* Registration and queries */
+#define EPMD_ALIVE2_REQ 'x'
+#define EPMD_PORT2_REQ 'z'
+#define EPMD_ALIVE2_RESP 'y'
+#define EPMD_PORT2_RESP 'w'
 #define EPMD_NAMES_REQ 'n'
+
+/* Interactive client command codes */
 #define EPMD_DUMP_REQ 'd'
 #define EPMD_KILL_REQ 'k'
 #define EPMD_STOP_REQ 's'
-
-/* New epmd messages */
-
-#define EPMD_ALIVE2_REQ 'x' /* 120 */
-#define EPMD_PORT2_REQ 'z' /* 122 */
-#define EPMD_ALIVE2_RESP 'y' /* 121 */
-#define EPMD_PORT2_RESP 'w' /* 119 */
diff --git a/erts/epmd/src/epmd_cli.c b/erts/epmd/src/epmd_cli.c
index c12f711bc5..ac55ba6bb6 100644
--- a/erts/epmd/src/epmd_cli.c
+++ b/erts/epmd/src/epmd_cli.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -42,7 +42,47 @@ void kill_epmd(EpmdVars *g)
 	epmd_cleanup_exit(g,1);
     }
     if ((rval = read_fill(fd,buf,2)) == 2) {
-	printf("Killed\n");
+	if (buf[0] == 'O' && buf[1] == 'K') {
+	    printf("Killed\n");
+	} else {
+	    printf("Killing not allowed - living nodes in database.\n");
+	}
+	epmd_cleanup_exit(g,0);
+    } else if (rval < 0) {
+	printf("epmd: failed to read answer from local epmd\n");
+	epmd_cleanup_exit(g,1);
+    } else { 			/* rval is now 0 or 1 */
+	buf[rval] = '\0';
+	printf("epmd: local epmd responded with <%s>\n", buf);
+	epmd_cleanup_exit(g,1);
+    }
+}
+
+void stop_cli(EpmdVars *g, char *name)
+{
+    char buf[1024];
+    int fd, rval, bsize;
+
+    bsize = strlen(name);
+    if (bsize > 1000) {
+	printf("epmd: Name too long!");
+	epmd_cleanup_exit(g, 1);
+    }
+
+    fd = conn_to_epmd(g);
+    bsize++;
+    put_int16(bsize, buf);
+    buf[2] = EPMD_STOP_REQ;
+    bsize += 2;
+    strcpy(buf+3, name);
+
+    if (write(fd, buf, bsize) != bsize) {
+	printf("epmd: Can't write to epmd\n");
+	epmd_cleanup_exit(g,1);
+    }
+    if ((rval = read_fill(fd,buf,7)) == 7) {
+	buf[7] = '\000';
+	printf("%s\n", buf);
 	epmd_cleanup_exit(g,0);
     } else if (rval < 0) {
 	printf("epmd: failed to read answer from local epmd\n");
@@ -97,7 +137,7 @@ static int conn_to_epmd(EpmdVars *g)
 
     { /* store port number in unsigned short */
       unsigned short sport = g->port;
-      SET_ADDR_LOOPBACK(address, FAMILY, sport);
+      SET_ADDR(address, EPMD_ADDR_LOOPBACK, sport);
     }
 
     if (connect(connect_sock, (struct sockaddr*)&address, sizeof address) < 0) 
diff --git a/erts/epmd/src/epmd_int.h b/erts/epmd/src/epmd_int.h
index 65fcf9bacb..2a0de4df9c 100644
--- a/erts/epmd/src/epmd_int.h
+++ b/erts/epmd/src/epmd_int.h
@@ -1,3 +1,4 @@
+/* -*- c-indent-level: 2; c-continued-statement-offset: 2 -*- */
 /*
  * %CopyrightBegin%
  *
@@ -37,19 +38,6 @@
 #define DONT_USE_MAIN
 #endif
 
-#ifdef _OSE_
-#define NO_SYSLOG
-#define NO_SYSCONF
-#define NO_DAEMON
-#define DONT_USE_MAIN
-#ifndef HAVE_SYS_TIME_H
-#define HAVE_SYS_TIME_H
-#endif
-#ifndef HAVE_UNISTD_H
-#define HAVE_UNISTD_H
-#endif
-#endif
-
 /* ************************************************************************ */
 /* Standard includes                                                        */
 
@@ -92,7 +80,7 @@
 #endif
 #endif /* ! VXWORKS */
 
-#if (!defined(__WIN32__) && !defined(_OSE_))
+#if !defined(__WIN32__)
 #  include <netinet/in.h>
 #  include <sys/socket.h>
 #  include <sys/stat.h>
@@ -105,10 +93,8 @@
 #  include <netinet/tcp.h>
 #endif /* ! WIN32 */
 
-#ifndef _OSE_
 #include <ctype.h>
 #include <signal.h>
-#endif
 
 #include <errno.h>
 
@@ -126,13 +112,6 @@
 
 #include <stdarg.h>
 
-#ifdef _OSE_
-#  include "ose.h"
-#  include "inet.h"
-#  include "sys/stat.h"
-#endif
-
-
 /* ************************************************************************ */
 /* Replace some functions by others by making the function name a macro */
 
@@ -148,10 +127,6 @@
 #define sleep(n) taskDelay((n) * sysClkRateGet())
 #endif /* VXWORKS */
 
-#ifdef _OSE_
-#define sleep(n) delay((n))
-#endif
-
 #ifdef USE_BCOPY
 #  define memcpy(a, b, c) bcopy((b), (a), (c))
 #  define memcmp(a, b, c) bcmp((a), (b), (c))
@@ -167,6 +142,10 @@
 #  define EADDRINUSE WSAEADDRINUSE
 #endif
 
+#if defined(__WIN32__) && !defined(ECONNABORTED)
+#  define ECONNABORTED WSAECONNABORTED
+#endif
+
 #ifndef SOMAXCONN
 #  define SOMAXCONN 128
 #endif
@@ -189,50 +168,40 @@
 #if defined(HAVE_IN6) && defined(AF_INET6) && defined(EPMD6)
 
 #define EPMD_SOCKADDR_IN sockaddr_in6
-#define FAMILY      AF_INET6
-
-#define SET_ADDR_LOOPBACK(addr, af, port) do { \
-    static u_int32_t __addr[4] = IN6ADDR_LOOPBACK_INIT; \
-    memset((char*)&(addr), 0, sizeof(addr)); \
-    (addr).sin6_family = (af); \
-    (addr).sin6_flowinfo = 0; \
-    (addr).sin6_addr.s6_addr32[0] = __addr[0]; \
-    (addr).sin6_addr.s6_addr32[1] = __addr[1]; \
-    (addr).sin6_addr.s6_addr32[2] = __addr[2]; \
-    (addr).sin6_addr.s6_addr32[3] = __addr[3]; \
-    (addr).sin6_port = htons(port); \
+#define EPMD_IN_ADDR in6_addr
+#define EPMD_S_ADDR s6_addr
+#define EPMD_ADDR_LOOPBACK in6addr_loopback.s6_addr
+#define EPMD_ADDR_ANY in6addr_any.s6_addr
+#define FAMILY AF_INET6
+
+#define SET_ADDR(dst, addr, port) do { \
+    memset((char*)&(dst), 0, sizeof(dst)); \
+    memcpy((char*)&(dst).sin6_addr.s6_addr, (char*)&(addr), 16); \
+    (dst).sin6_family = AF_INET6; \
+    (dst).sin6_flowinfo = 0; \
+    (dst).sin6_port = htons(port); \
  } while(0)
 
-#define SET_ADDR_ANY(addr, af, port) do { \
-    static u_int32_t __addr[4] = IN6ADDR_ANY_INIT; \
-    memset((char*)&(addr), 0, sizeof(addr)); \
-    (addr).sin6_family = (af); \
-    (addr).sin6_flowinfo = 0; \
-    (addr).sin6_addr.s6_addr32[0] = __addr[0]; \
-    (addr).sin6_addr.s6_addr32[1] = __addr[1]; \
-    (addr).sin6_addr.s6_addr32[2] = __addr[2]; \
-    (addr).sin6_addr.s6_addr32[3] = __addr[3]; \
-    (addr).sin6_port = htons(port); \
- } while(0)
+#define IS_ADDR_LOOPBACK(addr) \
+    (memcmp((addr).s6_addr, in6addr_loopback.s6_addr, 16) == 0)
 
 #else /* Not IP v6 */
 
 #define EPMD_SOCKADDR_IN sockaddr_in
-#define FAMILY      AF_INET
-
-#define SET_ADDR_LOOPBACK(addr, af, port) do { \
-    memset((char*)&(addr), 0, sizeof(addr)); \
-    (addr).sin_family = (af); \
-    (addr).sin_addr.s_addr = htonl(INADDR_LOOPBACK); \
-    (addr).sin_port = htons(port); \
+#define EPMD_IN_ADDR in_addr
+#define EPMD_S_ADDR s_addr
+#define EPMD_ADDR_LOOPBACK htonl(INADDR_LOOPBACK)
+#define EPMD_ADDR_ANY htonl(INADDR_ANY)
+#define FAMILY AF_INET
+
+#define SET_ADDR(dst, addr, port) do { \
+    memset((char*)&(dst), 0, sizeof(dst)); \
+    (dst).sin_family = AF_INET; \
+    (dst).sin_addr.s_addr = (addr); \
+    (dst).sin_port = htons(port); \
  } while(0)
 
-#define SET_ADDR_ANY(addr, af, port) do { \
-    memset((char*)&(addr), 0, sizeof(addr)); \
-    (addr).sin_family = (af); \
-    (addr).sin_addr.s_addr = htonl(INADDR_ANY); \
-    (addr).sin_port = htons(port); \
- } while(0)
+#define IS_ADDR_LOOPBACK(addr) ((addr).s_addr == htonl(INADDR_LOOPBACK))
 
 #endif /* Not IP v6 */
 
@@ -260,6 +229,8 @@
 /* Maximum length of a node name == atom name */
 #define MAXSYMLEN 255
 
+#define MAX_LISTEN_SOCKETS 16
+
 #define INBUF_SIZE 1024
 #define OUTBUF_SIZE 1024
 
@@ -275,8 +246,10 @@
 
 typedef struct {
   int fd;			/* File descriptor */
-  unsigned open:1;		/* TRUE if open */
-  unsigned keep:1;		/* Don't close when sent reply */
+  unsigned char open;		/* TRUE if open */
+  unsigned char keep;		/* Don't close when sent reply */
+  unsigned char local_peer;     /* The peer of this connection is via
+                                   loopback interface */
   unsigned got;			/* # of bytes we have got */
   unsigned want;		/* Number of bytes we want */
   char *buf;			/* The remaining buffer */
@@ -316,6 +289,7 @@ typedef struct {
   int debug;
   int silent; 
   int is_daemon;
+  int brutal_kill;
   unsigned packet_timeout;
   unsigned delay_accept;
   unsigned delay_write;
@@ -325,7 +299,8 @@ typedef struct {
   Connection *conn;
   Nodes nodes;
   fd_set orig_read_mask;
-  int listenfd;
+  int listenfd[MAX_LISTEN_SOCKETS];
+  char *addresses;
   char **argv;
 } EpmdVars;
 
@@ -337,6 +312,7 @@ void epmd_call(EpmdVars*,int);
 void run(EpmdVars*);
 void epmd_cleanup_exit(EpmdVars*, int);
 int epmd_conn_close(EpmdVars*,Connection*);
+void stop_cli(EpmdVars *g, char *name);
 
 #ifdef DONT_USE_MAIN
 int  start_epmd(char *,char *,char *,char *,char *,char *,char *,char *,char *,char *);
diff --git a/erts/epmd/src/epmd_srv.c b/erts/epmd/src/epmd_srv.c
index a033fab244..4d9b454f97 100644
--- a/erts/epmd/src/epmd_srv.c
+++ b/erts/epmd/src/epmd_srv.c
@@ -2,7 +2,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,6 +24,10 @@
 #include "epmd.h"     /* Renamed from 'epmd_r4.h' */
 #include "epmd_int.h"
 
+#ifndef INADDR_NONE
+#  define INADDR_NONE 0xffffffff
+#endif
+
 /*
  *  
  *  This server is a local name server for Erlang nodes. Erlang nodes can
@@ -39,46 +43,16 @@
  *  server keeps the socket open where the request for registration was
  *  made.
  *
- *  The protocol is briefly documented in "erl_ext_dist.txt". All requests
- *  to this server are done with a packet
+ *  The protocol is briefly documented in the ERTS User's Guide, see
+ *  http://www.erlang.org/doc/apps/erts/erl_dist_protocol.html
+ *
+ *  All requests to this server are done with a packet
  *
  *      2        n
  *  +--------+---------+
  *  | Length | Request |
  *  +--------+---------+
  *
- *  In all but one case there is only one request for each connection made
- *  to this server so we can safely close the socket after sending the
- *  reply.  The exception is ALIVE_REQ where we keep the connection
- *  open without sending any data. When we receive a "close" this is
- *  an indication that the Erlang node was terminated. The termination
- *  may have been "normal" or caused by a crash. The operating system
- *  ensure that the connection is closed either way.
- *  
- *  Reading is done non-blocking, i.e. we call a "read" only if we are
- *  told by the "select" function that there are data to read.
- *  
- *  Two databases are used: One node database where the registered names
- *  of the nodes are stored, and one connection database where the state
- *  of sockets and the data buffers is stored.
- *  
- *  Incomplete packets are thrown away after a timout. The Erlang node
- *  doing the request is responsible for completing in it in a reasonable time.
- *
- *  Note that if the server gets busy it may not have time to
- *  process all requests for connection. The "accept()" function
- *  will on most operating systems silently refuse to accept more
- *  than 5 outstanding requests. It is the client's responsibility
- *  to retry the request a number of times with random time interval.
- *  The "-debug" flag will insert a delay so you can test this 
- *  behaviour.
- *
- *  FIXME: In this code we assume that the packets we send on each
- *  socket is so small that a "write()" never block
- *
- *  FIXME: We never restarts a read or write that was terminated
- *  by an interrupt. Do we need to?
- *
  */
 
 /* We use separate data structures for node names and connections
@@ -98,7 +72,6 @@ static int conn_open(EpmdVars*,int);
 static int conn_close_fd(EpmdVars*,int);
 
 static void node_init(EpmdVars*);
-static Node *node_reg(EpmdVars*,char*,int,int);
 static Node *node_reg2(EpmdVars*,char*, int, int, unsigned char, unsigned char, int, int, int, char*);
 static int node_unreg(EpmdVars*,char*);
 static int node_unreg_sock(EpmdVars*,int);
@@ -110,106 +83,157 @@ static void print_names(EpmdVars*);
 
 void run(EpmdVars *g)
 {
-  int listensock;
+  struct EPMD_SOCKADDR_IN iserv_addr[MAX_LISTEN_SOCKETS];
+  int listensock[MAX_LISTEN_SOCKETS];
+  int num_sockets;
   int i;
   int opt;
-  struct EPMD_SOCKADDR_IN iserv_addr;
+  unsigned short sport = g->port;
 
   node_init(g);
   g->conn = conn_init(g);
 
   dbg_printf(g,2,"try to initiate listening port %d", g->port);
-  
-  if ((listensock = socket(FAMILY,SOCK_STREAM,0)) < 0) {
-    dbg_perror(g,"error opening stream socket");
-    epmd_cleanup_exit(g,1);
-  }
-  g->listenfd = listensock;
+
+  if (g->addresses != NULL)
+    {
+      char *tmp;
+      char *token;
+      int loopback_ok = 0;
+
+      if ((tmp = (char *)malloc(strlen(g->addresses) + 1)) == NULL)
+	{
+	  dbg_perror(g,"cannot allocate memory");
+	  epmd_cleanup_exit(g,1);
+	}
+      strcpy(tmp,g->addresses);
+
+      for(token = strtok(tmp,", "), num_sockets = 0;
+	  token != NULL;
+	  token = strtok(NULL,", "), num_sockets++)
+	{
+	  struct EPMD_IN_ADDR addr;
+#ifdef HAVE_INET_PTON
+	  int ret;
+
+	  if ((ret = inet_pton(FAMILY,token,&addr)) == -1)
+	    {
+	      dbg_perror(g,"cannot convert IP address to network format");
+	      epmd_cleanup_exit(g,1);
+	    }
+	  else if (ret == 0)
+#elif !defined(EPMD6)
+	  if ((addr.EPMD_S_ADDR = inet_addr(token)) == INADDR_NONE)
+#endif
+	    {
+	      dbg_tty_printf(g,0,"cannot parse IP address \"%s\"",token);
+	      epmd_cleanup_exit(g,1);
+	    }
+
+	  if (IS_ADDR_LOOPBACK(addr))
+	    loopback_ok = 1;
+
+	  if (num_sockets - loopback_ok == MAX_LISTEN_SOCKETS - 1)
+	    {
+	      dbg_tty_printf(g,0,"cannot listen on more than %d IP addresses",
+			     MAX_LISTEN_SOCKETS);
+	      epmd_cleanup_exit(g,1);
+	    }
+
+	  SET_ADDR(iserv_addr[num_sockets],addr.EPMD_S_ADDR,sport);
+	}
+
+      free(tmp);
+
+      if (!loopback_ok)
+	{
+	  SET_ADDR(iserv_addr[num_sockets],EPMD_ADDR_LOOPBACK,sport);
+	  num_sockets++;
+	}
+    }
+  else
+    {
+      SET_ADDR(iserv_addr[0],EPMD_ADDR_ANY,sport);
+      num_sockets = 1;
+    }
+
+#if !defined(__WIN32__)
+  /* We ignore the SIGPIPE signal that is raised when we call write
+     twice on a socket closed by the other end. */
+  signal(SIGPIPE, SIG_IGN);
+#endif
 
   /*
    * Initialize number of active file descriptors.
    * Stdin, stdout, and stderr are still open.
-   * One for the listen socket.
    */
-  g->active_conn = 3+1;
+  g->active_conn = 3 + num_sockets;
+  g->max_conn -= num_sockets;
+
+  FD_ZERO(&g->orig_read_mask);
+
+  for (i = 0; i < num_sockets; i++)
+    {
+      if ((listensock[i] = socket(FAMILY,SOCK_STREAM,0)) < 0)
+	{
+	  dbg_perror(g,"error opening stream socket");
+	  epmd_cleanup_exit(g,1);
+	}
+      g->listenfd[i] = listensock[i];
   
-  /*
-   * Note that we must not enable the SO_REUSEADDR on Windows,
-   * because addresses will be reused even if they are still in use.
-   */
+      /*
+       * Note that we must not enable the SO_REUSEADDR on Windows,
+       * because addresses will be reused even if they are still in use.
+       */
   
-#if (!defined(__WIN32__) && !defined(_OSE_))
-  /* We ignore the SIGPIPE signal that is raised when we call write
-     twice on a socket closed by the other end. */
-  signal(SIGPIPE, SIG_IGN);
-
-  opt = 1;			/* Set this option */
-  if (setsockopt(listensock,SOL_SOCKET,SO_REUSEADDR,(char* ) &opt,
-		 sizeof(opt)) <0) {
-    dbg_perror(g,"can't set sockopt");
-    epmd_cleanup_exit(g,1);
-  }
+#if !defined(__WIN32__)
+      opt = 1;
+      if (setsockopt(listensock[i],SOL_SOCKET,SO_REUSEADDR,(char* ) &opt,
+		     sizeof(opt)) <0)
+	{
+	  dbg_perror(g,"can't set sockopt");
+	  epmd_cleanup_exit(g,1);
+	}
 #endif
   
-  /* In rare cases select returns because there is someone
-     to accept but the request is withdrawn before the
-     accept function is called. We set the listen socket
-     to be non blocking to prevent us from being hanging
-     in accept() waiting for the next request. */
-#ifdef _OSE_  
-  opt = 1;
-  if (ioctl(listensock, FIONBIO, (char*)&opt) != 0)
-#else
+      /* In rare cases select returns because there is someone
+	 to accept but the request is withdrawn before the
+	 accept function is called. We set the listen socket
+	 to be non blocking to prevent us from being hanging
+	 in accept() waiting for the next request. */
 #if (defined(__WIN32__) || defined(NO_FCNTL))
-  opt = 1;
-  if (ioctl(listensock, FIONBIO, &opt) != 0) /* Gives warning in VxWorks */
+      opt = 1;
+      /* Gives warning in VxWorks */
+      if (ioctl(listensock[i], FIONBIO, &opt) != 0)
 #else
-  opt = fcntl(listensock, F_GETFL, 0);
-  if (fcntl(listensock, F_SETFL, opt | O_NONBLOCK) == -1)
+      opt = fcntl(listensock[i], F_GETFL, 0);
+      if (fcntl(listensock[i], F_SETFL, opt | O_NONBLOCK) == -1)
 #endif /* __WIN32__ || VXWORKS */
-#endif /* _OSE_ */
-    dbg_perror(g,"failed to set non-blocking mode of listening socket %d",
-	       listensock);
-
-  { /* store port number in unsigned short */
-    unsigned short sport = g->port;
-    SET_ADDR_ANY(iserv_addr, FAMILY, sport);
-  }
-  
-#ifdef _OSE_
-  {
-    int optlen = sizeof(opt);
-    opt = 1;
-    if(getsockopt(listensock, SOL_SOCKET, SO_REUSEADDR,
-		  (void*)&opt, &optlen) < 0)
-      fprintf(stderr, "\n\nGETSOCKOPT FAILS! %d\n\n", errno);
-    else if(opt == 1)
-      fprintf(stderr, "SO_REUSEADDR is set!\n");
-  }
-#endif
+	dbg_perror(g,"failed to set non-blocking mode of listening socket %d",
+		   listensock[i]);
 
-  if(bind(listensock,(struct sockaddr*) &iserv_addr, sizeof(iserv_addr)) < 0 )
-    {
-      if (errno == EADDRINUSE)
-	{
-	  dbg_tty_printf(g,1,"there is already a epmd running at port %d",
-			 g->port);
-	  epmd_cleanup_exit(g,0);
-	}
-      else
+      if (bind(listensock[i], (struct sockaddr*) &iserv_addr[i],
+	  sizeof(iserv_addr[i])) < 0)
 	{
-	  dbg_perror(g,"failed to bind socket");
-	  epmd_cleanup_exit(g,1);
+	  if (errno == EADDRINUSE)
+	    {
+	      dbg_tty_printf(g,1,"there is already a epmd running at port %d",
+			     g->port);
+	      epmd_cleanup_exit(g,0);
+	    }
+	  else
+	    {
+	      dbg_perror(g,"failed to bind socket");
+	      epmd_cleanup_exit(g,1);
+	    }
 	}
-    }
-
-  dbg_printf(g,2,"starting");
-
-  listen(listensock, SOMAXCONN);
 
-
-  FD_ZERO(&g->orig_read_mask);
-  FD_SET(listensock,&g->orig_read_mask);
+      if(listen(listensock[i], SOMAXCONN) < 0) {
+          dbg_perror(g,"failed to listen on socket");
+          epmd_cleanup_exit(g,1);
+      }
+      FD_SET(listensock[i],&g->orig_read_mask);
+    }
 
   dbg_tty_printf(g,2,"entering the main select() loop");
 
@@ -227,8 +251,16 @@ void run(EpmdVars *g)
       timeout.tv_sec = (g->packet_timeout < IDLE_TIMEOUT) ? 1 : IDLE_TIMEOUT;
       timeout.tv_usec = 0;
 
-      if ((ret = select(g->max_conn,&read_mask,(fd_set *)0,(fd_set *)0,&timeout)) < 0)
+      if ((ret = select(g->max_conn,&read_mask,(fd_set *)0,(fd_set *)0,&timeout)) < 0) {
 	dbg_perror(g,"error in select ");
+        switch (errno) {
+          case EAGAIN:
+          case EINTR:
+            break;
+          default:
+            epmd_cleanup_exit(g,1);
+        }
+      }
       else {
 	time_t now;
 	if (ret == 0) {
@@ -238,17 +270,18 @@ void run(EpmdVars *g)
 	  sleep(g->delay_accept);
 	}
 
-	if (FD_ISSET(listensock,&read_mask)) {
-	  if (do_accept(g, listensock) && g->active_conn < g->max_conn) {
-	    /*
-	     * The accept() succeeded, and we have at least one file
-	     * descriptor still free, which means that another accept()
-	     * could succeed. Go do do another select(), in case there
-	     * are more incoming connections waiting to be accepted.
-	     */
-	    goto select_again;
+	for (i = 0; i < num_sockets; i++)
+	  if (FD_ISSET(listensock[i],&read_mask)) {
+	    if (do_accept(g, listensock[i]) && g->active_conn < g->max_conn) {
+	      /*
+	       * The accept() succeeded, and we have at least one file
+	       * descriptor still free, which means that another accept()
+	       * could succeed. Go do do another select(), in case there
+	       * are more incoming connections waiting to be accepted.
+	       */
+	      goto select_again;
+	    }
 	  }
-	}
 	  
 	/* Check all open streams marked by select for data or a
 	   close.  We also close all open sockets except ALIVE
@@ -312,11 +345,9 @@ static void do_read(EpmdVars *g,Connection *s)
 		 s->fd,val);
 	  dbg_print_buf(g,s->buf,val);
 
-	  /* FIXME: Shouldn't be needed to close down.... */
 	  node_unreg_sock(g,s->fd);
 	  epmd_conn_close(g,s);
 	}
-      /* FIXME: We always close, probably the right thing to do */
       return;
     }
 
@@ -401,12 +432,21 @@ static int do_accept(EpmdVars *g,int listensock)
 
     if (msgsock < 0) {
         dbg_perror(g,"error in accept");
-	return EPMD_FALSE;
+        switch (errno) {
+            case EAGAIN:
+            case ECONNABORTED:
+            case EINTR:
+	            return EPMD_FALSE;
+            default:
+	            epmd_cleanup_exit(g,1);
+        }
     }
 
     return conn_open(g,msgsock);
 }
 
+/* buf is actually one byte larger than bsize,
+   giving place for null termination */
 static void do_request(g, fd, s, buf, bsize)
      EpmdVars *g;
      int fd;
@@ -417,117 +457,23 @@ static void do_request(g, fd, s, buf, bsize)
   char wbuf[OUTBUF_SIZE];	/* Buffer for writing */
   int i;
 
-  /*
-   * Terminate packet as a C string.  Needed for requests received from Erlang
-   * nodes with lower version than R3A.
-   */
-
-  buf[bsize] = '\0';
+  buf[bsize] = '\0'; /* Needed for strcmp in PORT2 and STOP requests
+			buf is always large enough */
 
   switch (*buf)
     {
-    case EPMD_ALIVE_REQ:
-      dbg_printf(g,1,"** got ALIVE_REQ");
-
-      /* The packet has the format "axxyyyyyy" where xx is port, given
-	 in network byte order, and yyyyyy is symname, possibly null
-	 terminated. */
-
-      if (buf[bsize - 1] == '\000') /* Skip null termination */
-	bsize--;
-	
-      if (bsize <= 3)
-	{
-	  dbg_printf(g,0,"packet to small for request ALIVE_REQ (%d)", bsize);
-	  return;
-	}
-
-      for (i = 3; i < bsize; i++)
-	if (buf[i] == '\000')
-	  {
-	    dbg_printf(g,0,"node name contains ascii 0 in ALIVE_REQ");
-	    return;
-	  }
-
-      {
-	Node *node;
-	int eport;
-	char *name  = &buf[3];	/* points to node name */
-
-	eport = get_int16(&buf[1]);
-
-	if ((node = node_reg(g, name, fd, eport)) == NULL)
-	  return;
-
-	wbuf[0] = EPMD_ALIVE_OK_RESP;
-	put_int16(node->creation, wbuf+1);
-  
-	if (g->delay_write)		/* Test of busy server */
-	  sleep(g->delay_write);
-
-	if (reply(g, fd, wbuf, 3) != 3)
-	  {
-	    dbg_tty_printf(g,1,"failed to send ALIVE_OK_RESP for \"%s\"",name);
-	    return;
-	  }
-
-	dbg_tty_printf(g,1,"** sent ALIVE_OK_RESP for \"%s\"",name);
-	s->keep = EPMD_TRUE;		/* Don't close on inactivity */
-      }
-      break;
-
-    case EPMD_PORT_REQ:
-      dbg_printf(g,1,"** got PORT_REQ");
-
-      if (buf[bsize - 1] == '\000') /* Skip null termination */
-	bsize--;
-	
-      if (bsize <= 1)
-	{
-	  dbg_printf(g,0,"packet to small for request PORT_REQ (%d)", bsize);
-	  return;
-	}
-
-      for (i = 1; i < bsize; i++)
-	if (buf[i] == '\000')
-	  {
-	    dbg_printf(g,0,"node name contains ascii 0 in PORT_REQ");
-	    return;
-	  }
-
-      {
-	char *name = &buf[1]; /* Points to node name */
-	Node *node;
-
-	for (node = g->nodes.reg; node; node = node->next)
-	  {
-	    if (strcmp(node->symname, name) == 0)
-	      {
-		put_int16(node->port,wbuf);
-		if (reply(g, fd, wbuf, 2) != 2)
-		  {
-		    dbg_tty_printf(g,1,"failed to send PORT_RESP for %s: %d",
-				   name,node->port);
-		    return;
-		  }
-		dbg_tty_printf(g,1,"** sent PORT_RESP for %s: %d",
-			       name,node->port);
-		return;
-	      }
-	  }
-	dbg_tty_printf(g,1,"Closed on PORT_REQ for %s",name);
-      }
-      /* FIXME: How about an answer if no port? Is a close enough? */
-      break;
-
     case EPMD_ALIVE2_REQ:
       dbg_printf(g,1,"** got ALIVE2_REQ");
+      if (!s->local_peer) {
+	   dbg_printf(g,0,"ALIVE2_REQ from non local address");
+	   return;
+      }
 
       /* The packet has the format "axxyyyyyy" where xx is port, given
 	 in network byte order, and yyyyyy is symname, possibly null
 	 terminated. */
 
-      if (bsize <= 13)
+      if (bsize <= 13) /* at least one character for the node name */
 	{
 	  dbg_printf(g,0,"packet to small for request ALIVE2_REQ (%d)",bsize);
 	  return;
@@ -550,7 +496,17 @@ static void do_request(g, fd, s, buf, bsize)
 	highvsn = get_int16(&buf[5]);
 	lowvsn = get_int16(&buf[7]);
 	namelen = get_int16(&buf[9]);
+	if (namelen + 13 > bsize) {
+	    dbg_printf(g,0,"Node name size error in ALIVE2_REQ");
+	    return;
+	}
 	extralen = get_int16(&buf[11+namelen]);
+
+	if (extralen + namelen + 13 > bsize) {
+	    dbg_printf(g,0,"Extra info size error in ALIVE2_REQ");
+	    return;
+	}
+
 	for (i = 11 ; i < 11 + namelen; i ++)
 	    if (buf[i] == '\000')  {
 		dbg_printf(g,0,"node name contains ascii 0 in ALIVE2_REQ");
@@ -593,7 +549,7 @@ static void do_request(g, fd, s, buf, bsize)
 	
       if (bsize <= 1)
 	{
-	  dbg_printf(g,0,"packet to small for request PORT2_REQ (%d)", bsize);
+	  dbg_printf(g,0,"packet too small for request PORT2_REQ (%d)", bsize);
 	  return;
 	}
 
@@ -681,6 +637,10 @@ static void do_request(g, fd, s, buf, bsize)
 
     case EPMD_DUMP_REQ:
       dbg_printf(g,1,"** got DUMP_REQ");
+      if (!s->local_peer) {
+	   dbg_printf(g,0,"DUMP_REQ from non local address");
+	   return;
+      }
       {
 	Node *node;
 
@@ -730,7 +690,19 @@ static void do_request(g, fd, s, buf, bsize)
       break;
 
     case EPMD_KILL_REQ:
+      if (!s->local_peer) {
+	   dbg_printf(g,0,"KILL_REQ from non local address");
+	   return;
+      }
       dbg_printf(g,1,"** got KILL_REQ");
+
+      if (!g->brutal_kill && (g->nodes.reg != NULL)) {
+	  dbg_printf(g,0,"Disallowed KILL_REQ, live nodes");
+	  if (reply(g, fd,"NO",2) != 2)
+	      dbg_printf(g,0,"failed to send reply to KILL_REQ");
+	  return;
+      }
+
       if (reply(g, fd,"OK",2) != 2)
 	dbg_printf(g,0,"failed to send reply to KILL_REQ");
       dbg_tty_printf(g,1,"epmd killed");
@@ -740,9 +712,18 @@ static void do_request(g, fd, s, buf, bsize)
 
     case EPMD_STOP_REQ:
       dbg_printf(g,1,"** got STOP_REQ");
+      if (!s->local_peer) {
+	   dbg_printf(g,0,"STOP_REQ from non local address");
+	   return;
+      }
+      if (!g->brutal_kill) {
+	  dbg_printf(g,0,"Disallowed STOP_REQ, no relaxed_command_check");
+	  return;
+      }
+
       if (bsize <= 1 )
 	{
-	  dbg_printf(g,0,"packet to small for request STOP_REQ (%d)",bsize);
+	  dbg_printf(g,0,"packet too small for request STOP_REQ (%d)",bsize);
 	  return;
 	}
 
@@ -827,6 +808,15 @@ static int conn_open(EpmdVars *g,int fd)
 
   for (i = 0; i < g->max_conn; i++) {
     if (g->conn[i].open == EPMD_FALSE) {
+      struct sockaddr_in si;
+      struct sockaddr_in di;
+#ifdef HAVE_SOCKLEN_T
+      socklen_t st;
+#else
+      int st;
+#endif
+      st = sizeof(si);
+
       g->active_conn++;
       s = &g->conn[i];
      
@@ -836,6 +826,24 @@ static int conn_open(EpmdVars *g,int fd)
       s->fd   = fd;
       s->open = EPMD_TRUE;
       s->keep = EPMD_FALSE;
+
+      /* Determine if connection is from localhost */
+      if (getpeername(s->fd,(struct sockaddr*) &si,&st) ||
+	  st < sizeof(si)) {
+	  /* Failure to get peername is regarded as non local host */
+	  s->local_peer = EPMD_FALSE;
+      } else {
+	  /* Only 127.x.x.x and connections from the host's IP address
+	     allowed, no false positives */
+	  s->local_peer =
+	      (((((unsigned) ntohl(si.sin_addr.s_addr)) & 0xFF000000U) ==
+	       0x7F000000U) ||
+	       (getsockname(s->fd,(struct sockaddr*) &di,&st) ?
+	       EPMD_FALSE : si.sin_addr.s_addr == di.sin_addr.s_addr));
+      }
+      dbg_tty_printf(g,2,(s->local_peer) ? "Local peer connected" :
+		     "Non-local peer connected");
+
       s->want = 0;		/* Currently unknown */
       s->got  = 0;
       s->mod_time = current_time(g); /* Note activity */
@@ -904,7 +912,7 @@ static void node_init(EpmdVars *g)
 
 
 /* We have got a close on a connection and it may be a
-   EPMD_ALIVE_CLOSE_REQ. Note that this call shouild be called
+   EPMD_ALIVE_CLOSE_REQ. Note that this call should be called
    *before* calling conn_close() */
 
 static int node_unreg(EpmdVars *g,char *name)
@@ -992,11 +1000,6 @@ static int node_unreg_sock(EpmdVars *g,int fd)
  *     Perhaps use the oldest or something.
  */
 
-static Node *node_reg(EpmdVars *g,char *name,int fd, int port)
-{
-    return node_reg2(g, name, fd, port, 0, 0, 0, 0, 0, NULL);
-}
-
 static Node *node_reg2(EpmdVars *g,
 		       char* name,
 		       int fd,
@@ -1022,6 +1025,11 @@ static Node *node_reg2(EpmdVars *g,
       dbg_printf(g,0,"node name is too long (%d) %s", strlen(name), name);
       return NULL;
     }
+  if (extralen > MAXSYMLEN)
+    {
+      dbg_printf(g,0,"extra data is too long (%d) %s", strlen(name), name);
+      return NULL;
+    }
 
   /* Fail if it is already registered */
 
diff --git a/erts/epmd/test/Makefile b/erts/epmd/test/Makefile
index 13dad09ae3..54688fd90b 100644
--- a/erts/epmd/test/Makefile
+++ b/erts/epmd/test/Makefile
@@ -74,7 +74,7 @@ release_tests_spec: opt
 	$(INSTALL_DIR) $(RELEPMDDIR)
 	$(INSTALL_DATA) epmd.spec epmd.spec.vxworks $(ERL_FILES) \
 		$(EMAKEFILE) $(RELEPMDDIR)
-	chmod -f -R u+w $(RELEPMDDIR)
+	chmod -R u+w $(RELEPMDDIR)
 
 release_docs_spec:
 
diff --git a/erts/epmd/test/epmd.spec b/erts/epmd/test/epmd.spec
index 0e2496bc72..e72272cf94 100644
--- a/erts/epmd/test/epmd.spec
+++ b/erts/epmd/test/epmd.spec
@@ -1 +1 @@
-{topcase, {dir, "../epmd_test"}}.
+{suites,"../epmd_test",all}.
diff --git a/erts/epmd/test/epmd_SUITE.erl b/erts/epmd/test/epmd_SUITE.erl
index 91e09faf75..6889ec0b34 100644
--- a/erts/epmd/test/epmd_SUITE.erl
+++ b/erts/epmd/test/epmd_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1998-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -17,7 +17,7 @@
 %% %CopyrightEnd%
 %%
 -module(epmd_SUITE).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -include_lib("kernel/include/file.hrl").
 
 
@@ -35,7 +35,9 @@
 -record(node_info, {port, node_type, prot, lvsn, hvsn, node_name, extra}).
 
 % Test server specific exports
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
 -export(
    [
@@ -63,7 +65,13 @@
     alive_req_too_large/1,
 
     returns_valid_empty_extra/1,
-    returns_valid_populated_extra_with_nulls/1
+    returns_valid_populated_extra_with_nulls/1,
+
+    buffer_overrun_1/1,
+    buffer_overrun_2/1,
+    no_nonlocal_register/1,
+    no_nonlocal_kill/1,
+    no_live_killing/1
    ]).
 
 
@@ -82,11 +90,8 @@
 -define(REG_REPEAT_LIM,1000).
 
 % Message codes in epmd protocol
--define(EPMD_ALIVE_REQ,		$a).
 -define(EPMD_ALIVE2_REQ,	$x).
--define(EPMD_ALIVE_OK_RESP,	$Y).
 -define(EPMD_ALIVE2_RESP,	$y).
--define(EPMD_PORT_REQ,		$p).
 -define(EPMD_PORT_PLEASE2_REQ,	$z).
 -define(EPMD_PORT2_RESP,	$w).
 -define(EPMD_NAMES_REQ,	$n).
@@ -98,45 +103,48 @@
 %% all/1
 %%
 
-all(suite) ->
-    [
-     register_name,
-     register_names_1,
-     register_names_2,
-     register_duplicate_name,
-     get_port_nr,
-     slow_get_port_nr,
-     unregister_others_name_1,
-     unregister_others_name_2,
-     register_overflow,
-     name_with_null_inside,
-     name_null_terminated,
-     stupid_names_req,
-
-     no_data,
-     one_byte,
-     two_bytes,
-     partial_packet,
-     zero_length,
-     too_large,
-     alive_req_too_small_1,
-     alive_req_too_small_2,
-     alive_req_too_large,
-
-     returns_valid_empty_extra,
-     returns_valid_populated_extra_with_nulls
-    ].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [register_name, register_names_1, register_names_2,
+     register_duplicate_name, get_port_nr, slow_get_port_nr,
+     unregister_others_name_1, unregister_others_name_2,
+     register_overflow, name_with_null_inside,
+     name_null_terminated, stupid_names_req, no_data,
+     one_byte, two_bytes, partial_packet, zero_length,
+     too_large, alive_req_too_small_1, alive_req_too_small_2,
+     alive_req_too_large, returns_valid_empty_extra,
+     returns_valid_populated_extra_with_nulls,
+     {group, buffer_overrun}, no_nonlocal_register,
+     no_nonlocal_kill, no_live_killing].
+
+groups() -> 
+    [{buffer_overrun, [],
+      [buffer_overrun_1, buffer_overrun_2]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 %%
 %% Run before and after each test case
 %%
 
 init_per_testcase(_Func, Config) ->
-    Dog = test_server:timetrap(?SHORT_TEST_TIMEOUT),
+    Dog = test_server:timetrap(?MEDIUM_TEST_TIMEOUT),
     cleanup(),
     [{watchdog, Dog} | Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     cleanup(),
     Dog = ?config(watchdog, Config),
     catch test_server:timetrap_cancel(Dog),	% We may have canceled already
@@ -148,7 +156,7 @@ register_name(doc) ->
     ["Register a name"];
 register_name(suite) ->
     [];
-register_name(Config) when list(Config) ->
+register_name(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = register_node("foobar"),
     ?line ok = close(Sock),			% Unregister
@@ -158,7 +166,7 @@ register_names_1(doc) ->
     ["Register and unregister two nodes"];
 register_names_1(suite) ->
     [];
-register_names_1(Config) when list(Config) ->
+register_names_1(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock1} = register_node("foobar"),
     ?line {ok,Sock2} = register_node("foozap"),
@@ -170,7 +178,7 @@ register_names_2(doc) ->
     ["Register and unregister two nodes"];
 register_names_2(suite) ->
     [];
-register_names_2(Config) when list(Config) ->
+register_names_2(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock1} = register_node("foobar"),
     ?line {ok,Sock2} = register_node("foozap"),
@@ -182,7 +190,7 @@ register_duplicate_name(doc) ->
     ["Two nodes with the same name"];
 register_duplicate_name(suite) ->
     [];
-register_duplicate_name(Config) when list(Config) ->
+register_duplicate_name(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = register_node("foobar"),
     ?line error = register_node("foobar"),
@@ -192,22 +200,9 @@ register_duplicate_name(Config) when list(Config) ->
 % Internal function to register a node name, no close, i.e. unregister
 
 register_node(Name) ->
-    register_node(Name,?DUMMY_PORT).
-
-register_node(Name, Port) ->
-    case send_req([?EPMD_ALIVE_REQ, put16(Port), Name]) of
-	{ok,Sock} ->
-	    case recv(Sock,3) of
-		{ok, [?EPMD_ALIVE_OK_RESP,_D1,_D0]} ->
-		    {ok,Sock};
-		Other ->
-		    test_server:format("recv on sock ~w: ~p~n",
-				       [Sock,Other]),
-		    error
-	    end;
-	error ->
-	    error
-    end.
+    register_node_v2(?DUMMY_PORT,$M,0,5,5,Name,"").
+register_node(Name,Port) ->
+    register_node_v2(Port,$M,0,5,5,Name,"").
 
 register_node_v2(Port, NodeType, Prot, HVsn, LVsn, Name, Extra) ->
     Req = [?EPMD_ALIVE2_REQ, put16(Port), NodeType, Prot,
@@ -254,7 +249,7 @@ parse_port2_resp(Resp) ->
 			    hvsn=HVsn,lvsn=LVsn,
 			    node_name=binary_to_list(NodeName),
 			    extra=binary_to_list(Extra)}};
-	Other ->
+	_Other ->
 	    test_server:format("invalid port2 resp: ~p~n",
 			       [Resp]),
 	    error
@@ -266,7 +261,7 @@ name_with_null_inside(doc) ->
     ["Register a name with a null char in it"];
 name_with_null_inside(suite) ->
     [];
-name_with_null_inside(Config) when list(Config) ->
+name_with_null_inside(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line error = register_node("foo\000bar"),
     ok.
@@ -277,11 +272,9 @@ name_null_terminated(doc) ->
     ["Register a name with terminating null byte"];
 name_null_terminated(suite) ->
     [];
-name_null_terminated(Config) when list(Config) ->
+name_null_terminated(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
-    ?line {ok,Sock} = register_node("foobar\000"),
-    ?line error = register_node("foobar"),
-    ?line ok = close(Sock),			% Unregister
+    ?line error = register_node("foobar\000"),
     ok.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -290,7 +283,7 @@ stupid_names_req(doc) ->
     ["Read names from epmd in a stupid way"];
 stupid_names_req(suite) ->
     [];
-stupid_names_req(Config) when list(Config) ->
+stupid_names_req(Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     LongDog = test_server:timetrap(?MEDIUM_TEST_TIMEOUT),
@@ -394,15 +387,15 @@ get_port_nr(doc) ->
     ["Register a name on a port and ask about port nr"];
 get_port_nr(suite) ->
     [];
-get_port_nr(Config) when list(Config) ->
-    port_request([?EPMD_PORT_REQ,"foo"]).
+get_port_nr(Config) when is_list(Config) ->
+    port_request([?EPMD_PORT_PLEASE2_REQ,"foo"]).
 
 slow_get_port_nr(doc) ->
     ["Register with slow write and ask about port nr"];
 slow_get_port_nr(suite) ->
     [];
-slow_get_port_nr(Config) when list(Config) ->
-    port_request([?EPMD_PORT_REQ,d,$f,d,$o,d,$o]).
+slow_get_port_nr(Config) when is_list(Config) ->
+    port_request([?EPMD_PORT_PLEASE2_REQ,d,$f,d,$o,d,$o]).
 
 
 % Internal function used above
@@ -413,9 +406,18 @@ port_request(M) ->
     ?line {ok,RSock} = register_node("foo", Port),
     ?line {ok,Sock} = connect(),
     ?line ok = send(Sock,[size16(M),M]),
-    R = put16(Port),
-    ?line {ok,R} = recv(Sock, length(R)),
-    ?line ok = close(RSock),
+    ?line case recv_until_sock_closes(Sock) of
+	      {ok, Resp} ->
+		  ?line close(RSock),
+		  ?line {ok,Rec} = parse_port2_resp(Resp),
+		  ?line Port = Rec#node_info.port,
+		  ok;
+	      Other ->
+		  ?line close(RSock),
+		  ?line test_server:format("recv on sock ~w: ~p~n",
+					   [Sock,Other]),
+		  ?line throw({error,Other})
+	  end,
     ok.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -424,8 +426,8 @@ unregister_others_name_1(doc) ->
     ["Unregister name of other node"];
 unregister_others_name_1(suite) ->
     [];
-unregister_others_name_1(Config) when list(Config) ->
-    ?line ok = epmdrun(),
+unregister_others_name_1(Config) when is_list(Config) ->
+    ?line ok = epmdrun("-relaxed_command_check"),
     ?line {ok,RSock} = register_node("foo"),
     ?line {ok,Sock} = connect(),
     M = [?EPMD_STOP_REQ,"foo"],
@@ -441,8 +443,8 @@ unregister_others_name_2(doc) ->
     ["Unregister name of other node"];
 unregister_others_name_2(suite) ->
     [];
-unregister_others_name_2(Config) when list(Config) ->
-    ?line ok = epmdrun(),
+unregister_others_name_2(Config) when is_list(Config) ->
+    ?line ok = epmdrun("-relaxed_command_check"),
     ?line {ok,Sock} = connect(),
     M = [?EPMD_STOP_REQ,"xxx42"],
     ?line ok = send(Sock,[size16(M),M]),
@@ -456,7 +458,7 @@ register_overflow(doc) ->
     ["Register too many, clean and redo 10 times"];
 register_overflow(suite) ->
     [];
-register_overflow(Config) when list(Config) ->
+register_overflow(Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     LongDog = test_server:timetrap(?LONG_TEST_TIMEOUT),
@@ -546,7 +548,7 @@ no_data(doc) ->
     ["Open but send no data"];
 no_data(suite) ->
     [];
-no_data(Config) when list(Config) ->
+no_data(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     sleep(?LONG_PAUSE),
@@ -559,7 +561,7 @@ one_byte(doc) ->
     ["Send one byte only"];
 one_byte(suite) ->
     [];
-one_byte(Config) when list(Config) ->
+one_byte(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     ?line ok = send(Sock,[0]),
@@ -573,7 +575,7 @@ two_bytes(doc) ->
     ["Send packet size only"];
 two_bytes(suite) ->
     [];
-two_bytes(Config) when list(Config) ->
+two_bytes(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     ?line ok = send(Sock,[put16(3)]),
@@ -587,7 +589,7 @@ partial_packet(doc) ->
     ["Got only part of a packet"];
 partial_packet(suite) ->
     [];
-partial_packet(Config) when list(Config) ->
+partial_packet(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     ?line ok = send(Sock,[put16(100),"only a few bytes"]),
@@ -601,7 +603,7 @@ zero_length(doc) ->
     ["Invalid zero packet size"];
 zero_length(suite) ->
     [];
-zero_length(Config) when list(Config) ->
+zero_length(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     ?line ok = send(Sock,[0,0,0,0,0,0,0,0,0,0]),
@@ -615,15 +617,20 @@ too_large(doc) ->
     ["Invalid large packet"];
 too_large(suite) ->
     [];
-too_large(Config) when list(Config) ->
+too_large(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     Size = 63000,
     M = lists:duplicate(Size, $z),
     ?line ok = send(Sock,[put16(Size),M]),
     sleep(?MEDIUM_PAUSE),
-    ?line closed = recv(Sock,1),
-    ok.
+    % With such a large packet, even the writes can fail as the
+    % daemon closes before everything is delivered -> econnaborted
+    case recv(Sock,1) of
+	closed -> ok;
+	{error,econnaborted} -> ok;
+	Other -> exit({unexpected,Other})
+    end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
@@ -631,10 +638,11 @@ alive_req_too_small_1(doc) ->
     ["Try to register but not enough data"];
 alive_req_too_small_1(suite) ->
     [];
-alive_req_too_small_1(Config) when list(Config) ->
+alive_req_too_small_1(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
-    M = [?EPMD_ALIVE_REQ, 42],
+    M = [?EPMD_ALIVE2_REQ, put16(?DUMMY_PORT),$M,0, put16(5),
+	 put16(5),put16(0)],
     ?line ok = send(Sock, [size16(M), M]),
     sleep(?MEDIUM_PAUSE),
     ?line closed = recv(Sock,1),
@@ -646,10 +654,11 @@ alive_req_too_small_2(doc) ->
     ["Try to register but not enough data"];
 alive_req_too_small_2(suite) ->
     [];
-alive_req_too_small_2(Config) when list(Config) ->
+alive_req_too_small_2(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
-    M = [?EPMD_ALIVE_REQ, put16(?DUMMY_PORT)],
+    M =  [?EPMD_ALIVE2_REQ, put16(?DUMMY_PORT),$M,0, put16(5),
+	  put16(5)],
     ?line ok = send(Sock, [size16(M), M]),
     sleep(?MEDIUM_PAUSE),
     ?line closed = recv(Sock,1),
@@ -661,7 +670,7 @@ alive_req_too_large(doc) ->
     ["Try to register but node name too large"];
 alive_req_too_large(suite) ->
     [];
-alive_req_too_large(Config) when list(Config) ->
+alive_req_too_large(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     L = [
@@ -678,10 +687,12 @@ alive_req_too_large(Config) when list(Config) ->
 	 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 	 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 	 ],
-    M = [?EPMD_ALIVE_REQ, put16(?DUMMY_PORT), L],
+    S = length(lists:flatten(L)),
+    M = [?EPMD_ALIVE2_REQ, put16(?DUMMY_PORT),$M,0, put16(5),
+	 put16(5), put16(S),L,put16(0)],
     ?line ok = send(Sock, [size16(M), M]),
     sleep(?MEDIUM_PAUSE),
-    ?line closed = recv(Sock,1),
+    ?line {ok,[?EPMD_ALIVE2_RESP,1]} = recv(Sock,2),
     ok.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -690,7 +701,7 @@ returns_valid_empty_extra(doc) ->
     ["Check that an empty extra is prefixed by a two byte length"];
 returns_valid_empty_extra(suite) ->
     [];
-returns_valid_empty_extra(Config) when list(Config) ->
+returns_valid_empty_extra(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = register_node_v2(4711, 72, 0, 5, 5, "foo", []),
     ?line {ok,#node_info{extra=[]}} = port_please_v2("foo"),
@@ -703,7 +714,7 @@ returns_valid_populated_extra_with_nulls(doc) ->
     ["Check a populated extra with embedded null characters"];
 returns_valid_populated_extra_with_nulls(suite) ->
     [];
-returns_valid_populated_extra_with_nulls(Config) when list(Config) ->
+returns_valid_populated_extra_with_nulls(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = register_node_v2(4711, 72, 0, 5, 5, "foo", "ABC\000\000"),
     ?line {ok,#node_info{extra="ABC\000\000"}} = port_please_v2("foo"),
@@ -711,6 +722,165 @@ returns_valid_populated_extra_with_nulls(Config) when list(Config) ->
     ok.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+buffer_overrun_1(suite) ->
+    [];
+buffer_overrun_1(doc) ->
+    ["Test security vulnerability in fake extra lengths in alive2_req"];
+buffer_overrun_1(Config) when is_list(Config) ->
+    ?line ok = epmdrun(),
+    ?line true = alltrue([hostile(N) || N <- lists:seq(1,10000)]),
+    ok.
+buffer_overrun_2(suite) ->
+    [];
+buffer_overrun_2(doc) ->
+    ["Test security vulnerability in fake extra lengths in alive2_req"];
+buffer_overrun_2(Config) when is_list(Config) ->
+    ?line ok = epmdrun(),
+    ?line [false | Rest] = [hostile2(N) || N <- lists:seq(255,10000)],
+    ?line true = alltrue(Rest),
+    ok.
+hostile(N) ->
+    try
+	Bin= <<$x:8,4747:16,$M:8,0:8,5:16,5:16,5:16,"gurka",N:16>>,
+	S = size(Bin),
+	{ok,E}=connect_sturdy(),
+	gen_tcp:send(E,[<<S:16>>,Bin]),
+	closed = recv(E,1),
+	gen_tcp:close(E),
+	true
+    catch
+	_:_ ->
+	    false
+    end.
+hostile2(N) ->
+    try
+	B2 = list_to_binary(lists:duplicate(N,255)),
+	Bin= <<$x:8,4747:16,$M:8,0:8,5:16,5:16,5:16,"gurka",N:16,B2/binary>>,
+	S = size(Bin),
+	{ok,E}=connect_sturdy(),
+	gen_tcp:send(E,[<<S:16>>,Bin]),
+	Z = recv(E,2),
+	gen_tcp:close(E),
+	(Z =:= closed) or (Z =:= {ok, [$y,1]})
+    catch
+	_A:_B ->
+	    false
+    end.
+
+alltrue([]) ->
+    true;
+alltrue([true|T]) ->
+    alltrue(T);
+alltrue([_|_]) ->
+    false.
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+no_nonlocal_register(suite) ->
+    [];
+no_nonlocal_register(doc) ->
+    ["Ensure that we cannot register throug a nonlocal connection"];
+no_nonlocal_register(Config) when is_list(Config) ->
+    ?line case {os:find_executable("ssh"),ct:get_config(ssh_proxy_host)} of
+	      {SSH,Name} when is_list(Name), is_list(SSH) ->
+		  do_no_nonlocal_register(Config,Name);
+	      {false,_} ->
+		  {skip, "No ssh command found to create proxy"};
+	      _ ->
+		  {skip, "No ssh_proxy_host configured in ts.config"}
+	  end.
+do_no_nonlocal_register(Config,SSHHost) when is_list(Config) ->
+    ?line ok = epmdrun(),
+    ?line ProxyPort = proxy_port(),
+    ?line ok = ssh_proxy(SSHHost,ProxyPort),
+    Res = try
+	      ?line Name = "gurka_"
+	      %++
+	      %integer_to_list(A1)++"_"++
+	      %integer_to_list(A2)++"_"++
+	      %integer_to_list(A3)++"_"++
+	      %integer_to_list(A4)
+	      ,
+	      ?line Bname = list_to_binary(Name),
+	      ?line NameS = byte_size(Bname),
+	      ?line Bin= <<$x:8,4747:16,$M:8,0:8,5:16,
+			  5:16,NameS:16,Bname/binary,
+			  0:16>>,
+	      ?line S = size(Bin),
+	      ?line {ok, E} = connect("localhost",ProxyPort,passive),
+	      ?line gen_tcp:send(E,[<<S:16>>,Bin]),
+	      ?line closed = recv(E,1),
+	      ?line gen_tcp:close(E),
+	      true
+	  catch
+	      _:_ ->
+		  false
+	  end,
+    %erlang:display(Res),
+    true = Res,
+    ok.
+
+no_nonlocal_kill(suite) ->
+    [];
+no_nonlocal_kill(doc) ->
+    ["Ensure that we cannot kill through nonlocal connection"];
+no_nonlocal_kill(Config) when is_list(Config) ->
+    ?line case {os:find_executable("ssh"),ct:get_config(ssh_proxy_host)} of
+	      {SSH,Name} when is_list(Name), is_list(SSH) ->
+		  do_no_nonlocal_kill(Config,Name);
+	      {false,_} ->
+		  {skip, "No ssh command found to create proxy"};
+	      _ ->
+		  {skip, "No ssh_proxy_host configured in ts.config"}
+	  end.
+do_no_nonlocal_kill(Config,SSHHost) when is_list(Config) ->
+    ?line ok = epmdrun(),
+    ?line ProxyPort = proxy_port(),
+    ?line ok = ssh_proxy(SSHHost,ProxyPort),
+    Res = try
+	      {ok, E} = connect("localhost",ProxyPort,passive),
+	      M = [?EPMD_KILL_REQ],
+	      send(E, [size16(M), M]),
+	      closed = recv(E,2),
+	      gen_tcp:close(E),
+	      sleep(?MEDIUM_PAUSE),
+	      {ok, E2} = connect("localhost",ProxyPort,passive),
+	      gen_tcp:close(E2),
+	      true
+	  catch
+	      _:_ ->
+		  false
+	  end,
+    %erlang:display(Res),
+    true = Res,
+    ok.
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+no_live_killing(doc) ->
+    ["Dont allow killing with live nodes or any unregistering w/o -relaxed_command_check"];
+no_live_killing(suite) ->
+    [];
+no_live_killing(Config) when is_list(Config) ->
+    ?line ok = epmdrun(),
+    ?line {ok,RSock} = register_node("foo"),
+    ?line {ok,Sock} = connect(),
+    ?line M = [?EPMD_KILL_REQ],
+    ?line ok = send(Sock,[size16(M),M]),
+    ?line {ok,"NO"} = recv(Sock,2),
+    ?line close(Sock),
+    ?line {ok,Sock2} = connect(),
+    ?line M2 = [?EPMD_STOP_REQ,"foo"],
+    ?line ok = send(Sock2,[size16(M2),M2]),
+    ?line closed = recv(Sock2,1),
+    ?line close(Sock2),
+    ?line close(RSock),
+    ?line sleep(?MEDIUM_PAUSE),
+    ?line {ok,Sock3} = connect(),
+    ?line M3 = [?EPMD_KILL_REQ],
+    ?line ok = send(Sock3,[size16(M3),M3]),
+    ?line {ok,"OK"} = recv(Sock3,2),
+    ?line close(Sock3),
+    ok.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 % Terminate all tests with killing epmd.
 
 cleanup() ->
@@ -726,21 +896,42 @@ cleanup() ->
 	    true
     end.
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Start an ssh channel to simulate remote access
+
+proxy_port() ->
+    ?PORT+1.
+
+ssh_proxy(SSHHost,ProxyPort) ->
+    ?line Host = lists:nth(2,string:tokens(atom_to_list(node()),"@")),
+    % Requires proxy to be a unix host with the command 'read' accessible
+    ?line osrun("ssh -L "++integer_to_list(ProxyPort)++":"++Host++":"
+		++integer_to_list(?PORT)++" "++SSHHost++" read").
+    
+
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 % Normal debug start of epmd
 
 epmdrun() ->
+    epmdrun([]).
+epmdrun(Args) ->
     case os:find_executable(epmd) of
 	false ->
 	    {error, {could_not_find_epmd_in_path}};
 	Path ->
-	    epmdrun(Path)
+	    epmdrun(Path,Args)
     end.
 
-epmdrun(Epmd) ->
+epmdrun(Epmd,Args0) ->
   %% test_server:format("epmdrun() => Epmd = ~p",[Epmd]),
-  osrun(Epmd ++ " " ?EPMDARGS " -port " ++ integer_to_list(?PORT)).
+    Args = case Args0 of
+	       [] ->
+		   [];
+	       O ->
+		   " "++O
+	   end,
+  osrun(Epmd ++ Args ++ " " ?EPMDARGS " -port " ++ integer_to_list(?PORT)).
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -753,20 +944,27 @@ osrun(Cmd) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 % Wrappers of TCP functions
 
-% These two functions is the interface for connect.
+% These functions is the interface for connect.
 % Passive mode is the default
 
 connect() ->
-    connect(?PORT, passive).
+    connect("localhost",?PORT, passive).
+
+connect(Addr) ->
+    connect(Addr,?PORT, passive).
 
 connect_active() ->
-    connect(?PORT, active).
+    connect("localhost",?PORT, active).
 
+%% Retry after 15 seconds, to avoid TIME_WAIT socket exhaust.
+connect_sturdy() ->
+    connect("localhost",?PORT, passive, 15000, 3).
 
 % Try a few times before giving up
-
-connect(Port, Mode) ->
-    case connect_repeat(?CONN_RETRY, Port, Mode) of
+connect(Addr, Port, Mode) ->
+    connect(Addr, Port, Mode, ?CONN_SLEEP, ?CONN_RETRY).
+connect(Addr, Port, Mode, Sleep, Retry) ->
+    case connect_repeat(Addr, Retry, Port, Mode, Sleep) of
 	{ok,Sock} ->
 	    {ok,Sock};
 	{error,timeout} ->
@@ -783,25 +981,25 @@ connect(Port, Mode) ->
 % Try a few times before giving up. Pause a small time between
 % each try.
 
-connect_repeat(1, Port, Mode) ->
-    connect_mode(Port, Mode);
-connect_repeat(Retry, Port, Mode) ->
-    case connect_mode(Port, Mode) of
+connect_repeat(Addr, 1, Port, Mode, _Sleep) ->
+    connect_mode(Addr,Port, Mode);
+connect_repeat(Addr,Retry, Port, Mode, Sleep) ->
+    case connect_mode(Addr,Port, Mode) of
 	{ok,Sock} ->
 	    {ok,Sock};
 	{error,Reason} ->
 	    test_server:format("connect: error: ~w~n",[Reason]),
-	    timer:sleep(?CONN_SLEEP),
-	    connect_repeat(Retry - 1, Port, Mode);
+	    timer:sleep(Sleep),
+	    connect_repeat(Addr, Retry - 1, Port, Mode, Sleep);
 	Any ->
 	    test_server:format("connect: unknown message: ~w~n",[Any]),
 	    exit(1)
     end.
 
-connect_mode(Port, active) ->
-    gen_tcp:connect("localhost", Port, [{packet, 0}], ?CONN_TIMEOUT);
-connect_mode(Port, passive) ->
-    gen_tcp:connect("localhost", Port, [{packet, 0}, {active, false}],
+connect_mode(Addr,Port, active) ->
+    gen_tcp:connect(Addr, Port, [{packet, 0}], ?CONN_TIMEOUT);
+connect_mode(Addr, Port, passive) ->
+    gen_tcp:connect(Addr, Port, [{packet, 0}, {active, false}],
 		    ?CONN_TIMEOUT).
 
 
@@ -858,9 +1056,9 @@ send(Sock, SendSpec) ->
 
 send([], RevBytes, _Sock) ->
     {ok,RevBytes};
-send([Byte | Spec], RevBytes, Sock) when integer(Byte) ->
+send([Byte | Spec], RevBytes, Sock) when is_integer(Byte) ->
     send(Spec, [Byte | RevBytes], Sock);
-send([List | Spec], RevBytes, Sock) when list(List) ->
+send([List | Spec], RevBytes, Sock) when is_list(List) ->
     case send(List, RevBytes, Sock) of
 	{ok,Left} ->
 	    send(Spec, Left, Sock);
diff --git a/erts/etc/common/Makefile.in b/erts/etc/common/Makefile.in
index 3db4fcba61..4754328c0b 100644
--- a/erts/etc/common/Makefile.in
+++ b/erts/etc/common/Makefile.in
@@ -96,9 +96,9 @@ endif
 
 # On windows we always need reentrant libraries.
 ifeq ($(TARGET),win32)
-ERLEXEC_XLIBS=-L../../lib/internal/$(TARGET) -lerts_internal_r$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
+ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal_r$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
 else
-ERLEXEC_XLIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
+ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ -lm
 endif
 
 # ----------------------------------------------------
@@ -178,7 +178,7 @@ MC_OUTPUTS= \
 	MT_FLAG="-MD"
 endif
 INET_GETHOST = $(BINDIR)/inet_gethost.exe
-INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer.exe $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe
+INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer.exe $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe $(BINDIR)/ct_run.exe
 INSTALL_SRC = $(WINETC)/start_erl.c $(WINETC)/Nmakefile.start_erl
 ERLEXECDIR=.
 INSTALL_LIBS =
@@ -211,7 +211,7 @@ ERLSRV_OBJECTS=
 MC_OUTPUTS=
 INET_GETHOST = $(BINDIR)/inet_gethost@EXEEXT@
 INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer@EXEEXT@ $(BINDIR)/dialyzer@EXEEXT@ \
- $(BINDIR)/erlc@EXEEXT@ $(BINDIR)/escript@EXEEXT@ \
+ $(BINDIR)/erlc@EXEEXT@ $(BINDIR)/escript@EXEEXT@ $(BINDIR)/ct_run@EXEEXT@ \
  $(BINDIR)/run_erl $(BINDIR)/to_erl $(BINDIR)/dyn_erl
 INSTALL_EMBEDDED_DATA = ../unix/start.src ../unix/start_erl.src
 INSTALL_TOP = Install
@@ -274,6 +274,7 @@ endif
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/dyn_erl.o
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/safe_string.o	
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/typer.o
+	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/ct_run.o
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/vxcall.o
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/erl.o
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/werl.o
@@ -295,7 +296,7 @@ $(OBJDIR)/inet_gethost.o: inet_gethost.c
 	$(CC) $(CFLAGS) -o $@ -c inet_gethost.c
 
 $(BINDIR)/inet_gethost@EXEEXT@: $(OBJDIR)/inet_gethost.o $(ENTRY_OBJ)
-	$(PURIFY) $(LD) $(LDFLAGS) $(ENTRY_LDFLAGS) -o $@ $(OBJDIR)/inet_gethost.o $(ENTRY_OBJ) $(LIBS) 
+	$(PURIFY) $(LD) $(LDFLAGS) $(ENTRY_LDFLAGS) -o $@ $(OBJDIR)/inet_gethost.o $(ENTRY_OBJ) $(LIBS) $(ERTS_INTERNAL_LIBS)
 
 $(BINDIR)/run_erl: $(OBJDIR)/safe_string.o $(OBJDIR)/run_erl.o
 	$(LD) $(LDFLAGS) -o $@ $(OBJDIR)/safe_string.o $(OBJDIR)/run_erl.o $(LIBS)
@@ -320,35 +321,42 @@ $(OBJDIR)/safe_string.o: ../unix/safe_string.c
 
 ifneq ($(TARGET),win32)
 $(BINDIR)/$(ERLEXEC): $(OBJDIR)/$(ERLEXEC).o
-	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/$(ERLEXEC).o $(ERLEXEC_XLIBS)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/$(ERLEXEC).o $(ERTS_INTERNAL_LIBS)
 
 $(OBJDIR)/$(ERLEXEC).o: $(ERLEXECDIR)/$(ERLEXEC).c
 	$(CC) -I$(EMUDIR) $(CFLAGS) -o $@ -c $(ERLEXECDIR)/$(ERLEXEC).c
 endif
 $(BINDIR)/erlc@EXEEXT@: $(OBJDIR)/erlc.o
-	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/erlc.o -L$(OBJDIR) $(LIBS)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/erlc.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS)
 
 $(OBJDIR)/erlc.o: erlc.c
 	$(CC) $(CFLAGS) -o $@ -c erlc.c
 
 $(BINDIR)/dialyzer@EXEEXT@: $(OBJDIR)/dialyzer.o
-	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/dialyzer.o -L$(OBJDIR) $(LIBS)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/dialyzer.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS)
 
 $(OBJDIR)/dialyzer.o: dialyzer.c
 	$(CC) $(CFLAGS) -o $@ -c dialyzer.c
 
 $(BINDIR)/typer@EXEEXT@: $(OBJDIR)/typer.o
-	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/typer.o -L$(OBJDIR) $(LIBS)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/typer.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS)
 
 $(OBJDIR)/typer.o: typer.c
 	$(CC) $(CFLAGS) -o $@ -c typer.c
 
 $(BINDIR)/escript@EXEEXT@: $(OBJDIR)/escript.o
-	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/escript.o -L$(OBJDIR) $(LIBS)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/escript.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS)
 
 $(OBJDIR)/escript.o: escript.c
 	$(CC) $(CFLAGS) -o $@ -c escript.c
 
+$(BINDIR)/ct_run@EXEEXT@: $(OBJDIR)/ct_run.o
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/ct_run.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS)
+
+$(OBJDIR)/ct_run.o: ct_run.c
+	$(CC) $(CFLAGS) -o $@ -c ct_run.c
+
+
 #------------------------------------------------------------------------
 # Windows specific targets
 # The windows platform is quite different from the others. erl/werl are small C programs
@@ -360,7 +368,7 @@ $(OBJDIR)/escript.o: escript.c
 ifeq ($(TARGET),win32)
 
 $(BINDIR)/$(ERLEXEC): $(OBJDIR)/erlexec.o $(OBJDIR)/win_erlexec.o $(OBJDIR)/init_file.o  $(OBJDIR)/$(ERLRES_OBJ)
-	$(LD) -dll $(LDFLAGS) -o $@ $(OBJDIR)/erlexec.o $(OBJDIR)/win_erlexec.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ) $(ERLEXEC_XLIBS)
+	$(LD) -dll $(LDFLAGS) -o $@ $(OBJDIR)/erlexec.o $(OBJDIR)/win_erlexec.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ) $(ERTS_INTERNAL_LIBS)
 
 $(BINDIR)/erl@EXEEXT@: $(OBJDIR)/erl.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ)
 	$(LD) $(LDFLAGS) -o $@ $(OBJDIR)/erl.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ) 
diff --git a/erts/etc/common/ct_run.c b/erts/etc/common/ct_run.c
new file mode 100644
index 0000000000..7aaab716f7
--- /dev/null
+++ b/erts/etc/common/ct_run.c
@@ -0,0 +1,545 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Purpose: Common Test front-end.
+ */
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#ifdef __WIN32__
+#include <winbase.h>
+#endif
+
+#include <ctype.h>
+
+#define NO 0
+#define YES 1
+
+#define ASIZE(a) (sizeof(a)/sizeof(a[0]))
+
+static int debug = 0;		/* Bit flags for debug printouts. */
+
+static char** eargv_base;	/* Base of vector. */
+static char** eargv;		/* First argument for erl. */
+
+static int eargc;		/* Number of arguments in eargv. */
+
+#ifdef __WIN32__
+#  define QUOTE(s) possibly_quote(s)
+#  define IS_DIRSEP(c) ((c) == '/' || (c) == '\\')
+#  define ERL_NAME "erl.exe"
+#else
+#  define QUOTE(s) s
+#  define IS_DIRSEP(c) ((c) == '/')
+#  define ERL_NAME "erl"
+#endif
+
+#define UNSHIFT(s) eargc++, eargv--; eargv[0] = QUOTE(s)
+#define PUSH(s) eargv[eargc++] = QUOTE(s)
+#define PUSH2(s, t) PUSH(s); PUSH(t)
+#define PUSH3(s, t, u) PUSH2(s, t); PUSH(u)
+#define PUSH4(s, t, u, v) PUSH2(s, t); PUSH2(u, v)
+
+/*
+ * The possible modes to start Common Test
+ */
+
+#define NORMAL_MODE 0
+#define VTS_MODE 1
+#define CT_SHELL_MODE 2
+#define MASTER_MODE 3
+#define ERL_SHELL_MODE 4
+
+/*
+ * Distribution
+ */
+
+#define SHORT_NAME 0
+#define FULL_NAME 1
+
+/*
+ * Local functions.
+ */
+
+static void error(char* format, ...);
+static char* emalloc(size_t size);
+static char* strsave(char* string);
+static void push_words(char* src);
+static int run_erlang(char* name, char** argv);
+static char* get_default_emulator(char* progname);
+static void print_deprecation_warning(char *progname); 
+#ifdef __WIN32__
+static char* possibly_quote(char* arg);
+#endif
+
+/*
+ * Supply a strerror() function if libc doesn't.
+ */
+#ifndef HAVE_STRERROR
+
+extern int sys_nerr;
+
+#ifndef SYS_ERRLIST_DECLARED
+extern const char * const sys_errlist[];
+#endif /* !SYS_ERRLIST_DECLARED */
+
+char *strerror(int errnum)
+{
+  static char *emsg[1024];
+
+  if (errnum != 0) {
+    if (errnum > 0 && errnum < sys_nerr)
+      sprintf((char *) &emsg[0], "(%s)", sys_errlist[errnum]);
+    else
+      sprintf((char *) &emsg[0], "errnum = %d ", errnum);
+  }
+  else {
+    emsg[0] = '\0';
+  }
+  return (char *) &emsg[0];
+}
+#endif /* !HAVE_STRERROR */
+
+int
+main(int argc, char** argv)
+{
+    int eargv_size;
+    int eargc_base;		/* How many arguments in the base of eargv. */
+    char* emulator;
+    char nodename[100];
+    char browser[100];
+    int ct_mode;
+    int dist_mode;
+    int cnt;
+    int erl_args;
+    char** argv0 = argv;
+
+    print_deprecation_warning(argv[0]);
+
+    emulator = get_default_emulator(argv[0]);
+
+    /*
+     * Allocate the argv vector to be used for arguments to Erlang.
+     * Arrange for starting to pushing information in the middle of
+     * the array, to allow easy addition of commands in the beginning.
+     */
+
+    eargv_size = argc*4+100;
+    eargv_base = (char **) emalloc(eargv_size*sizeof(char*));
+    eargv = eargv_base;
+    eargc = 0;
+    push_words(emulator);
+    eargc_base = eargc;
+    eargv = eargv + eargv_size/2;
+    eargc = 0;
+
+    strcpy(nodename, "ct");
+    dist_mode = SHORT_NAME;
+    browser[0] = '\0';
+    ct_mode = NORMAL_MODE;
+    erl_args = argc;
+    cnt = 1;
+
+    /*
+     * Check various flags before building command line
+     */
+
+    while (cnt < argc) {
+	if (strcmp(argv[1], "-erl_args") == 0) {
+	    erl_args = cnt;
+	}
+	else if (strcmp(argv[1], "-sname") == 0) {
+	    strncpy(nodename, argv[2], sizeof(nodename));
+	    nodename[sizeof(nodename)-1] = '\0';
+	    cnt++, argv++;
+	}
+	else if (strcmp(argv[1], "-name") == 0) {
+	    strncpy(nodename, argv[2], sizeof(nodename));
+	    nodename[sizeof(nodename)-1] = '\0';
+	    dist_mode = FULL_NAME;
+	    cnt++, argv++;
+	}
+	else {
+	    if (cnt < erl_args) {
+		if (strcmp(argv[1], "-vts") == 0) {
+		    ct_mode = VTS_MODE;
+		}
+		else if (strcmp(argv[1], "-browser") == 0) {
+		    strncpy(browser, argv[2], sizeof(browser));
+		    browser[sizeof(browser)-1] = '\0';
+		    cnt++, argv++;
+		}
+		else if (strcmp(argv[1], "-shell") == 0) {
+		    ct_mode = CT_SHELL_MODE;
+		}
+		else if (strcmp(argv[1], "-ctmaster") == 0) {
+		    strcpy(nodename, "ct_master");
+		    ct_mode = MASTER_MODE;
+		}
+		else if (strcmp(argv[1], "-ctname") == 0) {
+		    strncpy(nodename, argv[2], sizeof(nodename));
+		    nodename[sizeof(nodename)-1] = '\0';
+		    ct_mode = ERL_SHELL_MODE;
+		    cnt++, argv++;
+		}
+	    }
+	}
+	cnt++, argv++;
+    }
+
+    argv = argv0;
+
+    /*
+     * Push initial arguments.
+     */
+
+    if (dist_mode == FULL_NAME) {
+	PUSH2("-name", nodename);
+    }
+    else {
+	PUSH2("-sname", nodename);
+    }
+
+    /*
+     * Push everything else
+     */
+
+    if (ct_mode == VTS_MODE) {
+	PUSH4("-s", "webtool", "script_start", "vts");
+	if (browser[0] != '\0') PUSH(browser);
+	PUSH3("-s", "ct_run", "script_start");
+    }
+    else if (ct_mode == CT_SHELL_MODE) {
+	PUSH3("-s", "ct_run", "script_start");
+    }
+    else if (ct_mode == NORMAL_MODE) {
+	PUSH3("-s", "ct_run", "script_start");
+	PUSH3("-s", "erlang", "halt");
+    }
+
+    cnt = 1;
+    while (cnt < argc) {
+	if (strcmp(argv[1], "-erl_args") == 0) {
+	    PUSH("-ct_erl_args");
+	}
+	else if ((strcmp(argv[1], "-sname") == 0) || (strcmp(argv[1], "-name") == 0)) {
+	    cnt++, argv++;
+	}
+	else if (cnt < erl_args) {
+	    if (strcmp(argv[1], "-config") == 0)
+		PUSH("-ct_config");
+	    else if (strcmp(argv[1], "-decrypt_key") == 0)
+		PUSH("-ct_decrypt_key");
+	    else if (strcmp(argv[1], "-decrypt_file") == 0)
+		PUSH("-ct_decrypt_file");
+	    else
+		PUSH(argv[1]);
+	}
+	else {
+	    PUSH(argv[1]);
+	}
+	cnt++, argv++;
+    }
+
+    /*
+     * Move up the commands for invoking the emulator and adjust eargv
+     * accordingly.
+     */
+
+    while (--eargc_base >= 0) {
+	UNSHIFT(eargv_base[eargc_base]);
+    }
+
+    /*
+     * Invoke Erlang with the collected options.
+     */
+
+    PUSH(NULL);
+
+    return run_erlang(eargv[0], eargv);
+}
+
+static void
+push_words(char* src)
+{
+    char sbuf[MAXPATHLEN];
+    char* dst;
+
+    dst = sbuf;
+    while ((*dst++ = *src++) != '\0') {
+	if (isspace((int)*src)) {
+	    *dst = '\0';
+	    PUSH(strsave(sbuf));
+	    dst = sbuf;
+	    do {
+		src++;
+	    } while (isspace((int)*src));
+	}
+    }
+    if (sbuf[0])
+	PUSH(strsave(sbuf));
+}
+#ifdef __WIN32__
+char *make_commandline(char **argv)
+{
+    static char *buff = NULL;
+    static int siz = 0;
+    int num = 0;
+    char **arg, *p;
+
+    if (*argv == NULL) {
+	return "";
+    }
+    for (arg = argv; *arg != NULL; ++arg) {
+	num += strlen(*arg)+1;
+    }
+    if (!siz) {
+	siz = num;
+	buff = malloc(siz*sizeof(char));
+    } else if (siz < num) {
+	siz = num;
+	buff = realloc(buff,siz*sizeof(char));
+    }
+    p = buff;
+    for (arg = argv; *arg != NULL; ++arg) {
+	strcpy(p,*arg);
+	p+=strlen(*arg);
+	*p++=' ';
+    }
+    *(--p) = '\0';
+
+    if (debug) {
+	printf("Processed commandline:%s\n",buff);
+    }
+    return buff;
+}
+
+int my_spawnvp(char **argv)
+{
+    STARTUPINFO siStartInfo;
+    PROCESS_INFORMATION piProcInfo;
+    DWORD ec;
+
+    memset(&siStartInfo,0,sizeof(STARTUPINFO));
+    siStartInfo.cb = sizeof(STARTUPINFO);
+    siStartInfo.dwFlags = STARTF_USESTDHANDLES;
+    siStartInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
+    siStartInfo.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE);
+    siStartInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE);
+    siStartInfo.wShowWindow = SW_HIDE;
+    siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
+
+
+    if (!CreateProcess(NULL,
+		       make_commandline(argv),
+		       NULL,
+		       NULL,
+		       TRUE,
+		       0,
+		       NULL,
+		       NULL,
+		       &siStartInfo,
+		       &piProcInfo)) {
+	return -1;
+    }
+    CloseHandle(piProcInfo.hThread);
+
+    WaitForSingleObject(piProcInfo.hProcess,INFINITE);
+    if (!GetExitCodeProcess(piProcInfo.hProcess,&ec)) {
+	return 0;
+    }
+    return (int) ec;
+}
+#endif /* __WIN32__ */
+
+
+static int
+run_erlang(char* progname, char** argv)
+{
+#ifdef __WIN32__
+    int status;
+#endif
+
+    if (debug) {
+	int i = 0;
+	while (argv[i] != NULL)
+	    printf(" %s", argv[i++]);
+	printf("\n");
+    }
+
+#ifdef __WIN32__
+    /*
+     * Alas, we must wait here for the program to finish.
+     * Otherwise, the shell from which we were executed will think
+     * we are finished and print a prompt and read keyboard input.
+     */
+
+    status = my_spawnvp(argv)/*_spawnvp(_P_WAIT,progname,argv)*/;
+    if (status == -1) {
+	fprintf(stderr, "ct_run: Error executing '%s': %d", progname,
+		GetLastError());
+    }
+    return status;
+#else
+    execvp(progname, argv);
+    error("Error %d executing \'%s\'.", errno, progname);
+    return 2;
+#endif
+}
+
+static void
+error(char* format, ...)
+{
+    char sbuf[1024];
+    va_list ap;
+
+    va_start(ap, format);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
+    va_end(ap);
+    fprintf(stderr, "ct_run: %s\n", sbuf);
+    exit(1);
+}
+
+static char*
+emalloc(size_t size)
+{
+  char *p = malloc(size);
+  if (p == NULL)
+    error("Insufficient memory");
+  return p;
+}
+
+static char*
+strsave(char* string)
+{
+    char* p = emalloc(strlen(string)+1);
+    strcpy(p, string);
+    return p;
+}
+
+/* Instead of making sure basename exists, we do our own */
+static char *simple_basename(char *path)
+{
+    char *ptr;
+    for (ptr = path; *ptr != '\0'; ++ptr) {
+	if (*ptr == '/' || *ptr == '\\') {
+	    path = ptr + 1;
+	}
+    }
+    return path;
+}
+
+static void print_deprecation_warning(char* progpath)
+{
+  char *basename = simple_basename(progpath);
+  if(strcmp(basename,"run_test") == 0 ||
+       strcmp(basename, "run_test.exe") == 0) {
+    printf("---***---\nDeprecated: run_test is deprecated and will be removed in R16B,\n            please use ct_run instead\n---***---\n");
+  }
+}
+
+static char*
+get_default_emulator(char* progname)
+{
+    char sbuf[MAXPATHLEN];
+    char* s;
+
+    if (strlen(progname) >= sizeof(sbuf))
+        return ERL_NAME;
+
+    strcpy(sbuf, progname);
+    for (s = sbuf+strlen(sbuf); s >= sbuf; s--) {
+	if (IS_DIRSEP(*s)) {
+	    strcpy(s+1, ERL_NAME);
+#ifdef __WIN32__
+	    if (_access(sbuf, 0) != -1) {
+		return strsave(sbuf);
+	    }
+#else
+	    if (access(sbuf, 1) != -1) {
+		return strsave(sbuf);
+	    }
+#endif
+	    break;
+	}
+    }
+    return ERL_NAME;
+}
+
+#ifdef __WIN32__
+static char*
+possibly_quote(char* arg)
+{
+    int mustQuote = NO;
+    int n = 0;
+    char* s;
+    char* narg;
+
+    if (arg == NULL) {
+	return arg;
+    }
+
+    /*
+     * Scan the string to find out if it needs quoting and return
+     * the original argument if not.
+     */
+
+    for (s = arg; *s; s++, n++) {
+	switch(*s) {
+	case ' ':
+	    mustQuote = YES;
+	    continue;
+	case '"':
+	    mustQuote = YES;
+	    n++;
+	    continue;
+	case '\\':
+	    if(s[1] == '"')
+		n++;
+	    continue;
+	default:
+	    continue;
+	}
+    }
+    if (!mustQuote) {
+	return arg;
+    }
+
+    /*
+     * Insert the quotes and put a backslash in front of every quote
+     * inside the string.
+     */
+
+    s = narg = emalloc(n+2+1);
+    for (*s++ = '"'; *arg; arg++, s++) {
+	if (*arg == '"' || (*arg == '\\' && arg[1] == '"')) {
+	    *s++ = '\\';
+	}
+	*s = *arg;
+    }
+    if (s[-1] == '\\') {
+	*s++ ='\\';
+    }
+    *s++ = '"';
+    *s = '\0';
+    return narg;
+}
+#endif /* __WIN32__ */
diff --git a/erts/etc/common/dialyzer.c b/erts/etc/common/dialyzer.c
index 4b4c1124f1..04e9199ef3 100644
--- a/erts/etc/common/dialyzer.c
+++ b/erts/etc/common/dialyzer.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -147,6 +147,9 @@ main(int argc, char** argv)
     env = get_env("DIALYZER_EMULATOR");
     emulator = env ? env : get_default_emulator(argv[0]);
 
+    if (strlen(emulator) >= MAXPATHLEN)
+        error("Value of environment variable DIALYZER_EMULATOR is too large");
+
     /*
      * Allocate the argv vector to be used for arguments to Erlang.
      * Arrange for starting to pushing information in the middle of
@@ -228,7 +231,7 @@ main(int argc, char** argv)
 static void
 push_words(char* src)
 {
-    char sbuf[1024];
+    char sbuf[MAXPATHLEN];
     char* dst;
 
     dst = sbuf;
@@ -360,7 +363,7 @@ error(char* format, ...)
     va_list ap;
     
     va_start(ap, format);
-    vsprintf(sbuf, format, ap);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
     va_end(ap);
     fprintf(stderr, "dialyzer: %s\n", sbuf);
     exit(1);
@@ -389,6 +392,9 @@ get_default_emulator(char* progname)
     char sbuf[MAXPATHLEN];
     char* s;
 
+    if (strlen(progname) >= sizeof(sbuf))
+        return ERL_NAME;
+
     strcpy(sbuf, progname);
     for (s = sbuf+strlen(sbuf); s >= sbuf; s--) {
 	if (IS_DIRSEP(*s)) {
diff --git a/erts/etc/common/erlc.c b/erts/etc/common/erlc.c
index 09aca19e6c..35c360a99d 100644
--- a/erts/etc/common/erlc.c
+++ b/erts/etc/common/erlc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -28,6 +28,7 @@
 #include <winbase.h>
 /* FIXE ME config_win32.h? */
 #define HAVE_STRERROR 1
+#define snprintf _snprintf
 #endif
 
 #include <ctype.h>
@@ -148,10 +149,6 @@ int
 main(int argc, char** argv)
 {
     char cwd[MAXPATHLEN];	/* Current working directory. */
-    char** rpc_eargv;		/* Pointer to the beginning of arguments
-				 * if calling a running Erlang system
-				 * via erl_rpc().
-				 */
     int eargv_size;
     int eargc_base;		/* How many arguments in the base of eargv. */
     char* emulator;
@@ -160,6 +157,9 @@ main(int argc, char** argv)
     env = get_env("ERLC_EMULATOR");
     emulator = env ? env : get_default_emulator(argv[0]);
 
+    if (strlen(emulator) >= MAXPATHLEN)
+        error("Value of environment variable ERLC_EMULATOR is too large");
+
     /*
      * Allocate the argv vector to be used for arguments to Erlang.
      * Arrange for starting to pushing information in the middle of
@@ -170,7 +170,7 @@ main(int argc, char** argv)
      * base of the eargv vector, and move it up later.
      */
 
-    eargv_size = argc*4+100;
+    eargv_size = argc*6+100;
     eargv_base = (char **) emalloc(eargv_size*sizeof(char*));
     eargv = eargv_base;
     eargc = 0;
@@ -189,7 +189,6 @@ main(int argc, char** argv)
     PUSH2("-mode", "minimal");
     PUSH2("-boot", "start_clean");
     PUSH3("-s", "erl_compile", "compile_cmdline");
-    rpc_eargv = eargv+eargc;
 
     /*
      * Push standard arguments to Erlang.
@@ -262,6 +261,95 @@ main(int argc, char** argv)
 	    case 'I':
 		PUSH2("@i", process_opt(&argc, &argv, 0));
 		break;
+	    case 'M':
+		{
+		    char *buf, *key, *val;
+		    size_t buf_len;
+
+		    if (argv[1][2] == '\0') { /* -M */
+			/* Push the following options:
+			 *   o  'makedep'
+			 *   o  {makedep_output, standard_io}
+			 */
+			buf = strsave("makedep");
+			PUSH2("@option", buf);
+
+			key = "makedep_output";
+			val = "standard_io";
+			buf_len = 1 + strlen(key) + 1 + strlen(val) + 1 + 1;
+			buf = emalloc(buf_len);
+			snprintf(buf, buf_len, "{%s,%s}", key, val);
+			PUSH2("@option", buf);
+		    } else if (argv[1][3] == '\0') {
+			switch(argv[1][2]) {
+			case 'D': /* -MD */
+			    /* Push the following options:
+			     *   o  'makedep'
+			     */
+			    buf = strsave("makedep");
+			    PUSH2("@option", buf);
+			    break;
+			case 'F': /* -MF <file> */
+			    /* Push the following options:
+			     *   o  'makedep'
+			     *   o  {makedep_output, <file>}
+			     */
+			    buf = strsave("makedep");
+			    PUSH2("@option", buf);
+
+			    key = "makedep_output";
+			    val = process_opt(&argc, &argv, 1);
+			    buf_len = 1 + strlen(key) + 2 + strlen(val) + 2 + 1;
+			    buf = emalloc(buf_len);
+			    snprintf(buf, buf_len, "{%s,\"%s\"}", key, val);
+			    PUSH2("@option", buf);
+			    break;
+			case 'T': /* -MT <target> */
+			    /* Push the following options:
+			     *   o  {makedep_target, <target>}
+			     */
+			    key = "makedep_target";
+			    val = process_opt(&argc, &argv, 1);
+			    buf_len = 1 + strlen(key) + 2 + strlen(val) + 2 + 1;
+			    buf = emalloc(buf_len);
+			    snprintf(buf, buf_len, "{%s,\"%s\"}", key, val);
+			    PUSH2("@option", buf);
+			    break;
+			case 'Q': /* -MQ <target> */
+			    /* Push the following options:
+			     *   o  {makedep_target, <target>}
+			     *   o  makedep_quote_target
+			     */
+			    key = "makedep_target";
+			    val = process_opt(&argc, &argv, 1);
+			    buf_len = 1 + strlen(key) + 2 + strlen(val) + 2 + 1;
+			    buf = emalloc(buf_len);
+			    snprintf(buf, buf_len, "{%s,\"%s\"}", key, val);
+			    PUSH2("@option", buf);
+
+			    buf = strsave("makedep_quote_target");
+			    PUSH2("@option", buf);
+			    break;
+			case 'G': /* -MG */
+			    /* Push the following options:
+			     *   o  makedep_add_missing
+			     */
+			    buf = strsave("makedep_add_missing");
+			    PUSH2("@option", buf);
+			    break;
+			case 'P': /* -MP */
+			    /* Push the following options:
+			     *   o  makedep_phony
+			     */
+			    buf = strsave("makedep_add_missing");
+			    PUSH2("@option", buf);
+			    break;
+			default:
+			    goto error;
+			}
+		    }
+		}
+		break;
 	    case 'o':
 		PUSH2("@outdir", process_opt(&argc, &argv, 0));
 		break;
@@ -419,7 +507,7 @@ process_opt(int* pArgc, char*** pArgv, int offset)
 static void
 push_words(char* src)
 {
-    char sbuf[1024];
+    char sbuf[MAXPATHLEN];
     char* dst;
 
     dst = sbuf;
@@ -563,6 +651,15 @@ usage(void)
 	{"-hybrid", "compile using hybrid-heap emulator"},
 	{"-help", "shows this help text"},
 	{"-I path", "where to search for include files"},
+	{"-M", "generate a rule for make(1) describing the dependencies"},
+	{"-MF file", "write the dependencies to 'file'"},
+	{"-MT target", "change the target of the rule emitted by dependency "
+		"generation"},
+	{"-MQ target", "same as -MT but quote characters special to make(1)"},
+	{"-MG", "consider missing headers as generated files and add them to "
+		"the dependencies"},
+	{"-MP", "add a phony target for each dependency"},
+	{"-MD", "same as -M -MT file (with default 'file')"},
 	{"-o name", "name output directory or file"},
 	{"-pa path", "add path to the front of Erlang's code path"},
 	{"-pz path", "add path to the end of Erlang's code path"},
@@ -595,7 +692,7 @@ error(char* format, ...)
     va_list ap;
     
     va_start(ap, format);
-    vsprintf(sbuf, format, ap);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
     va_end(ap);
     fprintf(stderr, "erlc: %s\n", sbuf);
     exit(1);
@@ -624,6 +721,9 @@ get_default_emulator(char* progname)
     char sbuf[MAXPATHLEN];
     char* s;
 
+    if (strlen(progname) >= sizeof(sbuf))
+        return ERL_NAME;
+
     strcpy(sbuf, progname);
     for (s = sbuf+strlen(sbuf); s >= sbuf; s--) {
 	if (IS_DIRSEP(*s)) {
diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c
index f79f5cc978..7e04724f5a 100644
--- a/erts/etc/common/erlexec.c
+++ b/erts/etc/common/erlexec.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -120,6 +120,7 @@ static char *plusM_other_switches[] = {
 static char *pluss_val_switches[] = {
     "bt",
     "ct",
+    "wt",
     "ss",
     NULL
 };
@@ -131,6 +132,18 @@ static char *plush_val_switches[] = {
     NULL
 };
 
+/* +r arguments with values */
+static char *plusr_val_switches[] = {
+    "g",
+    NULL
+};
+
+/* +z arguments with values */
+static char *plusz_val_switches[] = {
+    "dbbl",
+    NULL
+};
+
 
 /*
  * Define sleep(seconds) in terms of Sleep() on Windows.
@@ -302,7 +315,7 @@ free_env_val(char *value)
 }
 
 /*
- * Add the arcitecture suffix to the program name if needed,
+ * Add the architecture suffix to the program name if needed,
  * except on Windows, where we insert it just before ".DLL".
  */
 static char*
@@ -381,6 +394,7 @@ int main(int argc, char **argv)
     int print_args_exit = 0;
     int print_qouted_cmd_exit = 0;
     erts_cpu_info_t *cpuinfo = NULL;
+    char* emu_name;
 
 #ifdef __WIN32__
     this_module_handle = module;
@@ -553,7 +567,8 @@ int main(int argc, char **argv)
 	    usage("+MYm");
     }
     emu = add_extra_suffixes(emu, emu_type);
-    sprintf(tmpStr, "%s" DIRSEP "%s" BINARY_EXT, bindir, emu);
+    emu_name = strsave(emu);
+    erts_snprintf(tmpStr, sizeof(tmpStr), "%s" DIRSEP "%s" BINARY_EXT, bindir, emu);
     emu = strsave(tmpStr);
 
     add_Eargs(emu);		/* Will be argv[0] -- necessary! */
@@ -564,12 +579,12 @@ int main(int argc, char **argv)
 
     s = get_env("PATH");
     if (!s) {
-	sprintf(tmpStr, "%s" PATHSEP "%s" DIRSEP "bin", bindir, rootdir);
+	erts_snprintf(tmpStr, sizeof(tmpStr), "%s" PATHSEP "%s" DIRSEP "bin", bindir, rootdir);
     } else if (strstr(s, bindir) == NULL) {
-	sprintf(tmpStr, "%s" PATHSEP "%s" DIRSEP "bin" PATHSEP "%s", bindir, 
+	erts_snprintf(tmpStr, sizeof(tmpStr), "%s" PATHSEP "%s" DIRSEP "bin" PATHSEP "%s", bindir,
 		rootdir, s);
     } else {
-	sprintf(tmpStr, "%s", s);
+	erts_snprintf(tmpStr, sizeof(tmpStr), "%s", s);
     }
     free_env_val(s);
     set_env("PATH", tmpStr);
@@ -669,6 +684,9 @@ int main(int argc, char **argv)
 			verbose = 1;
 		    } else if (strcmp(argv[i], "-emu_args_exit") == 0) {
 			print_args_exit = 1;
+		    } else if (strcmp(argv[i], "-emu_name_exit") == 0) {
+			printf("%s\n", emu_name);
+			exit(0);
 		    } else if (strcmp(argv[i], "-emu_qouted_cmd_exit") == 0) {
 			print_qouted_cmd_exit = 1;
 		    } else if (strcmp(argv[i], "-env") == 0) { /* -env VARNAME VARVALUE */
@@ -707,7 +725,7 @@ int main(int argc, char **argv)
 			error("-man not supported on Windows");
 #else
 			argv[i] = "man";
-			sprintf(tmpStr, "%s/man", rootdir);
+			erts_snprintf(tmpStr, sizeof(tmpStr), "%s/man", rootdir);
 			set_env("MANPATH", tmpStr);
 			execvp("man", argv+i);
 			error("Could not execute the 'man' command.");
@@ -872,6 +890,21 @@ int main(int argc, char **argv)
 			  i++;
 		      }
 		      break;
+		  case 'r':
+		      if (!is_one_of_strings(&argv[i][2],
+					     plusr_val_switches))
+			  goto the_default;
+		      else {
+			  if (i+1 >= argc
+			      || argv[i+1][0] == '-'
+			      || argv[i+1][0] == '+')
+			      usage(argv[i]);
+			  argv[i][0] = '-';
+			  add_Eargs(argv[i]);
+			  add_Eargs(argv[i+1]);
+			  i++;
+		      }
+		      break;
 		  case 's':
 		      if (!is_one_of_strings(&argv[i][2],
 					     pluss_val_switches))
@@ -887,6 +920,20 @@ int main(int argc, char **argv)
 			  i++;
 		      }
 		      break;
+		  case 'z':
+		      if (!is_one_of_strings(&argv[i][2], plusz_val_switches)) {
+			  goto the_default;
+		      } else {
+			  if (i+1 >= argc
+			      || argv[i+1][0] == '-'
+			      || argv[i+1][0] == '+')
+			      usage(argv[i]);
+			  argv[i][0] = '-';
+			  add_Eargs(argv[i]);
+			  add_Eargs(argv[i+1]);
+			  i++;
+		      }
+		      break;
 		  default:
 		  the_default:
 		    argv[i][0] = '-'; /* Change +option to -option. */
@@ -1069,11 +1116,12 @@ usage_aux(void)
 	  "[-hybrid] "
 #endif
 	  "[-make] [-man [manopts] MANPAGE] [-x] [-emu_args] "
-	  "[-args_file FILENAME] "
-	  "[+A THREADS] [+a SIZE] [+B[c|d|i]] [+c] [+h HEAP_SIZE_OPTION] [+K BOOLEAN] "
+	  "[-args_file FILENAME] [+A THREADS] [+a SIZE] [+B[c|d|i]] [+c] "
+	  "[+h HEAP_SIZE_OPTION] [+K BOOLEAN] "
 	  "[+l] [+M<SUBSWITCH> <ARGUMENT>] [+P MAX_PROCS] [+R COMPAT_REL] "
-	  "[+r] [+s SCHEDULER_OPTION] [+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] [+W<i|w>] "
-	  "[args ...]\n");
+	  "[+r] [+rg READER_GROUPS_LIMIT] [+s SCHEDULER_OPTION] "
+	  "[+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] "
+	  "[+W<i|w>] [+z MISC_OPTION] [args ...]\n");
   exit(1);
 }
 
@@ -1122,10 +1170,10 @@ start_epmd(char *epmd)
     if (!epmd) {
 	epmd = epmd_cmd;
 #ifdef __WIN32__
-	sprintf(epmd_cmd, "%s" DIRSEP "epmd", bindir);
+	erts_snprintf(epmd_cmd, sizeof(epmd_cmd), "%s" DIRSEP "epmd", bindir);
 	arg1 = "-daemon";
 #else
-	sprintf(epmd_cmd, "%s" DIRSEP "epmd -daemon", bindir);
+	erts_snprintf(epmd_cmd, sizeof(epmd_cmd), "%s" DIRSEP "epmd -daemon", bindir);
 #endif
     } 
 #ifdef __WIN32__
@@ -1201,7 +1249,7 @@ void error(char* format, ...)
     va_list ap;
 
     va_start(ap, format);
-    vsprintf(sbuf, format, ap);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
     va_end(ap);
     fprintf(stderr, "erlexec: %s\n", sbuf);
     exit(1);
@@ -1281,14 +1329,14 @@ static void get_start_erl_data(char *file)
     if (env)
 	reldir = strsave(env);
     else {
-	sprintf(tmpbuffer, "%s/releases", rootdir);
+	erts_snprintf(tmpbuffer, sizeof(tmpbuffer), "%s/releases", rootdir);
 	reldir = strsave(tmpbuffer);
     }
     free_env_val(env);
     if (file == NULL)
-       sprintf(start_erl_data, "%s/start_erl.data", reldir);
+       erts_snprintf(start_erl_data, sizeof(start_erl_data), "%s/start_erl.data", reldir);
     else
-       sprintf(start_erl_data, "%s", file);
+       erts_snprintf(start_erl_data, sizeof(start_erl_data), "%s", file);
     fp = _open(start_erl_data, _O_RDONLY );
     if( fp == -1 )
 	error( "open failed on %s",start_erl_data );
@@ -1318,16 +1366,16 @@ static void get_start_erl_data(char *file)
     }
 	
     bindir = emalloc(512);
-    sprintf(bindir,"%s/erts-%s/bin",rootdir,tmpbuffer);
+    erts_snprintf(bindir,512,"%s/erts-%s/bin",rootdir,tmpbuffer);
     /* BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin */
     tprogname = progname;
     progname = emalloc(strlen(tprogname) + 20);
-    sprintf(progname,"%s -start_erl",tprogname);
+    erts_snprintf(progname,strlen(tprogname) + 20,"%s -start_erl",tprogname);
 
     boot_script = emalloc(512);
     config_script = emalloc(512);
-    sprintf(boot_script, "%s/%s/start", reldir, otpstring);
-    sprintf(config_script, "%s/%s/sys", reldir, otpstring);
+    erts_snprintf(boot_script, 512, "%s/%s/start", reldir, otpstring);
+    erts_snprintf(config_script, 512, "%s/%s/sys", reldir, otpstring);
        
 }
 
@@ -1335,7 +1383,7 @@ static void get_start_erl_data(char *file)
 static char *replace_filename(char *path, char *new_base) 
 {
     int plen = strlen(path);
-    char *res = malloc((plen+strlen(new_base)+1)*sizeof(char));
+    char *res = emalloc((plen+strlen(new_base)+1)*sizeof(char));
     char *p;
 
     strcpy(res,path);
@@ -1350,7 +1398,7 @@ static char *path_massage(char *long_path)
 {
      char *p;
 
-     p = malloc(MAX_PATH+1);
+     p = emalloc(MAX_PATH+1);
      strcpy(p, long_path);
      GetShortPathName(p, p, MAX_PATH);
      return p;
@@ -1486,7 +1534,8 @@ get_parameters(int argc, char** argv)
 	/* Determine bindir from absolute path to executable */
 	char *p;
 	char buffer[PATH_MAX];
-	strcpy(buffer, argv[0]);
+	strncpy(buffer, argv[0], sizeof(buffer));
+	buffer[sizeof(buffer)-1] = '\0';
 	
 	for (p = buffer+strlen(buffer)-1 ; p >= buffer && *p != '/'; --p)
 	    ;
@@ -1499,7 +1548,8 @@ get_parameters(int argc, char** argv)
 	/* Determine rootdir from absolute path to bindir */
 	char *p;
 	char buffer[PATH_MAX];
-	strcpy(buffer, bindir);
+	strncpy(buffer, bindir, sizeof(buffer));
+	buffer[sizeof(buffer)-1] = '\0';
 	
 	for (p = buffer+strlen(buffer)-1; p >= buffer && *p != '/'; --p)
 	    ;
@@ -1925,6 +1975,11 @@ initial_argv_massage(int *argc, char ***argv)
      */
 
     vix = 0;
+
+    av = build_args_from_env("ERL_" OTP_SYSTEM_VERSION "_FLAGS");
+    if (av)
+	avv[vix++].argv = av;
+
     av = build_args_from_env("ERL_AFLAGS");
     if (av)
 	avv[vix++].argv = av;
@@ -1939,10 +1994,6 @@ initial_argv_massage(int *argc, char ***argv)
     if (av)
 	avv[vix++].argv = av;
 
-    av = build_args_from_env("ERL_" OTP_SYSTEM_VERSION "_FLAGS");
-    if (av)
-	avv[vix++].argv = av;
-
     av = build_args_from_env("ERL_ZFLAGS");
     if (av)
 	avv[vix++].argv = av;
diff --git a/erts/etc/common/escript.c b/erts/etc/common/escript.c
index 1bc5eb7651..6ed79c91e3 100644
--- a/erts/etc/common/escript.c
+++ b/erts/etc/common/escript.c
@@ -151,6 +151,9 @@ find_prog(char *origpath)
     char relpath[PMAX];
     char abspath[PMAX];
 
+    if (strlen(origpath) >= sizeof(relpath))
+        error("Path too long");
+
     strcpy(relpath, origpath);
 
     if (strstr(relpath, DIRSEPSTR) == NULL) {
@@ -180,19 +183,21 @@ find_prog(char *origpath)
                 end = strstr(beg, PATHSEPSTR);
                 if (end != NULL) {
                     sz = end - beg;
-                    strncpy(dir, beg, sz);
-                    dir[sz] = '\0';
                 } else {
                     sz = strlen(beg);
-                    strcpy(dir, beg);
                     look_for_sep = FALSE;
                 }
+                if (sz >= sizeof(dir)) {
+                    beg = end + 1;
+                    continue;
+                }
+                strncpy(dir, beg, sz);
+                dir[sz] = '\0';
                 beg = end + 1;
 
 #ifdef __WIN32__
-		strcpy(wildcard, dir);
-		strcat(wildcard, DIRSEPSTR);
-		strcat(wildcard, relpath); /* basename */
+		erts_snprintf(wildcard, sizeof(wildcard), "%s" DIRSEPSTR "%s",
+            dir, relpath /* basename */);
 		dir_handle = FindFirstFile(wildcard, &find_data);
 		if (dir_handle == INVALID_HANDLE_VALUE) {
 		    /* Try next directory in path */
@@ -217,9 +222,8 @@ find_prog(char *origpath)
 
                         if (strcmp(origpath, dirp->d_name) == 0) {
                             /* Wow we found the executable. */
-                            strcpy(relpath, dir);
-                            strcat(relpath, DIRSEPSTR);
-                            strcat(relpath, dirp->d_name);
+                            erts_snprintf(relpath, sizeof(relpath), "%s" DIRSEPSTR "%s",
+                                dir, dirp->d_name);
                             closedir(dp);
 			    look_for_sep = FALSE;
                             break;
@@ -291,7 +295,7 @@ append_shebang_args(char* scriptname)
 		    
 		    /* Find end of arg */
 		    end = beg;
-		    while (end && end[0] != ' ') {
+		    while (end && end < (linebuf+LINEBUFSZ-1) && end[0] != ' ') {
 			if (end[0] == '\n') {
 			    newline = TRUE;
 			    end[0]= '\0';
@@ -335,13 +339,16 @@ main(int argc, char** argv)
 	emulator = get_default_emulator(argv[0]);
     }
 
+    if (strlen(emulator) >= PMAX)
+        error("Value of environment variable ESCRIPT_EMULATOR is too large");
+
     /*
      * Allocate the argv vector to be used for arguments to Erlang.
      * Arrange for starting to pushing information in the middle of
      * the array, to allow easy addition of commands in the beginning.
      */
 
-    eargv_size = argc*4+1000;
+    eargv_size = argc*4+1000+LINEBUFSZ/2;
     eargv_base = (char **) emalloc(eargv_size*sizeof(char*));
     eargv = eargv_base;
     eargc = 0;
@@ -387,7 +394,8 @@ main(int argc, char** argv)
 	if (argc <= 1) {
 	    error("Missing filename\n");
 	}
-	strcpy(scriptname, argv[1]);
+	strncpy(scriptname, argv[1], sizeof(scriptname));
+	scriptname[sizeof(scriptname)-1] = '\0';
 	argc--;
 	argv++;
     } else {
@@ -395,16 +403,17 @@ main(int argc, char** argv)
 	int len;
 #endif
 	absname = find_prog(argv[0]);
-	strcpy(scriptname, absname);
-	efree(absname);
 #ifdef __WIN32__
-	len = strlen(scriptname);
-	if (len >= 4 && _stricmp(scriptname+len-4, ".exe") == 0) {
-	    scriptname[len-4] = '\0';
+	len = strlen(absname);
+	if (len >= 4 && _stricmp(absname+len-4, ".exe") == 0) {
+	    absname[len-4] = '\0';
 	}
 #endif
 
-	strcat(scriptname, ".escript");
+	erts_snprintf(scriptname, sizeof(scriptname), "%s.escript",
+        absname);
+	efree(absname);
+
     }
 
     /*
@@ -455,7 +464,7 @@ main(int argc, char** argv)
 static void
 push_words(char* src)
 {
-    char sbuf[1024];
+    char sbuf[PMAX];
     char* dst;
 
     dst = sbuf;
@@ -584,7 +593,7 @@ error(char* format, ...)
     va_list ap;
     
     va_start(ap, format);
-    vsprintf(sbuf, format, ap);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
     va_end(ap);
     fprintf(stderr, "escript: %s\n", sbuf);
     exit(1);
@@ -619,6 +628,9 @@ get_default_emulator(char* progname)
     char sbuf[MAXPATHLEN];
     char* s;
 
+    if (strlen(progname) >= sizeof(sbuf))
+        return ERL_NAME;
+
     strcpy(sbuf, progname);
     for (s = sbuf+strlen(sbuf); s >= sbuf; s--) {
 	if (IS_DIRSEP(*s)) {
diff --git a/erts/etc/common/heart.c b/erts/etc/common/heart.c
index 4f738947b7..7a5746e630 100644
--- a/erts/etc/common/heart.c
+++ b/erts/etc/common/heart.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -375,7 +375,8 @@ main(int argc, char **argv)
     _setmode(erlin_fd,_O_BINARY);
     _setmode(erlout_fd,_O_BINARY);
 #endif
-    strcpy(program_name, argv[0]);
+    strncpy(program_name, argv[0], sizeof(program_name));
+    program_name[sizeof(program_name)-1] = '\0';
     notify_ack(erlout_fd);
     cmd[0] = '\0';
     do_terminate(message_loop(erlin_fd,erlout_fd));
@@ -726,12 +727,16 @@ static int
 heart_cmd_reply(int fd, char *s)
 {
   struct msg m;
-  int len = strlen(s) + 1;	/* Include \0 */
+  int len = strlen(s);
 
-  /* FIXME if s >= MSG_BODY_SIZE error */
+  /* if s >= MSG_BODY_SIZE, return a write
+   * failure immediately.
+   */
+  if (len >= sizeof(m.fill))
+      return -1;
 
   m.op = HEART_CMD;
-  m.len = htons(len + 2);	/* Include Op */
+  m.len = htons(len + 1);	/* Include Op */
   strcpy((char*)m.fill, s);
 
   return write_message(fd, &m);
diff --git a/erts/etc/common/inet_gethost.c b/erts/etc/common/inet_gethost.c
index ff16ee02c4..77bfd5e2bc 100644
--- a/erts/etc/common/inet_gethost.c
+++ b/erts/etc/common/inet_gethost.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -52,20 +52,21 @@
 #  include "config.h"
 #endif
 
+#include "erl_printf.h"
+
 #ifdef WIN32
 
 #define WIN32_LEAN_AND_MEAN
 #include <winsock2.h>
 #include <windows.h>
+#include <ws2tcpip.h>
 #include <process.h>
 #include <stdio.h>
 #include <stdlib.h>
 
 /* These are not used even if they would exist which they should not */
-#undef HAVE_GETADDRINFO
 #undef HAVE_GETIPNODEBYNAME
 #undef HAVE_GETHOSTBYNAME2
-#undef HAVE_GETNAMEINFO
 #undef HAVE_GETIPNODEBYADDR
 
 #else /* Unix */
@@ -1296,7 +1297,7 @@ static int read_request(AddrByte **buff, size_t *buff_size)
     }
 
     if (siz > *buff_size) {
-	if (buff_size == 0) {
+	if (*buff_size == 0) {
 	    *buff = ALLOC((*buff_size = siz));
 	} else {
 	    *buff = REALLOC(*buff, (*buff_size = siz));
@@ -1759,7 +1760,7 @@ static int worker_loop(void)
 		struct addrinfo hints;
 		
 		memset(&hints, 0, sizeof(hints));
-		hints.ai_flags = (AI_CANONNAME|AI_V4MAPPED|AI_ADDRCONFIG);
+		hints.ai_flags = AI_CANONNAME;
 		hints.ai_socktype = SOCK_STREAM;
 		hints.ai_family = AF_INET6;
 		DEBUGF(5, ("Starting getaddrinfo(%s, ...)", data));
@@ -2552,7 +2553,7 @@ static void debugf(char *format, ...)
     sprintf(buff,"%s[%d] (DEBUG):",program_name,(int) getpid());
 #endif
     ptr = buff + strlen(buff);
-    vsprintf(ptr,format,ap);
+    erts_vsnprintf(ptr,sizeof(buff)-strlen(buff)-2,format,ap);
     strcat(ptr,"\r\n");
 #ifdef WIN32
     if (debug_console_allocated != INVALID_HANDLE_VALUE) {
@@ -2574,7 +2575,7 @@ static void warning(char *format, ...)
     va_start(ap,format);
     sprintf(buff,"%s[%d]: WARNING:",program_name, (int) getpid());
     ptr = buff + strlen(buff);
-    vsprintf(ptr,format,ap);
+    erts_vsnprintf(ptr,sizeof(buff)-strlen(buff)-2,format,ap);
     strcat(ptr,"\r\n");
 #ifdef WIN32
     {
@@ -2596,7 +2597,7 @@ static void fatal(char *format, ...)
     va_start(ap,format);
     sprintf(buff,"%s[%d]: FATAL ERROR:",program_name, (int) getpid());
     ptr = buff + strlen(buff);
-    vsprintf(ptr,format,ap);
+    erts_vsnprintf(ptr,sizeof(buff)-strlen(buff)-2,format,ap);
     strcat(ptr,"\r\n");
 #ifdef WIN32
     {
diff --git a/erts/etc/common/typer.c b/erts/etc/common/typer.c
index c2567cb8b4..c95959d52d 100644
--- a/erts/etc/common/typer.c
+++ b/erts/etc/common/typer.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -175,7 +175,7 @@ main(int argc, char** argv)
 static void
 push_words(char* src)
 {
-    char sbuf[1024];
+    char sbuf[MAXPATHLEN];
     char* dst;
 
     dst = sbuf;
@@ -307,7 +307,7 @@ error(char* format, ...)
     va_list ap;
     
     va_start(ap, format);
-    vsprintf(sbuf, format, ap);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
     va_end(ap);
     fprintf(stderr, "typer: %s\n", sbuf);
     exit(1);
@@ -336,6 +336,9 @@ get_default_emulator(char* progname)
     char sbuf[MAXPATHLEN];
     char* s;
 
+    if (strlen(progname) >= sizeof(sbuf))
+        return ERL_NAME;
+
     strcpy(sbuf, progname);
     for (s = sbuf+strlen(sbuf); s >= sbuf; s--) {
 	if (IS_DIRSEP(*s)) {
diff --git a/erts/etc/unix/Install.src b/erts/etc/unix/Install.src
index 83f9690782..8f40c43874 100644
--- a/erts/etc/unix/Install.src
+++ b/erts/etc/unix/Install.src
@@ -89,8 +89,12 @@ cp -p $ERL_ROOT/erts-%I_VSN%/bin/erl .
 cp -p $ERL_ROOT/erts-%I_VSN%/bin/erlc .
 cp -p $ERL_ROOT/erts-%I_VSN%/bin/dialyzer .
 cp -p $ERL_ROOT/erts-%I_VSN%/bin/typer .
+cp -p $ERL_ROOT/erts-%I_VSN%/bin/ct_run .
 cp -p $ERL_ROOT/erts-%I_VSN%/bin/escript .
 
+# Remove in R16B
+ln -s ct_run run_test
+
 #
 # Set a soft link to epmd
 # This should not be done for an embedded system!
diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src
index f81ef6b0fe..0b2d6512ea 100644
--- a/erts/etc/unix/cerl.src
+++ b/erts/etc/unix/cerl.src
@@ -1,20 +1,20 @@
 #!/bin/sh
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2003-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2003-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 #
@@ -62,12 +62,21 @@ cxargs_add() {
     done
 }
 
+eeargs=
+eeargs_add() {
+    while [ $# -gt 0 ]; do
+	cargs="$cargs $1"
+	eeargs="$eeargs $1"
+	shift
+    done
+}
+
 core=
 
 GDB=
 GDBBP=
+GDBARGS=
 TYPE=
-EMU_TYPE=
 debug=
 run_valgrind=no
 
@@ -126,34 +135,37 @@ while [ $# -gt 0 ]; do
 	    ;;
 	"-smp")
 	    shift
-	    cargs="$cargs -smp"
-	    EMU_TYPE=.smp
+	    if [ $# -le 0 ]; then
+		eeargs_add -smp
+	    else
+		case $1 in
+		    disable)
+			shift
+			eeargs_add -smpdisable
+			;;
+		    enable)
+			shift
+			eeargs_add -smp
+			;;
+		    *)
+			eeargs_add -smp
+		esac
+	    fi
+	    ;;
+	"-smpdisable")
+	    shift
+	    eeargs_add -smpdisable
 	    ;;
 	"-lcnt")
 	    shift
 	    cargs="$cargs -lcnt"
 	    TYPE=.lcnt
 	    ;;
-	"-frag")
-	    shift
-	    cargs="$cargs -frag"
-	    EMU_TYPE=.frag
-	    ;;
-	"-smp_frag")
-	    shift
-	    cargs="$cargs -smp_frag"
-	    EMU_TYPE=.smp_frag
-	    ;;
 	"-gprof")
 	    shift
 	    cargs="$cargs -gprof"
 	    TYPE=.gprof
 	    ;;
-	"-hybrid")
-	    shift
-	    cargs="$cargs -hybrid"
-	    EMU_TYPE=.hybrid
-	    ;;
 	"-debug")
 	    shift
 	    cargs="$cargs -debug"
@@ -179,11 +191,6 @@ while [ $# -gt 0 ]; do
 #	    shift
 #	    GDB=xxgdb
 #	    ;;
-	"-shared")
-	    shift
-	    cargs="$cargs -shared"
-	    TYPE=.shared
-	    ;;
 	"-purify")
 	    shift
 	    cargs="$cargs -purify"
@@ -221,23 +228,38 @@ PATH=$BINDIR:$ROOTDIR/bin:$PATH
 EXEC=$BINDIR/erlexec
 
 PROGNAME="$PROGNAME $cargs"
-EMU=$EMU$TYPE$EMU_TYPE
+EMU="$EMU$TYPE"
+EMU_NAME=`$EXEC -emu_name_exit $eeargs`
+
 if [ $run_valgrind != yes ]; then
     xargs="$xargs -pz $PRELOADED --"
 fi
 if [ "x$GDB" = "x" ]; then
     if [ $run_valgrind = yes ]; then
+	valversion=`valgrind --version`
+	valmajor=`echo $valversion | sed 's,[a-z]*\-\([0-9]*\).*,\1,'`
+        valminor=`echo $valversion | sed 's,[a-z]*\-[0-9]*.\([0-9]*\).*,\1,'`
 	emu_xargs=`echo $xargs | sed "s|+|-|g"`
-	if [ "x$VALGRIND_LOG_DIR" = "x" ]; then
-	    valgrind_log=
-	else
-	    valgrind_log="--log-file=$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU.log"
-	fi
 	if [ "x$VALGRIND_LOG_XML" = "x" ]; then
 	    valgrind_xml=
+	    log_file_prefix="--log-file="
 	else
 	    export VALGRIND_LOG_XML
 	    valgrind_xml="--xml=yes"
+	    if [ $valmajor -gt 2 -a $valminor -gt 4 ]; then
+		log_file_prefix="--xml-file="
+	    else
+		log_file_prefix="--log-file="
+	    fi
+	fi
+	if [ "x$VALGRIND_LOG_DIR" = "x" ]; then
+	    valgrind_log=
+	else
+	    if [ $valmajor -gt 2 -a $valminor -gt 4 ]; then
+		valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log.$$"
+	    else
+		valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log"
+	    fi
 	fi
 	if [ "x$VALGRIND_MISC_FLAGS" = "x" ]; then
 	    valgrind_misc_flags=
@@ -249,9 +271,9 @@ if [ "x$GDB" = "x" ]; then
 	early_beam_args=`echo $beam_args | sed "s|^\(.*-progname\).*$|\1|g"`
 	late_beam_args=`echo $beam_args | sed "s|^$pre_beam_args.*\(-- -home.*\)$|\1|g"`
 	
-	exec valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU $emu_xargs $early_beam_args "$PROGNAME" $late_beam_args -pz $PRELOADED
+	exec valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $emu_xargs $early_beam_args "$PROGNAME" $late_beam_args -pz $PRELOADED
     else
-	exec $EXEC $xargs ${1+"$@"}
+	exec $EXEC $eeargs $xargs ${1+"$@"}
     fi
 else
     if [ "x$EMACS" = "x" ]; then
@@ -277,9 +299,14 @@ else
 	    ;;
     esac
 
+    # Set annotation level for gdb in emacs 22 and higher.
+    emacs_major=`$EMACS --version | head -1 | sed 's,^[^0-9]*\([0-9]*\).*,\1,g'`
+    if [ '!' -z "$emacs_major" -a $emacs_major -gt 21 ]; then
+	GDBARGS="--annotate=3 "
+    fi
     gdbcmd="$gdbcmd $GDBBP \
             (insert-string \"source $ROOTDIR/erts/etc/unix/etp-commands\") \
             (comint-send-input)"
     # Fire up gdb in emacs...
-    exec $EMACS --eval "(progn (gdb \"gdb $EMU\") $gdbcmd)"
+    exec $EMACS --eval "(progn (gdb \"gdb $GDBARGS$EMU_NAME\") $gdbcmd)"
 fi
diff --git a/erts/etc/unix/format_man_pages b/erts/etc/unix/format_man_pages
index 2c4f6eee4f..93dcdcd8fa 100644
--- a/erts/etc/unix/format_man_pages
+++ b/erts/etc/unix/format_man_pages
@@ -3,7 +3,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1996-2009. All Rights Reserved.
+# Copyright Ericsson AB 1996-2010. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -59,34 +59,21 @@ esac
 # Create the 'cat' directories (probably not needed)
 #
 
-cd $ERL_ROOT
+cd $ERL_ROOT/man
 
-if [ ! -d man/cat1 ]
-then
-    mkdir man/cat1
-fi
+for d in 0 1 2 3 4 5 6 7 8 9
+do
+  if [ ! -d cat$d ]
+      then
+      mkdir cat$d
+  fi
 
-if [ ! -d man/cat3 ]
-then
-    mkdir man/cat3
-fi
-
-if [ ! -d man/cat4 ]
-then
-    mkdir man/cat4
-fi
-
-if [ ! -d man/cat6 ]
-then
-    mkdir man/cat6
-fi
+done
 
 #
 # Cleanup old formatting
 #
 
-cd $ERL_ROOT/man
-
 rm -f whatis windex
 
 # Remove old cat files
diff --git a/erts/etc/unix/run_erl.c b/erts/etc/unix/run_erl.c
index 4bb148df98..8db8e09bee 100644
--- a/erts/etc/unix/run_erl.c
+++ b/erts/etc/unix/run_erl.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -75,6 +75,9 @@
 #ifdef HAVE_SYS_IOCTL_H
 #  include <sys/ioctl.h>
 #endif
+#if defined(__sun) && defined(__SVR4)
+#  include <stropts.h>
+#endif
 
 #include "run_erl.h"
 #include "safe_string.h"    /* sn_printf, strn_cpy, strn_cat, etc */
@@ -218,6 +221,8 @@ int main(int argc, char **argv)
   char *p, *ptyslave=NULL;
   int i = 1;
   int off_argv;
+  int calculated_pipename = 0;
+  int highest_pipe_num = 0;
 
   program_name = argv[0];
 
@@ -295,10 +300,10 @@ int main(int argc, char **argv)
   if(*pipename && pipename[strlen(pipename)-1] == '/') {
     /* The user wishes us to find a unique pipe name in the specified */
     /* directory */
-    int highest_pipe_num = 0;
     DIR *dirp;
     struct dirent *direntp;
 
+    calculated_pipename = 1;
     dirp = opendir(pipename);
     if(!dirp) {
       ERRNO_ERR1(LOG_ERR,"Can't access pipe directory '%s'.", pipename);
@@ -319,28 +324,37 @@ int main(int argc, char **argv)
 	      PIPE_STUBNAME, highest_pipe_num+1);
   } /* if */
 
-  /* write FIFO - is read FIFO for `to_erl' program */
-  strn_cpy(fifo1, sizeof(fifo1), pipename);
-  strn_cat(fifo1, sizeof(fifo1), ".r");
-  if (create_fifo(fifo1, PERM) < 0) {
-    ERRNO_ERR1(LOG_ERR,"Cannot create FIFO %s for writing.", fifo1);
-    exit(1);
-  }
-
-  /* read FIFO - is write FIFO for `to_erl' program */
-  strn_cpy(fifo2, sizeof(fifo2), pipename);
-  strn_cat(fifo2, sizeof(fifo2), ".w");
-
-  /* Check that nobody is running run_erl already */
-  if ((fd = open (fifo2, O_WRONLY|DONT_BLOCK_PLEASE, 0)) >= 0) {
-    /* Open as client succeeded -- run_erl is already running! */
-    fprintf(stderr, "Erlang already running on pipe %s.\n", pipename);
-    close(fd);
-    exit(1);
-  }
-  if (create_fifo(fifo2, PERM) < 0) { 
-    ERRNO_ERR1(LOG_ERR,"Cannot create FIFO %s for reading.", fifo2);
-    exit(1);
+  for(;;) {
+      /* write FIFO - is read FIFO for `to_erl' program */
+      strn_cpy(fifo1, sizeof(fifo1), pipename);
+      strn_cat(fifo1, sizeof(fifo1), ".r");
+      if (create_fifo(fifo1, PERM) < 0) {
+	  ERRNO_ERR1(LOG_ERR,"Cannot create FIFO %s for writing.", fifo1);
+	  exit(1);
+      }
+      
+      /* read FIFO - is write FIFO for `to_erl' program */
+      strn_cpy(fifo2, sizeof(fifo2), pipename);
+      strn_cat(fifo2, sizeof(fifo2), ".w");
+      
+      /* Check that nobody is running run_erl already */
+      if ((fd = open (fifo2, O_WRONLY|DONT_BLOCK_PLEASE, 0)) >= 0) {
+	  /* Open as client succeeded -- run_erl is already running! */
+	  close(fd);
+	  if (calculated_pipename) {
+	      ++highest_pipe_num;
+	      strn_catf(pipename, sizeof(pipename), "%s.%d",
+			PIPE_STUBNAME, highest_pipe_num+1);
+	      continue;
+	  } 
+	  fprintf(stderr, "Erlang already running on pipe %s.\n", pipename);
+	  exit(1);
+      }
+      if (create_fifo(fifo2, PERM) < 0) { 
+	  ERRNO_ERR1(LOG_ERR,"Cannot create FIFO %s for reading.", fifo2);
+	  exit(1);
+      }
+      break;
   }
 
   /*
@@ -864,8 +878,12 @@ static int open_pty_master(char **ptyslave)
 
 /* Use the posix_openpt if working, as this guarantees creation of the 
    slave device properly. */
-#ifdef HAVE_WORKING_POSIX_OPENPT
+#if defined(HAVE_WORKING_POSIX_OPENPT) || (defined(__sun) && defined(__SVR4))
+#  ifdef HAVE_WORKING_POSIX_OPENPT
   if ((mfd = posix_openpt(O_RDWR)) >= 0) {
+#  elif defined(__sun) && defined(__SVR4)
+  if ((mfd = open("/dev/ptmx", O_RDWR)) >= 0) {
+#  endif
       if ((*ptyslave = ptsname(mfd)) != NULL &&
 	  grantpt(mfd) == 0 && 
 	  unlockpt(mfd) == 0) {
@@ -973,16 +991,53 @@ static int open_pty_master(char **ptyslave)
 static int open_pty_slave(char *name)
 {
   int sfd;
-#ifdef DEBUG
   struct termios tty_rmode;
-#endif
 
   if ((sfd = open(name, O_RDWR, 0)) < 0) {
     return -1;
   }
 
+#if defined(__sun) && defined(__SVR4)
+  /* Load the necessary STREAMS modules for Solaris */
+  if ((ioctl(sfd, I_FIND, "ldterm")) < 0) {
+    ERROR0(LOG_ERR, "Failed to find ldterm STREAMS module");
+    return -1;
+  }
+  if (ioctl(sfd, I_PUSH, "ptem") < 0) {
+    ERROR0(LOG_ERR, "Failed to push ptem STREAMS module");
+    return -1;
+  }
+  if (ioctl(sfd, I_PUSH, "ldterm") < 0) {
+    ERROR0(LOG_ERR, "Failed to push ldterm STREAMS module");
+    return -1;
+  }
+  if (ioctl(sfd, I_PUSH, "ttcompat") < 0) {
+    ERROR0(LOG_ERR, "Failed to push ttcompat STREAMS module");
+    return -1;
+  }
+#endif
+
+  if (getenv("RUN_ERL_DISABLE_FLOWCNTRL")) {
+    if (tcgetattr(sfd, &tty_rmode) < 0) {
+      fprintf(stderr, "Cannot get terminal's current mode\n");
+      exit(-1);
+    }
+
+    tty_rmode.c_iflag &= ~IXOFF;
+    if (tcsetattr(sfd, TCSANOW, &tty_rmode) < 0) {
+      fprintf(stderr, "Cannot disable terminal's flow control on input\n");
+      exit(-1);
+    }
+
+    tty_rmode.c_iflag &= ~IXON;
+    if (tcsetattr(sfd, TCSANOW, &tty_rmode) < 0) {
+      fprintf(stderr, "Cannot disable terminal's flow control on output\n");
+      exit(-1);
+    }
+  }
+
 #ifdef DEBUG
-  if (tcgetattr(sfd, &tty_rmode) , 0) {
+  if (tcgetattr(sfd, &tty_rmode) < 0) {
     fprintf(stderr, "Cannot get terminals current mode\n");
     exit(-1);
   }
diff --git a/erts/etc/unix/to_erl.c b/erts/etc/unix/to_erl.c
index 886b301997..11fa3ff4cc 100644
--- a/erts/etc/unix/to_erl.c
+++ b/erts/etc/unix/to_erl.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -125,7 +125,7 @@ static void usage(char *pname)
 int main(int argc, char **argv)
 {
     char  FIFO1[FILENAME_MAX], FIFO2[FILENAME_MAX];
-    int i, len, wfd, rfd, result = 0;
+    int i, len, wfd, rfd;
     fd_set readfds;
     char buf[BUFSIZ];
     char pipename[FILENAME_MAX];
@@ -367,7 +367,6 @@ int main(int argc, char **argv)
 	    }
 	    else {
 		fprintf(stderr, "Error in select.\n");
-		result = -1;
 		break;
 	    }
 	}
@@ -398,7 +397,6 @@ int main(int argc, char **argv)
 		close(wfd);
 		if (len < 0) {
 		    fprintf(stderr, "Error in reading from stdin.\n");
-		    result = -1;
 		} else {
 		    fprintf(stderr, "[EOF]\n\r");
 		}
@@ -420,7 +418,6 @@ int main(int argc, char **argv)
 		fprintf(stderr, "Error in writing to FIFO.\n");
 		close(rfd);
 		close(wfd);
-		result = -1;
 		break;
 	    }
 	    STATUS("\" OK\r\n");
@@ -447,7 +444,6 @@ int main(int argc, char **argv)
 		close(wfd);
 		if (len < 0) {
 		    fprintf(stderr, "Error in reading from FIFO.\n");
-		    result = -1;
 		} else
 		    fprintf(stderr, "[End]\n\r");
 		break;
@@ -456,7 +452,6 @@ int main(int argc, char **argv)
 		    if ((len=version_handshake(buf,len,wfd)) < 0) {
 			close(rfd);
 			close(wfd);
-			result = -1;
 			break;
 		    }
 		    if (protocol_ver >= 1) {
@@ -475,7 +470,6 @@ int main(int argc, char **argv)
 		    fprintf(stderr, "Error in writing to terminal.\n");
 		    close(rfd);
 		    close(wfd);
-		    result = -1;
 		    break;
 		}
 		STATUS("\" OK\r\n");
diff --git a/erts/etc/win32/Install.c b/erts/etc/win32/Install.c
index 4a559cd8a2..6e60512f6d 100644
--- a/erts/etc/win32/Install.c
+++ b/erts/etc/win32/Install.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -45,8 +45,8 @@ int main(int argc, char **argv)
     InitSection *ini_section;
     HANDLE module = GetModuleHandle(NULL);
     char *binaries[] = { "erl.exe", "werl.exe", "erlc.exe",
-			     "dialyzer.exe", "typer.exe",
-			     "escript.exe", NULL };
+			 "dialyzer.exe", "typer.exe",
+			 "escript.exe", "ct_run.exe", NULL };
     char *scripts[] = { "start_clean.boot", "start_sasl.boot", NULL };
     char fromname[MAX_PATH];
     char toname[MAX_PATH];
@@ -172,6 +172,20 @@ int main(int argc, char **argv)
 	}
     }
 
+    // Remove in R16B
+    sprintf(fromname,"%s\\%s",bin_dir,"ct_run.exe");
+    sprintf(toname,"%s\\%s",bin_dir,"run_test.exe");
+    if (GetFileAttributes(fromname) == 0xFFFFFFFF) {
+      fprintf(stderr,"Could not find file %s\n",
+	      fromname);
+      exit(1);
+    }
+    if (!CopyFile(fromname,toname,FALSE)) {
+      fprintf(stderr,"Could not copy file %s to %s\n",
+	      fromname,toname);
+      fprintf(stderr,"Continuing installation anyway...\n");
+    }
+    
     for (i = 0; scripts[i] != NULL; ++i) {
 	sprintf(fromname,"%s\\%s",release_dir,scripts[i]);
 	sprintf(toname,"%s\\%s",bin_dir,scripts[i]);
diff --git a/erts/etc/win32/cygwin_tools/vc/ld.sh b/erts/etc/win32/cygwin_tools/vc/ld.sh
index ac39bf871c..406c63ffee 100755
--- a/erts/etc/win32/cygwin_tools/vc/ld.sh
+++ b/erts/etc/win32/cygwin_tools/vc/ld.sh
@@ -3,7 +3,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2002-2009. All Rights Reserved.
+# Copyright Ericsson AB 2002-2010. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -53,7 +53,7 @@ while test -n "$1" ; do
 	    STDLIB_FORCED=true;
 	    STDLIB=LIBCMTD.LIB;; 
 	-lsocket)
-	    DEFAULT_LIBRARIES="$DEFAULT_LIBRARIES WS2_32.LIB";;
+	    DEFAULT_LIBRARIES="$DEFAULT_LIBRARIES WS2_32.LIB IPHLPAPI.LIB";;
 	-l*)
 	    y=`echo $x | sed 's,^-l\(.*\),\1,g'`;
 	    MPATH=`cygpath -m $y`;
@@ -158,7 +158,7 @@ else
 fi    
 	    
 p=$$
-CMD="$linktype -nologo -incremental:no $CMD $STDLIB $DEFAULT_LIBRARIES"
+CMD="$linktype -nologo -incremental:no -largeaddressaware $CMD $STDLIB $DEFAULT_LIBRARIES"
 if [ "X$LD_SH_DEBUG_LOG" != "X" ]; then
     echo ld.sh "$SAVE" >>$LD_SH_DEBUG_LOG
     echo link.exe $CMD >>$LD_SH_DEBUG_LOG
@@ -167,6 +167,10 @@ eval link.exe "$CMD"  >/tmp/link.exe.${p}.1 2>/tmp/link.exe.${p}.2
 RES=$?
 CMANIFEST=`cygpath $MANIFEST`
 if [ "$RES" = "0" -a -f "$CMANIFEST" ]; then
+    # Add stuff to manifest to turn off "virtualization"
+    sed -n -i '1h;1!H;${;g;s,<trustInfo.*</trustInfo>.,,g;p;}' $CMANIFEST
+    sed -i "s/<\/assembly>/ <ms_asmv2:trustInfo xmlns:ms_asmv2=\"urn:schemas-microsoft-com:asm.v2\">\n  <ms_asmv2:security>\n   <ms_asmv2:requestedPrivileges>\n    <ms_asmv2:requestedExecutionLevel level=\"AsInvoker\" uiAccess=\"false\"\/>\n   <\/ms_asmv2:requestedPrivileges>\n  <\/ms_asmv2:security>\n <\/ms_asmv2:trustInfo>\n<\/assembly>/" $CMANIFEST
+
     eval mt.exe -nologo -manifest "$MANIFEST" -outputresource:"$OUTPUTRES" >>/tmp/link.exe.${p}.1 2>>/tmp/link.exe.${p}.2
     RES=$?
     if [ "$RES" != "0" ]; then
diff --git a/erts/etc/win32/cygwin_tools/vc/rc.sh b/erts/etc/win32/cygwin_tools/vc/rc.sh
index 6a6921c49e..054c672e64 100755
--- a/erts/etc/win32/cygwin_tools/vc/rc.sh
+++ b/erts/etc/win32/cygwin_tools/vc/rc.sh
@@ -2,20 +2,20 @@
 # set -x
 # 
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2002-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2002-2010. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 # 
 # Save the command line for debug outputs
@@ -80,7 +80,7 @@ if [ "X$RC_SH_DEBUG_LOG" != "X" ]; then
 fi
 eval $RCC "$CMD"  >/tmp/rc.exe.${p}.1 2>/tmp/rc.exe.${p}.2
 RES=$?
-tail +2 /tmp/rc.exe.${p}.2 >&2
+tail -n +2 /tmp/rc.exe.${p}.2 >&2
 cat /tmp/rc.exe.${p}.1
 rm -f /tmp/rc.exe.${p}.2 /tmp/rc.exe.${p}.1
 exit $RES
diff --git a/erts/etc/win32/erlsrv/erlsrv_service.c b/erts/etc/win32/erlsrv/erlsrv_service.c
index a58ee862c5..242e2905a9 100644
--- a/erts/etc/win32/erlsrv/erlsrv_service.c
+++ b/erts/etc/win32/erlsrv/erlsrv_service.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -523,7 +523,7 @@ static BOOL start_a_service(ServerInfo *srvi){
 		    srvi->keys[WorkDir].data.bytes : NULL,
 		    &start,
 		    &(srvi->info))){
-    sprintf(errbuff,"Could not start erlang service"
+    sprintf(errbuff,"Could not start erlang service "
 	    "with commandline \"%s\".",
 	    service_name,
 	    execbuff
@@ -924,7 +924,7 @@ static VOID WINAPI service_main_loop(DWORD argc, char **argv){
     } else {
       DWORD ecode = NO_ERROR;
       if(success_wait == NO_SUCCESS_WAIT){
-	log_warning("Erlang machine volountarily stopped. "
+	log_warning("Erlang machine voluntarily stopped. "
 		    "The service is not restarted as OnFail "
 		    "is set to ignore.");
       } else {
diff --git a/erts/etc/win32/nsis/Makefile b/erts/etc/win32/nsis/Makefile
index ebb3ad9a96..ae2343b420 100644
--- a/erts/etc/win32/nsis/Makefile
+++ b/erts/etc/win32/nsis/Makefile
@@ -1,7 +1,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2003-2009. All Rights Reserved.
+# Copyright Ericsson AB 2003-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -45,6 +45,7 @@ WTARGET_DIR=$(shell (cygpath -d $(TARGET_DIR) 2>/dev/null || cygpath -d $(TARGET
 
 REDIST_FILE=$(shell (sh ./find_redist.sh || echo ""))
 REDIST_DLL_VERSION=$(shell (sh ./dll_version_helper.sh || echo ""))
+REDIST_DLL_NAME=$(shell (sh ./dll_version_helper.sh -n || echo ""))
 
 release_spec:
 	@NSIS_VER=`makensis /hdrinfo | head -1 | awk '{print $$2}'`; \
@@ -73,6 +74,7 @@ release_spec:
 		cp $(REDIST_FILE) $(RELEASE_PATH)/vcredist_x86.exe;\
 		echo '!define HAVE_REDIST_FILE 1'  >> $(VERSION_HEADER); \
 		echo '!define REDIST_DLL_VERSION "$(REDIST_DLL_VERSION)"' >> $(VERSION_HEADER);\
+		echo '!define REDIST_DLL_NAME "$(REDIST_DLL_NAME)"' >> $(VERSION_HEADER);\
 	  fi;\
 	  if [ -f $(RELEASE_PATH)/docs/doc/index.html ];\
 	  then \
diff --git a/erts/etc/win32/nsis/dll_version_helper.sh b/erts/etc/win32/nsis/dll_version_helper.sh
index e0047dea8b..eecd4a72b5 100755
--- a/erts/etc/win32/nsis/dll_version_helper.sh
+++ b/erts/etc/win32/nsis/dll_version_helper.sh
@@ -2,7 +2,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2007-2009. All Rights Reserved.
+# Copyright Ericsson AB 2007-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -26,6 +26,7 @@
 # exit 0
 
 cat > hello.c <<EOF
+#include <windows.h>
 #include <stdio.h>
 
 int main(void)
@@ -35,15 +36,81 @@ int main(void)
 }
 
 EOF
-cl /MD hello.c > /dev/null 2>&1
+cl /MD hello.c  > /dev/null 2>&1
 if [ '!' -f hello.exe.manifest ]; then
-    echo "This compiler does not generate manifest files - OK if using mingw" >&2
-    exit 0
+    # Gah - VC 2010 changes the way it handles DLL's and manifests... Again...
+    # need another way of getting the version
+    DLLNAME=`dumpbin.exe /imports hello.exe | egrep MSVCR.*dll`
+    DLLNAME=`echo $DLLNAME`
+    cat > helper.c <<EOF
+#include <windows.h>
+#include <stdio.h>
+
+#define REQ_MODULE "$DLLNAME"
+
+int main(void)
+{
+  DWORD dummy;
+  DWORD versize;
+  int i,n;
+  unsigned char *versinfo;
+  char buff[100];
+
+  char *vs_verinfo;
+  unsigned int vs_ver_size;
+  
+  struct LANGANDCODEPAGE {
+    WORD language;
+    WORD codepage;
+  } *translate;
+
+  unsigned int tr_size;
+  
+  if (!(versize = GetFileVersionInfoSize(REQ_MODULE,&dummy))) {
+    fprintf(stderr,"No version info size in %s!\n",REQ_MODULE);
+    exit(1);
+  }
+  versinfo=malloc(versize);
+  if (!GetFileVersionInfo(REQ_MODULE,dummy,versize,versinfo)) {
+    fprintf(stderr,"No version info in %s!\n",REQ_MODULE);
+    exit(2);
+  }
+  if (!VerQueryValue(versinfo,"\\\\VarFileInfo\\\\Translation",&translate,&tr_size)) {
+    fprintf(stderr,"No translation info in %s!\n",REQ_MODULE);
+    exit(3);
+  }
+  n = tr_size/sizeof(translate);
+  for(i=0; i < n; ++i) {
+    sprintf(buff,"\\\\StringFileInfo\\\\%04x%04x\\\\FileVersion",
+	    translate[i].language,translate[i].codepage);
+    if (VerQueryValue(versinfo,buff,&vs_verinfo,&vs_ver_size)) {
+      printf("%s\n",(char *) vs_verinfo);
+      return 0;
+    }
+  } 
+  fprintf(stderr,"Failed to find file version of %s\n",REQ_MODULE);
+  return 0;
+}
+EOF
+    cl /MD helper.c version.lib > /dev/null 2>&1
+    if [ '!' -f helper.exe ]; then
+	echo "Failed to build helper program." >&2
+	exit 1
+    fi
+    NAME=$DLLNAME
+    VERSION=`./helper`
+else
+    VERSION=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*version=.\([0-9\.]*\).*,\1,g' | grep -v '<'`
+    NAME=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*name=.[A-Za-z\.]*\([0-9]*\).*,msvcr\1.dll,g' | grep -v '<'`
+fi
+#rm -f hello.c hello.obj hello.exe hello.exe.manifest helper.c helper.obj helper.exe helper.exe.manifest
+if [ "$1" = "-n" ]; then
+    ASKEDFOR=$NAME
+else
+    ASKEDFOR=$VERSION
 fi
-VERSION=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*version=.\([0-9\.]*\).*,\1,g' | grep -v '<'`
-rm -f hello.c hello.obj hello.exe hello.exe.manifest
-if [ -z "$VERSION" ]; then
+if [ -z "$ASKEDFOR" ]; then
     exit 1
 fi
-echo $VERSION
+echo $ASKEDFOR
 exit 0
diff --git a/erts/etc/win32/nsis/erlang20.nsi b/erts/etc/win32/nsis/erlang20.nsi
index 43e5d91604..941e8e6f5d 100644
--- a/erts/etc/win32/nsis/erlang20.nsi
+++ b/erts/etc/win32/nsis/erlang20.nsi
@@ -311,23 +311,23 @@ FunctionEnd
 

 Function .onInit

    SectionGetFlags 0 $MYTEMP 

-;   MessageBox MB_YESNO "Found $SYSDIR\msvcr80.dll" IDYES FoundLbl

-   IfFileExists $SYSDIR\msvcr80.dll MaybeFoundInSystemLbl

+   ;MessageBox MB_YESNO "Found $SYSDIR\${REDIST_DLL_NAME}" IDYES FoundLbl

+   IfFileExists $SYSDIR\${REDIST_DLL_NAME} MaybeFoundInSystemLbl

    SearchSxsLbl:	

         FindFirst $0 $1 $WINDIR\WinSxS\x86*

         LoopLbl:

 	    StrCmp $1 "" NotFoundLbl

-	    IfFileExists $WINDIR\WinSxS\$1\msvcr80.dll MaybeFoundInSxsLbl

+	    IfFileExists $WINDIR\WinSxS\$1\${REDIST_DLL_NAME} MaybeFoundInSxsLbl

 	    FindNext $0 $1

 	    Goto LoopLbl

         MaybeFoundInSxsLbl:

-	    GetDllVersion $WINDIR\WinSxS\$1\msvcr80.dll $R0 $R1

+	    GetDllVersion $WINDIR\WinSxS\$1\${REDIST_DLL_NAME} $R0 $R1

 	    Call DllVersionGoodEnough

 	    FindNext $0 $1

 	    IntCmp 2 $R0 LoopLbl

 	    Goto FoundLbl  

    MaybeFoundInSystemLbl:

-	GetDllVersion $SYSDIR\msvcr80.dll $R0 $R1

+	GetDllVersion $SYSDIR\${REDIST_DLL_NAME} $R0 $R1

 	Call DllVersionGoodEnough

 	IntCmp 2 $R0 SearchSxSLbl  

    FoundLbl:

diff --git a/erts/etc/win32/nsis/find_redist.sh b/erts/etc/win32/nsis/find_redist.sh
index c5572839c5..bc4260ecba 100755
--- a/erts/etc/win32/nsis/find_redist.sh
+++ b/erts/etc/win32/nsis/find_redist.sh
@@ -2,7 +2,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2007-2009. All Rights Reserved.
+# Copyright Ericsson AB 2007-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -107,16 +107,81 @@ for x in cl bin vc; do
     fi
     BPATH="$NBPATH"
 done
-#echo $BPATH
-for x in sdk v2.0 bootstrapper packages vcredist_x86 vcredist_x86.exe; do
-    #echo "x=$x"
-    #echo "BPATH=$BPATH"
-    NBPATH=`add_path_element $x "$BPATH"`
-    if [ "$NBPATH" = "$BPATH" ]; then
-	echo "Failed to locate vcredist_x86.exe because directory structure was unexpected" >&2
-	exit 3
+BPATH_LIST=$BPATH
+
+# rc.exe is in the Microsoft SDK directory of VS2008
+RCPATH=`lookup_prog_in_path rc`
+fail=false
+if [ '!' -z "$RCPATH" ]; then 
+    BPATH=$RCPATH
+    allow_fail=false
+    for x in rc bin @ANY v6.0A v7.0A v7.1; do
+	if [ $x = @ANY ]; then
+	    allow_fail=true
+	else
+	    NBPATH=`remove_path_element $x "$BPATH"`
+	    if [ $allow_fail = false -a "$NBPATH" = "$BPATH" ]; then
+		fail=true
+		break;
+	    fi
+	    BPATH="$NBPATH"
+	fi
+    done
+    if [ $fail = false ]; then
+	BPATH_LIST="$BPATH_LIST $BPATH"
+    fi
+fi
+
+# Frantic search through two roots with different 
+# version directories. We want to be very specific about the
+# directory structures as we wouldnt want to find the wrong 
+# redistributables...
+
+#echo $BPATH_LIST
+for BP in $BPATH_LIST; do
+    for verdir in "sdk v2.0" "sdk v3.5" "v6.0A" "v7.0"  "v7.0A" "v7.1"; do
+	BPATH=$BP
+	fail=false
+	allow_fail=false
+	for x in $verdir @ANY bootstrapper packages vcredist_x86 Redist VC @ALL vcredist_x86.exe; do
+	    #echo "x=$x"
+	    #echo "BPATH=$BPATH"
+	    #echo "allow_fail=$allow_fail"
+	    if [ $x = @ANY ]; then
+		allow_fail=true
+	    elif [ $x = @ALL ]; then
+		allow_fail=false
+	    else
+		NBPATH=`add_path_element $x "$BPATH"`
+		if [ $allow_fail = false -a "$NBPATH" = "$BPATH" ]; then
+		    fail=true
+		    break;
+		fi
+		BPATH="$NBPATH"
+	    fi
+	done
+	if [ $fail = false ]; then
+	    break;
+	fi
+    done
+    if [ $fail = false ]; then
+	echo $BPATH
+	exit 0
     fi
-    BPATH="$NBPATH"
 done
-echo $BPATH
-exit 0
\ No newline at end of file
+
+# shortcut for locating vcredist_x86.exe is to put it into $ERL_TOP
+if [ -f $ERL_TOP/vcredist_x86.exe ]; then
+    echo $ERL_TOP/vcredist_x86.exe
+    exit 0
+fi
+
+# or $ERL_TOP/.. to share across multiple builds
+if [ -f $ERL_TOP/../vcredist_x86.exe ]; then
+    echo $ERL_TOP/../vcredist_x86.exe
+    exit 0
+fi
+
+echo "Failed to locate vcredist_x86.exe because directory structure was unexpected" >&2
+exit 3
+
diff --git a/erts/example/next_perm.cc b/erts/example/next_perm.cc
index ee81cb0404..d7c8f1ad97 100644
--- a/erts/example/next_perm.cc
+++ b/erts/example/next_perm.cc
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -120,7 +120,7 @@ static void ready_async(ErlDrvData drv_data, ErlDrvThreadData async_data)
     ErlDrvPort port = reinterpret_cast<ErlDrvPort>(drv_data);
     our_async_data* d = reinterpret_cast<our_async_data*>(async_data);
     int n = d->data.size(), result_n = n*2 + 5;
-    ErlDrvTermData* result = new ErlDrvTermData[result_n], * rp = result;
+    ErlDrvTermData *result = new ErlDrvTermData[result_n], *rp = result;
     *rp++ = ERL_DRV_PORT;
     *rp++ = driver_mk_port(port);
     for (vector<int>::iterator i = d->data.begin();
diff --git a/erts/include/erl_int_sizes_config.h.in b/erts/include/erl_int_sizes_config.h.in
index ef49995732..4b8a8e1b98 100644
--- a/erts/include/erl_int_sizes_config.h.in
+++ b/erts/include/erl_int_sizes_config.h.in
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -31,3 +31,6 @@
 
 /* The number of bytes in a long long.  */
 #undef SIZEOF_LONG_LONG
+
+/* Define if building a halfword-heap 64bit emulator (needed for NIF's) */
+#undef HALFWORD_HEAP_EMULATOR
diff --git a/erts/include/internal/erl_misc_utils.h b/erts/include/internal/erl_misc_utils.h
index 82e9ba3798..507e1726f4 100644
--- a/erts/include/internal/erl_misc_utils.h
+++ b/erts/include/internal/erl_misc_utils.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -34,7 +34,7 @@ typedef struct {
 
 erts_cpu_info_t *erts_cpu_info_create(void);
 void erts_cpu_info_destroy(erts_cpu_info_t *cpuinfo);
-void erts_cpu_info_update(erts_cpu_info_t *cpuinfo);
+int erts_cpu_info_update(erts_cpu_info_t *cpuinfo);
 int erts_get_cpu_configured(erts_cpu_info_t *cpuinfo);
 int erts_get_cpu_online(erts_cpu_info_t *cpuinfo);
 int erts_get_cpu_available(erts_cpu_info_t *cpuinfo);
@@ -50,4 +50,9 @@ int erts_unbind_from_cpu_str(char *str);
 
 int erts_milli_sleep(long);
 
+#ifdef __WIN32__
+int erts_map_win_error_to_errno(DWORD win_error);
+int erts_get_last_win_errno(void);
+#endif
+
 #endif /* #ifndef ERL_MISC_UTILS_H_ */
diff --git a/erts/include/internal/erl_printf_format.h b/erts/include/internal/erl_printf_format.h
index 45818079ea..064c4a5c09 100644
--- a/erts/include/internal/erl_printf_format.h
+++ b/erts/include/internal/erl_printf_format.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -40,7 +40,7 @@ extern int erts_printf_ulong(fmtfn_t, void*, char, int, int, unsigned long);
 extern int erts_printf_slong(fmtfn_t, void*, char, int, int, signed long);
 extern int erts_printf_double(fmtfn_t, void *, char, int, int, double);
 
-extern int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long);
+extern int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long, unsigned long*);
 
 #endif
 
diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h
new file mode 100644
index 0000000000..1caf4d0567
--- /dev/null
+++ b/erts/include/internal/ethr_atomics.h
@@ -0,0 +1,726 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: The ethread atomic API
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_ATOMIC_H__
+#define ETHR_ATOMIC_H__
+
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+#  define ETHR_NEED_ATOMIC_PROTOTYPES__
+#endif
+
+#ifndef ETHR_HAVE_NATIVE_ATOMICS
+/*
+ * No native atomic implementation available. :(
+ * Use fallback...
+ */
+typedef ethr_sint32_t ethr_atomic32_t;
+typedef ethr_sint_t ethr_atomic_t;
+#else
+/*
+ * Map ethread native atomics to ethread API atomics.
+ *
+ * We do at least have a native atomic implementation that
+ * can handle integers of a size larger than or equal to
+ * the size of pointers.
+ */
+
+/* -- Pointer size atomics -- */
+
+#undef ETHR_NAINT_T__
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_NATMC_ADDR_FUNC__
+#if ETHR_SIZEOF_PTR == 8
+#  if defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#    define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr
+typedef ethr_native_atomic64_t ethr_atomic_t;
+#    define ETHR_NAINT_T__ ethr_sint64_t
+#    define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#  else
+#    error "Missing native atomic implementation"
+#  endif
+#elif ETHR_SIZEOF_PTR == 4
+#  define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr
+#  ifdef ETHR_HAVE_NATIVE_ATOMIC32
+typedef ethr_native_atomic32_t ethr_atomic_t;
+#    define ETHR_NAINT_T__ ethr_sint32_t
+#    define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#  elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
+typedef ethr_native_atomic64_t ethr_atomic_t;
+#    define ETHR_NATMC_T__ ethr_native_atomic64_t
+#    define ETHR_NAINT_T__ ethr_sint64_t
+#    define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#  else
+#    error "Missing native atomic implementation"
+#  endif
+#endif
+
+/* -- 32-bit atomics -- */
+
+#undef ETHR_NAINT32_T__
+#undef ETHR_NATMC32_FUNC__
+#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
+typedef ethr_native_atomic32_t ethr_atomic32_t;
+#  define ETHR_NAINT32_T__ ethr_sint32_t
+#  define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X
+#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
+typedef ethr_native_atomic64_t ethr_atomic32_t;
+#  define ETHR_NAINT32_T__ ethr_sint64_t
+#  define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X
+#else
+#  error "Missing native atomic implementation"
+#endif
+
+#endif
+
+#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__
+ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *);
+void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t);
+void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_read(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *);
+void ethr_atomic_inc(ethr_atomic_t *);
+void ethr_atomic_dec(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t);
+void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
+ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *);
+void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t);
+void ethr_atomic_dec_relb(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
+ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
+
+ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *);
+void ethr_atomic32_init(ethr_atomic32_t *, ethr_sint32_t);
+void ethr_atomic32_set(ethr_atomic32_t *, ethr_sint32_t);
+ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *);
+ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *);
+ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *);
+void ethr_atomic32_inc(ethr_atomic32_t *);
+void ethr_atomic32_dec(ethr_atomic32_t *);
+ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *, ethr_sint32_t);
+void ethr_atomic32_add(ethr_atomic32_t *, ethr_sint32_t);
+ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *, ethr_sint32_t);
+ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *, ethr_sint32_t);
+ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *, ethr_sint32_t);
+ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *,
+				    ethr_sint32_t,
+				    ethr_sint32_t);
+ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *);
+ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *);
+void ethr_atomic32_set_relb(ethr_atomic32_t *, ethr_sint32_t);
+void ethr_atomic32_dec_relb(ethr_atomic32_t *);
+ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *);
+ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *,
+					 ethr_sint32_t,
+					 ethr_sint32_t);
+ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *,
+					 ethr_sint32_t,
+					 ethr_sint32_t);
+#endif
+
+int ethr_init_atomics(void);
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#ifndef ETHR_HAVE_NATIVE_ATOMICS
+/*
+ * Fallbacks for atomics used in absence of a native implementation.
+ */
+
+#define ETHR_ATOMIC_ADDR_BITS 10
+#define ETHR_ATOMIC_ADDR_SHIFT 6
+
+typedef struct {
+    union {
+	ethr_spinlock_t lck;
+	char buf[ETHR_CACHE_LINE_SIZE];
+    } u;
+} ethr_atomic_protection_t;
+
+extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
+
+#define ETHR_ATOMIC_PTR2LCK__(PTR) \
+(&ethr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \
+			& ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck)
+
+
+#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS)			\
+do {									\
+    ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP));		\
+    ethr_spin_lock(slp__);						\
+    { EXPS; }								\
+    ethr_spin_unlock(slp__);						\
+} while (0)
+
+#endif
+
+/*
+ * --- Pointer size atomics ---------------------------------------------------
+ */
+
+static ETHR_INLINE ethr_sint_t *
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_addr)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t *) ETHR_NATMC_ADDR_FUNC__(var);
+#else
+    return (ethr_sint_t *) var;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) i);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) i);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+    return res;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) incr);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr);
+#endif
+}   
+    
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) i);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var);
+    return res;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC_FUNC__(inc)(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC_FUNC__(dec)(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var,
+					      ethr_sint_t mask)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var,
+						       (ETHR_NAINT_T__) mask);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask);
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var,
+					     ethr_sint_t mask)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var,
+						      (ETHR_NAINT_T__) mask);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask);
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var,
+						 (ETHR_NAINT_T__) new);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new);
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
+					    ethr_sint_t new,
+                                            ethr_sint_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var,
+						    (ETHR_NAINT_T__) new,
+						    (ETHR_NAINT_T__) exp);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var,
+    {
+	res = *var;
+	if (__builtin_expect(res == exp, 1))
+	    *var = new;
+    });
+    return res;
+#endif
+}
+
+/*
+ * Important memory barrier requirements.
+ *
+ * The following atomic operations *must* supply a memory barrier of
+ * at least the type specified by its suffix:
+ *    _acqb = acquire barrier
+ *    _relb = release barrier
+ */
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var,
+					     ethr_sint_t val)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+#else
+    ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC_FUNC__(dec_relb)(var);
+#else
+    ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var,
+						 ethr_sint_t new,
+						 ethr_sint_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var,
+							 (ETHR_NAINT_T__) new,
+							 (ETHR_NAINT_T__) exp);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var,
+						 ethr_sint_t new,
+						 ethr_sint_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var,
+							 (ETHR_NAINT_T__) new,
+							 (ETHR_NAINT_T__) exp);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
+#endif
+}
+
+/*
+ * --- 32-bit atomics ---------------------------------------------------------
+ */
+
+static ETHR_INLINE ethr_sint32_t *
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_addr)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic32_addr(var);
+#else
+    return (ethr_sint32_t *) var;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_init)(ethr_atomic32_t *var,
+					   ethr_sint32_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) i);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(ethr_atomic32_t *var, ethr_sint32_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) i);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+#else
+    ethr_sint32_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+    return res;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add)(ethr_atomic32_t *var,
+					  ethr_sint32_t incr)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) incr);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr);
+#endif
+}   
+    
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add_read)(ethr_atomic32_t *var,
+					       ethr_sint32_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t)
+	ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) i);
+#else
+    ethr_sint32_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var);
+    return res;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC32_FUNC__(inc)(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC32_FUNC__(dec)(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+#else
+    ethr_sint32_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+#else
+    ethr_sint32_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_band)(ethr_atomic32_t *var,
+						ethr_sint32_t mask)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t)
+	ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) mask);
+#else
+    ethr_sint32_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask);
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_bor)(ethr_atomic32_t *var,
+					       ethr_sint32_t mask)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return
+	(ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var,
+						       (ETHR_NAINT32_T__) mask);
+#else
+    ethr_sint32_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask);
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_xchg)(ethr_atomic32_t *var,
+					   ethr_sint32_t new)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var,
+						     (ETHR_NAINT32_T__) new);
+#else
+    ethr_sint32_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new);
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(ethr_atomic32_t *var,
+					    ethr_sint32_t new,
+                                            ethr_sint32_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var,
+							(ETHR_NAINT32_T__) new,
+							(ETHR_NAINT32_T__) exp);
+#else
+    ethr_sint32_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var,
+    {
+	res = *var;
+	if (__builtin_expect(res == exp, 1))
+	    *var = new;
+    });
+    return res;
+#endif
+}
+
+/*
+ * Important memory barrier requirements.
+ *
+ * The following atomic operations *must* supply a memory barrier of
+ * at least the type specified by its suffix:
+ *    _acqb = acquire barrier
+ *    _relb = release barrier
+ */
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_acqb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read_acqb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(var);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set_relb)(ethr_atomic32_t *var,
+					       ethr_sint32_t val)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+#else
+    ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(var, val);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_relb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ETHR_NATMC32_FUNC__(dec_relb)(var);
+#else
+    ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read_relb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_acqb)(ethr_atomic32_t *var,
+						   ethr_sint32_t new,
+						   ethr_sint32_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t)
+	ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var,
+					  (ETHR_NAINT32_T__) new,
+					  (ETHR_NAINT32_T__) exp);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp);
+#endif
+}
+
+static ETHR_INLINE ethr_sint32_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_relb)(ethr_atomic32_t *var,
+						   ethr_sint32_t new,
+						   ethr_sint32_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return (ethr_sint32_t)
+	ETHR_NATMC32_FUNC__(cmpxchg_relb)(var,
+					  (ETHR_NAINT32_T__) new,
+					  (ETHR_NAINT32_T__) exp);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp);
+#endif
+}
+
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#undef ETHR_NAINT_T__
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_NATMC_ADDR_FUNC__
+
+#undef ETHR_NAINT32_T__
+#undef ETHR_NATMC32_FUNC__
+
+#endif
diff --git a/erts/include/internal/ethr_internal.h b/erts/include/internal/ethr_internal.h
new file mode 100644
index 0000000000..e9c3daf783
--- /dev/null
+++ b/erts/include/internal/ethr_internal.h
@@ -0,0 +1,67 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Internal ethread exports
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_INTERNAL_H__
+#define ETHR_INTERNAL_H__
+
+#include "erl_misc_utils.h"
+
+extern ethr_memory_allocators ethr_mem__;
+extern erts_cpu_info_t *ethr_cpu_info__;
+extern size_t ethr_pagesize__;
+extern size_t ethr_min_stack_size__; /* kilo words */
+extern size_t ethr_max_stack_size__; /* kilo words */
+extern int ethr_not_completely_inited__;
+extern int ethr_not_inited__;
+
+extern void *(*ethr_thr_prepare_func__)(void);
+extern void (*ethr_thr_parent_func__)(void *);
+extern void (*ethr_thr_child_func__)(void *);
+
+#define ETHR_PAGE_ALIGN(SZ) \
+  (((((size_t) (SZ)) - 1)/ethr_pagesize__ + 1)*ethr_pagesize__)
+#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1)
+#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024)
+
+#undef ETHR_STACK_GUARD_SIZE 
+#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE
+#  define ETHR_STACK_GUARD_SIZE (ethr_pagesize__)
+#endif
+
+/* implemented in lib_src/<thr-lib>/ethread.c */
+int ethr_set_tse__(ethr_ts_event *tsep);
+ethr_ts_event *ethr_get_tse__(void);
+ETHR_PROTO_NORETURN__ ethr_abort__(void);
+#ifdef ETHR_WIN32_THREADS
+int ethr_win_get_errno__(void);
+#endif
+
+/* implemented in lib_src/common/ethread_aux.c */
+int ethr_init_common__(ethr_init_data *id);
+int ethr_late_init_common__(ethr_late_init_data *lid);
+void ethr_run_exit_handlers__(void);
+void ethr_ts_event_destructor__(void *vtsep);
+
+
+#endif
diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h
new file mode 100644
index 0000000000..fadaf1e2a4
--- /dev/null
+++ b/erts/include/internal/ethr_mutex.h
@@ -0,0 +1,674 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Mutex, rwmutex and condition variable implementation
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_MUTEX_H__
+#define ETHR_MUTEX_H__
+
+#define ETHR_RWMUTEX_INITIALIZED 	0x99999999
+#define ETHR_MUTEX_INITIALIZED		0x77777777
+#define ETHR_COND_INITIALIZED		0x55555555
+
+#if 0
+#  define ETHR_MTX_HARD_DEBUG
+#endif
+
+#if 0
+#  define ETHR_MTX_CHK_EXCL
+#if 1
+#    define ETHR_MTX_CHK_NON_EXCL
+#endif
+#endif
+
+#ifdef ETHR_MTX_HARD_DEBUG
+#  ifdef __GNUC__
+#    warning ETHR_MTX_HARD_DEBUG
+#  endif
+/*#  define ETHR_MTX_HARD_DEBUG_LFS*/
+/*#  define ETHR_MTX_HARD_DEBUG_FENCE*/
+/*#  define ETHR_MTX_HARD_DEBUG_Q*/
+#  define ETHR_MTX_HARD_DEBUG_WSQ
+
+#  if !defined(ETHR_MTX_HARD_DEBUG_WSQ) && defined(ETHR_MTX_HARD_DEBUG_Q)
+#    define ETHR_MTX_HARD_DEBUG_WSQ
+#  endif
+#endif
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
+
+#ifdef ETHR_DEBUG
+#  ifndef ETHR_MTX_CHK_EXCL
+#    define ETHR_MTX_CHK_EXCL
+#  endif
+#  ifndef ETHR_MTX_CHK_NON_EXCL
+#    define ETHR_MTX_CHK_NON_EXCL
+#  endif
+#endif
+
+#if 0
+#  define ETHR_MTX_Q_LOCK_SPINLOCK__
+#  define ETHR_MTX_QLOCK_TYPE__ ethr_spinlock_t
+#elif defined(ETHR_PTHREADS)
+#  define ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__
+#  define ETHR_MTX_QLOCK_TYPE__ pthread_mutex_t
+#elif defined(ETHR_WIN32_THREADS)
+#  define ETHR_MTX_Q_LOCK_CRITICAL_SECTION__
+#  define ETHR_MTX_QLOCK_TYPE__ CRITICAL_SECTION
+#else
+#  error Need a qlock implementation
+#endif
+
+#define ETHR_RWMTX_W_FLG__		(((ethr_sint32_t) 1) << 31)
+#define ETHR_RWMTX_W_WAIT_FLG__		(((ethr_sint32_t) 1) << 30)
+#define ETHR_RWMTX_R_WAIT_FLG__		(((ethr_sint32_t) 1) << 29)
+
+/* frequent read kind */
+#define ETHR_RWMTX_R_FLG__		(((ethr_sint32_t) 1) << 28)
+#define ETHR_RWMTX_R_ABRT_UNLCK_FLG__	(((ethr_sint32_t) 1) << 27)
+#define ETHR_RWMTX_R_PEND_UNLCK_MASK__	(ETHR_RWMTX_R_ABRT_UNLCK_FLG__ - 1)
+
+/* normal kind */
+#define ETHR_RWMTX_RS_MASK__		(ETHR_RWMTX_R_WAIT_FLG__ - 1)
+
+#define ETHR_RWMTX_WAIT_FLGS__ \
+  (ETHR_RWMTX_W_WAIT_FLG__|ETHR_RWMTX_R_WAIT_FLG__)
+
+#define ETHR_CND_WAIT_FLG__		ETHR_RWMTX_R_WAIT_FLG__
+
+#ifdef ETHR_DEBUG
+#define ETHR_DBG_CHK_UNUSED_FLG_BITS(V)			\
+  ETHR_ASSERT(!((V) & ~(ETHR_RWMTX_W_FLG__		\
+			| ETHR_RWMTX_W_WAIT_FLG__	\
+			| ETHR_RWMTX_R_WAIT_FLG__	\
+			| ETHR_RWMTX_RS_MASK__)))
+#else
+#define ETHR_DBG_CHK_UNUSED_FLG_BITS(V)
+#endif
+
+#define ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(MTX)		\
+  ETHR_DBG_CHK_UNUSED_FLG_BITS(ethr_atomic32_read(&(MTX)->mtxb.flgs))
+
+struct ethr_mutex_base_ {
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+    long pre_fence;
+#endif
+    ethr_atomic32_t flgs;
+    short aux_scnt;
+    short main_scnt;
+    ETHR_MTX_QLOCK_TYPE__ qlck;
+    ethr_ts_event *q;
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    int ws;
+#endif
+#ifdef ETHR_MTX_CHK_EXCL
+    ethr_atomic32_t exclusive;
+#endif
+#ifdef ETHR_MTX_CHK_NON_EXCL
+    ethr_atomic32_t non_exclusive;
+#endif
+#ifdef ETHR_MTX_HARD_DEBUG_LFS
+    ethr_atomic32_t hdbg_lfs;
+#endif
+};
+
+#endif
+
+typedef struct {
+    int main_spincount;
+    int aux_spincount;
+} ethr_mutex_opt;
+
+typedef struct {
+    int main_spincount;
+    int aux_spincount;
+} ethr_cond_opt;
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+
+typedef struct ethr_mutex_ ethr_mutex;
+struct ethr_mutex_ {
+    struct ethr_mutex_base_ mtxb;
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+    long post_fence;
+#endif
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+typedef struct ethr_cond_ ethr_cond;
+struct ethr_cond_ {
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+    struct {
+	long pre_fence;
+    } mtxb; /* mtxb allows us to use same macro as for mutex and rwmutex... */
+#endif
+    ETHR_MTX_QLOCK_TYPE__ qlck;
+    ethr_ts_event *q;
+    short aux_scnt;
+    short main_scnt;
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+    long post_fence;
+#endif
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+#else /* pthread */
+
+typedef struct ethr_mutex_ ethr_mutex;
+struct ethr_mutex_ {
+    pthread_mutex_t pt_mtx;
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+typedef struct ethr_cond_ ethr_cond;
+struct ethr_cond_ {
+    pthread_cond_t pt_cnd;
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+#endif /* pthread */
+
+int ethr_mutex_init_opt(ethr_mutex *, ethr_mutex_opt *);
+int ethr_mutex_init(ethr_mutex *);
+int ethr_mutex_destroy(ethr_mutex *);
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
+int ethr_mutex_trylock(ethr_mutex *);
+void ethr_mutex_lock(ethr_mutex *);
+void ethr_mutex_unlock(ethr_mutex *);
+#endif
+int ethr_cond_init_opt(ethr_cond *, ethr_cond_opt *);
+int ethr_cond_init(ethr_cond *);
+int ethr_cond_destroy(ethr_cond *);
+void ethr_cond_signal(ethr_cond *);
+void ethr_cond_broadcast(ethr_cond *);
+int ethr_cond_wait(ethr_cond *, ethr_mutex *);
+
+typedef enum {
+    ETHR_RWMUTEX_TYPE_NORMAL,
+    ETHR_RWMUTEX_TYPE_FREQUENT_READ,
+    ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
+} ethr_rwmutex_type;
+
+typedef enum {
+    ETHR_RWMUTEX_LONG_LIVED,
+    ETHR_RWMUTEX_SHORT_LIVED,
+    ETHR_RWMUTEX_UNKNOWN_LIVED
+} ethr_rwmutex_lived;
+
+typedef struct {
+    ethr_rwmutex_type type;
+    ethr_rwmutex_lived lived;
+    int main_spincount;
+    int aux_spincount;
+} ethr_rwmutex_opt;
+
+#define ETHR_RWMUTEX_OPT_DEFAULT_INITER \
+  {ETHR_RWMUTEX_TYPE_NORMAL, ETHR_RWMUTEX_UNKNOWN_LIVED, -1, -1}
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+typedef union {
+    struct {
+	ethr_atomic32_t readers;
+	int waiting_readers;
+	int byte_offset;
+	ethr_rwmutex_lived lived;
+    } data;
+    char align__[ETHR_CACHE_LINE_SIZE];
+} ethr_rwmtx_readers_array__;
+
+typedef struct ethr_rwmutex_ ethr_rwmutex;
+struct ethr_rwmutex_ {
+    struct ethr_mutex_base_ mtxb;
+    ethr_rwmutex_type type;
+    ethr_ts_event *rq_end;
+    union {
+	ethr_rwmtx_readers_array__ *ra;
+	int rs;
+    } tdata;
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+    long post_fence;
+#endif
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+#else /* pthread_rwlock */
+
+typedef struct ethr_rwmutex_ ethr_rwmutex;
+struct ethr_rwmutex_ {
+    pthread_rwlock_t pt_rwlock;
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+#endif /* pthread_rwlock */
+
+int ethr_rwmutex_set_reader_group(int);
+int ethr_rwmutex_init_opt(ethr_rwmutex *, ethr_rwmutex_opt *);
+int ethr_rwmutex_init(ethr_rwmutex *);
+int ethr_rwmutex_destroy(ethr_rwmutex *);
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) \
+    || !defined(ETHR_TRY_INLINE_FUNCS) \
+    || defined(ETHR_MUTEX_IMPL__)
+int ethr_rwmutex_tryrlock(ethr_rwmutex *);
+void ethr_rwmutex_rlock(ethr_rwmutex *);
+void ethr_rwmutex_runlock(ethr_rwmutex *);
+int ethr_rwmutex_tryrwlock(ethr_rwmutex *);
+void ethr_rwmutex_rwlock(ethr_rwmutex *);
+void ethr_rwmutex_rwunlock(ethr_rwmutex *);
+#endif
+
+#ifdef ETHR_MTX_HARD_DEBUG
+#define ETHR_MTX_HARD_ASSERT(A) \
+  ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, #A)))
+#else
+#define ETHR_MTX_HARD_ASSERT(A) ((void) 1)
+#endif
+
+#ifdef ETHR_MTX_HARD_DEBUG_LFS
+#  define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB)				\
+do {									\
+    ethr_atomic32_init(&(MTXB)->hdbg_lfs, 0);				\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB)				\
+do {									\
+    ethr_sint32_t val__;						\
+    ETHR_COMPILER_BARRIER;						\
+    val__ = ethr_atomic32_inc_read(&(MTXB)->hdbg_lfs);			\
+    ETHR_MTX_HARD_ASSERT(val__ > 0);					\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES)			\
+do {									\
+    ETHR_COMPILER_BARRIER;						\
+    if ((RES) == 0)							\
+	ETHR_MTX_HARD_DEBUG_LFS_RLOCK((MTXB));				\
+    else								\
+	ETHR_MTX_HARD_ASSERT((RES) == EBUSY);				\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB)				\
+do {									\
+    ethr_sint32_t val__ = ethr_atomic32_dec_read(&(MTXB)->hdbg_lfs);	\
+    ETHR_MTX_HARD_ASSERT(val__ >= 0);					\
+    ETHR_COMPILER_BARRIER;						\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB)				\
+do {									\
+    ethr_sint32_t val__;						\
+    ETHR_COMPILER_BARRIER;						\
+    val__ = ethr_atomic32_dec_read(&(MTXB)->hdbg_lfs);			\
+    ETHR_MTX_HARD_ASSERT(val__ == -1);					\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES)			\
+do {									\
+    ETHR_COMPILER_BARRIER;						\
+    if ((RES) == 0)							\
+	ETHR_MTX_HARD_DEBUG_LFS_RWLOCK((MTXB));				\
+    else								\
+	ETHR_MTX_HARD_ASSERT((RES) == EBUSY);				\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB)			\
+do {									\
+    ethr_sint32_t val__ = ethr_atomic32_inctest(&(MTXB)->hdbg_lfs);	\
+    ETHR_MTX_HARD_ASSERT(val__ == 0);					\
+    ETHR_COMPILER_BARRIER;						\
+} while (0)
+#else
+#  define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB)
+#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB)
+#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB)
+#endif
+
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+
+#if ETHR_SIZEOF_PTR == 8
+#  define ETHR_MTX_HARD_DEBUG_PRE_FENCE		0xdeadbeefdeadbeefL
+#  define ETHR_MTX_HARD_DEBUG_POST_FENCE	0xdeaddeaddeaddeadL
+#else
+#  define ETHR_MTX_HARD_DEBUG_PRE_FENCE		0xdeaddeadL
+#  define ETHR_MTX_HARD_DEBUG_POST_FENCE	0xdeaddeadL
+#endif
+
+#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X) \
+do { \
+    ETHR_COMPILER_BARRIER; \
+    ETHR_MTX_HARD_ASSERT((X)->mtxb.pre_fence == ETHR_MTX_HARD_DEBUG_PRE_FENCE);\
+    ETHR_MTX_HARD_ASSERT((X)->post_fence == ETHR_MTX_HARD_DEBUG_POST_FENCE); \
+    ETHR_COMPILER_BARRIER; \
+} while (0)
+#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X) \
+do { \
+    (X)->mtxb.pre_fence = ETHR_MTX_HARD_DEBUG_PRE_FENCE; \
+    (X)->post_fence = ETHR_MTX_HARD_DEBUG_POST_FENCE; \
+} while (0)
+#else
+#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X)
+#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X)
+#endif
+
+#ifdef ETHR_MTX_CHK_EXCL
+
+#if !defined(ETHR_DEBUG) && defined(__GNUC__)
+#warning "check exclusive is enabled"
+#endif
+
+#  define ETHR_MTX_CHK_EXCL_INIT__(MTXB)		\
+    ethr_atomic32_init(&(MTXB)->exclusive, 0)
+
+#  define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB)		\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (!ethr_atomic32_read(&(MTXB)->exclusive))	\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is exclusive");		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#  define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB)		\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (ethr_atomic32_read(&(MTXB)->exclusive))		\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is not exclusive");		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#  define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB)		\
+do {							\
+    ETHR_MTX_CHK_EXCL_IS_NOT_EXCL((MTXB));		\
+    ethr_atomic32_set(&(MTXB)->exclusive, 1);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#  define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB)		\
+do {							\
+    ETHR_MTX_CHK_EXCL_IS_EXCL((MTXB));			\
+    ethr_atomic32_set(&(MTXB)->exclusive, 0);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+
+#ifdef ETHR_MTX_CHK_NON_EXCL
+
+#if !defined(ETHR_DEBUG) && defined(__GNUC__)
+#warning "check non-exclusive is enabled"
+#endif
+
+#    define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)		\
+    ethr_atomic32_init(&(MTXB)->non_exclusive, 0)
+#    define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)		\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (!ethr_atomic32_read(&(MTXB)->non_exclusive))	\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is non-exclusive");		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (ethr_atomic32_read(&(MTXB)->non_exclusive))	\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is not non-exclusive");	\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    ethr_atomic32_inc(&(MTXB)->non_exclusive);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    ethr_atomic32_add(&(MTXB)->non_exclusive, (NO));	\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    ethr_atomic32_dec(&(MTXB)->non_exclusive);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#else
+#    define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)
+#    define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)
+#    define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)
+#    define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)
+#endif
+
+#else
+#  define ETHR_MTX_CHK_EXCL_INIT__(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB)
+#  define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)
+#  define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)
+#endif
+
+#  define ETHR_MTX_CHK_EXCL_INIT(MTXB)			\
+do {							\
+    ETHR_MTX_CHK_EXCL_INIT__((MTXB));			\
+    ETHR_MTX_CHK_NON_EXCL_INIT__((MTXB));		\
+} while (0)
+
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE 800
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC 50
+#define ETHR_MTX_DEFAULT_AUX_SPINCOUNT 50
+
+#define ETHR_CND_DEFAULT_MAIN_SPINCOUNT 0
+#define ETHR_CND_DEFAULT_AUX_SPINCOUNT 0
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
+
+void ethr_mutex_lock_wait__(ethr_mutex *, ethr_sint32_t);
+void ethr_mutex_unlock_wake__(ethr_mutex *, ethr_sint32_t);
+
+static ETHR_INLINE int
+ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
+{
+    ethr_sint32_t act;
+    int res;
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+
+    act = ethr_atomic32_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0);
+    res = (act == 0) ? 0 : EBUSY;
+
+#ifdef ETHR_MTX_CHK_EXCL
+    if (res == 0)
+	ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
+#endif
+
+    ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&mtx->mtxb, res);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+
+    ETHR_COMPILER_BARRIER;
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
+{
+    ethr_sint32_t act;
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+
+    act = ethr_atomic32_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0);
+    if (act != 0)
+	ethr_mutex_lock_wait__(mtx, act);
+
+    ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
+
+    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+
+    ETHR_COMPILER_BARRIER;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
+{
+    ethr_sint32_t act;
+    ETHR_COMPILER_BARRIER;
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&mtx->mtxb);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+
+    ETHR_MTX_CHK_EXCL_UNSET_EXCL(&mtx->mtxb);
+
+    act = ethr_atomic32_cmpxchg_relb(&mtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__);
+    if (act != ETHR_RWMTX_W_FLG__)
+	ethr_mutex_unlock_wake__(mtx, act);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#else /* pthread_mutex */
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
+
+static ETHR_INLINE int
+ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
+{
+    int res;
+    res = pthread_mutex_trylock(&mtx->pt_mtx);
+    if (res != 0 && res != EBUSY)
+	ETHR_FATAL_ERROR__(res);
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
+{
+    int res = pthread_mutex_lock(&mtx->pt_mtx);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
+{
+    int res = pthread_mutex_unlock(&mtx->pt_mtx);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif /* pthread_mutex */
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE 800
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC 50
+#define ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT 50
+
+#else /* pthread_rwlock */
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
+
+static ETHR_INLINE int
+ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock);
+    if (res != 0 && res != EBUSY)
+	ETHR_FATAL_ERROR__(res);
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_rdlock(&rwmtx->pt_rwlock);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+static ETHR_INLINE int
+ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_trywrlock(&rwmtx->pt_rwlock);
+    if (res != 0 && res != EBUSY)
+	ETHR_FATAL_ERROR__(res);
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_wrlock(&rwmtx->pt_rwlock);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif /* pthread_rwlock */
+
+int ethr_mutex_lib_init(int);
+int ethr_mutex_lib_late_init(int, int);
+
+#endif /* #ifndef ETHR_MUTEX_H__ */
diff --git a/erts/include/internal/ethr_optimized_fallbacks.h b/erts/include/internal/ethr_optimized_fallbacks.h
new file mode 100644
index 0000000000..8e04692856
--- /dev/null
+++ b/erts/include/internal/ethr_optimized_fallbacks.h
@@ -0,0 +1,209 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: "Optimized" fallbacks used when native ops are missing
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_OPTIMIZED_FALLBACKS_H__
+#define ETHR_OPTIMIZED_FALLBACKS_H__
+
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1
+#endif
+
+#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
+#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1
+#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
+/* --- Optimized spinlocks using pthread spinlocks -------------------------- */
+#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1
+
+typedef pthread_spinlock_t ethr_opt_spinlock_t;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+
+static ETHR_INLINE int
+ethr_opt_spinlock_init(ethr_opt_spinlock_t *lock)
+{
+    return pthread_spin_init((pthread_spinlock_t *) lock, 0);
+}
+
+static ETHR_INLINE int
+ethr_opt_spinlock_destroy(ethr_opt_spinlock_t *lock)
+{
+    return pthread_spin_destroy((pthread_spinlock_t *) lock);
+}
+
+
+static ETHR_INLINE int
+ethr_opt_spin_unlock(ethr_opt_spinlock_t *lock)
+{
+    return pthread_spin_unlock((pthread_spinlock_t *) lock);
+}
+
+static ETHR_INLINE int
+ethr_opt_spin_lock(ethr_opt_spinlock_t *lock)
+{
+    return pthread_spin_lock((pthread_spinlock_t *) lock);
+}
+
+#endif
+
+#elif defined(ETHR_HAVE_NATIVE_ATOMICS)
+/* --- Native spinlocks using native atomics -------------------------------- */
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1
+
+#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
+typedef ethr_native_atomic32_t ethr_native_spinlock_t;
+#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
+typedef ethr_native_atomic64_t ethr_native_spinlock_t;
+#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#else
+#  error "Missing native atomic implementation"
+#endif
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+
+static ETHR_INLINE void
+ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
+{
+    ETHR_NATMC_FUNC__(init)(lock, 0);
+}
+
+static ETHR_INLINE void
+ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
+{
+    ETHR_COMPILER_BARRIER;
+    ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == 1);
+    ETHR_NATMC_FUNC__(set_relb)(lock, 0);
+}
+
+static ETHR_INLINE void
+ethr_native_spin_lock(ethr_native_spinlock_t *lock)
+{
+    while (ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, 1, 0) != 0) {
+	while (ETHR_NATMC_FUNC__(read)(lock) != 0)
+	    ETHR_SPIN_BODY;
+    }
+    ETHR_COMPILER_BARRIER;
+}
+
+#endif
+
+#undef ETHR_NATMC_FUNC__
+
+#endif
+
+
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1
+#elif defined(ETHR_HAVE_NATIVE_ATOMICS)
+/* --- Native rwspinlocks using native atomics ------------------------------ */
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
+#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1
+
+#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
+typedef ethr_native_atomic32_t ethr_native_rwlock_t;
+#  define ETHR_NAINT_T__ ethr_sint32_t
+#  define ETHR_WLOCK_FLAG__ (((ethr_sint32_t) 1) << 30)
+#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
+typedef ethr_native_atomic64_t ethr_native_rwlock_t;
+#  define ETHR_NAINT_T__ ethr_sint64_t
+#  define ETHR_WLOCK_FLAG__ (((ethr_sint64_t) 1) << 62)
+#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#else
+#  error "Missing native atomic implementation"
+#endif
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+
+static ETHR_INLINE void
+ethr_native_rwlock_init(ethr_native_rwlock_t *lock)
+{
+    ETHR_NATMC_FUNC__(init)(lock, 0);
+}
+
+static ETHR_INLINE void
+ethr_native_read_unlock(ethr_native_rwlock_t *lock)
+{
+    ETHR_COMPILER_BARRIER;
+#ifdef DEBUG
+    ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) >= 0);
+#endif
+    ETHR_NATMC_FUNC__(dec_relb)(lock);
+}
+
+static ETHR_INLINE void
+ethr_native_read_lock(ethr_native_rwlock_t *lock)
+{
+    ETHR_NAINT_T__ act, exp = 0;
+    while (1) {
+	act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp+1, exp);
+	if (act == exp)
+	    break;
+	while (act & ETHR_WLOCK_FLAG__) {
+	    ETHR_SPIN_BODY;
+	    act = ETHR_NATMC_FUNC__(read)(lock);
+	}
+	exp = act;
+    }
+    ETHR_COMPILER_BARRIER;
+}
+
+static ETHR_INLINE void
+ethr_native_write_unlock(ethr_native_rwlock_t *lock)
+{
+    ETHR_COMPILER_BARRIER;
+    ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == ETHR_WLOCK_FLAG__);
+    ETHR_NATMC_FUNC__(set_relb)(lock, 0);
+}
+
+static ETHR_INLINE void
+ethr_native_write_lock(ethr_native_rwlock_t *lock)
+{
+    ETHR_NAINT_T__ act, exp = 0;
+    while (1) {
+	act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp);
+	if (act == exp)
+	    break;
+	ETHR_SPIN_BODY;
+	exp = act & ~ETHR_WLOCK_FLAG__;
+    }
+    act |= ETHR_WLOCK_FLAG__;
+    /* Wait for readers to leave */
+    while (act != ETHR_WLOCK_FLAG__) {
+	ETHR_SPIN_BODY;
+	act = ETHR_NATMC_FUNC__(read_acqb)(lock);
+    }
+    ETHR_COMPILER_BARRIER;
+}
+
+#endif
+
+#undef ETHR_NAINT_T__
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_WLOCK_FLAG__
+
+#endif
+
+#endif
diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h
index 934a79c6f9..4cd95faf6a 100644
--- a/erts/include/internal/ethread.h
+++ b/erts/include/internal/ethread.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -33,27 +33,15 @@
 #include <stdlib.h>
 #include "erl_errno.h"
 
-/*
- * Extra memory barrier requirements:
- * - ethr_atomic_or_old() needs to enforce a memory barrier sufficient
- *   for a lock operation.
- * - ethr_atomic_and_old() needs to enforce a memory barrier sufficient
- *   for an unlock operation.
- * - ethr_atomic_cmpxchg() needs to enforce a memory barrier sufficient
- *   for a lock and unlock operation.
- */
-
-
-#undef ETHR_USE_RWMTX_FALLBACK
 #undef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-#undef ETHR_HAVE_OPTIMIZED_LOCKS
-
-typedef struct {
-    long tv_sec;
-    long tv_nsec;
-} ethr_timeval;
+#undef ETHR_HAVE_OPTIMIZED_SPINLOCK
+#undef ETHR_HAVE_OPTIMIZED_RWSPINLOCK
 
 #if defined(DEBUG)
+#  define ETHR_DEBUG
+#endif
+
+#if defined(ETHR_DEBUG)
 #  undef ETHR_XCHK
 #  define  ETHR_XCHK 1
 #else
@@ -68,47 +56,57 @@ typedef struct {
 #elif defined(__WIN32__)
 #  define ETHR_INLINE __forceinline
 #endif
-#if defined(DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \
+#if defined(ETHR_DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \
     || (defined(__GNUC__) && defined(ERTS_MIXED_CYGWIN_VC))
 #  undef ETHR_INLINE
 #  define ETHR_INLINE 
 #  undef ETHR_TRY_INLINE_FUNCS
 #endif
-#ifdef ETHR_FORCE_INLINE_FUNCS
-#  define ETHR_TRY_INLINE_FUNCS
-#endif
 
-#if !defined(ETHR_DISABLE_NATIVE_IMPLS) \
-    && (defined(PURIFY) || defined(VALGRIND) || defined(ERTS_MIXED_CYGWIN_VC))
+#if !defined(ETHR_DISABLE_NATIVE_IMPLS) && (defined(PURIFY)||defined(VALGRIND))
 #  define ETHR_DISABLE_NATIVE_IMPLS
 #endif
 
-#define ETHR_RWMUTEX_INITIALIZED 	0x99999999
-#define ETHR_MUTEX_INITIALIZED		0x77777777
-#define ETHR_COND_INITIALIZED		0x55555555
+/* Assume 64-byte cache line size */
+#define ETHR_CACHE_LINE_SIZE ((ethr_uint_t) 64)
+#define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1)
 
-#define ETHR_CACHE_LINE_SIZE 64
+#define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \
+  (((((SZ) - 1) / ETHR_CACHE_LINE_SIZE) + 1) * ETHR_CACHE_LINE_SIZE)
 
-#ifdef ETHR_INLINE_FUNC_NAME_
-#  define ETHR_CUSTOM_INLINE_FUNC_NAME_
-#else
+#ifndef ETHR_INLINE_FUNC_NAME_
 #  define ETHR_INLINE_FUNC_NAME_(X) X
 #endif
 
-#define ETHR_COMPILER_BARRIER ethr_compiler_barrier()
-#ifdef __GNUC__
-#  undef ETHR_COMPILER_BARRIER
-#  define ETHR_COMPILER_BARRIER __asm__ __volatile__("":::"memory")
+#if !defined(__func__)
+#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
+#    if !defined(__GNUC__) ||  __GNUC__ < 2
+#      define __func__ "[unknown_function]"
+#    else
+#      define __func__ __FUNCTION__
+#    endif
+#  endif
 #endif
 
-#ifdef DEBUG
+int ethr_assert_failed(const char *file, int line, const char *func, char *a);
+#ifdef ETHR_DEBUG
 #define ETHR_ASSERT(A) \
-  ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, #A)))
-int ethr_assert_failed(char *f, int l, char *a);
+  ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, __func__, #A)))
 #else
 #define ETHR_ASSERT(A) ((void) 1)
 #endif
 
+#if defined(__GNUC__)
+#  define ETHR_PROTO_NORETURN__ void __attribute__((noreturn))
+#  define ETHR_IMPL_NORETURN__ void
+#elif defined(__WIN32__) && defined(_MSC_VER)
+#  define ETHR_PROTO_NORETURN__ __declspec(noreturn) void
+#  define ETHR_IMPL_NORETURN__ __declspec(noreturn) void
+#else
+#  define ETHR_PROTO_NORETURN__ void
+#  define ETHR_IMPL_NORETURN__ void
+#endif
+
 #if defined(ETHR_PTHREADS)
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * The pthread implementation                                                *
@@ -118,7 +116,9 @@ int ethr_assert_failed(char *f, int l, char *a);
 #error "_GNU_SOURCE not defined. Please, compile all files with -D_GNU_SOURCE."
 #endif
 
-#if defined(ETHR_HAVE_MIT_PTHREAD_H)
+#if defined(ETHR_NEED_NPTL_PTHREAD_H)
+#include <nptl/pthread.h>
+#elif defined(ETHR_HAVE_MIT_PTHREAD_H)
 #include <pthread/mit/pthread.h>
 #elif defined(ETHR_HAVE_PTHREAD_H)
 #include <pthread.h>
@@ -128,130 +128,23 @@ int ethr_assert_failed(char *f, int l, char *a);
 
 typedef pthread_t ethr_tid;
 
-typedef struct ethr_mutex_ ethr_mutex;
-struct ethr_mutex_ {
-    pthread_mutex_t pt_mtx;
-    int is_rec_mtx;
-    ethr_mutex *prev;
-    ethr_mutex *next;
-#if ETHR_XCHK
-    int initialized;
-#endif
-};
-
-typedef struct ethr_cond_ ethr_cond;
-struct ethr_cond_ {
-    pthread_cond_t pt_cnd;
-#if ETHR_XCHK
-    int initialized;
-#endif
-};
+typedef pthread_key_t ethr_tsd_key;
 
-#ifndef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-#define ETHR_USE_RWMTX_FALLBACK
-#else
-typedef struct ethr_rwmutex_ ethr_rwmutex;
-struct ethr_rwmutex_ {
-    pthread_rwlock_t pt_rwlock;
-#if ETHR_XCHK
-    int initialized;
-#endif
-};
-#endif
+#define ETHR_HAVE_ETHR_SIG_FUNCS 1
 
-/* Static initializers */
-#if ETHR_XCHK
-#define ETHR_MUTEX_XCHK_INITER	, ETHR_MUTEX_INITIALIZED
-#define ETHR_COND_XCHK_INITER	, ETHR_COND_INITIALIZED
-#else
-#define ETHR_MUTEX_XCHK_INITER
-#define ETHR_COND_XCHK_INITER
+#if defined(PURIFY) || defined(VALGRIND)
+#  define ETHR_FORCE_PTHREAD_RWLOCK
+#  define ETHR_FORCE_PTHREAD_MUTEX
 #endif
 
-#define ETHR_MUTEX_INITER {PTHREAD_MUTEX_INITIALIZER, 0, NULL, NULL ETHR_MUTEX_XCHK_INITER}
-#define ETHR_COND_INITER {PTHREAD_COND_INITIALIZER ETHR_COND_XCHK_INITER}
-
-#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE) \
-    || defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP)
-#  define ETHR_HAVE_ETHR_REC_MUTEX_INIT 1
-#  ifdef PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
-#    define ETHR_REC_MUTEX_INITER \
-            {PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP, 1, NULL, NULL ETHR_MUTEX_XCHK_INITER}
-#  endif
-#else
-#  undef ETHR_HAVE_ETHR_REC_MUTEX_INIT
+#if !defined(ETHR_FORCE_PTHREAD_RWLOCK)
+#  define ETHR_USE_OWN_RWMTX_IMPL__
 #endif
 
-#ifndef ETHR_HAVE_PTHREAD_ATFORK
-#  define ETHR_NO_FORKSAFETY 1
+#if !defined(ETHR_FORCE_PTHREAD_MUTEX) && 0
+#  define ETHR_USE_OWN_MTX_IMPL__
 #endif
 
-typedef pthread_key_t ethr_tsd_key;
-
-#define ETHR_HAVE_ETHR_SIG_FUNCS 1
-
-#ifdef ETHR_TRY_INLINE_FUNCS
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
-{
-    return pthread_mutex_trylock(&mtx->pt_mtx);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
-{
-    return pthread_mutex_lock(&mtx->pt_mtx);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
-{
-    return pthread_mutex_unlock(&mtx->pt_mtx);
-}
-
-#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_rdlock(&rwmtx->pt_rwlock);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_unlock(&rwmtx->pt_rwlock);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_trywrlock(&rwmtx->pt_rwlock);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_wrlock(&rwmtx->pt_rwlock);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_unlock(&rwmtx->pt_rwlock);
-}
-
-#endif /* ETHR_HAVE_PTHREAD_RWLOCK_INIT */
-
-#endif /* ETHR_TRY_INLINE_FUNCS */
-
 #elif defined(ETHR_WIN32_THREADS)
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * The native win32 threads implementation                                   *
@@ -273,412 +166,89 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx)
 #  undef WIN32_LEAN_AND_MEAN
 #endif
 
-/* Types */
-typedef long ethr_tid; /* thread id type */
-typedef struct {
-    volatile int initialized;
-    CRITICAL_SECTION cs;
-#if ETHR_XCHK
-    int is_rec_mtx;
-#endif
-} ethr_mutex;
-
-typedef struct cnd_wait_event__ cnd_wait_event_;
-
-typedef struct {
-    volatile int initialized;
-    CRITICAL_SECTION cs;
-    cnd_wait_event_ *queue;
-    cnd_wait_event_ *queue_end;
-} ethr_cond;
-
-#define ETHR_USE_RWMTX_FALLBACK
-
-/* Static initializers */
-
-#define ETHR_MUTEX_INITER {0}
-#define ETHR_COND_INITER {0}
-
-#define ETHR_REC_MUTEX_INITER ETHR_MUTEX_INITER
-
-#define ETHR_HAVE_ETHR_REC_MUTEX_INIT 1
-
-typedef DWORD ethr_tsd_key;
-
-#undef ETHR_HAVE_ETHR_SIG_FUNCS
-
-#ifdef ETHR_TRY_INLINE_FUNCS
-int ethr_fake_static_mutex_init(ethr_mutex *mtx);
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
-{
-    if (!mtx->initialized) {
-	int res = ethr_fake_static_mutex_init(mtx);
-	if (res != 0)
-	    return res;
-    }
-    return TryEnterCriticalSection(&mtx->cs) ? 0 : EBUSY;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
-{
-    if (!mtx->initialized) {
-	int res = ethr_fake_static_mutex_init(mtx);
-	if (res != 0)
-	    return res;
-    }
-    EnterCriticalSection(&mtx->cs);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
-{
-    LeaveCriticalSection(&mtx->cs);
-    return 0;
-}
-
-#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */
+#if defined(_MSC_VER)
 
-#ifdef ERTS_MIXED_CYGWIN_VC
-
-/* atomics */
-
-#ifdef _MSC_VER
-#  if _MSC_VER < 1300
-#    define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 /* Dont trust really old compilers */
-#  else
-#    if defined(_M_IX86)
-#      define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1
-#    else /* I.e. IA64 */
-#      if _MSC_VER >= 1400 
-#        define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1
-#      else
-#        define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0
-#      endif
-#    endif
-#  endif
-#  if _MSC_VER >= 1400
-#    include <intrin.h>
-#    undef ETHR_COMPILER_BARRIER
-#    define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
-#  endif
-#pragma intrinsic(_ReadWriteBarrier)
-#pragma intrinsic(_InterlockedAnd)
-#pragma intrinsic(_InterlockedOr)
-#else
-#    define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 
+#if ETHR_SIZEOF_LONG == 4
+#define ETHR_HAVE_INT32_T 1
+typedef long ethr_sint32_t;
+typedef unsigned long ethr_uint32_t;
 #endif
 
-#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1
-#define ETHR_HAVE_OPTIMIZED_LOCKS 1
-
-typedef struct {
-    volatile LONG value;
-} ethr_atomic_t;
-
-typedef struct {
-    volatile LONG locked;
-} ethr_spinlock_t;
-
-typedef struct {
-    volatile LONG counter;
-} ethr_rwlock_t;
-#define ETHR_WLOCK_FLAG__ (((LONG) 1) << 30)
-
-#ifdef ETHR_TRY_INLINE_FUNCS
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i)
-{
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-    var->value = (LONG) i;
-#else
-    (void) InterlockedExchange(&var->value, (LONG) i);
+#if ETHR_SIZEOF___INT64 == 8
+#define ETHR_HAVE_INT64_T 1
+typedef __int64 ethr_sint64_t;
+typedef unsigned __int64 ethr_uint64_t;
 #endif
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i)
-{
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-    var->value = (LONG) i;
-#else
-    (void) InterlockedExchange(&var->value, (LONG) i);
 #endif
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i)
-{
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-    *i = var->value;
-#else
-    *i = InterlockedExchangeAdd(&var->value, (LONG) 0);
-#endif
-    return 0;
-}
+struct ethr_join_data_;
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr)
-{
-    (void) InterlockedExchangeAdd(&var->value, (LONG) incr);
-    return 0;
-}   
-    
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *var,
-					    long i,
-					    long *testp)
-{
-    *testp = InterlockedExchangeAdd(&var->value, (LONG) i);
-    *testp += i;
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var)
-{
-    (void) InterlockedIncrement(&var->value);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var)
-{
-    (void) InterlockedDecrement(&var->value);
-    return 0;
-}
+/* Types */
+typedef struct {
+    long id;
+    struct ethr_join_data_ *jdata;
+} ethr_tid; /* thread id type */
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *var, long *testp)
-{
-    *testp = (long) InterlockedIncrement(&var->value);
-    return 0;
-}
+typedef DWORD ethr_tsd_key;
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *var, long *testp)
-{
-    *testp = (long) InterlockedDecrement(&var->value);
-    return 0;
-}
+#undef ETHR_HAVE_ETHR_SIG_FUNCS
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var,
-					    long mask,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     *
-     * According to msdn _InterlockedAnd() provides a full
-     * memory barrier.
-     */
-    *old = (long) _InterlockedAnd(&var->value, mask);
-    return 0;
-}
+#define ETHR_USE_OWN_RWMTX_IMPL__
+#define ETHR_USE_OWN_MTX_IMPL__
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var,
-					   long mask,
-					   long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     *
-     * According to msdn _InterlockedOr() provides a full
-     * memory barrier.
-     */
-    *old = (long) _InterlockedOr(&var->value, mask);
-    return 0;
-}
+#define ETHR_YIELD() (Sleep(0), 0)
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
-					    long new,
-                                            long expected,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     *
-     * According to msdn _InterlockedCompareExchange() provides a full
-     * memory barrier.
-     */
-    *old = _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) expected);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var,
-					 long new,
-					 long *old)
-{
-    *old = (long) InterlockedExchange(&var->value, (LONG) new);
-    return 0;
-}
-
-/*
- * According to msdn InterlockedExchange() provides a full
- * memory barrier. 
- */
+#else /* No supported thread lib found */
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock)
-{
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-    lock->locked = (LONG) 0;
+#ifdef ETHR_NO_SUPP_THR_LIB_NOT_FATAL
+#define ETHR_NO_THREAD_LIB
 #else
-    (void) InterlockedExchange(&lock->locked, (LONG) 0);
+#error "No supported thread lib found"
 #endif
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock)
-{
-    return 0;
-}
-
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock)
-{
-    ETHR_COMPILER_BARRIER;
-    {
-#ifdef DEBUG
-	LONG old =
-#endif
-	    InterlockedExchange(&lock->locked, (LONG) 0);
-#ifdef DEBUG
-	ETHR_ASSERT(old == 1);
 #endif
-    }
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
-{
-    LONG old;
-    do {
-	old = InterlockedExchange(&lock->locked, (LONG) 1);
-    } while (old != (LONG) 0);
-    ETHR_COMPILER_BARRIER;
-    return 0;
-}
 
-/*
- * According to msdn InterlockedIncrement, InterlockedDecrement,
- * and InterlockedExchangeAdd(), _InterlockedAnd, and _InterlockedOr
- * provides full memory barriers.
- */
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock)
-{
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-    lock->counter = (LONG) 0;
-#else
-    (void) InterlockedExchange(&lock->counter, (LONG) 0);
+#ifndef ETHR_HAVE_INT32_T
+#if ETHR_SIZEOF_INT == 4
+#define ETHR_HAVE_INT32_T 1
+typedef int ethr_sint32_t;
+typedef unsigned int ethr_uint32_t;
+#elif ETHR_SIZEOF_LONG == 4
+#define ETHR_HAVE_INT32_T 1
+typedef long ethr_sint32_t;
+typedef unsigned long ethr_uint32_t;
 #endif
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock)
-{
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock)
-{
-    ETHR_COMPILER_BARRIER;
-    {
-#ifdef DEBUG
-	LONG old =
 #endif
-	    InterlockedDecrement(&lock->counter);
-	ETHR_ASSERT(old != 0);
-    }
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock)
-{
-    while (1) {
-	LONG old = InterlockedIncrement(&lock->counter);
-	if ((old & ETHR_WLOCK_FLAG__) == 0)
-	    break; /* Got read lock */
-	/* Restore and wait for writers to unlock */
-	old = InterlockedDecrement(&lock->counter);
-	while (old & ETHR_WLOCK_FLAG__) {
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-	    old = lock->counter;
-#else
-	    old = InterlockedExchangeAdd(&lock->counter, (LONG) 0);
+#ifndef ETHR_HAVE_INT64_T
+#if ETHR_SIZEOF_INT == 8
+#define ETHR_HAVE_INT64_T 1
+typedef int ethr_sint64_t;
+typedef unsigned int ethr_uint64_t;
+#elif ETHR_SIZEOF_LONG == 8
+#define ETHR_HAVE_INT64_T 1
+typedef long ethr_sint64_t;
+typedef unsigned long ethr_uint64_t;
+#elif ETHR_SIZEOF_LONG_LONG == 8
+#define ETHR_HAVE_INT64_T 1
+typedef long long ethr_sint64_t;
+typedef unsigned long long ethr_uint64_t;
 #endif
-	}
-    }
-    ETHR_COMPILER_BARRIER;
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock)
-{
-    ETHR_COMPILER_BARRIER;
-    {
-#ifdef DEBUG
-	LONG old =
 #endif
-	    _InterlockedAnd(&lock->counter, ~ETHR_WLOCK_FLAG__);
-	ETHR_ASSERT(old & ETHR_WLOCK_FLAG__);
-    }
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock)
-{
-    LONG old;
-    do {
-	old = _InterlockedOr(&lock->counter, ETHR_WLOCK_FLAG__);
-    } while (old & ETHR_WLOCK_FLAG__);
-    /* We got the write part of the lock; wait for readers to unlock */
-    while ((old & ~ETHR_WLOCK_FLAG__) != 0) {
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-	old = lock->counter;
-#else
-	old = InterlockedExchangeAdd(&lock->counter, (LONG) 0);
+#if ETHR_SIZEOF_PTR == 4
+#ifndef ETHR_HAVE_INT32_T
+#error "No 32-bit integer type found"
 #endif
-	ETHR_ASSERT(old & ETHR_WLOCK_FLAG__);
-    }
-    ETHR_COMPILER_BARRIER;
-    return 0;
-}
-
-#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */
-
-#endif /* #ifdef ERTS_MIXED_CYGWIN_VC */
-
-#else /* No supported thread lib found */
-
-#ifdef ETHR_NO_SUPP_THR_LIB_NOT_FATAL
-#define ETHR_NO_THREAD_LIB
-#else
-#error "No supported thread lib found"
+typedef ethr_sint32_t ethr_sint_t;
+typedef ethr_uint32_t ethr_uint_t;
+#elif ETHR_SIZEOF_PTR == 8
+#ifndef ETHR_HAVE_INT64_T
+#error "No 64-bit integer type found"
 #endif
-
+typedef ethr_sint64_t ethr_sint_t;
+typedef ethr_uint64_t ethr_uint_t;
 #endif
 
 /* __builtin_expect() is needed by both native atomics code 
@@ -688,131 +258,169 @@ ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock)
 #endif
 
 /* For CPU-optimised atomics, spinlocks, and rwlocks. */
-#if !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(__GNUC__)
-#  if ETHR_SIZEOF_PTR == 4
-#    if defined(__i386__)
-#      include "i386/ethread.h"
-#    elif (defined(__powerpc__) || defined(__ppc__)) && !defined(__powerpc64__)
-#      include "ppc32/ethread.h"
-#    elif defined(__sparc__)
-#      include "sparc32/ethread.h"
-#    elif defined(__tile__)
-#      include "tile/ethread.h"
+#if !defined(ETHR_DISABLE_NATIVE_IMPLS)
+#  if defined(__GNUC__)
+#    if defined(ETHR_PREFER_GCC_NATIVE_IMPLS)
+#      include "gcc/ethread.h"
+#    elif defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS)
+#      include "libatomic_ops/ethread.h"
 #    endif
-#  elif ETHR_SIZEOF_PTR == 8
-#    if defined(__x86_64__)
-#      include "x86_64/ethread.h"
-#    elif defined(__sparc__) && defined(__arch64__)
-#      include "sparc64/ethread.h"
+#    ifndef ETHR_HAVE_NATIVE_ATOMICS
+#      if ETHR_SIZEOF_PTR == 4
+#        if defined(__i386__)
+#          include "i386/ethread.h"
+#        elif (defined(__powerpc__)||defined(__ppc__))&&!defined(__powerpc64__)
+#          include "ppc32/ethread.h"
+#        elif defined(__sparc__)
+#          include "sparc32/ethread.h"
+#        elif defined(__tile__)
+#          include "tile/ethread.h"
+#        endif
+#      elif ETHR_SIZEOF_PTR == 8
+#        if defined(__x86_64__)
+#          include "x86_64/ethread.h"
+#        elif defined(__sparc__) && defined(__arch64__)
+#          include "sparc64/ethread.h"
+#        endif
+#      endif
+#      include "gcc/ethread.h"
+#      include "libatomic_ops/ethread.h"
 #    endif
+#  elif defined(ETHR_HAVE_LIBATOMIC_OPS)
+#    include "libatomic_ops/ethread.h"
+#  elif defined(ETHR_WIN32_THREADS)
+#    include "win/ethread.h"
 #  endif
-#endif /* !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(__GNUC__) */
-
-#ifdef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-#  undef ETHR_HAVE_NATIVE_ATOMICS
-#endif
-#ifdef ETHR_HAVE_OPTIMIZED_LOCKS
-#  undef ETHR_HAVE_NATIVE_LOCKS
-#endif
+#endif /* !ETHR_DISABLE_NATIVE_IMPLS */
 
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1
-#endif
-#ifdef ETHR_HAVE_NATIVE_LOCKS
-#define ETHR_HAVE_OPTIMIZED_LOCKS 1
+#if defined(__GNUC__)
+#  ifndef ETHR_COMPILER_BARRIER
+#    define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")
+#  endif
+#  ifndef ETHR_SPIN_BODY
+#    if defined(__i386__) || defined(__x86_64__)
+#      define ETHR_SPIN_BODY __asm__ __volatile__("rep;nop" : : : "memory")
+#    elif defined(__ia64__)
+#      define ETHR_SPIN_BODY __asm__ __volatile__("hint @pause" : : : "memory")
+#    elif defined(__sparc__)
+#      define ETHR_SPIN_BODY __asm__ __volatile__("membar #LoadLoad")
+#    else
+#      define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER
+#    endif
+#  endif
+#elif defined(ETHR_WIN32_THREADS)
+#  ifndef ETHR_COMPILER_BARRIER
+#    include <intrin.h>
+#    pragma intrinsic(_ReadWriteBarrier)
+#    define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
+#  endif
+#  ifndef ETHR_SPIN_BODY
+#    define ETHR_SPIN_BODY do {YieldProcessor();ETHR_COMPILER_BARRIER;} while(0)
+#  endif
 #endif
 
-typedef struct {
-    unsigned open;
-    ethr_mutex mtx;
-    ethr_cond cnd;
-} ethr_gate;
+#define ETHR_YIELD_AFTER_BUSY_LOOPS 50
 
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
+#ifndef ETHR_HAVE_NATIVE_ATOMICS
 /*
- * Map ethread native atomics to ethread API atomics.
+ * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only,
+ * i.e. when our lock based atomic fallback is used, a noop is sufficient.
  */
-typedef ethr_native_atomic_t ethr_atomic_t;
+#define ETHR_MEMORY_BARRIER do { } while (0)
+#define ETHR_WRITE_MEMORY_BARRIER do { } while (0)
+#define ETHR_READ_MEMORY_BARRIER do { } while (0)
+#define ETHR_READ_DEPEND_MEMORY_BARRIER do { } while (0)
 #endif
 
-#ifdef ETHR_HAVE_NATIVE_LOCKS
-/*
- * Map ethread native spinlocks to ethread API spinlocks.
- */
-typedef ethr_native_spinlock_t ethr_spinlock_t;
-/*
- * Map ethread native rwlocks to ethread API rwlocks.
- */
-typedef ethr_native_rwlock_t ethr_rwlock_t;
+#ifndef ETHR_WRITE_MEMORY_BARRIER
+#  define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+#  define ETHR_WRITE_MEMORY_BARRIER_IS_FULL
 #endif
-
-#ifdef ETHR_USE_RWMTX_FALLBACK
-typedef struct {
-    ethr_mutex mtx;
-    ethr_cond rcnd;
-    ethr_cond wcnd;
-    unsigned readers;
-    unsigned waiting_readers;
-    unsigned waiting_writers;
-#if ETHR_XCHK
-    int initialized;
+#ifndef ETHR_READ_MEMORY_BARRIER
+#  define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+#  define ETHR_READ_MEMORY_BARRIER_IS_FULL
 #endif
-} ethr_rwmutex;
+#ifndef ETHR_READ_DEPEND_MEMORY_BARRIER
+#  define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
+#  define ETHR_READ_DEPEND_MEMORY_BARRIER_IS_COMPILER_BARRIER
 #endif
 
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-typedef long ethr_atomic_t;
-#endif
+#define ETHR_FATAL_ERROR__(ERR) \
+  ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR))
 
-#ifndef ETHR_HAVE_OPTIMIZED_LOCKS
+ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file,
+					 int line,
+					 const char *func,
+					 int err);
 
-#if defined(ETHR_WIN32_THREADS)
-typedef struct {
-    CRITICAL_SECTION cs;
-} ethr_spinlock_t;
-typedef struct {
-    CRITICAL_SECTION cs;
-    unsigned counter;
-} ethr_rwlock_t;
+void ethr_compiler_barrier_fallback(void);
+#ifndef ETHR_COMPILER_BARRIER
+#  define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback()
+#endif
 
-int ethr_do_spinlock_init(ethr_spinlock_t *lock);
-int ethr_do_rwlock_init(ethr_rwlock_t *lock);
+#ifndef ETHR_SPIN_BODY
+#  define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER
+#endif
 
-#define ETHR_RWLOCK_WRITERS (((unsigned) 1) << 31)
+#ifndef ETHR_YIELD
+#  if defined(ETHR_HAVE_SCHED_YIELD)
+#    ifdef ETHR_HAVE_SCHED_H
+#      include <sched.h>
+#    endif
+#    include <errno.h>
+#    if defined(ETHR_SCHED_YIELD_RET_INT)
+#      define ETHR_YIELD() (sched_yield() < 0 ? errno : 0)
+#    else
+#      define ETHR_YIELD() (sched_yield(), 0)
+#    endif
+#  elif defined(ETHR_HAVE_PTHREAD_YIELD)
+#    if defined(ETHR_PTHREAD_YIELD_RET_INT)
+#      define ETHR_YIELD() pthread_yield()
+#    else
+#      define ETHR_YIELD() (pthread_yield(), 0)
+#    endif
+#  else
+#    define ETHR_YIELD() (ethr_compiler_barrier(), 0)
+#  endif
+#endif
+
+#include "ethr_optimized_fallbacks.h"
 
-#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-typedef struct {
-    pthread_spinlock_t spnlck;
-} ethr_spinlock_t;
 typedef struct {
-    pthread_spinlock_t spnlck;
-    unsigned counter;
-} ethr_rwlock_t;
-#define ETHR_RWLOCK_WRITERS (((unsigned) 1) << 31)
+    void *(*thread_create_prepare_func)(void);
+    void (*thread_create_parent_func)(void *);
+    void (*thread_create_child_func)(void *);
+} ethr_init_data;
 
-#else /* ethr mutex/rwmutex */
+#define ETHR_INIT_DATA_DEFAULT_INITER {NULL, NULL, NULL}
 
 typedef struct {
-    ethr_mutex mtx;
-} ethr_spinlock_t;
+    void *(*alloc)(size_t);
+    void *(*realloc)(void *, size_t);
+    void (*free)(void *);
+} ethr_memory_allocator;
+
+#define ETHR_MEM_ALLOC_DEF_INITER__ {NULL, NULL, NULL}
 
 typedef struct {
-    ethr_rwmutex rwmtx;
-} ethr_rwlock_t;
+    ethr_memory_allocator std;
+    ethr_memory_allocator sl;
+    ethr_memory_allocator ll;
+} ethr_memory_allocators;
 
-#endif /* end mutex/rwmutex */
-#endif /* ETHR_HAVE_OPTIMIZED_LOCKS */
+#define ETHR_MEM_ALLOCS_DEF_INITER__					\
+  {ETHR_MEM_ALLOC_DEF_INITER__,						\
+   ETHR_MEM_ALLOC_DEF_INITER__,						\
+   ETHR_MEM_ALLOC_DEF_INITER__}
 
 typedef struct {
-    void *(*alloc)(size_t);
-    void *(*realloc)(void *, size_t);
-    void (*free)(void *);
-    void *(*thread_create_prepare_func)(void);
-    void (*thread_create_parent_func)(void *);
-    void (*thread_create_child_func)(void *);
-} ethr_init_data;
+    ethr_memory_allocators mem;
+    int reader_groups;
+    int main_threads;
+} ethr_late_init_data;
 
-#define ETHR_INIT_DATA_DEFAULT_INITER {malloc, realloc, free, NULL, NULL, NULL}
+#define ETHR_LATE_INIT_DATA_DEFAULT_INITER				\
+  {ETHR_MEM_ALLOCS_DEF_INITER__, 0, 0}
 
 typedef struct {
     int detached;			/* boolean (default false) */
@@ -821,18 +429,14 @@ typedef struct {
 
 #define ETHR_THR_OPTS_DEFAULT_INITER {0, -1}
 
-#if defined(ETHR_CUSTOM_INLINE_FUNC_NAME_) || !defined(ETHR_TRY_INLINE_FUNCS)
-#  define ETHR_NEED_MTX_PROTOTYPES__
-#  define ETHR_NEED_RWMTX_PROTOTYPES__
-#  define ETHR_NEED_SPINLOCK_PROTOTYPES__
-#  define ETHR_NEED_ATOMIC_PROTOTYPES__
-#endif
 
-#if !defined(ETHR_NEED_RWMTX_PROTOTYPES__) && defined(ETHR_USE_RWMTX_FALLBACK)
-#  define ETHR_NEED_RWMTX_PROTOTYPES__
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#  define ETHR_NEED_SPINLOCK_PROTOTYPES__
+#  define ETHR_NEED_RWSPINLOCK_PROTOTYPES__
 #endif
 
 int ethr_init(ethr_init_data *);
+int ethr_late_init(ethr_late_init_data *);
 int ethr_install_exit_handler(void (*funcp)(void));
 int ethr_thr_create(ethr_tid *, void * (*)(void *), void *, ethr_thr_opts *);
 int ethr_thr_join(ethr_tid, void **);
@@ -840,79 +444,12 @@ int ethr_thr_detach(ethr_tid);
 void ethr_thr_exit(void *);
 ethr_tid ethr_self(void);
 int ethr_equal_tids(ethr_tid, ethr_tid);
-int ethr_mutex_init(ethr_mutex *);
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-int ethr_rec_mutex_init(ethr_mutex *);
-#endif
-int ethr_mutex_destroy(ethr_mutex *);
-int ethr_mutex_set_forksafe(ethr_mutex *);
-int ethr_mutex_unset_forksafe(ethr_mutex *);
-#ifdef ETHR_NEED_MTX_PROTOTYPES__
-int ethr_mutex_trylock(ethr_mutex *);
-int ethr_mutex_lock(ethr_mutex *);
-int ethr_mutex_unlock(ethr_mutex *);
-#endif
-int ethr_cond_init(ethr_cond *);
-int ethr_cond_destroy(ethr_cond *);
-int ethr_cond_signal(ethr_cond *);
-int ethr_cond_broadcast(ethr_cond *);
-int ethr_cond_wait(ethr_cond *, ethr_mutex *);
-int ethr_cond_timedwait(ethr_cond *, ethr_mutex *, ethr_timeval *);
-
-int ethr_rwmutex_init(ethr_rwmutex *);
-int ethr_rwmutex_destroy(ethr_rwmutex *);
-#ifdef ETHR_NEED_RWMTX_PROTOTYPES__
-int ethr_rwmutex_tryrlock(ethr_rwmutex *);
-int ethr_rwmutex_rlock(ethr_rwmutex *);
-int ethr_rwmutex_runlock(ethr_rwmutex *);
-int ethr_rwmutex_tryrwlock(ethr_rwmutex *);
-int ethr_rwmutex_rwlock(ethr_rwmutex *);
-int ethr_rwmutex_rwunlock(ethr_rwmutex *);
-#endif
-
-#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__
-int ethr_atomic_init(ethr_atomic_t *, long);
-int ethr_atomic_set(ethr_atomic_t *, long);
-int ethr_atomic_read(ethr_atomic_t *, long *);
-int ethr_atomic_inctest(ethr_atomic_t *, long *);
-int ethr_atomic_dectest(ethr_atomic_t *, long *);
-int ethr_atomic_inc(ethr_atomic_t *);
-int ethr_atomic_dec(ethr_atomic_t *);
-int ethr_atomic_addtest(ethr_atomic_t *, long, long *);
-int ethr_atomic_add(ethr_atomic_t *, long);
-int ethr_atomic_and_old(ethr_atomic_t *, long, long *);
-int ethr_atomic_or_old(ethr_atomic_t *, long, long *);
-int ethr_atomic_xchg(ethr_atomic_t *, long, long *);
-int ethr_atomic_cmpxchg(ethr_atomic_t *, long, long, long *);
-#endif
-
-#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__
-int ethr_spinlock_init(ethr_spinlock_t *);
-int ethr_spinlock_destroy(ethr_spinlock_t *);
-int ethr_spin_unlock(ethr_spinlock_t *);
-int ethr_spin_lock(ethr_spinlock_t *);
 
-int ethr_rwlock_init(ethr_rwlock_t *);
-int ethr_rwlock_destroy(ethr_rwlock_t *);
-int ethr_read_unlock(ethr_rwlock_t *);
-int ethr_read_lock(ethr_rwlock_t *);
-int ethr_write_unlock(ethr_rwlock_t *);
-int ethr_write_lock(ethr_rwlock_t *);
-#endif
-
-int ethr_time_now(ethr_timeval *);
 int ethr_tsd_key_create(ethr_tsd_key *);
 int ethr_tsd_key_delete(ethr_tsd_key);
 int ethr_tsd_set(ethr_tsd_key, void *);
 void *ethr_tsd_get(ethr_tsd_key);
 
-int ethr_gate_init(ethr_gate *);
-int ethr_gate_destroy(ethr_gate *);
-int ethr_gate_close(ethr_gate *);
-int ethr_gate_let_through(ethr_gate *, unsigned);
-int ethr_gate_wait(ethr_gate *);
-int ethr_gate_swait(ethr_gate *, int);
-
 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
 #include <signal.h>
 int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset);
@@ -921,528 +458,274 @@ int ethr_sigwait(const sigset_t *set, int *sig);
 
 void ethr_compiler_barrier(void);
 
-#ifdef ETHR_TRY_INLINE_FUNCS
-
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i)
-{
-    ethr_native_atomic_init(var, i);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i)
-{
-    ethr_native_atomic_set(var, i);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i)
-{
-    *i = ethr_native_atomic_read(var);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr)
-{
-    ethr_native_atomic_add(var, incr);
-    return 0;
-}   
-    
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *var,
-					    long i,
-					    long *testp)
-{
-    *testp = ethr_native_atomic_add_return(var, i);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var)
-{
-    ethr_native_atomic_inc(var);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var)
-{
-    ethr_native_atomic_dec(var);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *var, long *testp)
-{
-    *testp = ethr_native_atomic_inc_return(var);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *var, long *testp)
-{
-    *testp = ethr_native_atomic_dec_return(var);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var,
-					    long mask,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    *old = ethr_native_atomic_and_retold(var, mask);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var,
-					   long mask,
-					   long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    *old = ethr_native_atomic_or_retold(var, mask);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var,
-					 long new,
-					 long *old)
-{
-    *old = ethr_native_atomic_xchg(var, new);
-    return 0;
-}   
-
-/*
- * If *var == *old, replace *old with new, else do nothing.
- * In any case return the original value of *var in *old.
- */
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
-					    long new,
-                                            long expected,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    *old = ethr_native_atomic_cmpxchg(var, new, expected);
-    return 0;
-}
+#if defined(ETHR_HAVE_NATIVE_SPINLOCKS)
+typedef ethr_native_spinlock_t ethr_spinlock_t;
+#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS)
+typedef ethr_opt_spinlock_t ethr_spinlock_t;
+#elif defined(__WIN32__)
+typedef CRITICAL_SECTION ethr_spinlock_t;
+#else
+typedef pthread_mutex_t ethr_spinlock_t;
+#endif
 
-#endif /* ETHR_HAVE_NATIVE_ATOMICS */
+#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__
+int ethr_spinlock_init(ethr_spinlock_t *);
+int ethr_spinlock_destroy(ethr_spinlock_t *);
+void ethr_spin_unlock(ethr_spinlock_t *);
+void ethr_spin_lock(ethr_spinlock_t *);
+#endif
 
-#ifdef ETHR_HAVE_NATIVE_LOCKS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE int
 ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock)
 {
+#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
     ethr_native_spinlock_init(lock);
     return 0;
+#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS)
+    return ethr_opt_spinlock_init((ethr_opt_spinlock_t *) lock);
+#elif defined(__WIN32__)
+    if (!InitializeCriticalSectionAndSpinCount((CRITICAL_SECTION *) lock, INT_MAX))
+	return ethr_win_get_errno__();
+    return 0;
+#else
+    return pthread_mutex_init((pthread_mutex_t *) lock, NULL);
+#endif
 }
 
 static ETHR_INLINE int
 ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock)
 {
+#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
     return 0;
+#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS)
+    return ethr_opt_spinlock_destroy((ethr_opt_spinlock_t *) lock);
+#elif defined(__WIN32__)
+    DeleteCriticalSection((CRITICAL_SECTION *) lock);
+    return 0;
+#else
+    return pthread_mutex_destroy((pthread_mutex_t *) lock);
+#endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock)
 {
+#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
     ethr_native_spin_unlock(lock);
-    return 0;
+#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS)
+    int err = ethr_opt_spin_unlock((ethr_opt_spinlock_t *) lock);
+    if (err)
+	ETHR_FATAL_ERROR__(err);
+#elif defined(__WIN32__)
+    LeaveCriticalSection((CRITICAL_SECTION *) lock);
+#else
+    int err = pthread_mutex_unlock((pthread_mutex_t *) lock);
+    if (err)
+	ETHR_FATAL_ERROR__(err);
+#endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
 {
+#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
     ethr_native_spin_lock(lock);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock)
-{
-    ethr_native_rwlock_init(lock);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock)
-{
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock)
-{
-    ethr_native_read_unlock(lock);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock)
-{
-    ethr_native_read_lock(lock);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock)
-{
-    ethr_native_write_unlock(lock);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock)
-{
-    ethr_native_write_lock(lock);
-    return 0;
+#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS)
+    int err = ethr_opt_spin_lock((ethr_opt_spinlock_t *) lock);
+    if (err)
+	ETHR_FATAL_ERROR__(err);
+#elif defined(__WIN32__)
+    EnterCriticalSection((CRITICAL_SECTION *) lock);
+#else
+    int err = pthread_mutex_lock((pthread_mutex_t *) lock);
+    if (err)
+	ETHR_FATAL_ERROR__(err);
+#endif
 }
 
-#endif /* ETHR_HAVE_NATIVE_LOCKS */
-
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
-/*
- * Fallbacks for atomics used in absence of optimized implementation.
- */
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
+#include "ethr_atomics.h"
 
-#define ETHR_ATOMIC_ADDR_BITS 4
-#define ETHR_ATOMIC_ADDR_SHIFT 3
+typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */
 
-typedef struct {
-    union {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-	pthread_spinlock_t spnlck;
+#if defined(ETHR_WIN32_THREADS)
+#  include "win/ethr_event.h"
 #else
-	ethr_mutex mtx;
-#endif
-	char buf[ETHR_CACHE_LINE_SIZE];
-    } u;
-} ethr_atomic_protection_t;
-
-extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
-
-
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-
-#define ETHR_ATOMIC_PTR2LCK__(PTR) \
-(&ethr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \
-			& ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.spnlck)
-
-
-#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS)			\
-do {									\
-    pthread_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP));		\
-    int res__ = pthread_spin_lock(slp__);				\
-    if (res__ != 0)							\
-	return res__;							\
-    { EXPS; }								\
-    return pthread_spin_unlock(slp__);					\
-} while (0)
-
-#else /* ethread mutex */
+#  include "pthread/ethr_event.h"
+#endif
+
+int ethr_set_main_thr_status(int, int);
+int ethr_get_main_thr_status(int *);
+
+struct ethr_ts_event_ {
+    ethr_ts_event *next;
+    ethr_ts_event *prev;
+    ethr_event event;
+    void *udata;
+    ethr_atomic32_t uaflgs;
+    unsigned uflgs;
+    unsigned iflgs;		/* for ethr lib only */
+    short rgix;			/* for ethr lib only */
+    short mtix;			/* for ethr lib only */
+};
 
-#define ETHR_ATOMIC_PTR2LCK__(PTR) \
-(&ethr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \
-		     & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.mtx)
+#define ETHR_TS_EV_ETHREAD	(((unsigned) 1) << 0)
+#define ETHR_TS_EV_INITED	(((unsigned) 1) << 1)
+#define ETHR_TS_EV_TMP		(((unsigned) 1) << 2)
+#define ETHR_TS_EV_MAIN_THR	(((unsigned) 1) << 3)
 
-#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS)			\
-do {									\
-    ethr_mutex *mtxp__ = ETHR_ATOMIC_PTR2LCK__((AP));			\
-    int res__ = ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(mtxp__);	\
-    if (res__ != 0)							\
-	return res__;							\
-    { EXPS; }								\
-    return ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(mtxp__);		\
-} while (0)
+int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp);
+int ethr_free_ts_event__(ethr_ts_event *tsep);
+int ethr_make_ts_event__(ethr_ts_event **tsepp);
 
-#endif /* end ethread mutex */
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
+ethr_ts_event *ethr_get_ts_event(void);
+void ethr_leave_ts_event(ethr_ts_event *);
+#endif
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_PTHREADS)
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = (ethr_atomic_t) i);
-}
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = (ethr_atomic_t) i);
-}
+extern pthread_key_t ethr_ts_event_key__;
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i)
+static ETHR_INLINE ethr_ts_event *
+ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void)
 {
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *i = (long) *var);
+    ethr_ts_event *tsep = pthread_getspecific(ethr_ts_event_key__);
+    if (!tsep) {
+	int res = ethr_make_ts_event__(&tsep);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+	ETHR_ASSERT(tsep);
+    }
+    return tsep;
 }
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *incp, long *testp)
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep)
 {
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, *testp = (long) ++(*incp));
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *decp, long *testp)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(decp, *testp = (long) --(*decp));
 }
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr);
-}   
-    
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *incp,
-					    long i,
-					    long *testp)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, *incp += i; *testp = *incp);
-}
+#endif
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *incp)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, ++(*incp));
-}
+#elif defined(ETHR_WIN32_THREADS)
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *decp)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(decp, --(*decp));
-}
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var,
-					    long mask,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var &= mask);
-}
+extern DWORD ethr_ts_event_key__;
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var,
-					   long mask,
-					   long *old)
+static ETHR_INLINE ethr_ts_event *
+ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void)
 {
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var |= mask);
+    ethr_ts_event *tsep = TlsGetValue(ethr_ts_event_key__);
+    if (!tsep) {
+	int res = ethr_get_tmp_ts_event__(&tsep);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+	ETHR_ASSERT(tsep);
+    }
+    return tsep;
 }
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var,
-					 long new,
-					 long *old)
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep)
 {
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var = new);
-}   
-
-/*
- * If *var == *old, replace *old with new, else do nothing.
- * In any case return the original value of *var in *old.
- */
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
-					    long new,
-                                            long expected,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(
-      var,
-      long old_val = *var;
-      *old = old_val;
-      if (__builtin_expect(old_val == expected, 1))
-          *var = new;
-      );
-    return 0;
+    if (tsep->iflgs & ETHR_TS_EV_TMP) {
+	int res = ethr_free_ts_event__(tsep);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+    }
 }
 
-#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */
-#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */
-
-/*
- * Fallbacks for spin locks, and rw spin locks used in absence of
- * optimized implementation.
- */
-#ifndef ETHR_HAVE_OPTIMIZED_LOCKS
-
-#ifdef ETHR_TRY_INLINE_FUNCS
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock)
-{
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    return pthread_spin_init(&lock->spnlck, 0);
-#else
-    return ethr_mutex_init(&lock->mtx);
 #endif
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock)
-{
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    return pthread_spin_destroy(&lock->spnlck);
-#else
-    return ethr_mutex_destroy(&lock->mtx);
 #endif
-}
 
+#include "ethr_mutex.h" /* Need atomic declarations and tse */
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock)
-{
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    return pthread_spin_unlock(&lock->spnlck);
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+typedef ethr_native_rwlock_t ethr_rwlock_t;
 #else
-    return ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(&lock->mtx);
+typedef ethr_rwmutex ethr_rwlock_t;
 #endif
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
-{
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    return pthread_spin_lock(&lock->spnlck);
-#else
-    return ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(&lock->mtx);
+#ifdef ETHR_NEED_RWSPINLOCK_PROTOTYPES__
+int ethr_rwlock_init(ethr_rwlock_t *);
+int ethr_rwlock_destroy(ethr_rwlock_t *);
+void ethr_read_unlock(ethr_rwlock_t *);
+void ethr_read_lock(ethr_rwlock_t *);
+void ethr_write_unlock(ethr_rwlock_t *);
+void ethr_write_lock(ethr_rwlock_t *);
 #endif
-}
 
-#ifdef ETHR_USE_RWMTX_FALLBACK
-#define ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(X) X
-#else
-#define ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(X) ETHR_INLINE_FUNC_NAME_(X)
-#endif
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE int
 ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    lock->counter = 0;
-    return pthread_spin_init(&lock->spnlck, 0);
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    ethr_native_rwlock_init(lock);
+    return 0;
 #else
-    return ethr_rwmutex_init(&lock->rwmtx);
+    return ethr_rwmutex_init_opt((ethr_rwmutex *) lock, NULL);
 #endif
 }
 
 static ETHR_INLINE int
 ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    return pthread_spin_destroy(&lock->spnlck);
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    return 0;
 #else
-    return ethr_rwmutex_destroy(&lock->rwmtx);
+    return ethr_rwmutex_destroy((ethr_rwmutex *) lock);
 #endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    int res = pthread_spin_lock(&lock->spnlck);
-    if (res != 0)
-	return res;
-    lock->counter--;
-    return pthread_spin_unlock(&lock->spnlck);
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    ethr_native_read_unlock(lock);
 #else
-    return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_runlock)(&lock->rwmtx);
+    ethr_rwmutex_runlock((ethr_rwmutex *) lock);
 #endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    int locked = 0;
-    do {
-	int res = pthread_spin_lock(&lock->spnlck);
-	if (res != 0)
-	    return res;
-	if ((lock->counter & ETHR_RWLOCK_WRITERS) == 0) {
-	    lock->counter++;
-	    locked = 1;
-	}
-	res = pthread_spin_unlock(&lock->spnlck);
-	if (res != 0)
-	    return res;
-    } while (!locked);
-    return 0;
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    ethr_native_read_lock(lock);
 #else
-    return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rlock)(&lock->rwmtx);
+    ethr_rwmutex_rlock((ethr_rwmutex *) lock);
 #endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    lock->counter = 0;
-    return pthread_spin_unlock(&lock->spnlck);
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    ethr_native_write_unlock(lock);
 #else
-    return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rwunlock)(&lock->rwmtx);
+    ethr_rwmutex_rwunlock((ethr_rwmutex *) lock);
 #endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    while (1) {
-	int res = pthread_spin_lock(&lock->spnlck);
-	if (res != 0)
-	    return res;
-	lock->counter |= ETHR_RWLOCK_WRITERS;
-	if (lock->counter == ETHR_RWLOCK_WRITERS)
-	    return 0;
-	res = pthread_spin_unlock(&lock->spnlck);
-	if (res != 0)
-	    return res;
-    }
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    ethr_native_write_lock(lock);
 #else
-    return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rwlock)(&lock->rwmtx);
+    ethr_rwmutex_rwlock((ethr_rwmutex *) lock);
 #endif
 }
 
-#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */
-
-#endif /* ETHR_HAVE_OPTIMIZED_LOCKS */
-
-#if defined(ETHR_HAVE_OPTIMIZED_LOCKS) || defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-# define ETHR_HAVE_OPTIMIZED_SPINLOCK
-#endif
+#endif /* ETHR_TRY_INLINE_FUNCS */
 
 #endif /* #ifndef ETHREAD_H__ */
diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in
index e5b4946a53..f394d790d2 100644
--- a/erts/include/internal/ethread_header_config.h.in
+++ b/erts/include/internal/ethread_header_config.h.in
@@ -1,25 +1,40 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
 /* Define to the size of pointers */
 #undef ETHR_SIZEOF_PTR
 
+/* Define to the size of int */
+#undef ETHR_SIZEOF_INT
+
+/* Define to the size of long */
+#undef ETHR_SIZEOF_LONG
+
+/* Define to the size of long long */
+#undef ETHR_SIZEOF_LONG_LONG
+
+/* Define to the size of __int64 */
+#undef ETHR_SIZEOF___INT64
+
+/* Define if bigendian */
+#undef ETHR_BIGENDIAN
+
 /* Define if you want to disable native ethread implementations */
 #undef ETHR_DISABLE_NATIVE_IMPLS
 
@@ -29,26 +44,97 @@
 /* Define if you have pthreads */
 #undef ETHR_PTHREADS
 
+/* Define if you need the <nptl/pthread.h> header file. */
+#undef ETHR_NEED_NPTL_PTHREAD_H
+
 /* Define if you have the <pthread.h> header file. */
 #undef ETHR_HAVE_PTHREAD_H
 
 /* Define if the pthread.h header file is in pthread/mit directory. */
 #undef ETHR_HAVE_MIT_PTHREAD_H
 
-/* Define if you have the pthread_mutexattr_settype function. */
-#undef ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE
+/* Define if you have the pthread_spin_lock function. */
+#undef ETHR_HAVE_PTHREAD_SPIN_LOCK
+
+/* Define if you want to force usage of pthread rwlocks */
+#undef ETHR_FORCE_PTHREAD_RWLOCK
 
-/* Define if you have the pthread_mutexattr_setkind_np function. */
-#undef ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP
+/* Define if you have the pthread_rwlockattr_setkind_np() function. */
+#undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP
 
-/* Define if you have the pthread_atfork function. */
-#undef ETHR_HAVE_PTHREAD_ATFORK
+/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock
+   attribute. */
+#undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP
 
-/* Define if you have the pthread_spin_lock function. */
-#undef ETHR_HAVE_PTHREAD_SPIN_LOCK
+/* Define if you have a linux futex implementation. */
+#undef ETHR_HAVE_LINUX_FUTEX
+
+/* Define if you have gcc atomic operations */
+#undef ETHR_HAVE_GCC_ATOMIC_OPS
+
+/* Define if you prefer gcc native ethread implementations */
+#undef ETHR_PREFER_GCC_NATIVE_IMPLS
+
+/* Define if you have the <sched.h> header file. */
+#undef ETHR_HAVE_SCHED_H
+
+/* Define if you have the sched_yield() function. */
+#undef ETHR_HAVE_SCHED_YIELD
+
+/* Define if you have the pthread_yield() function. */
+#undef ETHR_HAVE_PTHREAD_YIELD
+
+/* Define if pthread_yield() returns an int. */
+#undef ETHR_PTHREAD_YIELD_RET_INT
+
+/* Define if sched_yield() returns an int. */
+#undef ETHR_SCHED_YIELD_RET_INT
+
+/* Define if you want compatibilty with x86 processors before pentium4. */
+#undef ETHR_PRE_PENTIUM4_COMPAT
+
+/* Define if you have the pthread_rwlockattr_setkind_np() function. */
+#undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP
+
+/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock
+   attribute. */
+#undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP
+
+/* Define if you have gcc atomic operations */
+#undef ETHR_HAVE_GCC_ATOMIC_OPS
+
+/* Define if you prefer gcc native ethread implementations */
+#undef ETHR_PREFER_GCC_NATIVE_IMPLS
+
+/* Define if you have libatomic_ops atomic operations */
+#undef ETHR_HAVE_LIBATOMIC_OPS
+
+/* Define if you prefer libatomic_ops native ethread implementations */
+#undef ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS
+
+/* Define to the size of AO_t if libatomic_ops is used */
+#undef ETHR_SIZEOF_AO_T
+
+/* Define if you have _InterlockedCompareExchange64() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
+
+/* Define if you have _InterlockedDecrement64() */
+#undef ETHR_HAVE__INTERLOCKEDDECREMENT64
+
+/* Define if you have _InterlockedIncrement64() */
+#undef ETHR_HAVE__INTERLOCKEDINCREMENT64
+
+/* Define if you have _InterlockedExchangeAdd64() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
+
+/* Define if you have _InterlockedExchange64() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGE64
+
+/* Define if you have _InterlockedAnd64() */
+#undef ETHR_HAVE__INTERLOCKEDAND64
 
-/* Define if you have a pthread_rwlock implementation that can be used */
-#undef ETHR_HAVE_PTHREAD_RWLOCK_INIT
+/* Define if you have _InterlockedOr64() */
+#undef ETHR_HAVE__INTERLOCKEDOR64
 
 /* Define if you want to turn on extra sanity checking in the ethread library */
 #undef ETHR_XCHK
diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h
new file mode 100644
index 0000000000..16935084b1
--- /dev/null
+++ b/erts/include/internal/gcc/ethr_atomic.h
@@ -0,0 +1,290 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomics ethread support using gcc's builtins
+ * Author: Rickard Green
+ */
+
+#undef ETHR_INCLUDE_ATOMIC_IMPL__
+#if !defined(ETHR_GCC_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
+#define ETHR_GCC_ATOMIC32_H__
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#elif !defined(ETHR_GCC_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
+#define ETHR_GCC_ATOMIC64_H__
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+#endif
+
+#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
+
+#ifndef ETHR_GCC_ATOMIC_COMMON__
+#define ETHR_GCC_ATOMIC_COMMON__
+
+#define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 0
+#if defined(__i386__) || defined(__x86_64__) || defined(__sparc__) \
+    || defined(__powerpc__) || defined(__ppc__) || defined(__mips__)
+#  undef ETHR_READ_AND_SET_WITHOUT_SYNC_OP__
+#  define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 1
+#endif
+
+#if defined(__x86_64__) || (defined(__i386__) \
+			    && !defined(ETHR_PRE_PENTIUM4_COMPAT))
+#  define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1
+#else
+#  define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0
+#endif
+
+/*
+ * According to the documentation this is what we want:
+ *   #define ETHR_MEMORY_BARRIER __sync_synchronize()
+ * However, __sync_synchronize() is known to erroneously be
+ * a noop on at least some platforms with some gcc versions.
+ * This has suposedly been fixed in some gcc version, but we
+ * don't know from which version. Therefore, we only use
+ * it when it has been verified to work. Otherwise
+ * we use a workaround.
+ */
+#if defined(__mips__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2))
+/* __sync_synchronize() has been verified to work here */
+#define ETHR_MEMORY_BARRIER __sync_synchronize()
+#define ETHR_READ_DEPEND_MEMORY_BARRIER __sync_synchronize()
+#elif defined(__x86_64__) || (defined(__i386__) \
+			      && !defined(ETHR_PRE_PENTIUM4_COMPAT))
+/* Use fence instructions directly instead of workaround */
+#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory")
+#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory")
+#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory")
+#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory")
+#else
+/* Workaround */
+#define ETHR_MEMORY_BARRIER \
+do { \
+    volatile ethr_sint32_t x___ = 0; \
+    (void) __sync_val_compare_and_swap(&x___, (ethr_sint32_t) 0, (ethr_sint32_t) 1); \
+} while (0)
+#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+#endif
+
+#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")
+
+#endif /* ETHR_GCC_ATOMIC_COMMON__ */
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic32_t
+#define ETHR_AINT_T__ ethr_sint32_t
+#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic64_t
+#define ETHR_AINT_T__ ethr_sint64_t
+#else
+#error "Unsupported integer size"
+#endif
+
+typedef struct {
+    volatile ETHR_AINT_T__ counter;
+} ETHR_ATMC_T__;
+
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->counter;
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+{
+#if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__
+    var->counter = value;
+#else
+    /*
+     * Unfortunately no __sync_store() or similar exist in the gcc atomic
+     * op interface. We therefore have to simulate it this way...
+     */
+    ETHR_AINT_T__ act = 0, exp;
+    do {
+	exp = act;
+	act = __sync_val_compare_and_swap(&var->counter, exp, value);
+    } while (act != exp);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+{
+    ETHR_NATMC_FUNC__(set)(var, value);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
+{
+#if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__
+    return var->counter;
+#else
+    /*
+     * Unfortunately no __sync_fetch() or similar exist in the gcc atomic
+     * op interface. We therefore have to simulate it this way...
+     */
+    return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+{
+    (void) __sync_add_and_fetch(&var->counter, incr);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+{
+    return __sync_add_and_fetch(&var->counter, incr);
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
+{
+    (void) __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1);
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
+{
+    (void) __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
+{
+    return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
+{
+    return __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+{
+    return __sync_fetch_and_and(&var->counter, mask);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+{
+    return (ETHR_AINT_T__) __sync_fetch_and_or(&var->counter, mask);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
+			   ETHR_AINT_T__ new,
+			   ETHR_AINT_T__ old)
+{
+    return __sync_val_compare_and_swap(&var->counter, old, new);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
+{
+    ETHR_AINT_T__ exp, act = 0;
+    do {
+	exp = act;
+	act = __sync_val_compare_and_swap(&var->counter, exp, new);
+    } while (act != exp);
+    return act;
+}
+
+/*
+ * Atomic ops with at least specified barriers.
+ */
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
+{
+#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
+    ETHR_AINT_T__ val = var->counter;
+    ETHR_COMPILER_BARRIER;
+    return val;
+#else
+    return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
+    ETHR_COMPILER_BARRIER;
+    var->counter = i;
+#else
+    (void) ETHR_NATMC_FUNC__(xchg)(var, i);
+#endif
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_NATMC_FUNC__(inc_return)(var);
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
+{
+    ETHR_NATMC_FUNC__(dec)(var);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_NATMC_FUNC__(dec_return)(var);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ old)
+{
+    return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ old)
+{
+    return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+}
+
+#endif
+
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_ATMC_T__
+#undef ETHR_AINT_T__
+#undef ETHR_AINT_SUFFIX__
+
+#endif
diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h
new file mode 100644
index 0000000000..392a1aa2b2
--- /dev/null
+++ b/erts/include/internal/gcc/ethread.h
@@ -0,0 +1,40 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomic ethread support when using gcc
+ * Author: Rickard Green
+ */
+
+#ifndef ETHREAD_GCC_H__
+#define ETHREAD_GCC_H__
+
+#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_GCC_ATOMIC_OPS)
+#define ETHR_HAVE_NATIVE_ATOMICS 1
+
+#define ETHR_ATOMIC_WANT_32BIT_IMPL__
+#include "ethr_atomic.h"
+#if ETHR_SIZEOF_PTR == 8
+#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
+#  include "ethr_atomic.h"
+#endif
+
+#endif
+
+#endif
diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h
index 3291ad38e5..4e402f261a 100644
--- a/erts/include/internal/i386/atomic.h
+++ b/erts/include/internal/i386/atomic.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -23,133 +23,265 @@
  *
  * This code requires a 486 or newer processor.
  */
-#ifndef ETHREAD_I386_ATOMIC_H
-#define ETHREAD_I386_ATOMIC_H
 
-/* An atomic is an aligned long accessed via locked operations.
- */
-typedef struct {
-    volatile long counter;
-} ethr_native_atomic_t;
+#undef ETHR_INCLUDE_ATOMIC_IMPL__
+#if !defined(ETHR_X86_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
+#define ETHR_X86_ATOMIC32_H__
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#elif !defined(ETHR_X86_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
+#define ETHR_X86_ATOMIC64_H__
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+#endif
+
+#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#ifndef ETHR_X86_ATOMIC_COMMON__
+#define ETHR_X86_ATOMIC_COMMON__
 
-#ifdef __x86_64__
-#define LONG_SUFFIX "q"
+#define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1
+
+#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
+#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory")
+#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory")
+#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory")
+#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory")
+#else
+#define ETHR_MEMORY_BARRIER \
+do { \
+    volatile ethr_sint32_t x___ = 0; \
+    __asm__ __volatile__("lock; incl %0" : "=m"(x___) : "m"(x___) : "memory"); \
+} while (0)
+#endif
+
+#endif /* ETHR_X86_ATOMIC_COMMON__ */
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic32_t
+#define ETHR_AINT_T__ ethr_sint32_t
+#define ETHR_AINT_SUFFIX__ "l"
+#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic64_t
+#define ETHR_AINT_T__ ethr_sint64_t
+#define ETHR_AINT_SUFFIX__ "q"
 #else
-#define LONG_SUFFIX "l"
+#error "Unsupported integer size"
 #endif
 
+/* An atomic is an aligned ETHR_AINT_T__ accessed via locked operations.
+ */
+typedef struct {
+    volatile ETHR_AINT_T__ counter;
+} ETHR_ATMC_T__;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->counter;
+}
+
 static ETHR_INLINE void
-ethr_native_atomic_init(ethr_native_atomic_t *var, long i)
+ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
 {
     var->counter = i;
 }
-#define ethr_native_atomic_set(v, i)	ethr_native_atomic_init((v), (i))
 
-static ETHR_INLINE long
-ethr_native_atomic_read(ethr_native_atomic_t *var)
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    var->counter = i;
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
 {
     return var->counter;
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_add(ethr_native_atomic_t *var, long incr)
+ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
     __asm__ __volatile__(
-       "lock; add" LONG_SUFFIX " %1, %0"
+       "lock; add" ETHR_AINT_SUFFIX__ " %1, %0"
        : "=m"(var->counter)
        : "ir"(incr), "m"(var->counter));
 }      
        
 static ETHR_INLINE void
-ethr_native_atomic_inc(ethr_native_atomic_t *var)
+ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
 {
     __asm__ __volatile__(
-	"lock; inc" LONG_SUFFIX " %0"
+	"lock; inc" ETHR_AINT_SUFFIX__ " %0"
 	: "=m"(var->counter)
 	: "m"(var->counter));
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_dec(ethr_native_atomic_t *var)
+ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
 {
     __asm__ __volatile__(
-	"lock; dec" LONG_SUFFIX " %0"
+	"lock; dec" ETHR_AINT_SUFFIX__ " %0"
 	: "=m"(var->counter)
 	: "m"(var->counter));
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
-    long tmp;
+    ETHR_AINT_T__ tmp;
 
     tmp = incr;
     __asm__ __volatile__(
-	"lock; xadd" LONG_SUFFIX " %0, %1" /* xadd didn't exist prior to the 486 */
+	"lock; xadd" ETHR_AINT_SUFFIX__ " %0, %1" /* xadd didn't exist prior to the 486 */
 	: "=r"(tmp)
 	: "m"(var->counter), "0"(tmp));
     /* now tmp is the atomic's previous value */
     return tmp + incr;
 }
 
-#define ethr_native_atomic_inc_return(var) ethr_native_atomic_add_return((var), 1)
-#define ethr_native_atomic_dec_return(var) ethr_native_atomic_add_return((var), -1)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) 1);
+}
 
-static ETHR_INLINE long
-ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) -1);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
+			   ETHR_AINT_T__ new,
+			   ETHR_AINT_T__ old)
 {
     __asm__ __volatile__(
-      "lock; cmpxchg" LONG_SUFFIX " %2, %3"
+      "lock; cmpxchg" ETHR_AINT_SUFFIX__ " %2, %3"
       : "=a"(old), "=m"(var->counter)
       : "r"(new), "m"(var->counter), "0"(old)
       : "cc", "memory"); /* full memory clobber to make this a compiler barrier */
     return old;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 {
-    long tmp, old;
+    ETHR_AINT_T__ tmp, old;
 
     tmp = var->counter;
     do {
 	old = tmp;
-        tmp = ethr_native_atomic_cmpxchg(var, tmp & mask, tmp);
+        tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp & mask, tmp);
     } while (__builtin_expect(tmp != old, 0));
     /* now tmp is the atomic's previous value */
     return tmp;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 {
-    long tmp, old;
+    ETHR_AINT_T__ tmp, old;
 
     tmp = var->counter;
     do {
 	old = tmp;
-        tmp = ethr_native_atomic_cmpxchg(var, tmp | mask, tmp);
+        tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp | mask, tmp);
     } while (__builtin_expect(tmp != old, 0));
     /* now tmp is the atomic's previous value */
     return tmp;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val)
 {   
-    long tmp = val;
+    ETHR_AINT_T__ tmp = val;
     __asm__ __volatile__(
-	"xchg" LONG_SUFFIX " %0, %1"
+	"xchg" ETHR_AINT_SUFFIX__ " %0, %1"
 	: "=r"(tmp)
 	: "m"(var->counter), "0"(tmp));
     /* now tmp is the atomic's previous value */ 
     return tmp;
 } 
 
-#undef LONG_SUFFIX
+/*
+ * Atomic ops with at least specified barriers.
+ */
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
+{
+    ETHR_AINT_T__ val;
+#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
+    val = var->counter;
+#else
+    val = ETHR_NATMC_FUNC__(add_return)(var, 0);
+#endif
+    __asm__ __volatile__("" : : : "memory");
+    return val;
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    __asm__ __volatile__("" : : : "memory");
+#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
+    var->counter = i;
+#else
+    (void) ETHR_NATMC_FUNC__(xchg)(var, i);
+#endif
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
+{
+    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var);
+    __asm__ __volatile__("" : : : "memory");
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
+{
+    __asm__ __volatile__("" : : : "memory");
+    ETHR_NATMC_FUNC__(dec)(var);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
+{
+    __asm__ __volatile__("" : : : "memory");
+    return ETHR_NATMC_FUNC__(dec_return)(var);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ old)
+{
+    return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ old)
+{
+    return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+}
 
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
-#endif /* ETHREAD_I386_ATOMIC_H */
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_ATMC_T__
+#undef ETHR_AINT_T__
+#undef ETHR_AINT_SUFFIX__
+
+#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */
diff --git a/erts/include/internal/i386/ethread.h b/erts/include/internal/i386/ethread.h
index fad8b108fa..b5a17caefb 100644
--- a/erts/include/internal/i386/ethread.h
+++ b/erts/include/internal/i386/ethread.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,11 +24,17 @@
 #ifndef ETHREAD_I386_ETHREAD_H
 #define ETHREAD_I386_ETHREAD_H
 
+#define ETHR_ATOMIC_WANT_32BIT_IMPL__
 #include "atomic.h"
+#if ETHR_SIZEOF_PTR == 8
+#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
+#  include "atomic.h"
+#endif
 #include "spinlock.h"
 #include "rwlock.h"
 
 #define ETHR_HAVE_NATIVE_ATOMICS 1
-#define ETHR_HAVE_NATIVE_LOCKS 1
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
 
 #endif /* ETHREAD_I386_ETHREAD_H */
diff --git a/erts/include/internal/i386/rwlock.h b/erts/include/internal/i386/rwlock.h
index c009be8ef1..be47f459ce 100644
--- a/erts/include/internal/i386/rwlock.h
+++ b/erts/include/internal/i386/rwlock.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -31,7 +31,7 @@ typedef struct {
     volatile int lock;
 } ethr_native_rwlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 #define ETHR_RWLOCK_OFFSET	(1<<24)
 
diff --git a/erts/include/internal/i386/spinlock.h b/erts/include/internal/i386/spinlock.h
index 2b4832e26a..0325324895 100644
--- a/erts/include/internal/i386/spinlock.h
+++ b/erts/include/internal/i386/spinlock.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -31,7 +31,7 @@ typedef struct {
     volatile unsigned int lock;
 } ethr_native_spinlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
@@ -46,7 +46,7 @@ ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
      * On i386 this needs to be a locked operation
      * to avoid Pentium Pro errata 66 and 92.
      */
-#if defined(__x86_64__)
+#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
     __asm__ __volatile__("" : : : "memory");
     *(unsigned char*)&lock->lock = 0;
 #else
diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h
new file mode 100644
index 0000000000..93bc06036f
--- /dev/null
+++ b/erts/include/internal/libatomic_ops/ethr_atomic.h
@@ -0,0 +1,346 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomics ethread support using libatomic_ops
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__
+#define ETHR_LIBATOMIC_OPS_ATOMIC_H__
+
+#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS)
+#define ETHR_HAVE_NATIVE_ATOMICS 1
+
+#if (defined(__i386__) && !defined(ETHR_PRE_PENTIUM4_COMPAT)) \
+    || defined(__x86_64__)
+#define AO_USE_PENTIUM4_INSTRS
+#endif
+
+#include "atomic_ops.h"
+
+/*
+ * libatomic_ops can be downloaded from:
+ *   http://www.hpl.hp.com/research/linux/atomic_ops/
+ *
+ * These operations need to be defined by libatomic_ops;
+ * otherwise, we won't compile:
+ * - AO_nop_full()
+ * - AO_load()
+ * - AO_store()
+ * - AO_compare_and_swap()
+ *
+ * The `AO_t' type also have to be at least as large as the `void *' type.
+ */
+
+#if ETHR_SIZEOF_AO_T < ETHR_SIZEOF_PTR
+#error The AO_t type is too small
+#endif
+
+#if ETHR_SIZEOF_AO_T == 4
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic32_t
+#define ETHR_AINT_T__ ethr_sint32_t
+#define ETHR_AINT_SUFFIX__ "l"
+#elif ETHR_SIZEOF_AO_T == 8
+#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic64_t
+#define ETHR_AINT_T__ ethr_sint64_t
+#define ETHR_AINT_SUFFIX__ "q"
+#else
+#error "Unsupported integer size"
+#endif
+
+#if ETHR_SIZEOF_AO_T == 8
+typedef union {
+    volatile AO_t counter;
+    ethr_sint32_t sint32[2];
+} ETHR_ATMC_T__;
+#else
+typedef struct {
+    volatile AO_t counter;
+} ETHR_ATMC_T__;
+#endif
+
+#define ETHR_MEMORY_BARRIER AO_nop_full()
+#ifdef AO_HAVE_nop_write
+# define ETHR_WRITE_MEMORY_BARRIER AO_nop_write()
+#else
+# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+#endif
+#ifdef AO_HAVE_nop_read
+#  define ETHR_READ_MEMORY_BARRIER AO_nop_read()
+#else
+#  define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+#endif
+#ifdef AO_NO_DD_ORDERING
+#  define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#else
+#  define ETHR_READ_DEPEND_MEMORY_BARRIER AO_compiler_barrier()
+#endif
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->counter;
+}
+
+#if ETHR_SIZEOF_AO_T == 8
+/*
+ * We also need to provide an ethr_native_atomic32_addr(), since
+ * this 64-bit implementation will be used implementing 32-bit
+ * native atomics.
+ */
+
+static ETHR_INLINE ethr_sint32_t *
+ethr_native_atomic32_addr(ETHR_ATMC_T__ *var)
+{
+    ETHR_ASSERT(((void *) &var->sint32[0]) == ((void *) &var->counter));
+#ifdef ETHR_BIGENDIAN
+    return &var->sint32[1];
+#else
+    return &var->sint32[0];
+#endif
+}
+
+#endif /* ETHR_SIZEOF_AO_T == 8 */
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+{
+    AO_store(&var->counter, (AO_t) value);
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+{
+    ETHR_NATMC_FUNC__(set)(var, value);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__) AO_load(&var->counter);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+{
+#ifdef AO_HAVE_fetch_and_add_full
+    return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr;
+#else
+    while (1) {
+	AO_t exp = AO_load(&var->counter);
+	AO_t new = exp + (AO_t) incr;
+	if (AO_compare_and_swap_full(&var->counter, exp, new))
+	    return (ETHR_AINT_T__) new;
+    }
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+{
+    (void) ETHR_NATMC_FUNC__(add_return)(var, incr);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
+{
+#ifdef AO_HAVE_fetch_and_add1_full
+    return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1;
+#else
+    return ETHR_NATMC_FUNC__(add_return)(var, 1);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
+{
+    (void) ETHR_NATMC_FUNC__(inc_return)(var);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
+{
+#ifdef AO_HAVE_fetch_and_sub1_full
+    return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1;
+#else
+    return ETHR_NATMC_FUNC__(add_return)(var, -1);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
+{
+    (void) ETHR_NATMC_FUNC__(dec_return)(var);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+{
+    while (1) {
+	AO_t exp = AO_load(&var->counter);
+	AO_t new = exp & ((AO_t) mask);
+	if (AO_compare_and_swap_full(&var->counter, exp, new))
+	    return (ETHR_AINT_T__) exp;
+    }
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+{
+    while (1) {
+	AO_t exp = AO_load(&var->counter);
+	AO_t new = exp | ((AO_t) mask);
+	if (AO_compare_and_swap_full(&var->counter, exp, new))
+	    return (ETHR_AINT_T__) exp;
+    }
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
+			   ETHR_AINT_T__ new,
+			   ETHR_AINT_T__ exp)
+{
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+    } while (act == exp);
+    return act;
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
+{
+    while (1) {
+	AO_t exp = AO_load(&var->counter);
+	if (AO_compare_and_swap_full(&var->counter, exp, (AO_t) new))
+	    return (ETHR_AINT_T__) exp;
+    }
+}
+
+/*
+ * Atomic ops with at least specified barriers.
+ */
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
+{
+#ifdef AO_HAVE_load_acquire
+    return (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#else
+    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var);
+    ETHR_MEMORY_BARRIER;
+    return res;
+#endif
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
+{
+#ifdef AO_HAVE_fetch_and_add1_acquire
+    return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1;
+#else
+    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1);
+    return res;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+{
+#ifdef AO_HAVE_store_release
+    AO_store_release(&var->counter, (AO_t) value);
+#else
+    ETHR_MEMORY_BARRIER;
+    ETHR_NATMC_FUNC__(set)(var, value);
+#endif
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
+{
+#ifdef AO_HAVE_fetch_and_sub1_release
+    return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1;
+#else
+    return ETHR_NATMC_FUNC__(dec_return)(var);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
+{
+    (void) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ exp)
+{
+#ifdef AO_HAVE_compare_and_swap_acquire
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+    } while (act == exp);
+    AO_nop_full();
+    return act;
+#else
+    ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp);
+    return act;
+#endif
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ exp)
+{
+#ifdef AO_HAVE_compare_and_swap_release
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+    } while (act == exp);
+    return act;
+#else
+    return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp);
+#endif
+}
+
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_ATMC_T__
+#undef ETHR_AINT_T__
+
+#endif /* !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) */
+
+#endif /* ETHR_LIBATOMIC_OPS_ATOMIC_H__ */
diff --git a/erts/include/internal/libatomic_ops/ethread.h b/erts/include/internal/libatomic_ops/ethread.h
new file mode 100644
index 0000000000..ee73ba73bc
--- /dev/null
+++ b/erts/include/internal/libatomic_ops/ethread.h
@@ -0,0 +1,30 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomics ethread support using libatomic_ops
+ * Author: Rickard Green
+ */
+
+#ifndef ETHREAD_LIBATOMIC_OPS_H__
+#define ETHREAD_LIBATOMIC_OPS_H__
+
+#include "ethr_atomic.h"
+
+#endif
diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h
index fa701c6a92..522f433649 100644
--- a/erts/include/internal/ppc32/atomic.h
+++ b/erts/include/internal/ppc32/atomic.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -28,30 +28,39 @@
 #ifndef ETHREAD_PPC_ATOMIC_H
 #define ETHREAD_PPC_ATOMIC_H
 
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+
 typedef struct {
-    volatile int counter;
-} ethr_native_atomic_t;
+    volatile ethr_sint32_t counter;
+} ethr_native_atomic32_t;
+
+#define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory")
 
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+static ETHR_INLINE ethr_sint32_t *
+ethr_native_atomic32_addr(ethr_native_atomic32_t *var)
+{
+    return (ethr_sint32_t *) &var->counter;
+}
 
 static ETHR_INLINE void
-ethr_native_atomic_init(ethr_native_atomic_t *var, int i)
+ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i)
 {
     var->counter = i;
 }
-#define ethr_native_atomic_set(v, i)	ethr_native_atomic_init((v), (i))
+#define ethr_native_atomic32_set(v, i)	ethr_native_atomic32_init((v), (i))
 
-static ETHR_INLINE int
-ethr_native_atomic_read(ethr_native_atomic_t *var)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_read(ethr_native_atomic32_t *var)
 {
     return var->counter;
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_add_return(ethr_native_atomic_t *var, int incr)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
-    int tmp;
+    ethr_sint32_t tmp;
 
     __asm__ __volatile__(
 	"eieio\n\t"
@@ -68,16 +77,16 @@ ethr_native_atomic_add_return(ethr_native_atomic_t *var, int incr)
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_add(ethr_native_atomic_t *var, int incr)
+ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
     /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic_add_return(var, incr);
+    (void)ethr_native_atomic32_add_return(var, incr);
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_inc_return(ethr_native_atomic_t *var)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var)
 {
-    int tmp;
+    ethr_sint32_t tmp;
 
     __asm__ __volatile__(
 	"eieio\n\t"
@@ -94,16 +103,16 @@ ethr_native_atomic_inc_return(ethr_native_atomic_t *var)
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_inc(ethr_native_atomic_t *var)
+ethr_native_atomic32_inc(ethr_native_atomic32_t *var)
 {
     /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic_inc_return(var);
+    (void)ethr_native_atomic32_inc_return(var);
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_dec_return(ethr_native_atomic_t *var)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
 {
-    int tmp;
+    ethr_sint32_t tmp;
 
     __asm__ __volatile__(
 	"eieio\n\t"
@@ -120,16 +129,16 @@ ethr_native_atomic_dec_return(ethr_native_atomic_t *var)
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_dec(ethr_native_atomic_t *var)
+ethr_native_atomic32_dec(ethr_native_atomic32_t *var)
 {
     /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic_dec_return(var);
+    (void)ethr_native_atomic32_dec_return(var);
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_and_retold(ethr_native_atomic_t *var, int mask)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
-    int old, new;
+    ethr_sint32_t old, new;
 
     __asm__ __volatile__(
 	"eieio\n\t"
@@ -145,10 +154,10 @@ ethr_native_atomic_and_retold(ethr_native_atomic_t *var, int mask)
     return old;
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_or_retold(ethr_native_atomic_t *var, int mask)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
-    int old, new;
+    ethr_sint32_t old, new;
 
     __asm__ __volatile__(
 	"eieio\n\t"
@@ -164,10 +173,10 @@ ethr_native_atomic_or_retold(ethr_native_atomic_t *var, int mask)
     return old;
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_xchg(ethr_native_atomic_t *var, int val)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val)
 {
-    int tmp;
+    ethr_sint32_t tmp;
 
     __asm__ __volatile__(
 	"eieio\n\t"
@@ -182,10 +191,12 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, int val)
     return tmp;
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, int new, int expected)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
+			     ethr_sint32_t new,
+			     ethr_sint32_t expected)
 {
-  int old;
+  ethr_sint32_t old;
 
   __asm__ __volatile__(
     "eieio\n\t"
@@ -204,6 +215,26 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, int new, int expected)
     return old;
 }
 
+/*
+ * Atomic ops with at least specified barriers.
+ */
+
+static ETHR_INLINE long
+ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var)
+{
+    long res = ethr_native_atomic32_read(var);
+    ETHR_MEMORY_BARRIER;
+    return res;
+}
+
+#define ethr_native_atomic32_set_relb ethr_native_atomic32_xchg
+#define ethr_native_atomic32_inc_return_acqb ethr_native_atomic32_inc_return
+#define ethr_native_atomic32_dec_relb ethr_native_atomic32_dec_return
+#define ethr_native_atomic32_dec_return_relb ethr_native_atomic32_dec_return
+
+#define ethr_native_atomic32_cmpxchg_acqb ethr_native_atomic32_cmpxchg
+#define ethr_native_atomic32_cmpxchg_relb ethr_native_atomic32_cmpxchg
+
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
 #endif /* ETHREAD_PPC_ATOMIC_H */
diff --git a/erts/include/internal/ppc32/ethread.h b/erts/include/internal/ppc32/ethread.h
index d2a72c3dc1..3b619e9d01 100644
--- a/erts/include/internal/ppc32/ethread.h
+++ b/erts/include/internal/ppc32/ethread.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -29,6 +29,7 @@
 #include "rwlock.h"
 
 #define ETHR_HAVE_NATIVE_ATOMICS 1
-#define ETHR_HAVE_NATIVE_LOCKS 1
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
 
 #endif /* ETHREAD_PPC32_ETHREAD_H */
diff --git a/erts/include/internal/ppc32/rwlock.h b/erts/include/internal/ppc32/rwlock.h
index 9bdab12826..19ec26ab68 100644
--- a/erts/include/internal/ppc32/rwlock.h
+++ b/erts/include/internal/ppc32/rwlock.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -34,7 +34,7 @@ typedef struct {
     volatile int lock;
 } ethr_native_rwlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_rwlock_init(ethr_native_rwlock_t *lock)
diff --git a/erts/include/internal/ppc32/spinlock.h b/erts/include/internal/ppc32/spinlock.h
index 034c20c143..c8460a3e8a 100644
--- a/erts/include/internal/ppc32/spinlock.h
+++ b/erts/include/internal/ppc32/spinlock.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -34,7 +34,7 @@ typedef struct {
     volatile unsigned int lock;
 } ethr_native_spinlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
diff --git a/erts/include/internal/pthread/ethr_event.h b/erts/include/internal/pthread/ethr_event.h
new file mode 100644
index 0000000000..4c29b28536
--- /dev/null
+++ b/erts/include/internal/pthread/ethr_event.h
@@ -0,0 +1,135 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Author: Rickard Green
+ */
+
+#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_NATIVE_ATOMICS)
+/* --- Linux futex implementation of ethread events ------------------------- */
+#define ETHR_LINUX_FUTEX_IMPL__
+
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <linux/futex.h>
+#include <sys/time.h>
+
+#define ETHR_EVENT_OFF_WAITER__		((ethr_sint32_t) -1)
+#define ETHR_EVENT_OFF__		((ethr_sint32_t) 1)
+#define ETHR_EVENT_ON__ 		((ethr_sint32_t) 0)
+
+#if defined(FUTEX_WAIT_PRIVATE) && defined(FUTEX_WAKE_PRIVATE)
+#  define ETHR_FUTEX_WAIT__ FUTEX_WAIT_PRIVATE
+#  define ETHR_FUTEX_WAKE__ FUTEX_WAKE_PRIVATE
+#else
+#  define ETHR_FUTEX_WAIT__ FUTEX_WAIT
+#  define ETHR_FUTEX_WAKE__ FUTEX_WAKE
+#endif
+
+typedef struct {
+    ethr_atomic32_t futex;
+} ethr_event;
+
+#define ETHR_FUTEX__(FTX, OP, VAL)			\
+  (-1 == syscall(__NR_futex,				\
+		 (void *) ethr_atomic32_addr((FTX)),	\
+		 (OP),					\
+		 (int) (VAL),				\
+		 NULL,					\
+		 NULL,					\
+		 0)					\
+   ? errno : 0)
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
+
+static void ETHR_INLINE
+ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
+{
+    ethr_sint32_t val;
+    ETHR_MEMORY_BARRIER;
+    val = ethr_atomic32_xchg(&e->futex, ETHR_EVENT_ON__);
+    if (val == ETHR_EVENT_OFF_WAITER__) {
+	int res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAKE__, 1);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+    }
+}
+
+static void ETHR_INLINE
+ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e)
+{
+    ethr_atomic32_set(&e->futex, ETHR_EVENT_OFF__);
+    ETHR_MEMORY_BARRIER;
+}
+
+#endif
+
+#elif defined(ETHR_PTHREADS)
+/* --- Posix mutex/cond implementation of events ---------------------------- */
+
+typedef struct {
+    ethr_atomic32_t state;
+    pthread_mutex_t mtx;
+    pthread_cond_t cnd;
+} ethr_event;
+
+#define ETHR_EVENT_OFF_WAITER__		-1L
+#define ETHR_EVENT_OFF__		1L
+#define ETHR_EVENT_ON__ 		0L
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
+
+static void ETHR_INLINE
+ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
+{
+    ethr_sint32_t val;
+    ETHR_MEMORY_BARRIER;
+    val = ethr_atomic32_xchg(&e->state, ETHR_EVENT_ON__);
+    if (val == ETHR_EVENT_OFF_WAITER__) {
+	int res = pthread_mutex_lock(&e->mtx);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+	res = pthread_cond_signal(&e->cnd);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+	res = pthread_mutex_unlock(&e->mtx);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+    }
+}
+
+static void ETHR_INLINE
+ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e)
+{
+    ethr_atomic32_set(&e->state, ETHR_EVENT_OFF__);
+    ETHR_MEMORY_BARRIER;
+}
+
+#endif
+
+#endif
+
+int ethr_event_init(ethr_event *e);
+int ethr_event_destroy(ethr_event *e);
+int ethr_event_wait(ethr_event *e);
+int ethr_event_swait(ethr_event *e, int spincount);
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
+void ethr_event_set(ethr_event *e);
+void ethr_event_reset(ethr_event *e);
+#endif
diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h
index d6fdc6b2a4..c297522ab1 100644
--- a/erts/include/internal/sparc32/atomic.h
+++ b/erts/include/internal/sparc32/atomic.h
@@ -1,19 +1,19 @@
 /* 
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -21,153 +21,250 @@
  * Native ethread atomics on SPARC V9.
  * Author: Mikael Pettersson.
  */
-#ifndef ETHR_SPARC32_ATOMIC_H
-#define ETHR_SPARC32_ATOMIC_H
 
-typedef struct {
-    volatile long counter;
-} ethr_native_atomic_t;
+#undef ETHR_INCLUDE_ATOMIC_IMPL__
+#if !defined(ETHR_SPARC_V9_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
+#define ETHR_SPARC_V9_ATOMIC32_H__
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#elif !defined(ETHR_SPARC_V9_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
+#define ETHR_SPARC_V9_ATOMIC64_H__
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+#endif
+
+#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#ifndef ETHR_SPARC_V9_ATOMIC_COMMON__
+#define ETHR_SPARC_V9_ATOMIC_COMMON__
 
-#if defined(__arch64__)
-#define CASX "casx"
+#define ETHR_MEMORY_BARRIER \
+  __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \
+                       : : : "memory")
+
+#endif /* ETHR_SPARC_V9_ATOMIC_COMMON__ */
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic32_t
+#define ETHR_AINT_T__ ethr_sint32_t
+#define ETHR_CAS__ "cas"
+#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic64_t
+#define ETHR_AINT_T__ ethr_sint64_t
+#define ETHR_CAS__ "casx"
 #else
-#define CASX "cas"
+#error "Unsupported integer size"
 #endif
 
+typedef struct {
+    volatile ETHR_AINT_T__ counter;
+} ETHR_ATMC_T__;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->counter;
+}
+
 static ETHR_INLINE void
-ethr_native_atomic_init(ethr_native_atomic_t *var, long i)
+ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
 {
     var->counter = i;
 }
-#define ethr_native_atomic_set(v, i)	ethr_native_atomic_init((v), (i))
 
-static ETHR_INLINE long
-ethr_native_atomic_read(ethr_native_atomic_t *var)
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    var->counter = i;
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
 {
     return var->counter;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
-    long old, tmp;
+    ETHR_AINT_T__ old, tmp;
 
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
+    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
     do {
 	old = var->counter;
 	tmp = old+incr;
 	__asm__ __volatile__(
-	    CASX " [%2], %1, %0"
+	    ETHR_CAS__ " [%2], %1, %0"
 	    : "=&r"(tmp)
 	    : "r"(old), "r"(&var->counter), "0"(tmp)
 	    : "memory");
     } while (__builtin_expect(old != tmp, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
     return old+incr;
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_add(ethr_native_atomic_t *var, long incr)
+ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
-    (void)ethr_native_atomic_add_return(var, incr);
+    (void)ETHR_NATMC_FUNC__(add_return)(var, incr);
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_inc_return(ethr_native_atomic_t *var)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
 {
-    return ethr_native_atomic_add_return(var, 1);
+    return ETHR_NATMC_FUNC__(add_return)(var, 1);
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_inc(ethr_native_atomic_t *var)
+ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
 {
-    (void)ethr_native_atomic_add_return(var, 1);
+    (void)ETHR_NATMC_FUNC__(add_return)(var, 1);
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_dec_return(ethr_native_atomic_t *var)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
 {
-    return ethr_native_atomic_add_return(var, -1);
+    return ETHR_NATMC_FUNC__(add_return)(var, -1);
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_dec(ethr_native_atomic_t *var)
+ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
 {
-    (void)ethr_native_atomic_add_return(var, -1);
+    (void)ETHR_NATMC_FUNC__(add_return)(var, -1);
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 {
-    long old, tmp;
+    ETHR_AINT_T__ old, tmp;
 
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
+    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
     do {
 	old = var->counter;
 	tmp = old & mask;
 	__asm__ __volatile__(
-	    CASX " [%2], %1, %0"
+	    ETHR_CAS__ " [%2], %1, %0"
 	    : "=&r"(tmp)
 	    : "r"(old), "r"(&var->counter), "0"(tmp)
 	    : "memory");
     } while (__builtin_expect(old != tmp, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
     return old;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 {
-    long old, tmp;
+    ETHR_AINT_T__ old, tmp;
 
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
+    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
     do {
 	old = var->counter;
 	tmp = old | mask;
 	__asm__ __volatile__(
-	    CASX " [%2], %1, %0"
+	    ETHR_CAS__ " [%2], %1, %0"
 	    : "=&r"(tmp)
 	    : "r"(old), "r"(&var->counter), "0"(tmp)
 	    : "memory");
     } while (__builtin_expect(old != tmp, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
     return old;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val)
 {
-    long old, new;
+    ETHR_AINT_T__ old, new;
 
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad");
+    __asm__ __volatile__("membar #LoadLoad|#StoreLoad" : : : "memory");
     do {
 	old = var->counter;
 	new = val;
 	__asm__ __volatile__(
-	    CASX " [%2], %1, %0"
+	    ETHR_CAS__ " [%2], %1, %0"
 	    : "=&r"(new)
 	    : "r"(old), "r"(&var->counter), "0"(new)
 	    : "memory");
     } while (__builtin_expect(old != new, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
     return old;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
 {
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
+    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
     __asm__ __volatile__(
-      CASX " [%2], %1, %0"
+      ETHR_CAS__ " [%2], %1, %0"
       : "=&r"(new)
       : "r"(old), "r"(&var->counter), "0"(new)
       : "memory");
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
     return new;
 }
 
+/*
+ * Atomic ops with at least specified barriers.
+ */
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
+{
+    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var);
+    __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory");
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory");
+    ETHR_NATMC_FUNC__(set)(var, i);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
+{
+    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var);
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
+{
+    ETHR_NATMC_FUNC__(dec)(var);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_NATMC_FUNC__(dec_return)(var);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
+{
+    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+    return res;
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
+{
+    return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+}
+
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
-#endif /* ETHR_SPARC32_ATOMIC_H */
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_ATMC_T__
+#undef ETHR_AINT_T__
+#undef ETHR_CAS__
+
+#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */
diff --git a/erts/include/internal/sparc32/ethread.h b/erts/include/internal/sparc32/ethread.h
index 1d55399640..aea9794390 100644
--- a/erts/include/internal/sparc32/ethread.h
+++ b/erts/include/internal/sparc32/ethread.h
@@ -1,7 +1,7 @@
 /* 
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,11 +24,17 @@
 #ifndef ETHREAD_SPARC32_ETHREAD_H
 #define ETHREAD_SPARC32_ETHREAD_H
 
+#define ETHR_ATOMIC_WANT_32BIT_IMPL__
 #include "atomic.h"
+#if ETHR_SIZEOF_PTR == 8
+#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
+#  include "atomic.h"
+#endif
 #include "spinlock.h"
 #include "rwlock.h"
 
 #define ETHR_HAVE_NATIVE_ATOMICS 1
-#define ETHR_HAVE_NATIVE_LOCKS 1
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
 
 #endif /* ETHREAD_SPARC32_ETHREAD_H */
diff --git a/erts/include/internal/sparc32/rwlock.h b/erts/include/internal/sparc32/rwlock.h
index 12448e0b06..465ec96866 100644
--- a/erts/include/internal/sparc32/rwlock.h
+++ b/erts/include/internal/sparc32/rwlock.h
@@ -1,19 +1,19 @@
 /* 
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -29,7 +29,7 @@ typedef struct {
     volatile int lock;
 } ethr_native_rwlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_rwlock_init(ethr_native_rwlock_t *lock)
diff --git a/erts/include/internal/sparc32/spinlock.h b/erts/include/internal/sparc32/spinlock.h
index b4fe48b714..493d514210 100644
--- a/erts/include/internal/sparc32/spinlock.h
+++ b/erts/include/internal/sparc32/spinlock.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -29,7 +29,7 @@ typedef struct {
     volatile unsigned char lock;
 } ethr_native_spinlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h
index 59a9250e7c..5697afda25 100644
--- a/erts/include/internal/tile/atomic.h
+++ b/erts/include/internal/tile/atomic.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,105 +24,179 @@
 #ifndef ETHREAD_TILE_ATOMIC_H
 #define ETHREAD_TILE_ATOMIC_H
 
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+
 #include <atomic.h>
 
 /* An atomic is an aligned int accessed via locked operations.
  */
 typedef struct {
-    volatile long counter;
-} ethr_native_atomic_t;
+    volatile ethr_sint32_t counter;
+} ethr_native_atomic32_t;
+
+#define ETHR_MEMORY_BARRIER __insn_mf()
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+static ETHR_INLINE ethr_sint32_t *
+ethr_native_atomic32_addr(ethr_native_atomic32_t *var)
+{
+    return (ethr_sint32_t *) &var->counter;
+}
 
 static ETHR_INLINE void
-ethr_native_atomic_init(ethr_native_atomic_t *var, long i)
+ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i)
 {
     var->counter = i;
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_set(ethr_native_atomic_t *var, long i)
+ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i)
 {
-    __insn_mf();
     atomic_exchange_acq(&var->counter, i);
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_read(ethr_native_atomic_t *var)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_read(ethr_native_atomic32_t *var)
 {
     return var->counter;
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_add(ethr_native_atomic_t *var, long incr)
+ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
-    __insn_mf();
+    ETHR_MEMORY_BARRIER;
     atomic_add(&var->counter, incr);
+    ETHR_MEMORY_BARRIER;
 }      
        
 static ETHR_INLINE void
-ethr_native_atomic_inc(ethr_native_atomic_t *var)
+ethr_native_atomic32_inc(ethr_native_atomic32_t *var)
 {
-    __insn_mf();
+    ETHR_MEMORY_BARRIER;
     atomic_increment(&var->counter);
+    ETHR_MEMORY_BARRIER;
 }
 
 static ETHR_INLINE void
-ethr_native_atomic_dec(ethr_native_atomic_t *var)
+ethr_native_atomic32_dec(ethr_native_atomic32_t *var)
 {
-    __insn_mf();
+    ETHR_MEMORY_BARRIER;
     atomic_decrement(&var->counter);
+    ETHR_MEMORY_BARRIER;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
-    __insn_mf();
-    return atomic_exchange_and_add(&var->counter, incr) + incr;
+    ethr_sint32_t res;
+    ETHR_MEMORY_BARRIER;
+    res = atomic_exchange_and_add(&var->counter, incr) + incr;
+    ETHR_MEMORY_BARRIER;
+    return res;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_inc_return(ethr_native_atomic_t *var)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var)
 {
-    return ethr_native_atomic_add_return(var, 1);
+    return ethr_native_atomic32_add_return(var, 1);
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_dec_return(ethr_native_atomic_t *var)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
 {
-    return ethr_native_atomic_add_return(var, -1);
+    return ethr_native_atomic32_add_return(var, -1);
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
-    /* Implement a barrier suitable for a mutex unlock. */
-    __insn_mf();
-    return atomic_and_val(&var->counter, mask);
+    ethr_sint32_t res;
+    ETHR_MEMORY_BARRIER;
+    res = atomic_and_val(&var->counter, mask);
+    ETHR_MEMORY_BARRIER;
+    return res;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
-    __insn_mf();
-    return atomic_or_val(&var->counter, mask);
+    ethr_sint32_t res;
+    ETHR_MEMORY_BARRIER;
+    res = atomic_or_val(&var->counter, mask);
+    ETHR_MEMORY_BARRIER;
+    return res;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val)
 {   
-    __insn_mf();
+    ETHR_MEMORY_BARRIER;
     return atomic_exchange_acq(&var->counter, val);
 } 
 
-static ETHR_INLINE long
-ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long expected)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
+			     ethr_sint32_t new,
+			     ethr_sint32_t expected)
 {
-    /* Implement a barrier suitable for a mutex unlock. */
-    __insn_mf();
+    ETHR_MEMORY_BARRIER;
     return atomic_compare_and_exchange_val_acq(&var->counter, new, expected);
 }
 
+/*
+ * Atomic ops with at least specified barriers.
+ */
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var)
+{
+    ethr_sint32_t res = ethr_native_atomic32_read(var);
+    ETHR_MEMORY_BARRIER;
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var)
+{
+    return ethr_native_atomic32_inc_return(var);
+}
+
+static ETHR_INLINE void
+ethr_native_atomic32_set_relb(ethr_native_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_MEMORY_BARRIER;
+    ethr_native_atomic32_set(var, val);
+}
+
+static ETHR_INLINE void
+ethr_native_atomic32_dec_relb(ethr_native_atomic32_t *var)
+{
+    ethr_native_atomic32_dec(var);
+}
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_dec_return_relb(ethr_native_atomic32_t *var)
+{
+    return ethr_native_atomic32_dec_return(var);
+}
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var,
+				  ethr_sint32_t new,
+				  ethr_sint32_t exp)
+{
+    return ethr_native_atomic32_cmpxchg(var, new, exp);
+}
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_cmpxchg_relb(ethr_native_atomic32_t *var,
+				  ethr_sint32_t new,
+				  ethr_sint32_t exp)
+{
+    return ethr_native_atomic32_cmpxchg(var, new, exp);
+}
+
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
 #endif /* ETHREAD_TILE_ATOMIC_H */
diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h
new file mode 100644
index 0000000000..60def01a7e
--- /dev/null
+++ b/erts/include/internal/win/ethr_atomic.h
@@ -0,0 +1,415 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomics ethread support when using VC++
+ * Author: Rickard Green
+ */
+
+#undef ETHR_INCLUDE_ATOMIC_IMPL__
+#if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
+#define ETHR_WIN_ATOMIC32_H__
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
+#define ETHR_WIN_ATOMIC64_H__
+#ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
+/* _InterlockedCompareExchange64() required... */
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#endif
+#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+#endif
+
+#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
+
+#if defined(_MSC_VER) && _MSC_VER >= 1400
+
+#ifndef ETHR_WIN_ATOMIC_COMMON__
+#define ETHR_WIN_ATOMIC_COMMON__
+
+#define ETHR_HAVE_NATIVE_ATOMICS 1
+
+#if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64)
+#  define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1
+#else
+#  define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0
+#endif
+
+#if defined(_M_AMD64) || (defined(_M_IX86) \
+			  && !defined(ETHR_PRE_PENTIUM4_COMPAT))
+#  define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1
+#else
+#  define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0
+#endif
+/*
+ * No configure test checking for interlocked acquire/release
+ * versions have been written, yet. It should define
+ * ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS if, and
+ * only if, all used interlocked operations with barriers
+ * exists.
+ *
+ * Note, that these are pure optimizations for the itanium
+ * processor.
+ */
+
+#include <intrin.h>
+#undef ETHR_COMPILER_BARRIER
+#define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
+#pragma intrinsic(_ReadWriteBarrier)
+#pragma intrinsic(_InterlockedCompareExchange)
+
+#if defined(_M_AMD64) || (defined(_M_IX86) \
+			  && !defined(ETHR_PRE_PENTIUM4_COMPAT))
+#include <emmintrin.h>
+#include <mmintrin.h>
+#pragma intrinsic(_mm_mfence)
+#define ETHR_MEMORY_BARRIER _mm_mfence()
+#pragma intrinsic(_mm_sfence)
+#define ETHR_WRITE_MEMORY_BARRIER _mm_sfence()
+#pragma intrinsic(_mm_lfence)
+#define ETHR_READ_MEMORY_BARRIER _mm_lfence()
+#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
+
+#else
+
+#define ETHR_MEMORY_BARRIER					\
+do {								\
+    volatile long x___ = 0;					\
+    _InterlockedCompareExchange(&x___, (long) 1, (long) 0);	\
+} while (0)
+
+#endif
+
+#endif /* ETHR_WIN_ATOMIC_COMMON__ */
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+
+/*
+ * All used operations available as 32-bit intrinsics
+ */
+
+#pragma intrinsic(_InterlockedDecrement)
+#pragma intrinsic(_InterlockedIncrement)
+#pragma intrinsic(_InterlockedExchangeAdd)
+#pragma intrinsic(_InterlockedExchange)
+#pragma intrinsic(_InterlockedAnd)
+#pragma intrinsic(_InterlockedOr)
+#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
+#pragma intrinsic(_InterlockedExchangeAdd_acq)
+#pragma intrinsic(_InterlockedIncrement_acq)
+#pragma intrinsic(_InterlockedDecrement_rel)
+#pragma intrinsic(_InterlockedCompareExchange_acq)
+#pragma intrinsic(_InterlockedCompareExchange_rel)
+#endif
+
+#define ETHR_ILCKD__(X) _Interlocked ## X
+#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
+#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq
+#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel
+#else
+#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X
+#define ETHR_ILCKD_REL__(X) _Interlocked ## X
+#endif
+
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic32_t
+#define ETHR_AINT_T__ ethr_sint32_t
+
+#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+
+#define ETHR_HAVE_NATIVE_ATOMIC64 1
+
+/*
+ * _InterlockedCompareExchange64() is required. The other may not
+ * be available, but if so, we can generate them.
+ */
+#pragma intrinsic(_InterlockedCompareExchange64)
+
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__		
+#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) *(PTR)
+#else
+#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) (__int64) 0
+#endif
+
+#define ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, PTR, NEW, ACT, EXP, OPS, RET)	\
+{									\
+    __int64 NEW, ACT, EXP;						\
+    ACT = ETHR_OWN_ILCKD_INIT_VAL__(PTR);				\
+    do {								\
+	EXP = ACT;							\
+	{ OPS; }							\
+	ACT = _InterlockedCompareExchange64(PTR, NEW, EXP);		\
+    } while (ACT != EXP);						\
+    return RET;								\
+}
+
+#define ETHR_OWN_ILCKD_1_IMPL__(FUNC, NEW, ACT, EXP, OPS, RET)		\
+static __forceinline __int64						\
+FUNC(__int64 volatile *ptr)						\
+ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET)
+
+#define ETHR_OWN_ILCKD_2_IMPL__(FUNC, NEW, ACT, EXP, OPS, ARG, RET)	\
+static __forceinline __int64						\
+FUNC(__int64 volatile *ptr, __int64 ARG)				\
+ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET)
+
+
+#ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64
+#pragma intrinsic(_InterlockedDecrement64)
+#else
+ETHR_OWN_ILCKD_1_IMPL__(_InterlockedDecrement64, new, act, exp,
+			new = act - 1, new)
+#endif
+#ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64
+#pragma intrinsic(_InterlockedIncrement64)
+#else
+ETHR_OWN_ILCKD_1_IMPL__(_InterlockedIncrement64, new, act, exp,
+			new = act + 1, new)
+#endif
+#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
+#pragma intrinsic(_InterlockedExchangeAdd64)
+#else
+ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchangeAdd64, new, act, exp,
+			new = act + arg, arg, act)
+#endif
+#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64
+#pragma intrinsic(_InterlockedExchange64)
+#else
+ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchange64, new, act, exp,
+			new = arg, arg, act)
+#endif
+#ifdef ETHR_HAVE__INTERLOCKEDAND64
+#pragma intrinsic(_InterlockedAnd64)
+#else
+ETHR_OWN_ILCKD_2_IMPL__(_InterlockedAnd64, new, act, exp,
+			new = act & arg, arg, act)
+#endif
+#ifdef ETHR_HAVE__INTERLOCKEDOR64
+#pragma intrinsic(_InterlockedOr64)
+#else
+ETHR_OWN_ILCKD_2_IMPL__(_InterlockedOr64, new, act, exp,
+			new = act | arg, arg, act)
+#endif
+#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
+#pragma intrinsic(_InterlockedExchangeAdd64_acq)
+#pragma intrinsic(_InterlockedIncrement64_acq)
+#pragma intrinsic(_InterlockedDecrement64_rel)
+#pragma intrinsic(_InterlockedCompareExchange64_acq)
+#pragma intrinsic(_InterlockedCompareExchange64_rel)
+#endif
+
+#define ETHR_ILCKD__(X) _Interlocked ## X ## 64
+#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
+#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq
+#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel
+#else
+#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64
+#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64
+#endif
+
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic64_t
+#define ETHR_AINT_T__ ethr_sint64_t
+
+#else
+#error "Unsupported integer size"
+#endif
+
+typedef struct {
+    volatile ETHR_AINT_T__ value;
+} ETHR_ATMC_T__;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->value;
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
+    var->value = i;
+#else
+    (void) ETHR_ILCKD__(Exchange)(&var->value, i);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
+    var->value = i;
+#else
+    (void) ETHR_ILCKD__(Exchange)(&var->value, i);
+#endif
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
+{
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
+    return var->value;
+#else
+    return ETHR_ILCKD__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+{
+    (void) ETHR_ILCKD__(ExchangeAdd)(&var->value, incr);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i;
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
+{
+    (void) ETHR_ILCKD__(Increment)(&var->value);
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
+{
+    (void) ETHR_ILCKD__(Decrement)(&var->value);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_ILCKD__(Increment)(&var->value);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_ILCKD__(Decrement)(&var->value);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+{
+    return ETHR_ILCKD__(And)(&var->value, mask);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+{
+    return ETHR_ILCKD__(Or)(&var->value, mask);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
+			   ETHR_AINT_T__ new,
+			   ETHR_AINT_T__ old)
+{
+    return ETHR_ILCKD__(CompareExchange)(&var->value, new, old);
+}
+
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
+{
+    return ETHR_ILCKD__(Exchange)(&var->value, new);
+}
+
+/*
+ * Atomic ops with at least specified barriers.
+ */
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
+{
+#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
+    ETHR_AINT_T__ val = var->value;
+    ETHR_COMPILER_BARRIER;
+    return val;
+#else
+    return ETHR_ILCKD_ACQ__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0);
+#endif
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_ILCKD_ACQ__(Increment)(&var->value);
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
+    ETHR_COMPILER_BARRIER;
+    var->value = i;
+#else
+    (void) ETHR_ILCKD_REL__(Exchange)(&var->value, i);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
+{
+    (void) ETHR_ILCKD_REL__(Decrement)(&var->value);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_ILCKD_REL__(Decrement)(&var->value);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ old)
+{
+    return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old);
+}
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ old)
+{
+    return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old);
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#undef ETHR_ILCKD__
+#undef ETHR_ILCKD_ACQ__
+#undef ETHR_ILCKD_REL__
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_ATMC_T__
+#undef ETHR_AINT_T__
+#undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
+#undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
+
+#endif /* _MSC_VER */
+
+#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */
diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h
new file mode 100644
index 0000000000..598816b2c6
--- /dev/null
+++ b/erts/include/internal/win/ethr_event.h
@@ -0,0 +1,64 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Author: Rickard Green
+ */
+
+#define ETHR_EVENT_OFF_WAITER__		((long) -1)
+#define ETHR_EVENT_OFF__		((long) 1)
+#define ETHR_EVENT_ON__ 		((long) 0)
+
+typedef struct {
+    volatile long state;
+    HANDLE handle;
+} ethr_event;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
+
+#pragma intrinsic(_InterlockedExchange)
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
+{
+    /* _InterlockedExchange() imply a full memory barrier which is important */
+    long state = _InterlockedExchange(&e->state, ETHR_EVENT_ON__);
+    if (state == ETHR_EVENT_OFF_WAITER__) {
+	if (!SetEvent(e->handle))
+	    ETHR_FATAL_ERROR__(ethr_win_get_errno__());
+    }
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e)
+{
+    /* _InterlockedExchange() imply a full memory barrier which is important */
+    InterlockedExchange(&e->state, ETHR_EVENT_OFF__);
+}
+
+#endif
+
+int ethr_event_init(ethr_event *e);
+int ethr_event_destroy(ethr_event *e);
+int ethr_event_wait(ethr_event *e);
+int ethr_event_swait(ethr_event *e, int spincount);
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
+void ethr_event_set(ethr_event *e);
+void ethr_event_reset(ethr_event *e);
+#endif
diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h
new file mode 100644
index 0000000000..c01b17cf14
--- /dev/null
+++ b/erts/include/internal/win/ethread.h
@@ -0,0 +1,35 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomic and spinlock ethread support when using VC++
+ * Author: Rickard Green
+ */
+
+#ifndef ETHREAD_WIN_H__
+#define ETHREAD_WIN_H__
+
+#define ETHR_ATOMIC_WANT_32BIT_IMPL__
+#include "ethr_atomic.h"
+#if ETHR_SIZEOF_PTR == 8
+#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
+#  include "ethr_atomic.h"
+#endif
+
+#endif
diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in
index 49f5b1f048..757b3b24e2 100644
--- a/erts/lib_src/Makefile.in
+++ b/erts/lib_src/Makefile.in
@@ -280,8 +280,14 @@ endif
 #
 # ethread library
 #
+ETHR_THR_LIB_BASE_DIR=@ETHR_THR_LIB_BASE_DIR@
 ifneq ($(strip $(ETHR_LIB_NAME)),)
-ETHREAD_LIB_SRC=common/ethread.c
+ETHREAD_LIB_SRC=common/ethr_aux.c \
+		common/ethr_atomics.c \
+		common/ethr_mutex.c \
+		common/ethr_cbf.c \
+		$(ETHR_THR_LIB_BASE_DIR)/ethread.c \
+		$(ETHR_THR_LIB_BASE_DIR)/ethr_event.c
 ETHREAD_LIB_NAME=ethread$(TYPE_SUFFIX)
 
 ifeq ($(USING_VC),yes)
@@ -376,16 +382,21 @@ $(ERTS_LIB): $(ERTS_LIB_OBJS)
 # Object files
 #
 
+ifeq ($(TYPE)-@GCC@,debug-yes)
+$(r_OBJ_DIR)/ethr_aux.o: common/ethr_aux.c
+	$(CC) $(THR_DEFS) $(CFLAGS) -Wno-unused-function $(INCLUDES) -c $< -o $@
+endif
+
 $(r_OBJ_DIR)/%.o: common/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@
 
-$(r_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(r_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@
 
 $(OBJ_DIR)/%.o: common/%.c
 	$(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@
 
-$(OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@
 
 # Win32 specific
@@ -393,25 +404,25 @@ $(OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
 $(MD_OBJ_DIR)/%.o: common/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MD $(INCLUDES) -c $< -o $@
 
-$(MD_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(MD_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MD $(INCLUDES) -c $< -o $@
 
 $(MDd_OBJ_DIR)/%.o: common/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MDd $(INCLUDES) -c $< -o $@
 
-$(MDd_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(MDd_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MDd $(INCLUDES) -c $< -o $@
 
 $(MT_OBJ_DIR)/%.o: common/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MT $(INCLUDES) -c $< -o $@
 
-$(MT_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(MT_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MT $(INCLUDES) -c $< -o $@
 
 $(MTd_OBJ_DIR)/%.o: common/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MTd $(INCLUDES) -c $< -o $@
 
-$(MTd_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(MTd_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MTd $(INCLUDES) -c $< -o $@
 
 #
@@ -438,6 +449,9 @@ RELEASE_LIBS=$(ERTS_LIBS)
 INTERNAL_RELEASE_INCLUDES= \
 	$(ERTS_INCL_INT)/README \
 	$(ERTS_INCL_INT)/ethread.h \
+	$(ERTS_INCL_INT)/ethr_mutex.h \
+	$(ERTS_INCL_INT)/ethr_optimized_fallbacks.h \
+	$(ERTS_INCL_INT)/ethr_atomics.h \
 	$(ERTS_INCL_INT)/$(TARGET)/ethread.mk \
 	$(ERTS_INCL_INT)/$(TARGET)/erts_internal.mk \
 	$(ERTS_INCL_INT)/$(TARGET)/ethread_header_config.h \
@@ -447,7 +461,8 @@ INTERNAL_RELEASE_INCLUDES= \
 	$(ERTS_INCL_INT)/erl_misc_utils.h \
 	$(ERTS_INCL_INT)/erl_errno.h
 
-INTERNAL_X_RELEASE_INCLUDE_DIRS= i386 x86_64 ppc32 sparc32 sparc64 tile
+INTERNAL_X_RELEASE_INCLUDE_DIRS= \
+        i386 x86_64 ppc32 sparc32 sparc64 tile gcc pthread win libatomic_ops
 
 INTERNAL_RELEASE_LIBS= \
 	../lib/internal/README \
diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c
index 9c25d33a3c..5dbf98c7d1 100644
--- a/erts/lib_src/common/erl_misc_utils.c
+++ b/erts/lib_src/common/erl_misc_utils.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -21,10 +21,13 @@
 #include "config.h"
 #endif
 
+#if defined(__WIN32__)
+#  include <windows.h>
+#endif
+
 #include "erl_misc_utils.h"
 
 #if defined(__WIN32__)
-#  include <windows.h>
 #elif defined(VXWORKS)
 #  include <selectLib.h>
 #else /* UNIX */
@@ -59,8 +62,38 @@
 #  endif
 #endif
 
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(HAVE_SCHED_xETAFFINITY)
 #  include <sched.h>
+#  define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP, CPUSET)			\
+     (sched_getaffinity((CPUINFOP)->pid,				\
+			sizeof(cpu_set_t),				\
+			(CPUSET)) != 0 ? -errno : 0)
+#define ERTS_MU_SET_THR_AFFINITY__(SETP)				\
+     (sched_setaffinity(0, sizeof(cpu_set_t), (SETP)) != 0 ? -errno : 0)
+#elif defined(HAVE_CPUSET_xETAFFINITY)
+#  include <sys/param.h>
+#  include <sys/cpuset.h>
+#  define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP, CPUSET)			\
+     (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, 	    \
+			sizeof(cpuset_t),				\
+			(CPUSET)) != 0 ? -errno : 0)
+#define ERTS_MU_SET_THR_AFFINITY__(CPUSETP)				\
+     (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1,    \
+            sizeof(cpuset_t),               \
+            (CPUSETP)) != 0 ? -errno : 0)
+#  define cpu_set_t cpuset_t
+#elif defined(__WIN32__)
+#  define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#  define cpu_set_t DWORD
+#  define CPU_SETSIZE (sizeof(DWORD)*8)
+#  define CPU_ZERO(SETP) (*(SETP) = (DWORD) 0)
+#  define CPU_SET(CPU, SETP) (*(SETP) |= (((DWORD) 1) << (CPU)))
+#  define CPU_CLR(CPU, SETP) (*(SETP) &= ~(((DWORD) 1) << (CPU)))
+#  define CPU_ISSET(CPU, SETP) ((*(SETP) & (((DWORD) 1) << (CPU))) != (DWORD) 0)
+#define ERTS_MU_GET_PROC_AFFINITY__ get_proc_affinity
+#define ERTS_MU_SET_THR_AFFINITY__ set_thr_affinity
 #endif
 #ifdef HAVE_PSET_INFO
 #  include <sys/pset.h>
@@ -80,8 +113,33 @@
 #  define ERTS_SYS_CPU_PATH	"/sys/devices/system/cpu"
 #endif
 
+#ifdef __FreeBSD__
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#endif
+
 static int read_topology(erts_cpu_info_t *cpuinfo);
 
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+static int
+cpu_sets_are_eq(cpu_set_t *x, cpu_set_t *y)
+{
+    int i;
+    for (i = 0; i < CPU_SETSIZE; i++) {
+	if (CPU_ISSET(i, x)) {
+	    if (!CPU_ISSET(i, y))
+		return 0;
+	}
+	else {
+	    if (CPU_ISSET(i, y))
+		return 0;
+	}
+    }
+    return 1;
+}
+
+#endif
+
 int
 erts_milli_sleep(long ms)
 {
@@ -105,30 +163,66 @@ struct erts_cpu_info_t_ {
     int available;
     int topology_size;
     erts_cpu_topology_t *topology;
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     char *affinity_str;
     char affinity_str_buf[CPU_SETSIZE/4+2];
     cpu_set_t cpuset;
+#if defined(HAVE_SCHED_xETAFFINITY)
     pid_t pid;
+#endif
 #elif defined(HAVE_PSET_INFO)
     processorid_t *cpuids;
 #endif
 };
 
+#if defined(__WIN32__)
+
+static __forceinline int
+get_proc_affinity(erts_cpu_info_t *cpuinfo, cpu_set_t *cpuset)
+{
+    DWORD pamask, samask;
+    if (GetProcessAffinityMask(GetCurrentProcess(), &pamask, &samask)) {
+	*cpuset = (cpu_set_t) pamask;
+	return 0;
+    }
+    else {
+	*cpuset = (cpu_set_t) 0;
+	return -erts_get_last_win_errno();
+    }
+}
+
+static __forceinline int
+set_thr_affinity(cpu_set_t *set)
+{
+    if (*set == (cpu_set_t) 0)
+	return -ENOTSUP;
+    if (SetThreadAffinityMask(GetCurrentThread(), *set) == 0)
+	return -erts_get_last_win_errno();
+    else
+	return 0;
+}
+
+#endif
+
 erts_cpu_info_t *
 erts_cpu_info_create(void)
 {
     erts_cpu_info_t *cpuinfo = malloc(sizeof(erts_cpu_info_t));
     if (!cpuinfo)
 	return NULL;
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     cpuinfo->affinity_str = NULL;
+#if defined(HAVE_SCHED_xETAFFINITY)
     cpuinfo->pid = getpid();
+#endif
 #elif defined(HAVE_PSET_INFO)
     cpuinfo->cpuids = NULL;
 #endif
     cpuinfo->topology_size = 0;
     cpuinfo->topology = NULL;
+    cpuinfo->configured = -1;
+    cpuinfo->online = -1;
+    cpuinfo->available = -1;
     erts_cpu_info_update(cpuinfo);
     return cpuinfo;
 }
@@ -153,31 +247,40 @@ erts_cpu_info_destroy(erts_cpu_info_t *cpuinfo)
     }
 }
 
-void
+int
 erts_cpu_info_update(erts_cpu_info_t *cpuinfo)
 {
-    cpuinfo->configured = 0;
-    cpuinfo->online = 0;
-    cpuinfo->available = 0;
+    int changed = 0;
+    int configured = 0;
+    int online = 0;
+    int available = 0;
+    erts_cpu_topology_t *old_topology;
+    int old_topology_size;
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+    cpu_set_t cpuset;
+#endif
 
 #ifdef __WIN32__
     {
+	int i;
 	SYSTEM_INFO sys_info;
 	GetSystemInfo(&sys_info);
-	cpuinfo->configured = (int) sys_info.dwNumberOfProcessors;
-	
+	configured = (int) sys_info.dwNumberOfProcessors;
+	for (i = 0; i < sizeof(DWORD)*8; i++)
+	    if (sys_info.dwActiveProcessorMask & (((DWORD) 1) << i))
+		online++;
     }
 #elif !defined(NO_SYSCONF) && (defined(_SC_NPROCESSORS_CONF) \
 			       || defined(_SC_NPROCESSORS_ONLN))
 #ifdef _SC_NPROCESSORS_CONF
-    cpuinfo->configured = (int) sysconf(_SC_NPROCESSORS_CONF);
-    if (cpuinfo->configured < 0)
-	cpuinfo->configured = 0;
+    configured = (int) sysconf(_SC_NPROCESSORS_CONF);
+    if (configured < 0)
+	configured = 0;
 #endif
 #ifdef _SC_NPROCESSORS_ONLN
-    cpuinfo->online = (int) sysconf(_SC_NPROCESSORS_ONLN);
-    if (cpuinfo->online < 0)
-	cpuinfo->online = 0;
+    online = (int) sysconf(_SC_NPROCESSORS_ONLN);
+    if (online < 0)
+	online = 0;
 #endif
 #elif defined(HAVE_SYS_SYSCTL_H) && defined(CTL_HW) && (defined(HW_NCPU) \
 							|| defined(HW_AVAILCPU))
@@ -189,71 +292,138 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo)
 	len = sizeof(int);
 	mib[0] = CTL_HW;
 	mib[1] = HW_NCPU;
-	if (sysctl(&mib[0], 2, &cpuinfo->configured, &len, NULL, 0) < 0)
-	    cpuinfo->configured = 0;
+	if (sysctl(&mib[0], 2, &configured, &len, NULL, 0) < 0)
+	    configured = 0;
 #endif
 #ifdef HW_AVAILCPU
 	len = sizeof(int);
 	mib[0] = CTL_HW;
 	mib[1] = HW_AVAILCPU;
-	if (sysctl(&mib[0], 2, &cpuinfo->online, &len, NULL, 0) < 0)
-	    cpuinfo->online = 0;
+	if (sysctl(&mib[0], 2, &online, &len, NULL, 0) < 0)
+	    online = 0;
 #endif
     }
 #endif
 
-    if (cpuinfo->online > cpuinfo->configured)
-	cpuinfo->online = cpuinfo->configured;
+    if (online > configured)
+	online = configured;
+
+    if (cpuinfo->configured != configured)
+	changed = 1;
+    if (cpuinfo->online != online)
+	changed = 1;
 
-#ifdef HAVE_SCHED_xETAFFINITY
-    if (sched_getaffinity(cpuinfo->pid, sizeof(cpu_set_t), &cpuinfo->cpuset) == 0) {
-	int i, c, cn, si;
-	c = cn = 0;
-	si = sizeof(cpuinfo->affinity_str_buf) - 1;
-	cpuinfo->affinity_str_buf[si] = '\0';
-	for (i = 0; i < CPU_SETSIZE; i++) {
-	    if (CPU_ISSET(i, &cpuinfo->cpuset)) {
-		c |= 1 << cn;
-		cpuinfo->available++;
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+    if (ERTS_MU_GET_PROC_AFFINITY__(cpuinfo, &cpuset) == 0) {
+	if (!changed && !cpu_sets_are_eq(&cpuset, &cpuinfo->cpuset))
+	    changed = 1;
+
+	if (!changed)
+	    available = cpuinfo->available;
+	else {
+	    int i, c, cn, si;
+
+	    memcpy((void *) &cpuinfo->cpuset,
+		   (void *) &cpuset,
+		   sizeof(cpu_set_t));
+
+	    c = cn = 0;
+	    si = sizeof(cpuinfo->affinity_str_buf) - 1;
+	    cpuinfo->affinity_str_buf[si] = '\0';
+	    for (i = 0; i < CPU_SETSIZE; i++) {
+		if (CPU_ISSET(i, &cpuinfo->cpuset)) {
+		    c |= 1 << cn;
+		    available++;
+		}
+		cn++;
+		if (cn == 4) {
+		    cpuinfo->affinity_str_buf[--si] = (c < 10
+						       ? '0' + c
+						       : 'A' + c - 10);
+		    c = cn = 0;
+		}
 	    }
-	    cn++;
-	    if (cn == 4) {
+	    if (c)
 		cpuinfo->affinity_str_buf[--si] = (c < 10
 						   ? '0' + c
 						   : 'A' + c - 10);
-		c = cn = 0;
-	    }
+	    while (cpuinfo->affinity_str_buf[si] == '0')
+		si++;
+	    cpuinfo->affinity_str = &cpuinfo->affinity_str_buf[si];
 	}
-	if (c)
-	    cpuinfo->affinity_str_buf[--si] = (c < 10
-					       ? '0' + c
-					       : 'A' + c - 10);
-	while (cpuinfo->affinity_str_buf[si] == '0')
-	    si++;
-	cpuinfo->affinity_str = &cpuinfo->affinity_str_buf[si];
     }
 #elif defined(HAVE_PSET_INFO)
     {
-	uint_t numcpus = cpuinfo->configured;
-	if (cpuinfo->cpuids)
-	    free(cpuinfo->cpuids);
-	cpuinfo->cpuids = malloc(sizeof(processorid_t)*numcpus);
-	if (cpuinfo->cpuids) {
-	    if (pset_info(PS_MYID, NULL, &numcpus, &cpuinfo->cpuids) == 0)
-		cpuinfo->available = (int) numcpus;
-	    if (cpuinfo->available < 0) {
-		free(cpuinfo->cpuid);
-		cpuinfo->available = 0;
+	processorid_t *cpuids;
+	uint_t numcpus = configured;
+	cpuids = malloc(sizeof(processorid_t)*numcpus);
+	if (cpuids) {
+	    if (pset_info(PS_MYID, NULL, &numcpus, &cpuids) == 0)
+		available = (int) numcpus;
+	    if (available < 0) {
+		free(cpuids);
+		cpuids = NULL;
+		available = 0;
 	    }
 	}
+	if (!cpuids) {
+	    if (cpuinfo->cpuids)
+		changed = 1;
+	}
+	else {
+	    if (cpuinfo->cpuids)
+		changed = 1;
+	    if (memcmp((void *) cpuinfo->cpuids,
+		       (void *) cpuids,
+		       sizeof(processorid_t)*numcpus) != 0)
+		changed = 1;
+
+	}
+	if (!changed) {
+	    if (cpuids)
+		free(cpuids);
+	}
+	else {
+	    if (cpuinfo->cpuids)
+		free(cpuinfo->cpuids);
+	    cpuinfo->cpuids = cpuids;
+	}
     }
 #endif
 
-    if (cpuinfo->available > cpuinfo->online)
-	cpuinfo->available = cpuinfo->online;
+    if (available > online)
+	available = online;
+
+    if (cpuinfo->available != available)
+	changed = 1;
+
+    cpuinfo->configured = configured;
+    cpuinfo->online = online;
+    cpuinfo->available = available;
+
+    old_topology = cpuinfo->topology;
+    old_topology_size = cpuinfo->topology_size;
+    cpuinfo->topology = NULL;
 
     read_topology(cpuinfo);
 
+    if (cpuinfo->topology_size != old_topology_size
+	|| (old_topology_size != 0
+	    && memcmp((void *) cpuinfo->topology,
+		      (void *) old_topology,
+		      (sizeof(erts_cpu_topology_t)
+		       * old_topology_size)) != 0)) {
+	changed = 1;
+	if (old_topology)
+	    free(old_topology);
+    }
+    else {
+	if (cpuinfo->topology)
+	    free(cpuinfo->topology);
+	cpuinfo->topology = old_topology;
+    }
+
+    return changed;
 }
 
 int
@@ -289,7 +459,7 @@ erts_get_cpu_available(erts_cpu_info_t *cpuinfo)
 char *
 erts_get_unbind_from_cpu_str(erts_cpu_info_t *cpuinfo)
 {
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     if (!cpuinfo)
 	return "false";
     return cpuinfo->affinity_str;
@@ -303,7 +473,7 @@ erts_get_available_cpu(erts_cpu_info_t *cpuinfo, int no)
 {
     if (!cpuinfo || no < 1 || cpuinfo->available < no)
 	return -EINVAL;
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     {
 	cpu_set_t *allowed = &cpuinfo->cpuset;
 	int ix, n;
@@ -335,8 +505,8 @@ int
 erts_is_cpu_available(erts_cpu_info_t *cpuinfo, int id)
 {
     if (cpuinfo && 0 <= id) {
-#ifdef HAVE_SCHED_xETAFFINITY
-	if (id <= CPU_SETSIZE)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+	if (id < CPU_SETSIZE)
 	    return CPU_ISSET(id, &cpuinfo->cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
 	int no;
@@ -373,8 +543,8 @@ erts_get_cpu_topology(erts_cpu_info_t *cpuinfo,
 	return 0;
     memcpy((void *) topology,
 	   (void *) cpuinfo->topology,
-	   cpuinfo->configured*sizeof(erts_cpu_topology_t));
-    return cpuinfo->configured;
+	   cpuinfo->topology_size*sizeof(erts_cpu_topology_t));
+    return cpuinfo->topology_size;
 }
 
 int
@@ -388,7 +558,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu)
      */
     if (!cpuinfo)
 	return -EINVAL;
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     {
 	cpu_set_t bind_set;
 	if (cpu < 0)
@@ -398,9 +568,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu)
 
 	CPU_ZERO(&bind_set);
 	CPU_SET(cpu, &bind_set);
-	if (sched_setaffinity(0, sizeof(cpu_set_t), &bind_set) != 0)
-	    return -errno;
-	return 0;
+	return ERTS_MU_SET_THR_AFFINITY__(&bind_set);
     }
 #elif defined(HAVE_PROCESSOR_BIND)
     if (cpu < 0)
@@ -418,10 +586,8 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo)
 {
     if (!cpuinfo)
 	return -EINVAL;
-#if defined(HAVE_SCHED_xETAFFINITY)
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuinfo->cpuset) != 0)
-	return -errno;
-    return 0;
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+    return ERTS_MU_SET_THR_AFFINITY__(&cpuinfo->cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
     if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0)
 	return -errno;
@@ -434,7 +600,7 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo)
 int
 erts_unbind_from_cpu_str(char *str)
 {
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     char *c = str;
     int cpus = 0;
     int shft = 0;
@@ -486,9 +652,7 @@ erts_unbind_from_cpu_str(char *str)
     if (!cpus)
 	return -EINVAL;
 
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0)
-	return -errno;
-    return 0;
+    return ERTS_MU_SET_THR_AFFINITY__(&cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
     if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0)
 	return -errno;
@@ -541,6 +705,56 @@ cpu_cmp(const void *vx, const void *vy)
     return 0;
 }
 
+static void
+adjust_processor_nodes(erts_cpu_info_t *cpuinfo, int no_nodes)
+{
+    erts_cpu_topology_t *prev, *this, *last;
+    if (no_nodes > 1) {
+	int processor = -1;
+	int processor_node = 0;
+	int node = -1;
+
+	qsort(cpuinfo->topology,
+	      cpuinfo->topology_size,
+	      sizeof(erts_cpu_topology_t),
+	      pn_cmp);
+
+	prev = NULL;
+	this = &cpuinfo->topology[0];
+	last = &cpuinfo->topology[cpuinfo->configured-1];
+	while (1) {
+	    if (processor == this->processor) {
+		if (node != this->node)
+		    processor_node = 1;
+	    }
+	    else {
+		if (processor_node) {
+		make_processor_node:
+		    while (prev->processor == processor) {
+			prev->processor_node = prev->node;
+			prev->node = -1;
+			if (prev == &cpuinfo->topology[0])
+			    break;
+			prev--;
+		    }
+		    processor_node = 0;
+		}
+		processor = this->processor;
+		node = this->node;
+	    }
+	    if (this == last) {
+		if (processor_node) {
+		    prev = this;
+		    goto make_processor_node;
+		}
+		break;
+	    }
+	    prev = this++;
+	}
+    }
+}
+
+
 #ifdef __linux__
 
 static int
@@ -594,15 +808,12 @@ read_topology(erts_cpu_info_t *cpuinfo)
 
     errno = 0;
 
-    if (cpuinfo->topology)
-	free(cpuinfo->topology);
-
     if (cpuinfo->configured < 1)
 	goto error;
 
     cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
 			       * cpuinfo->configured);
-    if (!cpuinfo)
+    if (!cpuinfo->topology)
 	goto error;
 
     for (ix = 0; ix < cpuinfo->configured; ix++) {
@@ -710,49 +921,7 @@ read_topology(erts_cpu_info_t *cpuinfo)
 		cpuinfo->topology = t;
 	}
 
-	if (no_nodes > 1) {
-	    int processor = -1;
-	    int processor_node = 0;
-	    int node = -1;
-
-	    qsort(cpuinfo->topology,
-		  cpuinfo->topology_size,
-		  sizeof(erts_cpu_topology_t),
-		  pn_cmp);
-
-	    prev = NULL;
-	    this = &cpuinfo->topology[0];
-	    last = &cpuinfo->topology[cpuinfo->configured-1];
-	    while (1) {
-		if (processor == this->processor) {
-		    if (node != this->node)
-			processor_node = 1;
-		}
-		else {
-		    if (processor_node) {
-		    make_processor_node:
-			while (prev->processor == processor) {
-			    prev->processor_node = prev->node;
-			    prev->node = -1;
-			    if (prev == &cpuinfo->topology[0])
-				break;
-			    prev--;
-			}
-			processor_node = 0;
-		    }
-		    processor = this->processor;
-		    node = this->node;
-		}
-		if (this == last) {
-		    if (processor_node) {
-			prev = this;
-			goto make_processor_node;
-		    }
-		    break;
-		}
-		prev = this++;
-	    }
-	}
+	adjust_processor_nodes(cpuinfo, no_nodes);
 
 	qsort(cpuinfo->topology,
 	      cpuinfo->topology_size,
@@ -849,15 +1018,12 @@ read_topology(erts_cpu_info_t *cpuinfo)
 
     errno = 0;
 
-    if (cpuinfo->topology)
-	free(cpuinfo->topology);
-
     if (cpuinfo->configured < 1)
 	goto error;
 
     cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
 			       * cpuinfo->configured);
-    if (!cpuinfo)
+    if (!cpuinfo->topology)
 	goto error;
 
     for (ix = 0; ix < cpuinfo->configured; ix++) {
@@ -938,6 +1104,8 @@ read_topology(erts_cpu_info_t *cpuinfo)
 	}
     }
 
+    adjust_processor_nodes(cpuinfo, 1);
+
  error:
 
     if (res == 0) {
@@ -956,6 +1124,519 @@ read_topology(erts_cpu_info_t *cpuinfo)
 
 }
 
+#elif defined(__WIN32__)
+
+/*
+ * We cannot use Relation* out of the box since all of them are not
+ * always part of the LOGICAL_PROCESSOR_RELATIONSHIP enum. They are
+ * however documented as follows...
+ */
+#define ERTS_MU_RELATION_PROCESSOR_CORE       0 /* RelationProcessorCore */
+#define ERTS_MU_RELATION_NUMA_NODE            1 /* RelationNumaNode */
+#define ERTS_MU_RELATION_CACHE                2 /* RelationCache */
+#define ERTS_MU_RELATION_PROCESSOR_PACKAGE    3 /* RelationProcessorPackage */
+
+static __forceinline int
+rel_cmp_val(int r)
+{
+    switch (r) {
+    case ERTS_MU_RELATION_NUMA_NODE:         return 0;
+    case ERTS_MU_RELATION_PROCESSOR_PACKAGE: return 1;
+    case ERTS_MU_RELATION_PROCESSOR_CORE:    return 2;
+    default: /* currently not used */        return 3;
+    }
+}
+
+static int
+slpi_cmp(const void *vx, const void *vy)
+{
+    PSYSTEM_LOGICAL_PROCESSOR_INFORMATION x, y;
+    x = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION) vx;
+    y = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION) vy;
+
+    if ((int) x->Relationship != (int) y->Relationship)
+	return (rel_cmp_val((int) x->Relationship)
+		- rel_cmp_val((int) y->Relationship));
+
+    switch ((int) x->Relationship) {
+    case ERTS_MU_RELATION_NUMA_NODE:
+	if (x->NumaNode.NodeNumber == y->NumaNode.NodeNumber)
+	    break;
+	return ((int) x->NumaNode.NodeNumber) - ((int) y->NumaNode.NodeNumber);
+    case ERTS_MU_RELATION_PROCESSOR_CORE:
+    case ERTS_MU_RELATION_PROCESSOR_PACKAGE:
+    default:
+	break;
+    }
+
+    if (x->ProcessorMask == y->ProcessorMask)
+	return 0;
+    return x->ProcessorMask < y->ProcessorMask ? -1 : 1;
+}
+
+typedef BOOL (WINAPI *glpi_t)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
+
+static int
+read_topology(erts_cpu_info_t *cpuinfo)
+{
+    int res = 0;
+    glpi_t glpi;
+    int *core_id = NULL;
+    PSYSTEM_LOGICAL_PROCESSOR_INFORMATION slpip = NULL;
+    int wix, rix, max_l, l, packages, nodes, no_slpi;
+    DWORD slpi_size = 0;
+
+
+    glpi = (glpi_t) GetProcAddress(GetModuleHandle("kernel32"),
+				   "GetLogicalProcessorInformation");
+    if (!glpi)
+	return -ENOTSUP;
+
+    cpuinfo->topology = NULL;
+
+    if (cpuinfo->configured < 1 || sizeof(ULONG_PTR)*8 < cpuinfo->configured)
+	goto error;
+
+    while (1) {
+	DWORD werr;
+	if (TRUE == glpi(slpip, &slpi_size))
+	    break;
+	werr = GetLastError();
+	if (werr != ERROR_INSUFFICIENT_BUFFER) {
+	    res = -erts_map_win_error_to_errno(werr);
+	    goto error;
+	}
+	if (slpip)
+	    free(slpip);
+	slpip = malloc(slpi_size);
+	if (!slpip) {
+	    res = -ENOMEM;
+	    goto error;
+	}
+    }
+
+    no_slpi = (int) slpi_size/sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
+
+    qsort(slpip,
+	  no_slpi,
+	  sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION),
+	  slpi_cmp);
+
+    /*
+     * Now numa node relations appear before package relations which
+     * appear before core relations which appear before relations
+     * we aren't interested in...
+     */
+
+    max_l = 0;
+    packages = 0;
+    nodes = 0;
+    for (rix = 0; rix < no_slpi; rix++) {
+	PSYSTEM_LOGICAL_PROCESSOR_INFORMATION this = &slpip[rix];
+	for (l = sizeof(ULONG_PTR)*8 - 1; l > 0; l--) {
+	    if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) {
+		if (max_l < l)
+		    max_l = l;
+		break;
+	    }
+	}
+	if ((int) slpip[rix].Relationship == ERTS_MU_RELATION_PROCESSOR_PACKAGE)
+	    packages++;
+	if ((int) slpip[rix].Relationship == ERTS_MU_RELATION_NUMA_NODE)
+	    nodes++;
+    }
+
+    if (!packages) {
+      packages = 1;
+    }
+    core_id = malloc(sizeof(int)*packages);
+    if (!core_id) {
+	res = -ENOMEM;
+	goto error;
+    }
+
+    for (rix = 0; rix < packages; rix++)
+	core_id[rix] = 0;
+
+    cpuinfo->topology_size = max_l + 1;
+    cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
+			       * cpuinfo->topology_size);
+    if (!cpuinfo->topology) {
+	res = -ENOMEM;
+	goto error;
+    }
+
+    for (wix = 0; wix < cpuinfo->topology_size; wix++) {
+	cpuinfo->topology[wix].node = -1;
+	cpuinfo->topology[wix].processor = -1;
+	cpuinfo->topology[wix].processor_node = -1;
+	cpuinfo->topology[wix].core = -1;
+	cpuinfo->topology[wix].thread = -1;
+	cpuinfo->topology[wix].logical = -1;
+    }
+
+    nodes = 0;
+    packages = 0;
+
+    for (rix = 0; rix < no_slpi; rix++) {
+
+        switch ((int) slpip[rix].Relationship) {
+        case ERTS_MU_RELATION_NUMA_NODE:
+	    for (l = 0; l < sizeof(ULONG_PTR)*8; l++) {
+		if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) {
+		    cpuinfo->topology[l].logical = l;
+		    cpuinfo->topology[l].node = slpip[rix].NumaNode.NodeNumber;
+		}
+	    }
+	    nodes++;
+            break;
+        case ERTS_MU_RELATION_PROCESSOR_PACKAGE:
+	    for (l = 0; l < sizeof(ULONG_PTR)*8; l++) {
+		if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) {
+		    cpuinfo->topology[l].logical = l;
+		    cpuinfo->topology[l].processor = packages;
+		}
+	    }
+	    packages++;
+            break;
+        case ERTS_MU_RELATION_PROCESSOR_CORE: {
+	    int thread = 0;
+	    int processor = -1;
+	    for (l = 0; l < sizeof(ULONG_PTR)*8; l++) {
+		/*
+		 * Nodes and packages may not be supported; pretend
+		 * that there are one if this is the case...
+		 */
+		if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) {
+		    if (!nodes) {
+		      cpuinfo->topology[l].node = 0;
+		    }
+		    if (!packages) {
+		      cpuinfo->topology[l].processor = 0;
+		    }
+		    if (processor < 0) {
+			processor = cpuinfo->topology[l].processor;
+			if (processor < 0) {
+			    res = -EINVAL;
+			    goto error;
+			}
+		    }
+		    else if (processor != cpuinfo->topology[l].processor) {
+			res = -EINVAL;
+			goto error;
+		    }
+		    cpuinfo->topology[l].logical = l;
+		    cpuinfo->topology[l].thread = thread;
+		    cpuinfo->topology[l].core = core_id[processor];
+		    thread++;
+		}
+	    }
+	    core_id[processor]++;
+            break;
+	}
+        default:
+	    /*
+	     * We have reached the end of the relationships
+	     * that we (currently) are interested in...
+	     */
+	    goto relationships_done;
+        }
+    }
+
+ relationships_done:
+
+    /*
+     * There may be unused entries; remove them...
+     */
+    for (rix = wix = 0; rix < cpuinfo->topology_size; rix++) {
+	if (cpuinfo->topology[rix].logical >= 0) {
+	    if (wix != rix)
+		cpuinfo->topology[wix] = cpuinfo->topology[rix];
+	    wix++;
+	}
+    }
+
+    if (cpuinfo->topology_size != wix) {
+	erts_cpu_topology_t *new = cpuinfo->topology;
+	new = realloc(cpuinfo->topology,
+		      sizeof(erts_cpu_topology_t)*wix);
+	if (!new) {
+	    res = -ENOMEM;
+	    goto error;
+	}
+	cpuinfo->topology = new;
+	cpuinfo->topology_size = wix;
+    }
+
+    res = wix;
+
+    adjust_processor_nodes(cpuinfo, nodes);
+
+    qsort(cpuinfo->topology,
+	  cpuinfo->topology_size,
+	  sizeof(erts_cpu_topology_t),
+	  cpu_cmp);
+
+    if (res < cpuinfo->online)
+	res = -EINVAL;
+
+ error:
+
+    if (res <= 0) {
+	cpuinfo->topology_size = 0;
+	if (cpuinfo->topology) {
+	    free(cpuinfo->topology);
+	    cpuinfo->topology = NULL;
+	}
+    }
+
+    if (slpip)
+	free(slpip);
+    if (core_id)
+	free(core_id);
+
+    return res;
+}
+
+#elif defined(__FreeBSD__)
+
+/**
+ * FreeBSD topology detection is based on kern.sched.topology_spec XML as
+ * exposed by the ULE scheduler and described in SMP(4). It is available in
+ * 8.0 and higher.
+ *
+ * Threads are identified in this XML chunk with a THREAD flag. The function
+ * (simplistically) distinguishes cores and processors by the amount of cache
+ * they share (0 => processor, otherwise => core). Nodes are not identified
+ * (ULE doesn't handle NUMA yet, I believe).
+ */
+
+/**
+ * Recursively parse a topology_spec <group> tag.
+ */
+static
+const char* parse_topology_spec_group(erts_cpu_info_t *cpuinfo, const char* xml, int parentCacheLevel, int* processor_p, int* core_p, int* index_procs_p) {
+    int error = 0;
+    int cacheLevel = parentCacheLevel;
+    const char* next_group_start = strstr(xml + 1, "<group");
+    int is_thread_group = 0;
+    const char* next_cache_level;
+    const char* next_thread_flag;
+    const char* next_group_end;
+    const char* next_children;
+    const char* next_children_end;
+
+    /* parse the cache level */
+    next_cache_level = strstr(xml, "cache-level=\"");
+    if (next_cache_level && (next_group_start == NULL || next_cache_level < next_group_start)) {
+	sscanf(next_cache_level, "cache-level=\"%i\"", &cacheLevel);
+    }
+
+    /* parse the threads flag */
+    next_thread_flag = strstr(xml, "THREAD");
+    if (next_thread_flag && (next_group_start == NULL || next_thread_flag < next_group_start))
+	is_thread_group = 1;
+
+    /* Determine if it's a leaf with the position of the next children tag */
+    next_group_end = strstr(xml, "</group>");
+    next_children = strstr(xml, "<children>");
+    next_children_end = strstr(xml, "</children>");
+    if (next_children == NULL || next_group_end < next_children) {
+	do {
+	    const char* next_cpu_start;
+	    const char* next_cpu_cdata;
+	    const char* next_cpu_end;
+	    int cpu_str_size;
+	    char* cpu_str;
+	    char* cpu_crsr;
+	    char* brkb;
+	    int thread = 0;
+	    int index_procs = *index_procs_p;
+
+	    next_cpu_start = strstr(xml, "<cpu");
+	    if (!next_cpu_start) {
+		error = 1;
+		break;
+	    }
+	    next_cpu_cdata = strstr(next_cpu_start, ">") + 1;
+	    if (!next_cpu_cdata) {
+		error = 1;
+		break;
+	    }
+	    next_cpu_end = strstr(next_cpu_cdata, "</cpu>");
+	    if (!next_cpu_end) {
+		error = 1;
+		break;
+	    }
+	    cpu_str_size = next_cpu_end - next_cpu_cdata;
+	    cpu_str = (char*) malloc(cpu_str_size + 1);
+	    memcpy(cpu_str, (const char*) next_cpu_cdata, cpu_str_size);
+	    cpu_str[cpu_str_size] = 0;
+	    for (cpu_crsr = strtok_r(cpu_str, " \t,", &brkb); cpu_crsr; cpu_crsr = strtok_r(NULL, " \t,", &brkb)) {
+		int cpu_id;
+		if (index_procs >= cpuinfo->configured) {
+		    void* t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t) * (index_procs + 1)));
+		    if (t) {
+			cpuinfo->topology = t;
+		    } else {
+			error = 1;
+			break;
+		    }
+		}
+		cpu_id = atoi(cpu_crsr);
+		cpuinfo->topology[index_procs].node = -1;
+		cpuinfo->topology[index_procs].processor = *processor_p;
+		cpuinfo->topology[index_procs].processor_node = -1;
+		cpuinfo->topology[index_procs].core = *core_p;
+		cpuinfo->topology[index_procs].thread = thread;
+		cpuinfo->topology[index_procs].logical = cpu_id;
+		if (is_thread_group) {
+		    thread++;
+		} else {
+		    *core_p = (*core_p)++;
+		}
+		index_procs++;
+	    }
+	    *index_procs_p = index_procs;
+	    free(cpu_str);
+	} while (0);
+	xml = next_group_end;
+    } else {
+	while (next_group_start != NULL && next_group_start < next_children_end) {
+	    xml = parse_topology_spec_group(cpuinfo, next_group_start, cacheLevel, processor_p, core_p, index_procs_p);
+	    if (!xml)
+		break;
+	    next_group_start = strstr(xml, "<group");
+	    next_children_end = strstr(xml, "</children>");
+	}
+    }
+
+    if (cacheLevel == 0) {
+	*core_p = 0;
+	*processor_p = (*processor_p)++;
+    } else {
+	*core_p = (*core_p)++;
+    }
+
+    if (error)
+	xml = NULL;
+
+    return xml;
+}
+
+/**
+ * Parse the topology_spec. Return the number of CPUs or 0 if parsing failed.
+ */
+static
+int parse_topology_spec(erts_cpu_info_t *cpuinfo, const char* xml) {
+    int res = 1;
+    int index_procs = 0;
+    int core = 0;
+    int processor = 0;
+    xml = strstr(xml, "<groups");
+    if (!xml)
+	return -1;
+
+    xml += 7;
+    xml = strstr(xml, "<group");
+    while (xml) {
+	xml = parse_topology_spec_group(cpuinfo, xml, 0, &processor, &core, &index_procs);
+	if (!xml) {
+	    res = 0;
+	    break;
+	}
+	xml = strstr(xml, "<group");
+    }
+
+    if (res)
+	res = index_procs;
+
+    return res;
+}
+
+static int
+read_topology(erts_cpu_info_t *cpuinfo)
+{
+    int ix;
+    int res = 0;
+    size_t topology_spec_size = 0;
+    void* topology_spec = NULL;
+
+    errno = 0;
+
+    if (cpuinfo->configured < 1)
+	goto error;
+
+    cpuinfo->topology_size = cpuinfo->configured;
+    cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
+			       * cpuinfo->configured);
+    if (!cpuinfo->topology) {
+	res = -ENOMEM;
+	goto error;
+    }
+
+    for (ix = 0; ix < cpuinfo->configured; ix++) {
+	cpuinfo->topology[ix].node = -1;
+	cpuinfo->topology[ix].processor = -1;
+	cpuinfo->topology[ix].processor_node = -1;
+	cpuinfo->topology[ix].core = -1;
+	cpuinfo->topology[ix].thread = -1;
+	cpuinfo->topology[ix].logical = -1;
+    }
+
+    if (!sysctlbyname("kern.sched.topology_spec", NULL, &topology_spec_size, NULL, 0)) {
+	topology_spec = malloc(topology_spec_size);
+	if (!topology_spec) {
+	    res = -ENOMEM;
+	    goto error;
+	}
+
+	if (sysctlbyname("kern.sched.topology_spec", topology_spec, &topology_spec_size, NULL, 0)) {
+	    goto error;
+	}
+
+	res = parse_topology_spec(cpuinfo, topology_spec);
+	if (!res || res < cpuinfo->online)
+	    res = 0;
+	else {
+	    cpuinfo->topology_size = res;
+
+	    if (cpuinfo->topology_size != cpuinfo->configured) {
+		void *t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t)
+						  * cpuinfo->topology_size));
+		if (t)
+		    cpuinfo->topology = t;
+	    }
+
+	    adjust_processor_nodes(cpuinfo, 1);
+
+	    qsort(cpuinfo->topology,
+	        cpuinfo->topology_size,
+	        sizeof(erts_cpu_topology_t),
+	        cpu_cmp);
+	}
+    }
+
+error:
+
+    if (res == 0) {
+	cpuinfo->topology_size = 0;
+	if (cpuinfo->topology) {
+	    free(cpuinfo->topology);
+	    cpuinfo->topology = NULL;
+	}
+	if (errno)
+	    res = -errno;
+	else
+	    res = -EINVAL;
+    }
+
+    if (topology_spec)
+	free(topology_spec);
+
+    return res;
+}
+
 #else
 
 static int
@@ -965,3 +1646,98 @@ read_topology(erts_cpu_info_t *cpuinfo)
 }
 
 #endif
+
+#if defined(__WIN32__)
+
+int
+erts_map_win_error_to_errno(DWORD win_error)
+{
+    switch (win_error) {
+    case ERROR_INVALID_FUNCTION:		return EINVAL;	/* 1	*/
+    case ERROR_FILE_NOT_FOUND:			return ENOENT;	/* 2	*/
+    case ERROR_PATH_NOT_FOUND:			return ENOENT;	/* 3	*/
+    case ERROR_TOO_MANY_OPEN_FILES:		return EMFILE;	/* 4	*/
+    case ERROR_ACCESS_DENIED:			return EACCES;	/* 5	*/
+    case ERROR_INVALID_HANDLE:			return EBADF;	/* 6	*/
+    case ERROR_ARENA_TRASHED:			return ENOMEM;	/* 7	*/
+    case ERROR_NOT_ENOUGH_MEMORY:		return ENOMEM;	/* 8	*/
+    case ERROR_INVALID_BLOCK:			return ENOMEM;	/* 9	*/
+    case ERROR_BAD_ENVIRONMENT:			return E2BIG;	/* 10	*/
+    case ERROR_BAD_FORMAT:			return ENOEXEC;	/* 11	*/
+    case ERROR_INVALID_ACCESS:			return EINVAL;	/* 12	*/
+    case ERROR_INVALID_DATA:			return EINVAL;	/* 13	*/
+    case ERROR_OUTOFMEMORY:			return ENOMEM;	/* 14	*/
+    case ERROR_INVALID_DRIVE:			return ENOENT;	/* 15	*/
+    case ERROR_CURRENT_DIRECTORY:		return EACCES;	/* 16	*/
+    case ERROR_NOT_SAME_DEVICE:			return EXDEV;	/* 17	*/
+    case ERROR_NO_MORE_FILES:			return ENOENT;	/* 18	*/
+    case ERROR_WRITE_PROTECT:			return EACCES;	/* 19	*/
+    case ERROR_BAD_UNIT:			return EACCES;	/* 20	*/
+    case ERROR_NOT_READY:			return EACCES;	/* 21	*/
+    case ERROR_BAD_COMMAND:			return EACCES;	/* 22	*/
+    case ERROR_CRC:				return EACCES;	/* 23	*/
+    case ERROR_BAD_LENGTH:			return EACCES;	/* 24	*/
+    case ERROR_SEEK:				return EACCES;	/* 25	*/
+    case ERROR_NOT_DOS_DISK:			return EACCES;	/* 26	*/
+    case ERROR_SECTOR_NOT_FOUND:		return EACCES;	/* 27	*/
+    case ERROR_OUT_OF_PAPER:			return EACCES;	/* 28	*/
+    case ERROR_WRITE_FAULT:			return EACCES;	/* 29	*/
+    case ERROR_READ_FAULT:			return EACCES;	/* 30	*/
+    case ERROR_GEN_FAILURE:			return EACCES;	/* 31	*/
+    case ERROR_SHARING_VIOLATION:		return EACCES;	/* 32	*/
+    case ERROR_LOCK_VIOLATION:			return EACCES;	/* 33	*/
+    case ERROR_WRONG_DISK:			return EACCES;	/* 34	*/
+    case ERROR_SHARING_BUFFER_EXCEEDED:		return EACCES;	/* 36	*/
+    case ERROR_BAD_NETPATH:			return ENOENT;	/* 53	*/
+    case ERROR_NETWORK_ACCESS_DENIED:		return EACCES;	/* 65	*/
+    case ERROR_BAD_NET_NAME:			return ENOENT;	/* 67	*/
+    case ERROR_FILE_EXISTS:			return EEXIST;	/* 80	*/
+    case ERROR_CANNOT_MAKE:			return EACCES;	/* 82	*/
+    case ERROR_FAIL_I24:			return EACCES;	/* 83	*/
+    case ERROR_INVALID_PARAMETER:		return EINVAL;	/* 87	*/
+    case ERROR_NO_PROC_SLOTS:			return EAGAIN;	/* 89	*/
+    case ERROR_DRIVE_LOCKED:			return EACCES;	/* 108	*/
+    case ERROR_BROKEN_PIPE:			return EPIPE;	/* 109	*/
+    case ERROR_DISK_FULL:			return ENOSPC;	/* 112	*/
+    case ERROR_INVALID_TARGET_HANDLE:		return EBADF;	/* 114	*/
+    case ERROR_WAIT_NO_CHILDREN:		return ECHILD;	/* 128	*/
+    case ERROR_CHILD_NOT_COMPLETE:		return ECHILD;	/* 129	*/
+    case ERROR_DIRECT_ACCESS_HANDLE:		return EBADF;	/* 130	*/
+    case ERROR_NEGATIVE_SEEK:			return EINVAL;	/* 131	*/
+    case ERROR_SEEK_ON_DEVICE:			return EACCES;	/* 132	*/
+    case ERROR_DIR_NOT_EMPTY:			return ENOTEMPTY;/* 145	*/
+    case ERROR_NOT_LOCKED:			return EACCES;	/* 158	*/
+    case ERROR_BAD_PATHNAME:			return ENOENT;	/* 161	*/
+    case ERROR_MAX_THRDS_REACHED:		return EAGAIN;	/* 164	*/
+    case ERROR_LOCK_FAILED:			return EACCES;	/* 167	*/
+    case ERROR_ALREADY_EXISTS:			return EEXIST;	/* 183	*/
+    case ERROR_INVALID_STARTING_CODESEG:	return ENOEXEC;	/* 188	*/
+    case ERROR_INVALID_STACKSEG:		return ENOEXEC;	/* 189	*/
+    case ERROR_INVALID_MODULETYPE:		return ENOEXEC;	/* 190	*/
+    case ERROR_INVALID_EXE_SIGNATURE:		return ENOEXEC;	/* 191	*/
+    case ERROR_EXE_MARKED_INVALID:		return ENOEXEC;	/* 192	*/
+    case ERROR_BAD_EXE_FORMAT:			return ENOEXEC;	/* 193	*/
+    case ERROR_ITERATED_DATA_EXCEEDS_64k:	return ENOEXEC;	/* 194	*/
+    case ERROR_INVALID_MINALLOCSIZE:		return ENOEXEC;	/* 195	*/
+    case ERROR_DYNLINK_FROM_INVALID_RING:	return ENOEXEC;	/* 196	*/
+    case ERROR_IOPL_NOT_ENABLED:		return ENOEXEC;	/* 197	*/
+    case ERROR_INVALID_SEGDPL:			return ENOEXEC;	/* 198	*/
+    case ERROR_AUTODATASEG_EXCEEDS_64k:		return ENOEXEC;	/* 199	*/
+    case ERROR_RING2SEG_MUST_BE_MOVABLE:	return ENOEXEC;	/* 200	*/
+    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:	return ENOEXEC;	/* 201	*/
+    case ERROR_INFLOOP_IN_RELOC_CHAIN:		return ENOEXEC;	/* 202	*/
+    case ERROR_FILENAME_EXCED_RANGE:		return ENOENT;	/* 206	*/
+    case ERROR_NESTING_NOT_ALLOWED:		return EAGAIN;	/* 215	*/
+    case ERROR_NOT_ENOUGH_QUOTA:		return ENOMEM;	/* 1816	*/
+    default:					return EINVAL;
+    }
+}
+
+int
+erts_get_last_win_errno(void)
+{
+    return erts_map_win_error_to_errno(GetLastError());
+}
+
+
+#endif
diff --git a/erts/lib_src/common/erl_printf_format.c b/erts/lib_src/common/erl_printf_format.c
index bd3d38e649..fba3fd723c 100644
--- a/erts/lib_src/common/erl_printf_format.c
+++ b/erts/lib_src/common/erl_printf_format.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -25,9 +25,9 @@
  *  width:      [0-9]+ | '*' 
  *  precision:  [0-9]+ | '*' 
  *  length: hh | h | l | ll | L | j | t | b<sz>
- *  conversion: d,i | o,u,x,X | e,E | f,F | g,G | a,A | c | s | T |
+ *  conversion: d,i | o,u,x,X | e,E | f,F | g,G | a,A | c | s | T | R |
  *              p | n | %
- *  sz: 8 | 16 | 32 | 64 | p
+ *  sz: 8 | 16 | 32 | 64 | p | e
  */
 
 /* Without this, variable argument lists break on VxWorks */
@@ -76,6 +76,18 @@
 #endif
 #endif
 
+#ifndef ERTS_SIZEOF_ETERM
+#  ifdef HALFWORD_HEAP_EMULATOR
+#    if SIZEOF_VOID_P == 8
+#      define ERTS_SIZEOF_ETERM 4
+#    else
+#      error "HALFWORD_HEAP_EMULATOR only allowed on 64-bit architecture"
+#    endif
+#  else
+#    define ERTS_SIZEOF_ETERM SIZEOF_VOID_P
+#  endif
+#endif
+
 #if defined(__GNUC__)
 #  undef inline
 #  define inline __inline__
@@ -89,7 +101,7 @@
 #endif
 
 #define FMTC_d    0x0000
-#define FMTC_i    0x0001
+#define FMTC_R    0x0001
 #define FMTC_o    0x0002
 #define FMTC_u    0x0003
 #define FMTC_x    0x0004
@@ -153,7 +165,7 @@ static char heX[] = "0123456789ABCDEF";
 #define SIGN(X) ((X) > 0 ? 1 : ((X) < 0 ? -1 : 0)) 
 #define USIGN(X) ((X) == 0 ? 0 : 1)
 
-int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long) = NULL;
+int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long, unsigned long*) = NULL;
 
 static int
 noop_fn(void *vfp, char* buf, size_t len)
@@ -171,8 +183,8 @@ static int fmt_fld(fmtfn_t fn,void* arg,
     int len;
 
     /* format the prefix */
-    if ((sign || (fmt & (FMTF_sgn|FMTF_blk))) &&
-	(((fmt & FMTC_MASK) == FMTC_d) || ((fmt & FMTC_MASK) == FMTC_i))) {
+    if ((sign || (fmt & (FMTF_sgn|FMTF_blk)))
+	&& (fmt & FMTC_MASK) == FMTC_d) {
 	if (sign < 0)
 	    *pp++ = '-';
 	else if ((fmt & FMTF_sgn))
@@ -233,7 +245,6 @@ static int fmt_long(fmtfn_t fn,void* arg,int sign,unsigned long uval,
 
     switch(fmt & FMTC_MASK) {
     case FMTC_d:
-    case FMTC_i:
     case FMTC_u:
 	break;
     case FMTC_o: 
@@ -286,7 +297,6 @@ static int fmt_long_long(fmtfn_t fn,void* arg,int sign,
 
     switch(fmt & FMTC_MASK) {
     case FMTC_d:
-    case FMTC_i:
     case FMTC_u:
 	break;
     case FMTC_o: 
@@ -520,6 +530,17 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 #error No integer datatype with the same size as 'void *' found
 #endif
 		}
+		else if (*ptr == 'e') {
+		    ptr++;
+#if SIZEOF_INT == ERTS_SIZEOF_ETERM
+#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM
+		    fmt |= FMTL_l;
+#elif SIZEOF_LONG_LONG == ERTS_SIZEOF_ETERM
+		    fmt |= FMTL_ll;
+#else
+#error No integer datatype with the same size as Eterm found
+#endif
+		}
 		else {
 		    int bits = 0;
 		    while(isdigit((int) *ptr))
@@ -599,7 +620,7 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 	    /* specifier */
 	    switch(*ptr) {
 	    case 'd': ptr++; fmt |= FMTC_d; break;
-	    case 'i': ptr++; fmt |= FMTC_i; break;
+	    case 'i': ptr++; fmt |= FMTC_d; break;
 	    case 'o': ptr++; fmt |= FMTC_o; break;
 	    case 'u': ptr++; fmt |= FMTC_u; break;
 	    case 'x': ptr++; fmt |= FMTC_x; break;
@@ -614,6 +635,7 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 	    case 'p': ptr++; fmt |= FMTC_p; break;
 	    case 'n': ptr++; fmt |= FMTC_n; break;
 	    case 'T': ptr++; fmt |= FMTC_T; break;
+	    case 'R': ptr++; fmt |= FMTC_R; break;
 	    case '%':
 		FMT(fn,arg,ptr,1,count);
 		ptr++;
@@ -627,7 +649,6 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 
 	    switch(fmt & FMTC_MASK) {
 	    case FMTC_d:
-	    case FMTC_i:
 		switch(fmt & FMTL_MASK) {
 		case FMTL_hh: {
 		    signed char tval = (signed char) va_arg(ap,int);
@@ -791,9 +812,12 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 		default: *va_arg(ap,int*) = count; break;
 		}
 		break;
-	    case FMTC_T: {
+	    case FMTC_T:    /* Eterm */
+	    case FMTC_R: {  /* Eterm, Eterm* base  (base ignored if !HALFWORD_HEAP) */
 		long prec;
 		unsigned long eterm;
+		unsigned long* eterm_base;
+		
 		if (!erts_printf_eterm_func)
 		    return -EINVAL;
 		if (precision < 0)
@@ -803,14 +827,16 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 		else
 		    prec = (long) precision;
 		eterm = va_arg(ap, unsigned long);
+		eterm_base = ((fmt & FMTC_MASK) == FMTC_R) ?
+		    va_arg(ap, unsigned long*) : NULL;
 		if (width > 0 && !(fmt & FMTF_adj)) {
-		    res = (*erts_printf_eterm_func)(noop_fn, NULL, eterm, prec);
+		    res = (*erts_printf_eterm_func)(noop_fn, NULL, eterm, prec, eterm_base);
 		    if (res < 0)
 			return res;
 		    if (width > res)
 			BLANKS(fn, arg, width - res, count);
 		}
-		res = (*erts_printf_eterm_func)(fn, arg, eterm, prec);
+		res = (*erts_printf_eterm_func)(fn, arg, eterm, prec, eterm_base);
 		if (res < 0)
 		    return res;
 		count += res;
@@ -901,7 +927,7 @@ erts_printf_slong(fmtfn_t fn, void *arg, char conv, int pad, int width,
     unsigned long ul_val;
     switch (conv) {
     case 'd': fmt |= FMTC_d; break;
-    case 'i': fmt |= FMTC_i; break;
+    case 'i': fmt |= FMTC_d; break;
     case 'o': fmt |= FMTC_o; break;
     case 'x': fmt |= FMTC_x; break;
     case 'X': fmt |= FMTC_X; break;
diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c
new file mode 100644
index 0000000000..94557d904a
--- /dev/null
+++ b/erts/lib_src/common/ethr_atomics.c
@@ -0,0 +1,402 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: The ethread atomic API
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_ATOMIC_IMPL__
+
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#ifndef ETHR_HAVE_NATIVE_ATOMICS
+ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
+#endif
+
+int
+ethr_init_atomics(void)
+{
+#ifndef ETHR_HAVE_NATIVE_ATOMICS
+    {
+	int i;
+	for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) {
+	    int res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
+	    if (res != 0)
+		return res;
+	}
+    }
+#endif
+    return 0;
+}
+
+/*
+ * --- Pointer size atomics ---------------------------------------------------
+ */
+
+ethr_sint_t *
+ethr_atomic_addr(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(var);
+    return ethr_atomic_addr__(var);
+}
+
+void
+ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t i)
+{
+    ETHR_ASSERT(var);
+    ethr_atomic_init__(var, i);
+}
+
+void
+ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t i)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_set__(var, i);
+}
+
+ethr_sint_t
+ethr_atomic_read(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read__(var);
+}
+
+ethr_sint_t
+ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t incr)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_add_read__(var, incr);
+}   
+
+ethr_sint_t
+ethr_atomic_inc_read(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_inc_read__(var);
+}
+
+ethr_sint_t
+ethr_atomic_dec_read(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_dec_read__(var);
+}
+
+void
+ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t incr)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_add__(var, incr);
+}   
+    
+void
+ethr_atomic_inc(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_inc__(var);
+}
+
+void
+ethr_atomic_dec(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_dec__(var);
+}
+
+ethr_sint_t
+ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t mask)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read_band__(var, mask);
+}
+
+ethr_sint_t
+ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t mask)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read_bor__(var, mask);
+}
+
+ethr_sint_t
+ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t new)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_xchg__(var, new);
+}   
+
+ethr_sint_t
+ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t expected)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_cmpxchg__(var, new, expected);
+}
+
+ethr_sint_t
+ethr_atomic_read_acqb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read_acqb__(var);
+}
+
+ethr_sint_t
+ethr_atomic_inc_read_acqb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_inc_read_acqb__(var);
+}
+
+void
+ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t i)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_set_relb__(var, i);
+}
+
+void
+ethr_atomic_dec_relb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_dec_relb__(var);
+}
+
+ethr_sint_t
+ethr_atomic_dec_read_relb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_dec_read_relb__(var);
+}
+
+ethr_sint_t
+ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_cmpxchg_acqb__(var, new, exp);
+}
+
+ethr_sint_t
+ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_cmpxchg_relb__(var, new, exp);
+}
+
+
+/*
+ * --- 32-bit atomics ---------------------------------------------------------
+ */
+
+ethr_sint32_t *
+ethr_atomic32_addr(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(var);
+    return ethr_atomic32_addr__(var);
+}
+
+void
+ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t i)
+{
+    ETHR_ASSERT(var);
+    ethr_atomic32_init__(var, i);
+}
+
+void
+ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t i)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic32_set__(var, i);
+}
+
+ethr_sint32_t
+ethr_atomic32_read(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_read__(var);
+}
+
+
+ethr_sint32_t
+ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t incr)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_add_read__(var, incr);
+}   
+
+ethr_sint32_t
+ethr_atomic32_inc_read(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_inc_read__(var);
+}
+
+ethr_sint32_t
+ethr_atomic32_dec_read(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_dec_read__(var);
+}
+
+void
+ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t incr)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic32_add__(var, incr);
+}   
+    
+void
+ethr_atomic32_inc(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic32_inc__(var);
+}
+
+void
+ethr_atomic32_dec(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic32_dec__(var);
+}
+
+ethr_sint32_t
+ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t mask)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_read_band__(var, mask);
+}
+
+ethr_sint32_t
+ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t mask)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_read_bor__(var, mask);
+}
+
+ethr_sint32_t
+ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t new)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_xchg__(var, new);
+}   
+
+ethr_sint32_t
+ethr_atomic32_cmpxchg(ethr_atomic32_t *var,
+		      ethr_sint32_t new,
+		      ethr_sint32_t expected)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_cmpxchg__(var, new, expected);
+}
+
+ethr_sint32_t
+ethr_atomic32_read_acqb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_read_acqb__(var);
+}
+
+ethr_sint32_t
+ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_inc_read_acqb__(var);
+}
+
+void
+ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t i)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic32_set_relb__(var, i);
+}
+
+void
+ethr_atomic32_dec_relb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic32_dec_relb__(var);
+}
+
+ethr_sint32_t
+ethr_atomic32_dec_read_relb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_dec_read_relb__(var);
+}
+
+ethr_sint32_t
+ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var,
+			   ethr_sint32_t new,
+			   ethr_sint32_t exp)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_cmpxchg_acqb__(var, new, exp);
+}
+
+ethr_sint32_t
+ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var,
+			   ethr_sint32_t new,
+			   ethr_sint32_t exp)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic32_cmpxchg_relb__(var, new, exp);
+}
+
diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c
new file mode 100644
index 0000000000..2c3e25a805
--- /dev/null
+++ b/erts/lib_src/common/ethr_aux.c
@@ -0,0 +1,582 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: A Thread library for use in the ERTS and other OTP
+ *              applications.
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_AUX_IMPL__
+#define ETHR_ATOMIC_IMPL__ /* Needed in order to pull in
+			      native atomic implementations
+			      for optimized fallbacks of
+			      spinlocks and rwspinlocks */
+#include "ethread.h"
+#include "ethr_internal.h"
+#include <string.h>
+#include <limits.h>
+
+#ifndef __WIN32__
+#include <unistd.h>
+#endif
+
+#define ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE 100
+#define ERTS_TS_EV_ALLOC_POOL_SIZE 25
+
+erts_cpu_info_t *ethr_cpu_info__;
+
+int ethr_not_completely_inited__ = 1;
+int ethr_not_inited__ = 1;
+
+ethr_memory_allocators ethr_mem__ = ETHR_MEM_ALLOCS_DEF_INITER__;
+
+void *(*ethr_thr_prepare_func__)(void) = NULL;
+void (*ethr_thr_parent_func__)(void *) = NULL;
+void (*ethr_thr_child_func__)(void *) = NULL;
+
+typedef struct ethr_xhndl_list_ ethr_xhndl_list;
+struct ethr_xhndl_list_ {
+    ethr_xhndl_list *next;
+    void (*funcp)(void);
+};
+
+size_t ethr_pagesize__;
+size_t ethr_min_stack_size__; /* kilo words */
+size_t ethr_max_stack_size__; /* kilo words */
+
+ethr_rwmutex xhndl_rwmtx;
+ethr_xhndl_list *xhndl_list;
+
+static int main_threads;
+
+static int init_ts_event_alloc(void);
+
+int
+ethr_init_common__(ethr_init_data *id)
+{
+    int res;
+    if (id) {
+	ethr_thr_prepare_func__	= id->thread_create_prepare_func;
+	ethr_thr_parent_func__	= id->thread_create_parent_func;
+	ethr_thr_child_func__	= id->thread_create_child_func;
+    }
+
+    ethr_cpu_info__ = erts_cpu_info_create();
+    if (!ethr_cpu_info__)
+	return ENOMEM;
+
+#ifdef _SC_PAGESIZE
+    ethr_pagesize__ = (size_t) sysconf(_SC_PAGESIZE);
+#elif defined(HAVE_GETPAGESIZE)
+    ethr_pagesize__ = (size_t) getpagesize();
+#else
+    ethr_pagesize__ = (size_t) 4*1024; /* Guess 4 KB */
+#endif
+
+    /* User needs at least 4 KB */
+    ethr_min_stack_size__ = 4*1024;
+#if SIZEOF_VOID_P == 8
+    /* Double that on 64-bit archs */
+    ethr_min_stack_size__ *= 2;
+#endif
+    /* On some systems as much as about 4 KB is used by the system */
+    ethr_min_stack_size__ += 4*1024;
+    /* There should be room for signal handlers */
+#ifdef SIGSTKSZ
+    ethr_min_stack_size__ += SIGSTKSZ;
+#else
+    ethr_min_stack_size__ += ethr_pagesize__;
+#endif
+    /* The system may think that we need more stack */
+#if defined(PTHREAD_STACK_MIN)
+    if (ethr_min_stack_size__ < PTHREAD_STACK_MIN)
+	ethr_min_stack_size__ = PTHREAD_STACK_MIN;
+#elif defined(_SC_THREAD_STACK_MIN)
+    {
+	size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN);
+	if (ethr_min_stack_size__ < thr_min_stk_sz)
+	    ethr_min_stack_size__ = thr_min_stk_sz;
+    }
+#endif
+    /* The guard is at least on some platforms included in the stack size
+       passed when creating threads */
+#ifdef ETHR_STACK_GUARD_SIZE
+    ethr_min_stack_size__ += ETHR_STACK_GUARD_SIZE;
+#endif
+    ethr_min_stack_size__ = ETHR_PAGE_ALIGN(ethr_min_stack_size__);
+
+    ethr_min_stack_size__ = ETHR_B2KW(ethr_min_stack_size__);
+
+    ethr_max_stack_size__ = 32*1024*1024;
+#if SIZEOF_VOID_P == 8
+    ethr_max_stack_size__ *= 2;
+#endif
+    ethr_max_stack_size__ = ETHR_B2KW(ethr_max_stack_size__);
+
+    res = ethr_init_atomics();
+    if (res != 0)
+	return res;
+
+    res = ethr_mutex_lib_init(erts_get_cpu_configured(ethr_cpu_info__));
+    if (res != 0)
+	return res;
+
+    xhndl_list = NULL;
+
+    return 0;
+}
+
+int
+ethr_late_init_common__(ethr_late_init_data *lid)
+{
+    ethr_ts_event *tsep = NULL;
+    int reader_groups;
+    int res;
+    int i;
+    ethr_memory_allocator *m[] = {&ethr_mem__.std,
+				  &ethr_mem__.sl,
+				  &ethr_mem__.ll};
+    if (lid)
+	ethr_mem__ = lid->mem;
+    if (!ethr_mem__.std.alloc
+	|| !ethr_mem__.std.realloc
+	|| !ethr_mem__.std.free) {
+	ethr_mem__.std.alloc = malloc;
+	ethr_mem__.std.realloc = realloc;
+	ethr_mem__.std.free = free;
+    }
+    for (i = 0; i < sizeof(m)/sizeof(m[0]); i++) {
+	if (!m[i]->alloc || !m[i]->realloc || !m[i]->free) {
+	    m[i]->alloc = ethr_mem__.std.alloc;
+	    m[i]->realloc = ethr_mem__.std.realloc;
+	    m[i]->free = ethr_mem__.std.free;
+	}
+
+    }
+    res = init_ts_event_alloc();
+    if (res != 0)
+	return res;
+    res = ethr_make_ts_event__(&tsep);
+    if (res == 0)
+	tsep->iflgs |= ETHR_TS_EV_ETHREAD;
+    if (!lid) {
+	main_threads = 0;
+	reader_groups = 0;
+    }
+    else {
+	if (lid->main_threads < 0 || USHRT_MAX < lid->main_threads)
+	    return res;
+	main_threads = lid->main_threads;
+	reader_groups = lid->reader_groups;
+    }
+    res = ethr_mutex_lib_late_init(reader_groups, main_threads);
+    if (res != 0)
+	return res;
+    ethr_not_completely_inited__ = 0; /* Need it for
+					 rwmutex_init */
+    res = ethr_rwmutex_init(&xhndl_rwmtx);
+    ethr_not_completely_inited__ = 1;
+    if (res != 0)
+	return res;
+    return 0;
+}
+
+int
+ethr_install_exit_handler(void (*funcp)(void))
+{
+    ethr_xhndl_list *xhp;
+
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+
+    if (!funcp)
+	return EINVAL;
+
+    xhp = (ethr_xhndl_list *) ethr_mem__.std.alloc(sizeof(ethr_xhndl_list));
+    if (!xhp)
+	return ENOMEM;
+
+    ethr_rwmutex_rwlock(&xhndl_rwmtx);
+
+    xhp->funcp = funcp;
+    xhp->next = xhndl_list;
+    xhndl_list = xhp;
+
+    ethr_rwmutex_rwunlock(&xhndl_rwmtx);
+
+    return 0;
+}
+
+void
+ethr_run_exit_handlers__(void)
+{
+    ethr_xhndl_list *xhp;
+
+    ethr_rwmutex_rlock(&xhndl_rwmtx);
+
+    xhp = xhndl_list;
+
+    ethr_rwmutex_runlock(&xhndl_rwmtx);
+
+    for (; xhp; xhp = xhp->next)
+	(*xhp->funcp)();
+}
+
+/*
+ * Thread specific event alloc, etc.
+ *
+ * Note that we don't know when it is safe to destroy an event, but
+ * we know when it is safe to reuse it. ts_event_free() therefore
+ * never destroys an event (but makes freed events available for
+ * reuse).
+ *
+ * We could easily keep track of the usage of events, and by this
+ * make it possible to destroy events. We would however suffer a
+ * performance penalty for this and save very little memory.
+ */
+
+typedef union {
+    ethr_ts_event ts_ev;
+    char align[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_ts_event))];
+} ethr_aligned_ts_event;
+
+static ethr_spinlock_t ts_ev_alloc_lock;
+static ethr_ts_event *free_ts_ev;
+
+static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp)
+{
+    int i;
+    ethr_aligned_ts_event *atsev;
+    atsev = ethr_mem__.std.alloc(sizeof(ethr_aligned_ts_event) * size
+				 + ETHR_CACHE_LINE_SIZE);
+    if (!atsev)
+	return NULL;
+    if ((((ethr_uint_t) atsev) & ETHR_CACHE_LINE_MASK) == 0)
+	atsev = ((ethr_aligned_ts_event *)
+		 ((((ethr_uint_t) atsev) & ~ETHR_CACHE_LINE_MASK)
+		  + ETHR_CACHE_LINE_SIZE));
+    for (i = 1; i < size; i++) {
+	atsev[i-1].ts_ev.next = &atsev[i].ts_ev;
+	ethr_atomic32_init(&atsev[i-1].ts_ev.uaflgs, 0);
+	atsev[i-1].ts_ev.iflgs = 0;
+    }
+    ethr_atomic32_init(&atsev[size-1].ts_ev.uaflgs, 0);
+    atsev[size-1].ts_ev.iflgs = 0;
+    atsev[size-1].ts_ev.next = NULL;
+    if (endpp)
+	*endpp = &atsev[size-1].ts_ev;
+    return &atsev[0].ts_ev;
+}
+
+static int init_ts_event_alloc(void)
+{
+    free_ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE,
+			       NULL);
+    if (!free_ts_ev)
+	return ENOMEM;
+    return ethr_spinlock_init(&ts_ev_alloc_lock);
+}
+
+static ethr_ts_event *ts_event_alloc(void)
+{
+    ethr_ts_event *ts_ev;
+    ethr_spin_lock(&ts_ev_alloc_lock);
+    if (free_ts_ev) {
+	ts_ev = free_ts_ev;
+	free_ts_ev = ts_ev->next;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+    }
+    else {
+	ethr_ts_event *ts_ev_pool_end;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+
+	ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_POOL_SIZE, &ts_ev_pool_end);
+	if (!ts_ev)
+	    return NULL;
+
+	ethr_spin_lock(&ts_ev_alloc_lock);
+	ts_ev_pool_end->next = free_ts_ev;
+	free_ts_ev = ts_ev->next;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+    }
+    return ts_ev;
+}
+
+static void ts_event_free(ethr_ts_event *ts_ev)
+{
+    ETHR_ASSERT(!ts_ev->udata);
+    ethr_spin_lock(&ts_ev_alloc_lock);
+    ts_ev->next = free_ts_ev;
+    free_ts_ev = ts_ev;
+    ethr_spin_unlock(&ts_ev_alloc_lock);
+}
+
+int ethr_make_ts_event__(ethr_ts_event **tsepp)
+{
+    int res;
+    ethr_ts_event *tsep = *tsepp;
+
+    if (!tsep) {
+	tsep = ts_event_alloc();
+	if (!tsep)
+	    return ENOMEM;
+    }
+
+    if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) {
+	res = ethr_event_init(&tsep->event);
+	if (res != 0) {
+	    ts_event_free(tsep);
+	    return res;
+	}
+    }
+
+    tsep->iflgs = ETHR_TS_EV_INITED;
+    tsep->udata = NULL;
+    tsep->rgix = 0;
+    tsep->mtix = 0;
+
+    res = ethr_set_tse__(tsep);
+    if (res != 0 && tsepp && *tsepp) {
+	ts_event_free(tsep);
+	return res;
+    }
+
+    if (tsepp)
+	*tsepp = tsep;
+
+    return 0;
+}
+
+int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp)
+{
+    int res;
+    ethr_ts_event *tsep = *tsepp;
+
+    if (!tsep) {
+	tsep = ts_event_alloc();
+	if (!tsep)
+	    return ENOMEM;
+    }
+
+    if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) {
+	res = ethr_event_init(&tsep->event);
+	if (res != 0) {
+	    ts_event_free(tsep);
+	    return res;
+	}
+    }
+
+    tsep->iflgs = ETHR_TS_EV_INITED|ETHR_TS_EV_TMP;
+    tsep->udata = NULL;
+
+    if (tsepp)
+	*tsepp = tsep;
+
+    return 0;
+}
+
+int ethr_free_ts_event__(ethr_ts_event *tsep)
+{
+    ts_event_free(tsep);
+    return 0;
+}
+
+void ethr_ts_event_destructor__(void *vtsep)
+{
+    if (vtsep) {
+	ethr_ts_event *tsep = (ethr_ts_event *) vtsep;
+	ts_event_free(tsep);
+	ethr_set_tse__(NULL);
+    }
+}
+
+int ethr_set_main_thr_status(int on, int no)
+{
+    ethr_ts_event *tsep = ethr_get_tse__();
+    if (!tsep)
+	return EINVAL;
+    if (on) {
+	if (no < 1 || main_threads < no)
+	    return EINVAL;
+	tsep->mtix = (unsigned short) no;
+	tsep->iflgs |= ETHR_TS_EV_MAIN_THR;
+    }
+    else {
+	tsep->iflgs &= ~ETHR_TS_EV_MAIN_THR;
+	tsep->mtix = (unsigned short) 0;
+    }
+    return 0;
+}
+
+int ethr_get_main_thr_status(int *on)
+{
+    ethr_ts_event *tsep = ethr_get_tse__();
+    if (!tsep)
+	*on = 0;
+    else {
+	if (tsep->iflgs & ETHR_TS_EV_MAIN_THR)
+	    *on = 1;
+	else
+	    *on = 0;
+    }
+    return 0;
+}
+
+/* Spinlocks and rwspinlocks */
+
+int
+ethr_spinlock_init(ethr_spinlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_spinlock_init__(lock);
+}
+
+int
+ethr_spinlock_destroy(ethr_spinlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_spinlock_destroy__(lock);
+}
+
+void
+ethr_spin_unlock(ethr_spinlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_spin_unlock__(lock);
+}
+
+void
+ethr_spin_lock(ethr_spinlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_spin_lock__(lock);
+}
+
+int
+ethr_rwlock_init(ethr_rwlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_rwlock_init__(lock);
+}
+
+int
+ethr_rwlock_destroy(ethr_rwlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_rwlock_destroy__(lock);
+}
+
+void
+ethr_read_unlock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_read_unlock__(lock);
+}
+
+void
+ethr_read_lock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_read_lock__(lock);
+}
+
+void
+ethr_write_unlock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_write_unlock__(lock);
+}
+
+void
+ethr_write_lock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_write_lock__(lock);
+}
+
+ETHR_IMPL_NORETURN__ ethr_fatal_error__(const char *file,
+					int line,
+					const char *func,
+					int err)
+{
+    char *errstr;
+    if (err == ENOTSUP)
+	errstr = "Operation not supported";
+    else {
+	errstr = strerror(err);
+	if (!errstr)
+	    errstr = "Unknown error";
+    }
+    fprintf(stderr, "%s:%d: Fatal error in %s(): %s (%d)\n",
+	    file, line, func, errstr, err);
+    ethr_abort__();
+}
+
+int ethr_assert_failed(const char *file, int line, const char *func, char *a)
+{
+    fprintf(stderr, "%s:%d: %s(): Assertion failed: %s\n", file, line, func, a);
+    ethr_abort__();
+    return 0;
+}
diff --git a/erts/lib_src/common/ethr_cbf.c b/erts/lib_src/common/ethr_cbf.c
new file mode 100644
index 0000000000..04feceec89
--- /dev/null
+++ b/erts/lib_src/common/ethr_cbf.c
@@ -0,0 +1,36 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+
+/*
+ * We keep this function alone in a separate file so the
+ * compiler wont optimize it away.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ethread.h"
+
+void
+ethr_compiler_barrier_fallback(void)
+{
+
+}
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
new file mode 100644
index 0000000000..2ddef32dfc
--- /dev/null
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -0,0 +1,2912 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Mutex, rwmutex and condition variable implementation
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_MUTEX_IMPL__
+
+#include <limits.h>
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#define ETHR_SPIN_WITH_WAITERS 1
+
+#define ETHR_MTX_MAX_FLGS_SPIN 10
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+static int default_rwmtx_main_spincount;
+static int default_rwmtx_aux_spincount;
+#endif
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+static int default_mtx_main_spincount;
+static int default_mtx_aux_spincount;
+static int default_cnd_main_spincount;
+static int default_cnd_aux_spincount;
+#endif
+
+static int no_spin;
+
+#ifndef ETHR_USE_OWN_RWMTX_IMPL__
+static pthread_rwlockattr_t write_pref_attr_data;
+static pthread_rwlockattr_t *write_pref_attr;
+#endif
+
+#if defined(ETHR_MTX_Q_LOCK_SPINLOCK__)
+#  define ETHR_MTX_QLOCK_INIT ethr_spinlock_init
+#  define ETHR_MTX_QLOCK_DESTROY ethr_spinlock_destroy
+#  define ETHR_MTX_Q_LOCK ethr_spin_lock
+#  define ETHR_MTX_Q_UNLOCK ethr_spin_unlock
+#elif defined(ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__)
+#  define ETHR_MTX_QLOCK_INIT(QL) pthread_mutex_init((QL), NULL)
+#  define ETHR_MTX_QLOCK_DESTROY pthread_mutex_destroy
+#  define ETHR_MTX_Q_LOCK(L)					\
+do {								\
+    int res__ = pthread_mutex_lock(L);				\
+    if (res__ != 0)						\
+	ETHR_FATAL_ERROR__(res__);				\
+} while (0)
+#  define ETHR_MTX_Q_UNLOCK(L)					\
+do {								\
+    int res__ = pthread_mutex_unlock(L);			\
+    if (res__ != 0)						\
+	ETHR_FATAL_ERROR__(res__);				\
+} while (0)
+#elif defined(ETHR_MTX_Q_LOCK_CRITICAL_SECTION__)
+#  define ETHR_MTX_QLOCK_INIT(QL) (InitializeCriticalSection((QL)), 0)
+#  define ETHR_MTX_QLOCK_DESTROY(QL) (DeleteCriticalSection((QL)), 0)
+#  define ETHR_MTX_Q_LOCK(QL) EnterCriticalSection((QL))
+#  define ETHR_MTX_Q_UNLOCK(QL) LeaveCriticalSection((QL))
+#endif
+
+int
+ethr_mutex_lib_init(int cpu_conf)
+{
+    int res = 0;
+
+    no_spin = cpu_conf == 1;
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+    default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE;
+    default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT;
+    default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT;
+    default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT;
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+    default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE;
+    default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT;
+
+#else
+
+#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \
+    && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
+    res = pthread_rwlockattr_init(&write_pref_attr_data);
+    if (res != 0)
+	return res;
+    res = pthread_rwlockattr_setkind_np(
+	&write_pref_attr_data,
+	PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
+    write_pref_attr = &write_pref_attr_data;
+#else
+    write_pref_attr = NULL;
+#endif
+
+#endif
+
+    return res;
+}
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+#ifdef ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS
+#if 0 /*
+       * When inc and dec are real atomic instructions as on x86, the
+       * ETHR_RLOCK_WITH_INC_DEC implementations performs better with
+       * lots of read locks compared to the cmpxchg based implementation.
+       * It, however, performs worse with lots of mixed reads and writes.
+       * It could be used for rwlocks that are known to be read locked
+       * much, but the readers array based implementation outperforms it
+       * by far. Therefore, it has been disabled, and will probably be
+       * removed some time in the future.
+       */
+#  define ETHR_RLOCK_WITH_INC_DEC
+#endif
+#endif
+
+static int reader_groups_array_size = 0;
+static int main_threads_array_size = 0;
+
+#endif
+
+int
+ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads)
+{
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+    default_mtx_main_spincount += (no_main_threads
+				   * ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC);
+    if (default_mtx_main_spincount > ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+	default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+    default_rwmtx_main_spincount += (no_main_threads
+				     * ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC);
+    if (default_rwmtx_main_spincount > ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+	default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+
+    reader_groups_array_size = (no_reader_groups <= 1
+				? 1
+				: no_reader_groups + 1);
+    main_threads_array_size = (no_main_threads <= 1
+			       ? 1
+			       : no_main_threads + 1);
+#endif
+    return 0;
+}
+
+int
+ethr_rwmutex_set_reader_group(int ix)
+{
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+    ethr_ts_event *tse;
+
+    if (ix < 0 || reader_groups_array_size <= ix)
+	return EINVAL;
+
+    tse = ethr_get_ts_event();
+
+    if ((tse->iflgs & ETHR_TS_EV_ETHREAD) == 0) {
+	ethr_leave_ts_event(tse);
+	return EINVAL;
+    }
+
+    tse->rgix = ix;
+    
+    ethr_leave_ts_event(tse);
+#endif
+    return 0;
+}
+
+#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ)
+static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int);
+#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) hard_debug_chk_q__(&(RWMTX)->mtxb,1)
+#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) hard_debug_chk_q__(&(MTX)->mtxb, 0)
+#else
+#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX)
+#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX)
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+static void
+rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
+			   ethr_sint32_t initial,
+			   int q_locked);
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx,
+		    int have_w,
+		    ethr_sint32_t initial,
+		    int transfer_read_lock);
+static int
+rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
+			     ethr_sint32_t initial,
+			     ethr_ts_event *tse,
+			     int start_next_ix,
+			     int check_before_try,
+			     int try_write_lock);
+#endif
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
+
+/* -- Utilities operating both on ordinary mutexes and read write mutexes -- */
+
+static ETHR_INLINE void
+rwmutex_freqread_wtng_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix = (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ
+	      ? tse->rgix
+	      : tse->mtix);
+    rwmtx->tdata.ra[ix].data.waiting_readers++;
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx,
+			  ethr_rwmutex_type type,
+			  int ix,
+			  int inc)
+{
+    if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0)
+	ethr_atomic32_add(&rwmtx->tdata.ra[ix].data.readers, inc);
+    else {
+	ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+	ETHR_ASSERT(inc == 1);
+	ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1);
+    }
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_inc:
+	ethr_atomic32_inc(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_inc;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+	ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1);
+    }
+}
+
+#if 0 /* Not used */
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec:
+	ethr_atomic32_dec(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0);
+    }
+}
+
+#endif
+
+static ETHR_INLINE ethr_sint32_t
+rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec_read:
+	return ethr_atomic32_dec_read(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec_read;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0);
+	return (ethr_sint32_t) 0;
+    }
+}
+
+static ETHR_INLINE ethr_sint32_t
+rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec_read:
+	return ethr_atomic32_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec_read;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic32_set_relb(&rwmtx->tdata.ra[ix].data.readers,
+			     (ethr_sint32_t) 0);
+	return (ethr_sint32_t) 0;
+    }
+}
+
+static ETHR_INLINE ethr_sint32_t
+rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix)
+{
+    ethr_sint32_t res = ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers);
+#ifdef ETHR_DEBUG
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+	ETHR_ASSERT(res >= 0);
+	break;
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+	ETHR_ASSERT(ix == 0 ? res >= 0 : (res == 0 || res == 1));
+	break;
+    default:
+	ETHR_ASSERT(0);
+	break;
+    }
+#endif
+    return res;
+}
+
+
+static ETHR_INLINE void
+enqueue(ethr_ts_event **queue,
+	ethr_ts_event *tse_start,
+	ethr_ts_event *tse_end)
+{
+    if (!*queue) {
+	*queue = tse_start;
+	tse_start->prev = tse_end;
+	tse_end->next = tse_start;
+    }
+    else {
+	tse_end->next = *queue;
+	tse_start->prev = (*queue)->prev;
+	(*queue)->prev->next = tse_start;
+	(*queue)->prev = tse_end;
+    }
+}
+
+static ETHR_INLINE void
+insert(ethr_ts_event *tse_pred, ethr_ts_event *tse)
+{
+    tse->next = tse_pred->next;
+    tse->prev = tse_pred;
+    tse_pred->next->prev = tse;
+    tse_pred->next = tse;
+}
+
+static ETHR_INLINE void
+dequeue(ethr_ts_event **queue,
+	ethr_ts_event *tse_start,
+	ethr_ts_event *tse_end)
+{
+    if (tse_start->prev == tse_end) {
+	ETHR_ASSERT(*queue == tse_start && tse_end->next == tse_start);
+	*queue = NULL;
+    }
+    else {
+	if (*queue == tse_start)
+	    *queue = tse_end->next;
+	tse_end->next->prev = tse_start->prev;
+	tse_start->prev->next = tse_end->next;
+    }
+}
+
+static void
+event_wait(struct ethr_mutex_base_ *mtxb,
+	   ethr_ts_event *tse,
+	   int spincount,
+	   ethr_sint32_t type,
+	   int is_rwmtx,
+	   int is_freq_read)
+{
+    int locked = 0;
+    ethr_sint32_t act;
+    int need_try_complete_runlock = 0;
+    int transfer_read_lock = 0;
+
+    /* Need to enqueue and wait... */
+
+    tse->uflgs = type;
+    ethr_atomic32_set(&tse->uaflgs, type);
+
+    ETHR_MTX_Q_LOCK(&mtxb->qlck);
+    locked = 1;
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    act = ethr_atomic32_read(&mtxb->flgs);
+
+    if (act & type) {
+
+	/* Wait bit already there; enqueue... */
+
+	ETHR_ASSERT(mtxb->q);
+	if (type == ETHR_RWMTX_W_WAIT_FLG__) {
+	    enqueue(&mtxb->q, tse, tse);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	    mtxb->ws++;
+#endif
+	}
+	else {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_ASSERT(is_rwmtx);
+	    ETHR_ASSERT(rwmtx->rq_end);
+	    insert(rwmtx->rq_end, tse);
+	    rwmtx->rq_end = tse;
+	    if (is_freq_read)
+		rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse);
+	    else
+		rwmtx->tdata.rs++;
+	}
+    }
+    else {
+
+	/* Set wait bit */
+
+	while (1) {
+	    ethr_sint32_t new, exp = act;
+	    need_try_complete_runlock = 0;
+	    transfer_read_lock = 0;
+
+	    if (type == ETHR_RWMTX_W_WAIT_FLG__) {
+		if (is_freq_read && act == ETHR_RWMTX_R_FLG__)
+		    need_try_complete_runlock = 1;
+		if (act != 0)
+		    new = act | ETHR_RWMTX_W_WAIT_FLG__;
+		else
+		    new = ETHR_RWMTX_W_FLG__; /* Try to get it */
+	    }
+	    else {
+		ETHR_ASSERT(is_rwmtx);
+
+		if (!is_freq_read) {
+		    if (act & (ETHR_RWMTX_W_FLG__| ETHR_RWMTX_W_WAIT_FLG__))
+			new = act | ETHR_RWMTX_R_WAIT_FLG__;
+		    else
+			new = act + 1; /* Try to get it */
+		}
+		else {
+		    new = act | ETHR_RWMTX_R_WAIT_FLG__;
+		    if ((act & (ETHR_RWMTX_W_FLG__
+				| ETHR_RWMTX_W_WAIT_FLG__)) == 0) {
+			/* Transfer read lock to this thread. */
+			transfer_read_lock = 1;
+		    }
+		}
+	    }
+
+	    act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs, new, exp);
+	    if (exp == act) {
+		if (new & type) {
+		    act = new;
+		    break;
+		}
+		else {
+		    /* Got it */
+		    goto done;
+		}
+	    }
+	}
+
+	/* Enqueue */
+
+	if (type == ETHR_RWMTX_R_WAIT_FLG__) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_ASSERT(is_rwmtx);
+	    ETHR_ASSERT(!rwmtx->rq_end);
+	    rwmtx->rq_end =  tse;
+	    if (is_freq_read)
+		rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse);
+	    else
+		rwmtx->tdata.rs++;
+	}
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	else {
+	    mtxb->ws++;
+	}
+#endif
+
+	enqueue(&mtxb->q, tse, tse);
+    }
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    /* Wait */
+    locked = 0;
+
+    ETHR_ASSERT(!(transfer_read_lock && need_try_complete_runlock));
+
+    if (transfer_read_lock) {
+	ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
+		    != ETHR_RWMUTEX_TYPE_NORMAL);
+	/*
+	 * We are the only one in the queue and we are not write
+	 * locked; rwmutex_transfer_read_lock() will:
+	 * - transfer a read lock to us (since we're first in q)
+	 * - unlock the Q-lock
+	 */
+	rwmutex_transfer_read_lock(((ethr_rwmutex *) mtxb), act, 1);
+    }
+    else {
+	ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+	if (need_try_complete_runlock) {
+	    ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
+			!= ETHR_RWMUTEX_TYPE_NORMAL);
+	    /*
+	     * We were the only one in queue when we enqueued, and it
+	     * was seemingly read locked. We need to try to complete a
+	     * runlock otherwise we might be hanging forever. If the
+	     * runlock could be completed we will be dequeued and
+	     * woken by ourselves.
+	     */
+	    rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
+					 act, tse, 0, 1, 0);
+	}
+    }
+
+    while (1) {
+	ethr_event_reset(&tse->event);
+
+	act = ethr_atomic32_read_acqb(&tse->uaflgs);
+	if (!act)
+	    goto done; /* Got it */
+
+	ETHR_ASSERT(act == type);
+	ethr_event_swait(&tse->event, spincount);
+	/* swait result: 0 || EINTR */
+
+	act = ethr_atomic32_read_acqb(&tse->uaflgs);
+	if (!act)
+	    goto done; /* Got it */
+    }
+
+ done:
+    if (locked)
+	ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+}
+
+static void
+wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    ethr_ts_event *tse;
+
+    tse = mtxb->q;
+    ETHR_ASSERT(tse);
+    dequeue(&mtxb->q, tse, tse);
+
+    ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+    ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    mtxb->ws--;
+#endif
+#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ)
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+    ethr_atomic32_set(&tse->uaflgs, 0);
+    ethr_event_set(&tse->event);
+}
+
+static ETHR_INLINE int
+initial_spincount(struct ethr_mutex_base_ *mtxb)
+{
+    return (mtxb->aux_scnt < ETHR_MTX_MAX_FLGS_SPIN
+	    ? mtxb->aux_scnt
+	    : ETHR_MTX_MAX_FLGS_SPIN);
+}
+
+static ETHR_INLINE int
+update_spincount(struct ethr_mutex_base_ *mtxb,
+		 ethr_ts_event *tse,
+		 int *scnt_state,
+		 int *scnt)
+{
+    int state = *scnt_state;
+    if (state <= 0) {
+	/* Here state is max spincount to do on event negated */
+	*scnt = -state;
+    }
+    else {
+	/* Here state is initial spincount made on flags */
+	*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
+		 ? mtxb->main_scnt
+		 : mtxb->aux_scnt);
+	if (*scnt <= state)
+	    *scnt = 0;
+	else {
+	    if (*scnt <= ETHR_MTX_MAX_FLGS_SPIN)
+		*scnt_state = 0; /* No spin on event */
+	    else {
+		 /* Spin on event after... */
+		*scnt_state = -1*(*scnt - ETHR_MTX_MAX_FLGS_SPIN);
+		/* ... we have spun on flags */
+		*scnt = ETHR_MTX_MAX_FLGS_SPIN;
+	    }
+	    *scnt -= state;
+	    return 0;
+	}
+    }
+    return 1;
+}
+
+int check_readers_array(ethr_rwmutex *rwmtx,
+			int start_rix,
+			int length);
+
+static ETHR_INLINE void
+write_lock_wait(struct ethr_mutex_base_ *mtxb,
+		ethr_sint32_t initial,
+		int is_rwmtx,
+		int is_freq_read)
+{
+    ethr_sint32_t act = initial;
+    int scnt, start_scnt;
+    ethr_ts_event *tse = NULL;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+    int res;
+
+    ETHR_ASSERT(!is_freq_read || is_rwmtx);
+
+    start_scnt = scnt = initial_spincount(mtxb);
+
+    /*
+     * Spin trying to write lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+	while (act != 0) {
+
+	    if (is_freq_read && act == ETHR_RWMTX_R_FLG__) {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		scnt--;
+		if (!tse)
+		    tse = ethr_get_ts_event();
+		res = rwmutex_try_complete_runlock(rwmtx, act,
+						   tse, 0, 0,
+						   1);
+		if (res != EBUSY)
+		    goto done; /* Got it */
+		if (scnt <= 0)
+		    goto chk_spin;
+		if (--until_yield == 0) {
+		    until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		    ETHR_YIELD();
+		}
+	    }
+
+	    if (scnt <= 0) {
+	    chk_spin:
+		scnt = 0;
+
+		if (!tse)
+		    tse = ethr_get_ts_event();
+		if (update_spincount(mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(mtxb, tse, scnt, ETHR_RWMTX_W_WAIT_FLG__,
+			       is_rwmtx, is_freq_read);
+		    goto done;
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic32_read(&mtxb->flgs);
+	    scnt--;
+	}
+
+	act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs,
+					 ETHR_RWMTX_W_FLG__,
+					 0);
+	if (act == 0)
+	    goto done; /* Got it */
+    }
+
+ done:
+    if (tse)
+	ethr_leave_ts_event(tse);
+}
+
+static int
+mtxb_init(struct ethr_mutex_base_ *mtxb,
+	  int def_main_scnt,
+	  int main_scnt,
+	  int def_aux_scnt,
+	  int aux_scnt)
+{
+    ETHR_MTX_HARD_DEBUG_LFS_INIT(mtxb);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    mtxb->ws = 0;
+#endif
+    ETHR_MTX_CHK_EXCL_INIT(mtxb);
+    if (no_spin) {
+	mtxb->main_scnt = 0;
+	mtxb->aux_scnt = 0;
+    }
+    else {
+	if (main_scnt > SHRT_MAX)
+	    mtxb->main_scnt = SHRT_MAX;
+	else if (main_scnt < 0)
+	    mtxb->main_scnt = def_main_scnt;
+	else
+	    mtxb->main_scnt = (short) main_scnt;
+	if (aux_scnt > SHRT_MAX)
+	    mtxb->aux_scnt = SHRT_MAX;
+	else if (aux_scnt < 0)
+	    mtxb->aux_scnt = def_aux_scnt;
+	else
+	    mtxb->aux_scnt = (short) aux_scnt;
+	if (mtxb->main_scnt < mtxb->aux_scnt)
+	    mtxb->main_scnt = mtxb->aux_scnt;
+
+    }
+    mtxb->q = NULL;
+    ethr_atomic32_init(&mtxb->flgs, 0);
+    return ETHR_MTX_QLOCK_INIT(&mtxb->qlck);
+}
+
+static int
+mtxb_destroy(struct ethr_mutex_base_ *mtxb)
+{
+    ethr_sint32_t act;
+    ETHR_MTX_Q_LOCK(&mtxb->qlck);
+    act = ethr_atomic32_read(&mtxb->flgs);
+    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+    if (act != 0)
+	return EINVAL;
+    return ETHR_MTX_QLOCK_DESTROY(&mtxb->qlck);
+}
+
+
+#endif /* ETHR_USE_OWN_RWMTX_IMPL__ || ETHR_USE_OWN_MTX_IMPL__ */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Mutex and condition variable implementation                               *
+\*                                                                           */
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+
+/* -- Mutex ---------------------------------------------------------------- */
+
+int
+ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt)
+{
+    int res;
+#if ETHR_XCHK
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = ETHR_MUTEX_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(mtx);
+    res = mtxb_init(&mtx->mtxb,
+		    default_mtx_main_spincount,
+		    opt ? opt->main_spincount : -1,
+		    default_mtx_aux_spincount,
+		    opt ? opt->aux_spincount : -1);
+#if ETHR_XCHK
+    if (res != 0)
+	mtx->initialized = 0;
+#endif
+    return res;
+}
+
+int
+ethr_mutex_init(ethr_mutex *mtx)
+{
+    return ethr_mutex_init_opt(mtx, NULL);
+}
+
+int
+ethr_mutex_destroy(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = 0;
+#endif
+    return mtxb_destroy(&mtx->mtxb);
+}
+
+void
+ethr_mutex_lock_wait__(ethr_mutex *mtx, ethr_sint32_t initial)
+{
+    write_lock_wait(&mtx->mtxb, initial, 0, 0);
+}
+
+void
+ethr_mutex_unlock_wake__(ethr_mutex *mtx, ethr_sint32_t initial)
+{
+    ethr_ts_event *tse;
+
+    ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck);
+    tse = mtx->mtxb.q;
+
+    ETHR_ASSERT(tse);
+    ETHR_ASSERT(ethr_atomic32_read(&mtx->mtxb.flgs)
+		== (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
+    ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__);
+    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+
+    /*
+     * If we have multiple waiters, there is no need to modify
+     * mtxb->flgs; otherwise, we need to clear the write wait bit...
+     */
+    if (tse->next == mtx->mtxb.q)
+	ethr_atomic32_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__);
+
+    wake_writer(&mtx->mtxb, 0);
+}
+
+/* -- Condition variables -------------------------------------------------- */
+
+static void
+enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
+{
+    ethr_sint32_t act;
+
+    /*
+     * `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx'
+     * is not currently locked by current thread, we almost certainly have a
+     * hard to debug race condition. There might however be some (strange)
+     * use for it. POSIX also allow a call to `pthread_cond_signal' or
+     * `pthread_cond_broadcast' even though the the associated mutex isn't
+     * locked by the caller. Therefore, we also allow this kind of strange
+     * usage, but optimize for the case where the mutex is locked by the
+     * calling thread.
+     */
+
+    ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck);
+
+    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    {
+	int dbg_nws__ = 0;
+	ethr_ts_event *dbg_tse__;
+	for (dbg_tse__ = tse_start;
+	     dbg_tse__ != tse_end;
+	     dbg_tse__ = dbg_tse__->next)
+	    dbg_nws__++;
+	mtx->mtxb.ws += dbg_nws__ + 1;
+    }
+#endif
+
+    act = ethr_atomic32_read(&mtx->mtxb.flgs);
+    ETHR_ASSERT(act == 0
+		|| act == ETHR_RWMTX_W_FLG__
+		|| act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
+    if (act & ETHR_RWMTX_W_FLG__) {
+	/* The normal sane case */
+	if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) {
+	    ETHR_ASSERT(!mtx->mtxb.q);
+	    act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs,
+					(ETHR_RWMTX_W_FLG__
+					 | ETHR_RWMTX_W_WAIT_FLG__),
+					ETHR_RWMTX_W_FLG__);
+	    if (act != ETHR_RWMTX_W_FLG__) {
+		/*
+		 * Sigh... this wasn't so sane after all since, the mutex was
+		 * obviously not locked by the current thread....
+		 */
+		ETHR_ASSERT(act == 0);
+		goto mtx_unlocked;
+	    }
+	}
+
+#ifdef ETHR_DEBUG
+	if (act & ETHR_RWMTX_W_WAIT_FLG__)
+	    ETHR_ASSERT(mtx->mtxb.q);
+	else
+	    ETHR_ASSERT(!mtx->mtxb.q);
+#endif
+
+	enqueue(&mtx->mtxb.q, tse_start, tse_end);
+
+	ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+	ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+    }
+    else {
+	int multi;
+    mtx_unlocked:
+	/* Sigh... mutex isn't locked... */
+
+	multi = tse_start != tse_end;
+
+	while (1) {
+	    ethr_sint32_t new, exp = act;
+
+	    if (multi || (act & ETHR_RWMTX_W_FLG__))
+		new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__;
+	    else
+		new = ETHR_RWMTX_W_FLG__;
+
+	    act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs, new, exp);
+	    if (exp == act) {
+		ETHR_ASSERT(!mtx->mtxb.q);
+		if (act & ETHR_RWMTX_W_FLG__) {
+		    enqueue(&mtx->mtxb.q, tse_start, tse_end);
+
+		    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+		    ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+		}
+		else {
+		    ETHR_ASSERT(!mtx->mtxb.q);
+		    /*
+		     * Acquired the mutex on behalf of the
+		     * first thread in the queue; wake
+		     * it and enqueue the rest...
+		     */
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+		    mtx->mtxb.ws--;
+#endif
+		    if (multi) {
+			enqueue(&mtx->mtxb.q, tse_start->next, tse_end);
+			ETHR_ASSERT(mtx->mtxb.q);
+		    }
+
+		    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+		    ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+		    ethr_atomic32_set(&tse_start->uaflgs, 0);
+		    ethr_event_set(&tse_start->event);
+		}
+		break;
+	    }
+	}
+    }
+}
+
+int
+ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
+{
+#if ETHR_XCHK
+    if (!cnd) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = ETHR_COND_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(cnd);
+    cnd->q = NULL;
+    if (no_spin) {
+	cnd->main_scnt = 0;
+	cnd->aux_scnt = 0;
+    }
+    else {
+	if (!opt || opt->main_spincount < 0)
+	    cnd->main_scnt = default_cnd_main_spincount;
+	else if (opt->main_spincount > SHRT_MAX)
+	    cnd->main_scnt = SHRT_MAX;
+	else
+	    cnd->main_scnt = (short) opt->main_spincount;
+	if (!opt || opt->aux_spincount < 0)
+	    cnd->aux_scnt = default_cnd_aux_spincount;
+	else if (opt->aux_spincount > SHRT_MAX)
+	    cnd->aux_scnt = SHRT_MAX;
+	else
+	    cnd->aux_scnt = (short) opt->aux_spincount;
+	if (cnd->main_scnt < cnd->aux_scnt)
+	    cnd->main_scnt = cnd->aux_scnt;
+    }
+    ETHR_MTX_QLOCK_INIT(&cnd->qlck);
+    return 0;
+}
+
+int
+ethr_cond_init(ethr_cond *cnd)
+{
+    return ethr_cond_init_opt(cnd, NULL);
+}
+
+int
+ethr_cond_destroy(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = 0;
+#endif
+    return ETHR_MTX_QLOCK_DESTROY(&cnd->qlck);
+}
+
+void
+ethr_cond_signal(ethr_cond *cnd)
+{
+    ethr_ts_event *tse;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+
+    ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+    tse = cnd->q;
+
+    if (!tse) {
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+    }
+    else {
+	ethr_mutex *mtx = (ethr_mutex *) tse->udata;
+
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+	ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+	ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__);
+
+	ethr_atomic32_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
+
+	dequeue(&cnd->q, tse, tse);
+
+	ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+	tse->next = tse->prev = NULL;
+
+	enqueue_mtx(mtx, tse, tse);
+
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    }
+}
+
+void
+ethr_cond_broadcast(ethr_cond *cnd)
+{
+    int got_all;
+    ethr_ts_event *tse;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    do {
+	got_all = 1;
+
+	ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+	tse = cnd->q;
+
+	if (!tse) {
+	    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+	}
+	else {
+	    ethr_mutex *mtx = (ethr_mutex *) tse->udata;
+	    ethr_ts_event *tse_tmp, *tse_end;
+
+	    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+	    tse_end = cnd->q->prev;
+
+	    tse_tmp = tse;
+
+	    do {
+
+		if (mtx == (ethr_mutex *) tse_tmp->udata) {
+		    /* The normal case */
+
+		    ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+		    ETHR_ASSERT(ethr_atomic32_read(&tse_tmp->uaflgs)
+				== ETHR_CND_WAIT_FLG__);
+
+		    ethr_atomic32_set(&tse_tmp->uaflgs,
+				      ETHR_RWMTX_W_WAIT_FLG__);
+		}
+		else {
+		    /* Should be very unusual */
+		    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+		    tse_end = tse_tmp->prev;
+		    got_all = 0;
+		    break;
+		}
+
+		tse_tmp = tse_tmp->next;
+
+	    } while (tse_tmp != cnd->q);
+
+	    dequeue(&cnd->q, tse, tse_end);
+
+	    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+	    enqueue_mtx(mtx, tse, tse_end);
+	}
+
+    } while (!got_all);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+}
+
+int
+ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
+{
+    int woken;
+    int scnt;
+    void *udata = NULL;
+    ethr_ts_event *tse;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    tse = ethr_get_ts_event();
+
+    scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
+	    ? cnd->main_scnt
+	    : cnd->aux_scnt);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+
+    udata = tse->udata; /* Got to restore udata before returning */
+    tse->udata = (void *) mtx;
+
+    tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */
+    ethr_atomic32_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__);
+
+    ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+    enqueue(&cnd->q, tse, tse);
+
+    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+    ethr_mutex_unlock(mtx);
+
+    /* Wait */
+    woken = 0;
+    while (1) {
+	ethr_sint32_t act;
+
+	ethr_event_reset(&tse->event);
+
+	act = ethr_atomic32_read_acqb(&tse->uaflgs);
+	if (!act)
+	    break; /* Mtx locked */
+
+	/* First time, got EINTR, or spurious wakeup... */
+
+	ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
+		    || act == ETHR_RWMTX_W_WAIT_FLG__);
+
+	if (woken) {
+	    /*
+	     * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already been enqueued
+	     * on the mutex; continue wait until locked...
+	     */
+	    if (act == ETHR_CND_WAIT_FLG__) {
+		ETHR_MTX_Q_LOCK(&cnd->qlck);
+		act = ethr_atomic32_read(&tse->uaflgs);
+		ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
+			    || act == ETHR_RWMTX_W_WAIT_FLG__);
+		/*
+		 * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already
+		 * enqueued on the mutex; continue wait until locked...
+		 */
+		if (act == ETHR_CND_WAIT_FLG__)
+		    dequeue(&cnd->q, tse, tse);
+
+		ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+		if (act == ETHR_CND_WAIT_FLG__) {
+		    tse->udata = udata;
+		    ethr_leave_ts_event(tse);
+		    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+		    ethr_mutex_lock(mtx);
+		    return EINTR;
+		}
+	    }
+	    ETHR_ASSERT(act == ETHR_RWMTX_W_WAIT_FLG__);
+	}
+	ethr_event_swait(&tse->event, scnt);
+	/* swait result: 0 || EINTR */
+	woken = 1;
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
+    tse->udata = udata;
+    ethr_leave_ts_event(tse);
+    return 0;
+}
+
+#else
+/* -- pthread mutex and condition variables -------------------------------- */
+
+int
+ethr_mutex_init(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = ETHR_MUTEX_INITIALIZED;
+#endif
+    return pthread_mutex_init(&mtx->pt_mtx, NULL);
+}
+
+int
+ethr_mutex_destroy(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+#if ETHR_XCHK
+    mtx->initialized = 0;
+#endif
+    return pthread_mutex_destroy(&mtx->pt_mtx);
+}
+
+int
+ethr_cond_init(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (!cnd) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = ETHR_COND_INITIALIZED;
+#endif
+    return pthread_cond_init(&cnd->pt_cnd, NULL);
+}
+
+int
+ethr_cond_destroy(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = 0;
+#endif
+    return pthread_cond_destroy(&cnd->pt_cnd);
+}
+
+void
+ethr_cond_signal(ethr_cond *cnd)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    res = pthread_cond_signal(&cnd->pt_cnd);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+void
+ethr_cond_broadcast(ethr_cond *cnd)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    res = pthread_cond_broadcast(&cnd->pt_cnd);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+int
+ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    res = pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx);
+    if (res != 0 && res != EINTR)
+	ETHR_FATAL_ERROR__(res);
+    return res;
+}
+
+#endif /* pthread_mutex */
+
+/* -- Exported symbols of inline functions --------------------------------- */
+
+int
+ethr_mutex_trylock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    return ethr_mutex_trylock__(mtx);
+}
+
+void
+ethr_mutex_lock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    ethr_mutex_lock__(mtx);
+}
+
+void
+ethr_mutex_unlock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    ethr_mutex_unlock__(mtx);
+}
+
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Read/Write Mutex                                                          *
+\*                                                                           */
+
+static void
+wake_readers(ethr_rwmutex *rwmtx, int rs)
+{
+    ethr_ts_event *tse;
+#ifdef ETHR_DEBUG
+    int drs = 0;
+#endif
+
+    tse = rwmtx->mtxb.q;
+    ETHR_ASSERT(tse);
+    ETHR_ASSERT(rwmtx->rq_end);
+    dequeue(&rwmtx->mtxb.q, tse, rwmtx->rq_end);
+    rwmtx->rq_end->next = NULL;
+    rwmtx->rq_end = NULL;
+
+    ETHR_ASSERT(!rwmtx->mtxb.q
+		|| (ethr_atomic32_read(&rwmtx->mtxb.q->uaflgs)
+		    == ETHR_RWMTX_W_WAIT_FLG__));
+
+    ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+
+    while (tse) {
+	ethr_ts_event *tse_next;
+
+#ifdef ETHR_DEBUG
+	ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+	ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs)
+		    == ETHR_RWMTX_R_WAIT_FLG__);
+	drs++;
+#endif
+
+	tse_next = tse->next; /* we aren't allowed to read tse->next
+				 after we have reset uaflgs */
+
+	ethr_atomic32_set(&tse->uaflgs, 0);
+	ethr_event_set(&tse->event);
+	tse = tse_next;
+    }
+
+    ETHR_ASSERT(rs == drs);
+}
+
+static ETHR_INLINE int
+is_w_waiter(ethr_ts_event *tse)
+{
+    ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__
+		|| tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+    return tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__;
+}
+
+static ETHR_INLINE int
+multiple_w_waiters(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(rwmtx->mtxb.q);
+    ETHR_ASSERT(rwmtx->mtxb.q->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+
+    if (!rwmtx->rq_end)
+	return rwmtx->mtxb.q->next != rwmtx->mtxb.q;
+    else {
+	ETHR_ASSERT(rwmtx->mtxb.q->next != rwmtx->mtxb.q);
+	if (rwmtx->mtxb.q->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__)
+	    return 1;
+	ETHR_ASSERT(rwmtx->rq_end->next == rwmtx->mtxb.q
+		    || rwmtx->rq_end->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+	return rwmtx->rq_end->next != rwmtx->mtxb.q;
+    }
+}
+
+int check_readers_array(ethr_rwmutex *rwmtx,
+			int start_rix,
+			int length)
+{
+    int ix = start_rix;
+
+    ETHR_MEMORY_BARRIER;
+
+    do {
+	ethr_sint32_t act = rwmutex_freqread_rdrs_read(rwmtx, ix);
+	if (act != 0)
+	    return EBUSY;
+	ix++;
+	if (ix == length)
+	    ix = 0;
+    } while (ix != start_rix);
+
+    return 0;
+}
+
+static void
+rwmutex_freqread_rdrs_dec_chk_wakeup(ethr_rwmutex *rwmtx,
+				     ethr_ts_event *tse,
+				     ethr_sint32_t initial)
+{
+    ethr_sint32_t act = initial;
+
+    if ((act & (ETHR_RWMTX_W_FLG__|
+		ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) == 0) {
+	if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	    if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
+		/*
+		 * We *need* to try to complete the runlock.
+		 * A writer that just enqueued (not seen by us
+		 * in flag field) may depend on someone else
+		 * completing the runlock. We just took over
+		 * that responsibilty since we modified reader
+		 * groups.
+		 */
+		rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	    }
+	}
+	else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_R_WAIT_FLG__)
+	    rwmutex_transfer_read_lock(rwmtx, act, 0);
+	else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_W_WAIT_FLG__)
+	    rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	else {
+	    /*
+	     * Don't know if we got readers or writers
+	     * first in queue; need to peek
+	     */
+	    ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+	    if (!rwmtx->mtxb.q)
+		ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+	    else if (is_w_waiter(rwmtx->mtxb.q)) {
+		act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+		ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+		if ((act & ETHR_RWMTX_W_FLG__) == 0)
+		    rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	    }
+	    else {
+		/*
+		 * rwmutex_transfer_read_lock() will
+		 * unlock Q lock.
+		 */
+		act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+		if (act & ETHR_RWMTX_W_FLG__)
+		    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+		else
+		    rwmutex_transfer_read_lock(rwmtx, act, 1);
+	    }
+	}
+    }
+}
+
+static void
+rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx,
+					 ethr_ts_event *tse)
+{
+    ethr_sint32_t act;
+    /*
+     * Restore failed increment
+     */
+    act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse);
+
+    ETHR_MEMORY_BARRIER;
+
+    if (act == 0) {
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
+    }
+}
+
+static int
+rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
+			     ethr_sint32_t initial,
+			     ethr_ts_event *tse,
+			     int start_next_ix,
+			     int check_before_try,
+			     int try_write_lock)
+{
+    ethr_ts_event *tse_tmp;
+    ethr_sint32_t act = initial;
+    int six, res, length;
+
+    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+
+    if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__)
+	return try_write_lock ? EBUSY : 0;
+
+    tse_tmp = tse;
+    if (!tse_tmp)
+	tse_tmp = ethr_get_ts_event();
+
+    if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
+	goto check_waiters;
+
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	length = reader_groups_array_size;
+	six = tse_tmp->rgix;
+    }
+    else {
+	length = main_threads_array_size;
+	six = tse_tmp->mtix;
+    }
+    if (start_next_ix) {
+	six++;
+	if (six >= length)
+	    six = 0;
+    }
+
+    if (!tse)
+	ethr_leave_ts_event(tse_tmp);
+
+    if (check_before_try) {
+	res = check_readers_array(rwmtx, six, length);
+
+	ETHR_MEMORY_BARRIER;
+
+	if (res == EBUSY)
+	    return try_write_lock ? EBUSY : 0;
+    }
+
+ restart:
+
+    while (1) {
+	ethr_sint32_t exp = act;
+	ethr_sint32_t new = act+1;
+
+	ETHR_ASSERT((act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) == 0);
+
+	ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
+		    < ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+	act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (exp == act) {
+	    act = new;
+	    break;
+	}
+
+	if (!try_write_lock) {
+	    if (act == 0 || (act & (ETHR_RWMTX_W_FLG__
+				    | ETHR_RWMTX_R_ABRT_UNLCK_FLG__)))
+		return 0;
+	    if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+		if ((act & ETHR_RWMTX_R_FLG__) == 0)
+		    return 0;
+	    }
+	    else if ((act & ETHR_RWMTX_R_FLG__) == 0) {
+		if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
+		    return 0;
+		goto check_waiters;
+	    }
+	}
+	else {
+	    if (act == 0)
+		goto tryrwlock;
+	    if (act & (ETHR_RWMTX_W_FLG__
+		       | ETHR_RWMTX_R_ABRT_UNLCK_FLG__))
+		return EBUSY;
+	}
+    }
+
+    res = check_readers_array(rwmtx, six, length);
+
+    ETHR_MEMORY_BARRIER;
+
+    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+
+    while (1) {
+	int finished_abort = 0;
+	ethr_sint32_t exp = act;
+	ethr_sint32_t new = act;
+
+	new--;
+	if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
+	    if ((new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) == 0) {
+		new &= ~ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+		finished_abort = 1;
+	    }
+	    ETHR_ASSERT(act & ETHR_RWMTX_R_FLG__);
+	}
+	else if ((act & ETHR_RWMTX_R_FLG__) && res != EBUSY) {
+	    new &= ~ETHR_RWMTX_R_FLG__;
+	}
+
+	ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+	act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (exp == act) {
+	    act = new;
+	    if (act & ETHR_RWMTX_W_FLG__)
+		return try_write_lock ? EBUSY : 0;
+	    if (finished_abort && (act & ETHR_RWMTX_WAIT_FLGS__))
+		goto restart;
+	    if (act & (ETHR_RWMTX_R_FLG__
+		       | ETHR_RWMTX_R_ABRT_UNLCK_FLG__
+		       | ETHR_RWMTX_R_PEND_UNLCK_MASK__))
+		return try_write_lock ? EBUSY : 0;
+	    /* Read unlock completed */
+	    break;
+	}
+    }
+
+    /*
+     * Read unlock completed, but we have to check if
+     * threads have to be woken (or if we should try
+     * to write lock it).
+     */
+
+    if (act & ETHR_RWMTX_WAIT_FLGS__) {
+    check_waiters:
+	rwmutex_unlock_wake(rwmtx, 0, act, 0);
+	return try_write_lock ? EBUSY : 0;
+    }
+
+    if (!try_write_lock)
+	return 0;
+
+ tryrwlock:
+    /* Try to write lock it */
+
+    act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+				     ETHR_RWMTX_W_FLG__,
+				     0);
+    return act == 0 ? 0 : EBUSY;
+}
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+
+static ETHR_INLINE void
+rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx)
+{
+    ethr_sint32_t act;
+    /*
+     * Restore failed increment
+     */
+    act = ethr_atomic32_dec_read(&rwmtx->mtxb.flgs);
+    if ((act & ETHR_RWMTX_WAIT_FLGS__)
+	&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	rwmutex_unlock_wake(rwmtx, 0, act, 0);
+    }
+}
+
+#endif
+
+static void
+rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
+{
+    ethr_sint32_t act = initial, exp;
+    int scnt, start_scnt;
+    ethr_ts_event *tse = NULL;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    start_scnt = scnt = initial_spincount(&rwmtx->mtxb);
+
+    /*
+     * Spin trying to read lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	rwmutex_incdec_restore_failed_tryrlock(rwmtx);
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+#endif
+
+	while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+	    if (scnt <= 0) {
+		tse = ethr_get_ts_event();
+		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(&rwmtx->mtxb, tse, scnt,
+			       ETHR_RWMTX_R_WAIT_FLG__, 1, 0);
+		    goto done;
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	    scnt--;
+	}
+	exp = act;
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic32_inc_read(&rwmtx->mtxb.flgs);
+	if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0)
+	    goto done; /* Got it */
+#else
+	act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	if (act == exp)
+	    goto done; /* Got it */
+#endif
+    }
+
+ done:
+    if (tse)
+	ethr_leave_ts_event(tse);
+}
+
+static void
+rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
+			    ethr_ts_event *tse);
+
+static int
+rwmutex_freqread_rlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse, int trylock)
+{
+    int res = 0;
+    ethr_sint32_t act;
+
+    rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+    ETHR_MEMORY_BARRIER;
+
+    act = ethr_atomic32_read_acqb(&rwmtx->mtxb.flgs);
+
+    if (act != ETHR_RWMTX_R_FLG__) {
+	int wake_other_readers;
+
+	while (1) {
+	    ethr_sint32_t exp, new;
+
+	    wake_other_readers = 0;
+
+	    if (act == 0)
+		new = act | ETHR_RWMTX_R_FLG__;
+	    else if (act == ETHR_RWMTX_R_FLG__)
+		break; /* Got it */
+	    else if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+		if (trylock)
+		    res = EBUSY;
+		else
+		    rwmutex_freqread_rlock_wait(rwmtx, tse);
+		break;
+	    }
+	    else if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
+		if ((act & ETHR_RWMTX_R_FLG__) == 0)
+		    ETHR_FATAL_ERROR__(EFAULT);
+		/*
+		 * An aborted runlock, not write locked, and no write
+		 * waiters, i.e., we got it...
+		 */
+		if (act & ETHR_RWMTX_R_WAIT_FLG__)
+		    wake_other_readers = 1;
+		break;
+	    }
+	    else {
+		new = act | ETHR_RWMTX_R_FLG__;
+		if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
+		    /*
+		     * Someone is doing tryrwlock (no writer and no
+		     * write waiters); we will try to abort that...
+		     */
+		    new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+		}
+
+		if (act & ETHR_RWMTX_R_WAIT_FLG__)
+		    wake_other_readers = 1;
+	    }
+
+	    exp = act;
+	    act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
+	    if (act == exp)
+		break;
+	}
+
+	if (wake_other_readers)
+	    rwmutex_transfer_read_lock(rwmtx, act, 0);
+    }
+
+    return res;
+}
+
+static void
+rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
+			    ethr_ts_event *tse)
+{
+    ethr_sint32_t act;
+    int scnt, start_scnt;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    start_scnt = scnt = initial_spincount(&rwmtx->mtxb);
+
+    /*
+     * Spin trying to read lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+
+	while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+	    if (scnt <= 0) {
+		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(&rwmtx->mtxb, tse, scnt,
+			       ETHR_RWMTX_R_WAIT_FLG__, 1, 1);
+		    return; /* Got it */
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	    scnt--;
+	}
+
+	if (rwmutex_freqread_rlock(rwmtx, tse, 1) != EBUSY)
+	    break; /* Got it */
+    }
+}
+
+static void
+rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
+{
+    write_lock_wait(&rwmtx->mtxb, initial, 1, 0);
+}
+
+static void
+rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
+{
+    write_lock_wait(&rwmtx->mtxb, initial, 1, 1);
+}
+
+static ETHR_INLINE void
+rwlock_wake_set_flags(ethr_rwmutex *rwmtx,
+		      ethr_sint32_t new_initial,
+		      ethr_sint32_t act_initial)
+{
+    ethr_sint32_t act, act_mask;
+    int chk_abrt_flg;
+
+    ETHR_MEMORY_BARRIER;
+
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
+	/* r pend unlock mask may vary and must be retained */
+	act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
+	if (new_initial & ETHR_RWMTX_R_FLG__)
+	    chk_abrt_flg = 1;
+	else
+	    chk_abrt_flg = 0;
+    }
+    else {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	/* rs mask may vary and must be retained */
+	act_mask = ETHR_RWMTX_RS_MASK__;
+	chk_abrt_flg = 0;
+#else
+	/* rs mask always zero */
+	ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0);
+	ethr_atomic32_set(&rwmtx->mtxb.flgs, new_initial);
+	return;
+#endif
+    }
+
+    act = act_initial;
+    while (1) {
+	ethr_sint32_t exp = act;
+	ethr_sint32_t new = new_initial + (act & act_mask);
+	if (chk_abrt_flg && (act & act_mask))
+	    new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+	act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (act == exp)
+	    break;
+	exp = act;
+    }
+}
+
+#ifdef ETHR_DEBUG
+
+static void
+dbg_unlock_wake(ethr_rwmutex *rwmtx,
+		int have_w,
+		ethr_ts_event *tse)
+{
+    ethr_sint32_t exp, act, imask;
+
+    exp = have_w ? ETHR_RWMTX_W_FLG__ : 0;
+
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+	imask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
+    else {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	imask = ETHR_RWMTX_RS_MASK__;
+#else
+	imask = 0;
+#endif
+    }
+
+    ETHR_ASSERT(tse);
+
+    if (is_w_waiter(tse)) {
+
+	exp |= ETHR_RWMTX_W_WAIT_FLG__;
+	if (rwmtx->rq_end) {
+	    exp |= ETHR_RWMTX_R_WAIT_FLG__;
+	}
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	ETHR_ASSERT((exp & ~imask) == (act & ~imask));
+
+	ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+
+    }
+    else {
+
+	exp |= ETHR_RWMTX_R_WAIT_FLG__;
+	if (rwmtx->rq_end->next != rwmtx->mtxb.q)
+	    exp |= ETHR_RWMTX_W_WAIT_FLG__;
+	else if (exp == ETHR_RWMTX_R_WAIT_FLG__) {
+	    if (!have_w) {
+		if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+		    imask |= ETHR_RWMTX_R_FLG__;
+		else
+		    imask |= ETHR_RWMTX_RS_MASK__;
+	    }
+	}
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	ETHR_ASSERT((exp & ~imask) == (act & ~imask));
+
+	ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+
+    }
+}
+
+#endif
+
+static void
+rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
+			   ethr_sint32_t initial,
+			   int q_locked)
+{
+    ethr_sint32_t act = initial;
+
+    if (!q_locked) {
+	ethr_ts_event *tse;
+	ETHR_ASSERT(initial & ETHR_RWMTX_R_WAIT_FLG__);
+	ETHR_ASSERT((initial & ETHR_RWMTX_W_FLG__) == 0);
+	ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	tse = rwmtx->mtxb.q;
+	if ((act & ETHR_RWMTX_W_FLG__) || !tse || is_w_waiter(tse)) {
+	    /* Someone else woke the readers up... */
+	    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+	    return;
+	}
+    }
+
+    rwmutex_unlock_wake(rwmtx, 0, initial, 1);
+}
+
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, ethr_sint32_t initial,
+		    int transfer_read_lock)
+{
+    ethr_sint32_t new, act = initial;
+    ethr_ts_event *tse;
+
+    if (transfer_read_lock) {
+	/*
+	 * - Q already locked
+	 * - Got R waiters first in Q
+	 * - Not W locked
+	 */
+	tse = rwmtx->mtxb.q;
+
+	ETHR_ASSERT(act & ETHR_RWMTX_R_WAIT_FLG__);
+	ETHR_ASSERT((act & (ETHR_RWMTX_W_FLG__)) == 0);
+	ETHR_ASSERT(tse && !is_w_waiter(tse));
+    }
+    else {
+
+	if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	    if (!have_w)
+		return;
+	    else {
+		while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+		    ethr_sint32_t exp = act;
+		    new = exp & ~ETHR_RWMTX_W_FLG__;
+		    act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+		    if (act == exp)
+			return;
+		}
+	    }
+	}
+
+	ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+	tse = rwmtx->mtxb.q;
+
+	if (!have_w) {
+	    if (!tse) {
+#ifdef ETHR_DEBUG
+		act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+		ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
+#endif
+		goto already_served;
+	    }
+	    act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	    if (act == (ETHR_RWMTX_R_WAIT_FLG__|ETHR_RWMTX_R_FLG__)) {
+		ETHR_ASSERT(tse && !is_w_waiter(tse));
+	    }
+	    else if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
+	    already_served:
+		ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+		return;
+	    }
+	}
+    }
+
+#ifdef ETHR_DEBUG
+    dbg_unlock_wake(rwmtx, have_w, tse);
+#endif
+
+    if (is_w_waiter(tse)) {
+
+	if (!have_w) {
+	    act = ethr_atomic32_read_bor(&rwmtx->mtxb.flgs,
+				       ETHR_RWMTX_W_FLG__);
+	    ETHR_ASSERT((act & ~(ETHR_RWMTX_WAIT_FLGS__
+				 | (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL
+				    ? 0
+				    : ETHR_RWMTX_R_PEND_UNLCK_MASK__))) == 0);
+	    ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__);
+	    act |= ETHR_RWMTX_W_FLG__;
+	}
+
+	/*
+	 * If we have multiple write waiters, there
+	 * is no need to modify mtxb->flgs; otherwise,
+	 * we need to clear the write wait bit...
+	 */
+	if (!multiple_w_waiters(rwmtx)) {
+	    new = ETHR_RWMTX_W_FLG__;
+	    if (tse->next != rwmtx->mtxb.q) {
+		ETHR_ASSERT(tse->next->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+		new |= ETHR_RWMTX_R_WAIT_FLG__;
+	    }
+
+	    rwlock_wake_set_flags(rwmtx, new, act);
+	}
+
+	wake_writer(&rwmtx->mtxb, 1);
+    }
+    else {
+	int rs;
+
+	if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) {
+	    rs = rwmtx->tdata.rs;
+	    new = (ethr_sint32_t) rs;
+	    rwmtx->tdata.rs = 0;
+	}
+	else {
+	    ethr_rwmutex_type type = rwmtx->type;
+	    int length = (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ
+			  ? reader_groups_array_size
+			  : main_threads_array_size);
+	    int ix;
+
+	    rs = 0;
+	    for (ix = 0; ix < length; ix++) {
+		int wrs = rwmtx->tdata.ra[ix].data.waiting_readers;
+		rwmtx->tdata.ra[ix].data.waiting_readers = 0;
+		ETHR_ASSERT(wrs >= 0);
+		if (wrs) {
+		    rs += wrs;
+		    rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs);
+		}
+	    }
+
+	    new = ETHR_RWMTX_R_FLG__;
+	}
+
+	if (rwmtx->rq_end->next != rwmtx->mtxb.q)
+	    new |= ETHR_RWMTX_W_WAIT_FLG__;
+
+	rwlock_wake_set_flags(rwmtx, new, act);
+
+	wake_readers(rwmtx, rs);
+    }
+}
+
+static ethr_rwmtx_readers_array__ *
+alloc_readers_array(int length, ethr_rwmutex_lived lived)
+{
+    ethr_rwmtx_readers_array__ *ra;
+    size_t sz;
+    void *mem;
+
+    sz = sizeof(ethr_rwmtx_readers_array__) * (length + 1);
+
+    switch (lived) {
+    case ETHR_RWMUTEX_LONG_LIVED:
+	mem = ethr_mem__.ll.alloc(sz);
+	break;
+    case ETHR_RWMUTEX_SHORT_LIVED:
+	mem = ethr_mem__.sl.alloc(sz);
+	break;
+    default:
+	mem = ethr_mem__.std.alloc(sz);
+	break;
+    }
+    if (!mem)
+	return NULL;
+
+    if ((((ethr_uint_t) mem) & ETHR_CACHE_LINE_MASK) == 0) {
+	ra = (ethr_rwmtx_readers_array__ *) mem;
+	ra->data.byte_offset = 0;
+    }
+    else {
+	ra = ((ethr_rwmtx_readers_array__ *)
+	      ((((ethr_uint_t) mem) & ~ETHR_CACHE_LINE_MASK)
+	       + ETHR_CACHE_LINE_SIZE));
+	ra->data.byte_offset = (int) ((ethr_uint_t) ra
+				      - (ethr_uint_t) mem);
+    }
+    ra->data.lived = lived;
+    return ra;
+}
+
+static void
+free_readers_array(ethr_rwmtx_readers_array__ *ra)
+{
+    void *ptr = (void *) (((char *) ra) - ra->data.byte_offset);
+    switch (ra->data.lived) {
+    case ETHR_RWMUTEX_LONG_LIVED:
+	ethr_mem__.ll.free(ptr);
+	break;
+    case ETHR_RWMUTEX_SHORT_LIVED:
+	ethr_mem__.sl.free(ptr);
+	break;
+    default:
+	ethr_mem__.std.free(ptr);
+	break;
+    }
+}
+
+int
+ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
+{
+    int res;
+    ethr_rwmtx_readers_array__ *ra = NULL;
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(rwmtx);
+    rwmtx->rq_end = NULL;
+    rwmtx->type = opt ? opt->type : ETHR_RWMUTEX_TYPE_NORMAL;
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+	if (main_threads_array_size <= reader_groups_array_size) {
+	    /* No point using reader groups... */
+	    rwmtx->type = ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ;
+	}
+	/* Fall through */
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	int length;
+
+	length = (rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
+		  ? main_threads_array_size
+		  : reader_groups_array_size);
+
+	if (length == 1) {
+	    /* No point using a frequent reader type... */
+	    rwmtx->type = ETHR_RWMUTEX_TYPE_NORMAL;
+	}
+	else {
+	    int ix;
+	    ra = alloc_readers_array(length,
+				     (opt
+				      ? opt->lived
+				      : ETHR_RWMUTEX_UNKNOWN_LIVED));
+	    if (!ra) {
+		res = ENOMEM;
+		goto error;
+	    }
+
+	    rwmtx->tdata.ra = ra;
+
+	    for (ix = 0; ix < length; ix++) {
+		ethr_atomic32_init(&rwmtx->tdata.ra[ix].data.readers, 0);
+		rwmtx->tdata.ra[ix].data.waiting_readers = 0;
+	    }
+	    break;
+	}
+    }
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	rwmtx->tdata.rs = 0;
+	break;
+    default:
+	res = EINVAL;
+	goto error;
+    }
+    res = mtxb_init(&rwmtx->mtxb,
+		    default_rwmtx_main_spincount,
+		    opt ? opt->main_spincount : -1,
+		    default_rwmtx_aux_spincount,
+		    opt ? opt->aux_spincount : -1);
+    if (res == 0)
+	return 0;
+
+ error:
+
+    if (ra)
+	free_readers_array(ra);
+
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return res;
+}
+
+int
+ethr_rwmutex_init(ethr_rwmutex *rwmtx)
+{
+    return ethr_rwmutex_init_opt(rwmtx, NULL);
+}
+
+int
+ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
+{
+    int res;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
+	ethr_sint32_t act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	if (act == ETHR_RWMTX_R_FLG__)
+	    rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0);
+    }
+    res = mtxb_destroy(&rwmtx->mtxb);
+    if (res != 0)
+	return res;
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+	free_readers_array(rwmtx->tdata.ra);
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return 0;
+}
+
+#define ETHR_MAX_TRYRLOCK_TRIES 5
+
+int
+ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
+{
+    int res = 0;
+    ethr_sint32_t act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL: {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
+	    res = EBUSY;
+	else {
+	    act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs);
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_incdec_restore_failed_tryrlock(rwmtx);
+		res = EBUSY;
+	    }
+	}
+#else
+	ethr_sint32_t exp = 0;
+	int tries = 0;
+
+	while (1) {
+	    act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	    if (act == exp) {
+		res = 0;
+		break;
+	    }
+	    if (tries > ETHR_MAX_TRYRLOCK_TRIES
+		|| (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))) {
+		res = EBUSY;
+		break;
+	    }
+	    tries++;
+	    exp = act;
+	}
+#endif
+	break;
+    }
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+	res = rwmutex_freqread_rlock(rwmtx, tse, 1);
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+#ifdef ETHR_MTX_CHK_EXCL
+    if (res == 0) {
+	ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
+	ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+    }
+#endif
+
+    ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+    return res;
+}
+
+void
+ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
+{
+    ethr_sint32_t act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL: {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs);
+	if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
+	    rwmutex_normal_rlock_wait(rwmtx, act);
+#else
+	ethr_sint32_t exp = 0;
+
+	while (1) {
+	    act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	    if (act == exp)
+		break;
+
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_normal_rlock_wait(rwmtx, act);
+		break;
+	    }
+	    exp = act;
+	}
+#endif
+	break;
+    }
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+	rwmutex_freqread_rlock(rwmtx, tse, 0);
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+    ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+}
+
+void
+ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
+{
+    ethr_sint32_t act;
+
+    ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(&rwmtx->mtxb);
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic32_dec_read_relb(&rwmtx->mtxb.flgs);
+	if ((act & ETHR_RWMTX_WAIT_FLGS__)
+	    && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+	    rwmutex_unlock_wake(rwmtx, 0, act, 0);
+	}
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+
+	act = rwmutex_freqread_rdrs_dec_read_relb(rwmtx, tse);
+
+	ETHR_ASSERT(act >= 0);
+
+	ETHR_MEMORY_BARRIER;
+
+	if (act == 0) {
+	    act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	    if (act != ETHR_RWMTX_R_FLG__)
+		rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
+	}
+
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+}
+
+int
+ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
+{
+    int res = 0;
+    ethr_sint32_t act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					 ETHR_RWMTX_W_FLG__, 0);
+	if (act != 0)
+	    res = EBUSY;
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+
+	res = 0;
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+
+	do {
+
+	    if (act == 0)
+		act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+						 ETHR_RWMTX_W_FLG__, 0);
+	    else if (act == ETHR_RWMTX_R_FLG__) {
+		res = rwmutex_try_complete_runlock(rwmtx, act, NULL,
+						   0, 1, 1);
+		break;
+	    }
+	    else {
+		res = EBUSY;
+		break;
+	    }
+
+	} while (act != 0);
+
+	break;
+    }
+
+#ifdef ETHR_MTX_CHK_EXCL
+    if (res == 0) {
+	ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
+	ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+    }
+#endif
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+    ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    return res;
+}
+
+void
+ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
+{
+    ethr_sint32_t act;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					 ETHR_RWMTX_W_FLG__, 0);
+	if (act != 0)
+	    rwmutex_normal_rwlock_wait(rwmtx, act);
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+
+	do {
+
+	    if (act != 0) {
+		rwmutex_freqread_rwlock_wait(rwmtx, act);
+		break;
+	    }
+
+	    act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					     ETHR_RWMTX_W_FLG__, 0);
+
+	} while (act != 0);
+
+	break;
+    }
+
+    ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+}
+
+void
+ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
+{
+    ethr_sint32_t act;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb);
+
+    ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_UNSET_EXCL(&rwmtx->mtxb);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs,
+					 0, ETHR_RWMTX_W_FLG__);
+	if (act != ETHR_RWMTX_W_FLG__)
+	    rwmutex_unlock_wake(rwmtx, 1, act, 0);
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+	act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs, 0,
+					 ETHR_RWMTX_W_FLG__);
+	if (act != ETHR_RWMTX_W_FLG__)
+	    rwmutex_unlock_wake(rwmtx, 1, act, 0);
+	break;
+    }
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+}
+
+#else
+/* -- pthread read/write mutex --------------------------------------------- */
+
+int
+ethr_rwmutex_init(ethr_rwmutex *rwmtx)
+{
+#if ETHR_XCHK
+    if (!rwmtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
+#endif
+    return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr);
+}
+
+int
+ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
+{
+    return ethr_rwmutex_init(rwmtx);
+}
+
+int
+ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
+{
+    int res;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    res = pthread_rwlock_destroy(&rwmtx->pt_rwlock);
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return res;
+}
+
+/* -- Exported symbols of inline functions --------------------------------- */
+
+int
+ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_tryrlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_rlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_runlock__(rwmtx);
+}
+
+int
+ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_tryrwlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_rwlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_rwunlock__(rwmtx);
+}
+
+#endif /* pthread */
+
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+static void
+hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    int res;
+    ethr_sint32_t flgs = ethr_atomic32_read(&mtxb->flgs);
+
+    ETHR_MTX_HARD_ASSERT(res == 0);
+
+    ETHR_MTX_HARD_ASSERT(!(flgs & ETHR_RWMTX_R_WAIT_FLG__) || is_rwmtx);
+
+    if (!(flgs & ETHR_RWMTX_WAIT_FLGS__)) {
+	ETHR_MTX_HARD_ASSERT(!mtxb->q);
+	if (is_rwmtx) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end);
+	    ETHR_MTX_HARD_ASSERT(!rwmtx->rs);
+	}
+    }
+    else {
+	ethr_ts_event *tse;
+	int ws = 0, rs = 0, rsf = 0, ref = 0;
+
+	ETHR_MTX_HARD_ASSERT(mtxb->q);
+
+	tse = mtxb->q;
+
+	do {
+	    ethr_sint32_t type;
+
+	    ETHR_MTX_HARD_ASSERT(tse->next->prev == tse);
+	    ETHR_MTX_HARD_ASSERT(tse->prev->next == tse);
+
+	    type = ethr_atomic32_read(&tse->uaflgs);
+	    ETHR_MTX_HARD_ASSERT(type == tse->uflgs);
+	    switch (type) {
+	    case ETHR_RWMTX_W_WAIT_FLG__:
+		ws++;
+		break;
+	    case ETHR_RWMTX_R_WAIT_FLG__: {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		ETHR_MTX_HARD_ASSERT(is_rwmtx);
+		if (!rsf)
+		    rsf = 1;
+		ETHR_MTX_HARD_ASSERT(!ref);
+		if (rwmtx->rq_end == tse) {
+		    ETHR_MTX_HARD_ASSERT(
+			tse->next == rwmtx->mtxb.q
+			|| tse->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+		    ref = 1;
+		}
+		rs++;
+		break;
+	    }
+	    default:
+		ETHR_MTX_HARD_ASSERT(! "invalid wait type found");
+	    }
+
+	    tse = tse->next;
+	} while (tse != mtxb->q);
+
+	if (is_rwmtx) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(rs == rwmtx->rs);
+	}
+
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	ETHR_MTX_HARD_ASSERT(ws == mtxb->ws);
+#endif
+
+	if (flgs & ETHR_RWMTX_W_WAIT_FLG__)
+	    ETHR_MTX_HARD_ASSERT(ws);
+	else
+	    ETHR_MTX_HARD_ASSERT(!ws);
+
+	if (flgs & ETHR_RWMTX_R_WAIT_FLG__) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(is_rwmtx);
+	    ETHR_MTX_HARD_ASSERT(rwmtx->rq_end);
+	    ETHR_MTX_HARD_ASSERT(rsf);
+	    ETHR_MTX_HARD_ASSERT(ref);
+	    ETHR_MTX_HARD_ASSERT(rs);
+	}
+	else {
+	    if (is_rwmtx) {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end);
+	    }
+	    ETHR_MTX_HARD_ASSERT(!rsf);
+	    ETHR_MTX_HARD_ASSERT(!ref);
+	    ETHR_MTX_HARD_ASSERT(!rs);
+	}
+    }
+}
+
+#elif defined(ETHR_MTX_HARD_DEBUG_WSQ)
+
+static void
+hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    int ws = 0;
+    int rs = 0;
+
+    if (mtxb->q) {
+	ethr_ts_event *tse = mtxb->q;
+	do {
+	    switch (tse->uflgs) {
+	    case ETHR_RWMTX_W_WAIT_FLG__:
+		ws++;
+		break;
+	    case ETHR_RWMTX_R_WAIT_FLG__:
+		rs++;
+		break;
+	    default:
+		ETHR_MTX_HARD_ASSERT(0);
+		break;
+	    }
+	    tse = tse->next;
+	} while (tse != mtxb->q);
+    }
+
+    ETHR_MTX_HARD_ASSERT(mtxb->ws == ws);
+    if (is_rwmtx) {
+	ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	ETHR_MTX_HARD_ASSERT(rwmtx->rs == rs);
+    }
+}
+
+#endif
+
+#endif
diff --git a/erts/lib_src/common/ethread.c b/erts/lib_src/common/ethread.c
deleted file mode 100644
index eb4d0cad20..0000000000
--- a/erts/lib_src/common/ethread.c
+++ /dev/null
@@ -1,3346 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-/*
- * Description: A Thread library for use in the ERTS and other OTP
- *              applications.
- * Author: Rickard Green
- */
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#undef ETHR_STACK_GUARD_SIZE 
-
-#if defined(ETHR_PTHREADS)
-
-#ifdef ETHR_TIME_WITH_SYS_TIME
-#  include <time.h>
-#  include <sys/time.h>
-#else
-#  ifdef ETHR_HAVE_SYS_TIME_H
-#    include <sys/time.h>
-#  else
-#    include <time.h>
-#  endif
-#endif
-#include <sys/types.h>
-#include <unistd.h>
-#include <signal.h>
-
-#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE
-#  define ETHR_STACK_GUARD_SIZE (pagesize)
-#endif
-
-#elif defined(ETHR_WIN32_THREADS)
-
-#undef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#include <process.h>
-#include <winerror.h>
-
-#else
-#error "Missing thread implementation"
-#endif
-
-#include <limits.h>
-
-#define ETHR_FORCE_INLINE_FUNCS
-#define ETHR_INLINE_FUNC_NAME_(X) X ## __
-#include "ethread.h"
-
-#ifndef ETHR_HAVE_ETHREAD_DEFINES
-#error Missing configure defines
-#endif
-
-/*
- * ----------------------------------------------------------------------------
- * Common stuff
- * ----------------------------------------------------------------------------
- */
-
-#define ETHR_MAX_THREADS 2048 /* Has to be an even power of 2 */
-
-static int ethr_not_inited = 1;
-
-#define ASSERT(A) ETHR_ASSERT((A))
-
-static void *(*allocp)(size_t) = malloc;
-static void *(*reallocp)(void *, size_t) = realloc;
-static void (*freep)(void *) = free;
-
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
-#endif
-
-void *(*thread_create_prepare_func)(void) = NULL;
-void (*thread_create_parent_func)(void *) = NULL;
-void (*thread_create_child_func)(void *) = NULL;
-
-typedef struct ethr_xhndl_list_ ethr_xhndl_list;
-struct ethr_xhndl_list_ {
-    ethr_xhndl_list *next;
-    void (*funcp)(void);
-};
-
-static size_t pagesize;
-#define ETHR_PAGE_ALIGN(SZ) (((((size_t) (SZ)) - 1)/pagesize + 1)*pagesize)
-static size_t min_stack_size; /* kilo words */
-static size_t max_stack_size; /* kilo words */
-#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1)
-#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024)
-
-ethr_mutex xhndl_mtx;
-ethr_xhndl_list *xhndl_list;
-
-static int
-init_common(ethr_init_data *id)
-{
-    int res;
-    if (id) {
-	allocp				= id->alloc;
-	reallocp			= id->realloc;
-	freep				= id->free;
-	thread_create_prepare_func	= id->thread_create_prepare_func;
-	thread_create_parent_func	= id->thread_create_parent_func;
-	thread_create_child_func	= id->thread_create_child_func;
-    }
-    if (!allocp || !reallocp || !freep)
-	return EINVAL;
-
-#ifdef _SC_PAGESIZE
-    pagesize = (size_t) sysconf(_SC_PAGESIZE);
-#elif defined(HAVE_GETPAGESIZE)
-    pagesize = (size_t) getpagesize();
-#else
-    pagesize = (size_t) 4*1024; /* Guess 4 KB */
-#endif
-
-    /* User needs at least 4 KB */
-    min_stack_size = 4*1024;
-#if SIZEOF_VOID_P == 8
-    /* Double that on 64-bit archs */
-    min_stack_size *= 2;
-#endif
-    /* On some systems as much as about 4 KB is used by the system */
-    min_stack_size += 4*1024;
-    /* There should be room for signal handlers */
-#ifdef SIGSTKSZ
-    min_stack_size += SIGSTKSZ;
-#else
-    min_stack_size += pagesize;
-#endif
-    /* The system may think that we need more stack */
-#if defined(PTHREAD_STACK_MIN)
-    if (min_stack_size < PTHREAD_STACK_MIN)
-	min_stack_size = PTHREAD_STACK_MIN;
-#elif defined(_SC_THREAD_STACK_MIN)
-    {
-	size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN);
-	if (min_stack_size < thr_min_stk_sz)
-	    min_stack_size = thr_min_stk_sz;
-    }
-#endif
-    /* The guard is at least on some platforms included in the stack size
-       passed when creating threads */
-#ifdef ETHR_STACK_GUARD_SIZE
-    min_stack_size += ETHR_STACK_GUARD_SIZE;
-#endif
-    min_stack_size = ETHR_PAGE_ALIGN(min_stack_size);
-
-    min_stack_size = ETHR_B2KW(min_stack_size);
-
-    max_stack_size = 32*1024*1024;
-#if SIZEOF_VOID_P == 8
-    max_stack_size *= 2;
-#endif
-    max_stack_size = ETHR_B2KW(max_stack_size);
-
-    xhndl_list = NULL;
-
-    res = ethr_mutex_init(&xhndl_mtx);
-    if (res != 0)
-	return res;
-
-    res = ethr_mutex_set_forksafe(&xhndl_mtx);
-    if (res != 0 && res != ENOTSUP)
-	return res;
-
-    return 0;
-}
-
-int
-ethr_install_exit_handler(void (*funcp)(void))
-{
-    ethr_xhndl_list *xhp;
-    int res;
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    if (!funcp)
-	return EINVAL;
-
-    xhp = (ethr_xhndl_list *) (*allocp)(sizeof(ethr_xhndl_list));
-    if (!xhp)
-	return ENOMEM;
-
-    res = ethr_mutex_lock__(&xhndl_mtx);
-    if (res != 0) {
-	(*freep)((void *) xhp);
-	return res;
-    }
-
-    xhp->funcp = funcp;
-    xhp->next = xhndl_list;
-    xhndl_list = xhp;
-
-    res = ethr_mutex_unlock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    return res;
-}
-
-static void
-run_exit_handlers(void)
-{
-    int res;
-    ethr_xhndl_list *xhp;
-
-    res = ethr_mutex_lock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    xhp = xhndl_list;
-
-    res = ethr_mutex_unlock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    for (; xhp; xhp = xhp->next)
-	(*xhp->funcp)();
-}
-
-#if defined(ETHR_PTHREADS)
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * pthread implementation                                                    *
-\*                                                                           */
-
-typedef struct {
-    pthread_mutex_t mtx;
-    pthread_cond_t cnd;
-    int initialized;
-    void *(*thr_func)(void *);
-    void *arg;
-    void *prep_func_res;
-} thr_wrap_data_;
-
-static int no_ethreads;
-static ethr_mutex no_ethrs_mtx;
-
-#ifndef ETHR_HAVE_PTHREAD_ATFORK
-#define ETHR_HAVE_PTHREAD_ATFORK 0
-#endif
-
-#if !ETHR_HAVE_PTHREAD_ATFORK
-#warning "Cannot enforce fork-safety"
-#endif
-
-/*
- * ----------------------------------------------------------------------------
- * Static functions
- * ----------------------------------------------------------------------------
- */
-
-/*
- * Functions with safe_ prefix aborts on failure. To be used when
- * we cannot recover after failure.
- */
-
-static ETHR_INLINE void
-safe_mutex_lock(pthread_mutex_t *mtxp)
-{
-    int res = pthread_mutex_lock(mtxp);
-    if (res != 0)
-	abort();
-}
-
-static ETHR_INLINE void
-safe_mutex_unlock(pthread_mutex_t *mtxp)
-{
-    int res = pthread_mutex_unlock(mtxp);
-    if (res != 0)
-	abort();
-}
-
-static ETHR_INLINE void
-safe_cond_signal(pthread_cond_t *cndp)
-{
-    int res = pthread_cond_signal(cndp);
-    if (res != 0)
-	abort();
-}
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-static volatile int rec_mtx_attr_need_init = 1;
-static pthread_mutexattr_t rec_mtx_attr;
-
-static int init_rec_mtx_attr(void);
-
-#endif
-
-#if ETHR_HAVE_PTHREAD_ATFORK
-
-static ethr_mutex forksafe_mtx = ETHR_MUTEX_INITER;
-
-static void lock_mutexes(void)
-{
-    ethr_mutex *m = &forksafe_mtx;
-    do {
-
-	safe_mutex_lock(&m->pt_mtx);
-
-	m = m->next;
-
-    } while (m != &forksafe_mtx);
-}
-
-static void unlock_mutexes(void)
-{
-    ethr_mutex *m = forksafe_mtx.prev;
-    do {
-
-	safe_mutex_unlock(&m->pt_mtx);
-
-	m = m->prev;
-
-    } while (m->next != &forksafe_mtx);
-}
-
-#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK
-
-static void reinit_mutexes(void)
-{
-    ethr_mutex *m = forksafe_mtx.prev;
-    do {
-	pthread_mutexattr_t *attrp = NULL;
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-	if (m->is_rec_mtx) {
-	    if (rec_mtx_attr_need_init) {
-		int res = init_rec_mtx_attr();
-		if (res != 0)
-		    abort();
-	    }
-	    attrp = &rec_mtx_attr;
-	}
-#endif
-	if (pthread_mutex_init(&m->pt_mtx, attrp) != 0)
-	    abort();
-
-	m = m->prev;
-
-    } while (m->next != &forksafe_mtx);
-}
-
-#endif
-
-static int
-init_forksafe(void)
-{
-    static int init_done = 0;
-    int res = 0;
-
-    if (init_done)
-	return res;
-
-    forksafe_mtx.prev = &forksafe_mtx;
-    forksafe_mtx.next = &forksafe_mtx;
-
-    res = pthread_atfork(lock_mutexes,
-			 unlock_mutexes,
-#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK
-			 reinit_mutexes
-#else
-			 unlock_mutexes
-#endif
-	);
-
-    init_done = 1;
-    return res;
-}
-
-#endif
-
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE)
-
-#define SET_REC_MUTEX_ATTR(AP) \
-  pthread_mutexattr_settype((AP), PTHREAD_MUTEX_RECURSIVE);
-
-#elif defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP)
-
-#define SET_REC_MUTEX_ATTR(AP) \
-  pthread_mutexattr_setkind_np((AP), PTHREAD_MUTEX_RECURSIVE_NP);
-
-#else
-
-#error "Don't know how to set recursive mutex attributes"
-
-#endif
-
-static int
-init_rec_mtx_attr(void)
-{
-    int res, mres;
-    static pthread_mutex_t attrinit_mtx = PTHREAD_MUTEX_INITIALIZER;
-
-    mres = pthread_mutex_lock(&attrinit_mtx);
-    if (mres != 0)
-	return mres;
-    /* Got here under race conditions; check again ... */
-    if (!rec_mtx_attr_need_init)
-	res = 0;
-    else {
-	res = pthread_mutexattr_init(&rec_mtx_attr);
-	if (res == 0) {
-	    res = SET_REC_MUTEX_ATTR(&rec_mtx_attr);
-	    if (res == 0)
-		rec_mtx_attr_need_init = 0;
-	    else
-		(void) pthread_mutexattr_destroy(&rec_mtx_attr);
-	}
-    }
-
-    mres = pthread_mutex_unlock(&attrinit_mtx);
-    if (mres != 0)
-	return mres;
-    return res;
-}
-
-#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-
-static ETHR_INLINE void thr_exit_cleanup(void)
-{
-    run_exit_handlers();
-    safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-    ASSERT(no_ethreads > 0);
-    no_ethreads--;
-    safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-}
-
-static void *thr_wrapper(void *vtwd)
-{
-    void *res;
-    thr_wrap_data_ *twd = (thr_wrap_data_ *) vtwd;
-    void *(*thr_func)(void *) = twd->thr_func;
-    void *arg = twd->arg;
-
-    safe_mutex_lock(&twd->mtx);
-
-    if (thread_create_child_func)
-	(*thread_create_child_func)(twd->prep_func_res);
-
-    twd->initialized = 1;
-
-    safe_cond_signal(&twd->cnd);
-    safe_mutex_unlock(&twd->mtx);
-
-    res = (*thr_func)(arg);
-    thr_exit_cleanup();
-    return res;
-}
-
-
-/*
- * ----------------------------------------------------------------------------
- * Exported functions
- * ----------------------------------------------------------------------------
- */
-
-int
-ethr_init(ethr_init_data *id)
-{
-    int res;
-
-    if (!ethr_not_inited)
-	return EINVAL;
-
-    ethr_not_inited = 0;
-
-    res = init_common(id);
-    if (res != 0)
-	goto error;
-
-#if ETHR_HAVE_PTHREAD_ATFORK
-    init_forksafe();
-#endif
-
-    no_ethreads = 1;
-    res = ethr_mutex_init(&no_ethrs_mtx);
-    if (res != 0)
-	goto error;
-    res = ethr_mutex_set_forksafe(&no_ethrs_mtx);
-    if (res != 0 && res != ENOTSUP)
-	goto error;
-
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-    {
-	int i;
-	for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) {
-#ifdef ETHR_HAVE_PTHREAD_SPIN_LOCK
-	    res = pthread_spin_init(&ethr_atomic_protection__[i].u.spnlck, 0);
-#else
-	    res = ethr_mutex_init(&ethr_atomic_protection__[i].u.mtx);
-#endif
-	    if (res != 0)
-		goto error;
-	}
-    }
-#endif
-
-    return 0;
-
- error:
-    ethr_not_inited = 1;
-    return res;
-
-}
-
-int
-ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
-		ethr_thr_opts *opts)
-{
-    thr_wrap_data_ twd;
-    pthread_attr_t attr;
-    int res, dres;
-    int use_stack_size = (opts && opts->suggested_stack_size >= 0
-			  ? opts->suggested_stack_size
-			  : -1 /* Use system default */);
-
-#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
-    if (use_stack_size < 0)
-	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
-#endif
-
-    twd.initialized = 0;
-    twd.thr_func = func;
-    twd.arg = arg;
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!tid || !func) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    /* Call prepare func if it exist */
-    if (thread_create_prepare_func)
-	twd.prep_func_res = (*thread_create_prepare_func)();
-    else
-	twd.prep_func_res = NULL;
-
-    /* Set som thread attributes */
-    res = pthread_attr_init(&attr);
-    if (res != 0)
-	goto cleanup_parent_func;
-    res = pthread_mutex_init(&twd.mtx, NULL);
-    if (res != 0)
-	goto cleanup_attr_destroy;
-    res = pthread_cond_init(&twd.cnd, NULL);
-    if (res != 0)
-	goto cleanup_mutex_destroy;
-
-    /* Schedule child thread in system scope (if possible) ... */
-    res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
-    if (res != 0 && res != ENOTSUP)
-	goto cleanup_cond_destroy;
-
-    if (use_stack_size >= 0) {
-	size_t suggested_stack_size = (size_t) use_stack_size;
-	size_t stack_size;
-#ifdef DEBUG
-	suggested_stack_size /= 2; /* Make sure we got margin */
-#endif
-#ifdef ETHR_STACK_GUARD_SIZE
-	/* The guard is at least on some platforms included in the stack size
-	   passed when creating threads */
-	suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE);
-#endif
-	if (suggested_stack_size < min_stack_size)
-	    stack_size = ETHR_KW2B(min_stack_size);
-	else if (suggested_stack_size > max_stack_size)
-	    stack_size = ETHR_KW2B(max_stack_size);
-	else
-	    stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
-	(void) pthread_attr_setstacksize(&attr, stack_size);
-    }
-
-#ifdef ETHR_STACK_GUARD_SIZE
-    (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE);
-#endif
-
-    /* Detached or joinable... */
-    res = pthread_attr_setdetachstate(&attr,
-				      (opts && opts->detached
-				       ? PTHREAD_CREATE_DETACHED
-				       : PTHREAD_CREATE_JOINABLE));
-    if (res != 0)
-	goto cleanup_cond_destroy;
-    
-    res = pthread_mutex_lock(&twd.mtx);
-
-    if (res != 0)
-	goto cleanup_cond_destroy;
-
-    safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-    if (no_ethreads < ETHR_MAX_THREADS) {
-	no_ethreads++;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-    }
-    else {
-	res = EAGAIN;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-	goto cleanup_mutex_unlock;
-    }
-
-    res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void *) &twd);
-
-    if (res != 0) {
-	safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-	ASSERT(no_ethreads > 0);
-	no_ethreads--;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-    }
-    else {
-
-	/* Wait for child to initialize... */
-	while (!twd.initialized) {
-	    res = pthread_cond_wait(&twd.cnd, &twd.mtx);
-	    if (res != 0 && res != EINTR)
-		break;
-	}
-
-    }
-
-    /* Cleanup... */
- cleanup_mutex_unlock:
-    dres = pthread_mutex_unlock(&twd.mtx);
-    if (res == 0)
-	res = dres;
- cleanup_cond_destroy:
-    dres = pthread_cond_destroy(&twd.cnd);
-    if (res == 0)
-	res = dres;
- cleanup_mutex_destroy:
-    dres = pthread_mutex_destroy(&twd.mtx);
-    if (res == 0)
-	res = dres;
- cleanup_attr_destroy:
-    dres = pthread_attr_destroy(&attr);
-    if (res == 0)
-	res = dres;
- cleanup_parent_func:
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    return res;
-}
-
-int
-ethr_thr_join(ethr_tid tid, void **res)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_join((pthread_t) tid, res);
-}
-
-int
-ethr_thr_detach(ethr_tid tid)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_detach((pthread_t) tid);
-}
-
-void
-ethr_thr_exit(void *res)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return;
-    }
-#endif
-    thr_exit_cleanup();
-    pthread_exit(res);
-}
-
-ethr_tid
-ethr_self(void)
-{
-    return (ethr_tid) pthread_self();
-}
-
-int
-ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
-{
-    return pthread_equal((pthread_t) tid1, (pthread_t) tid2);
-}
-
-
-/*
- * Mutex functions
- */
-
-
-int
-ethr_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#endif
-    mtx->prev = NULL;
-    mtx->next = NULL;
-    mtx->is_rec_mtx = 0;
-    return pthread_mutex_init(&mtx->pt_mtx, NULL);
-}
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-int
-ethr_rec_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#endif
-    if (rec_mtx_attr_need_init)
-	init_rec_mtx_attr();
-
-    mtx->prev = NULL;
-    mtx->next = NULL;
-    mtx->is_rec_mtx = 1;
-    return pthread_mutex_init(&mtx->pt_mtx, &rec_mtx_attr);
-}
-
-#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-
-int
-ethr_mutex_destroy(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (mtx->next) {
-	ASSERT(mtx->prev);
-	ethr_mutex_unset_forksafe(mtx);
-    }
-#if ETHR_XCHK
-    mtx->initialized = 0;
-#endif
-    return pthread_mutex_destroy(&mtx->pt_mtx);
-}
-
-int ethr_mutex_set_forksafe(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-#if ETHR_HAVE_PTHREAD_ATFORK
-    res = pthread_mutex_lock(&forksafe_mtx.pt_mtx);
-    if (res != 0)
-	return res;
-    if (!forksafe_mtx.next) {
-	ASSERT(!forksafe_mtx.prev);
-	init_forksafe();
-    }
-    if (mtx->next) {
-	/* forksafe already set for this mutex */ 
-	ASSERT(mtx->prev);
-    }
-    else {
-	mtx->next = forksafe_mtx.next;
-	mtx->prev = &forksafe_mtx;
-	forksafe_mtx.next->prev = mtx;
-	forksafe_mtx.next = mtx;
-    }
-
-    res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx);
-
-#else /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    res = ENOTSUP;
-#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    return res;
-}
-
-int ethr_mutex_unset_forksafe(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-#if ETHR_HAVE_PTHREAD_ATFORK
-    res = pthread_mutex_lock(&forksafe_mtx.pt_mtx);
-    if (res != 0)
-	return res;
-    if (!forksafe_mtx.next) {
-	ASSERT(!forksafe_mtx.prev);
-	init_forksafe();
-    }
-    if (!mtx->next) {
-	/* forksafe already unset for this mutex */ 
-	ASSERT(!mtx->prev);
-    }
-    else {
-	mtx->prev->next = mtx->next;
-	mtx->next->prev = mtx->prev;
-	mtx->next = NULL;
-	mtx->prev = NULL;
-    }
-    res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx);
-
-#else /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    res = ENOTSUP;
-#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    return res;
-}
-
-int
-ethr_mutex_trylock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_trylock__(mtx);
-}
-
-int
-ethr_mutex_lock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_lock__(mtx);
-}
-
-int
-ethr_mutex_unlock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_unlock__(mtx);
-}
-
-/*
- * Condition variable functions
- */
-
-int
-ethr_cond_init(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    cnd->initialized = ETHR_COND_INITIALIZED;
-#endif
-    return pthread_cond_init(&cnd->pt_cnd, NULL);
-}
-
-int
-ethr_cond_destroy(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    cnd->initialized = 0;
-#endif
-    return pthread_cond_destroy(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_signal(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_signal(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_broadcast(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_broadcast(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| cnd->initialized != ETHR_COND_INITIALIZED
-	|| !mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx);
-}
-
-int
-ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout)
-{
-    struct timespec to;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| cnd->initialized != ETHR_COND_INITIALIZED
-	|| !mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED
-	|| !timeout) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    to.tv_sec = timeout->tv_sec;
-    to.tv_nsec = timeout->tv_nsec;
-
-    return pthread_cond_timedwait(&cnd->pt_cnd, &mtx->pt_mtx, &to);
-}
-
-
-#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-
-int
-ethr_rwmutex_init(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
-#endif
-    return pthread_rwlock_init(&rwmtx->pt_rwlock, NULL);
-}
-
-int
-ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = pthread_rwlock_destroy(&rwmtx->pt_rwlock);
-#if ETHR_XCHK
-    rwmtx->initialized = 0;
-#endif
-    return res;
-}
-
-int
-ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_tryrlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_runlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_tryrwlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rwlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rwunlock__(rwmtx);
-}
-
-#endif /* #ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT */
-
-/*
- * Current time
- */
-
-int
-ethr_time_now(ethr_timeval *time)
-{
-    int res;
-    struct timeval tv;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!time) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    res = gettimeofday(&tv, NULL);
-    time->tv_sec = (long) tv.tv_sec;
-    time->tv_nsec = ((long) tv.tv_usec)*1000;
-    return res;
-}
-
-/*
- * Thread specific data
- */
-
-int
-ethr_tsd_key_create(ethr_tsd_key *keyp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!keyp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_key_create((pthread_key_t *) keyp, NULL);
-}
-
-int
-ethr_tsd_key_delete(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_key_delete((pthread_key_t) key);
-}
-
-int
-ethr_tsd_set(ethr_tsd_key key, void *value)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_setspecific((pthread_key_t) key, value);
-}
-
-void *
-ethr_tsd_get(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return NULL;
-    }
-#endif
-    return pthread_getspecific((pthread_key_t) key);
-}
-
-/*
- * Signal functions
- */
-
-#if ETHR_HAVE_ETHR_SIG_FUNCS
-
-int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!set && !oset) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-  return pthread_sigmask(how, set, oset);
-}
-
-int ethr_sigwait(const sigset_t *set, int *sig)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!set || !sig) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (sigwait(set, sig) < 0)
-	return errno;
-    return 0;
-}
-
-#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */
-
-#elif defined(ETHR_WIN32_THREADS)
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * Native win32 threads implementation                                       *
-\*                                                                           */
-
-#define INVALID_TID -1
-
-/* The spin count values are more or less taken out of the blue */
-#define ETHR_MUTEX_SPIN_COUNT	5000
-#define ETHR_COND_SPIN_COUNT	1000
-
-ethr_tid serial_shift; /* Bits to shift serial when constructing a tid */
-ethr_tid last_serial; /* Last thread table serial used */
-ethr_tid last_ix; /* Last thread table index used */
-ethr_tid thr_ix_mask; /* Mask used to mask out thread table index from a tid */
-
-/* Event used for conditional variables. On per thread. */
-/*typedef struct cnd_wait_event__ cnd_wait_event_;*/
-struct cnd_wait_event__ {
-    HANDLE handle;
-    cnd_wait_event_ *prev;
-    cnd_wait_event_ *next;
-    int in_queue;
-};
-
-/* Thread specific data. Stored in the thread table */
-typedef struct {
-    ethr_tid thr_id;
-    HANDLE thr_handle;
-    ethr_tid joiner;
-    void *result;
-    cnd_wait_event_ wait_event;
-} thr_data_;
-
-/* Argument passed to thr_wrapper() */
-typedef struct {
-    void * (*func)(void *);
-    void * arg;
-    thr_data_ *ptd;
-    thr_data_ *td;
-    int res;
-    void *prep_func_res;
-} thr_wrap_data_;
-
-
-static CRITICAL_SECTION thr_table_cs; /* Critical section used to protect
-					 the thread table from concurrent
-					 accesses. */
-static CRITICAL_SECTION fake_static_init_cs; /* Critical section used to protect
-					initialazition of 'statically
-					initialized' mutexes */
-static thr_data_ * thr_table[ETHR_MAX_THREADS]; /* The thread table */
-
-static DWORD tls_own_thr_data;
-
-static thr_data_ main_thr_data;
-
-#define THR_IX(TID)	((TID) & thr_ix_mask)
-#define OWN_THR_DATA	((thr_data_ *) TlsGetValue(tls_own_thr_data))
-
-/*
- * ----------------------------------------------------------------------------
- * Static functions
- * ----------------------------------------------------------------------------
- */
-
-static int
-get_errno(void)
-{
-    switch (GetLastError()) {
-    case ERROR_INVALID_FUNCTION:		return EINVAL;	/* 1	*/
-    case ERROR_FILE_NOT_FOUND:			return ENOENT;	/* 2	*/
-    case ERROR_PATH_NOT_FOUND:			return ENOENT;	/* 3	*/
-    case ERROR_TOO_MANY_OPEN_FILES:		return EMFILE;	/* 4	*/
-    case ERROR_ACCESS_DENIED:			return EACCES;	/* 5	*/
-    case ERROR_INVALID_HANDLE:			return EBADF;	/* 6	*/
-    case ERROR_ARENA_TRASHED:			return ENOMEM;	/* 7	*/
-    case ERROR_NOT_ENOUGH_MEMORY:		return ENOMEM;	/* 8	*/
-    case ERROR_INVALID_BLOCK:			return ENOMEM;	/* 9	*/
-    case ERROR_BAD_ENVIRONMENT:			return E2BIG;	/* 10	*/
-    case ERROR_BAD_FORMAT:			return ENOEXEC;	/* 11	*/
-    case ERROR_INVALID_ACCESS:			return EINVAL;	/* 12	*/
-    case ERROR_INVALID_DATA:			return EINVAL;	/* 13	*/
-    case ERROR_OUTOFMEMORY:			return ENOMEM;	/* 14	*/
-    case ERROR_INVALID_DRIVE:			return ENOENT;	/* 15	*/
-    case ERROR_CURRENT_DIRECTORY:		return EACCES;	/* 16	*/
-    case ERROR_NOT_SAME_DEVICE:			return EXDEV;	/* 17	*/
-    case ERROR_NO_MORE_FILES:			return ENOENT;	/* 18	*/
-    case ERROR_WRITE_PROTECT:			return EACCES;	/* 19	*/
-    case ERROR_BAD_UNIT:			return EACCES;	/* 20	*/
-    case ERROR_NOT_READY:			return EACCES;	/* 21	*/
-    case ERROR_BAD_COMMAND:			return EACCES;	/* 22	*/
-    case ERROR_CRC:				return EACCES;	/* 23	*/
-    case ERROR_BAD_LENGTH:			return EACCES;	/* 24	*/
-    case ERROR_SEEK:				return EACCES;	/* 25	*/
-    case ERROR_NOT_DOS_DISK:			return EACCES;	/* 26	*/
-    case ERROR_SECTOR_NOT_FOUND:		return EACCES;	/* 27	*/
-    case ERROR_OUT_OF_PAPER:			return EACCES;	/* 28	*/
-    case ERROR_WRITE_FAULT:			return EACCES;	/* 29	*/
-    case ERROR_READ_FAULT:			return EACCES;	/* 30	*/
-    case ERROR_GEN_FAILURE:			return EACCES;	/* 31	*/
-    case ERROR_SHARING_VIOLATION:		return EACCES;	/* 32	*/
-    case ERROR_LOCK_VIOLATION:			return EACCES;	/* 33	*/
-    case ERROR_WRONG_DISK:			return EACCES;	/* 34	*/
-    case ERROR_SHARING_BUFFER_EXCEEDED:		return EACCES;	/* 36	*/
-    case ERROR_BAD_NETPATH:			return ENOENT;	/* 53	*/
-    case ERROR_NETWORK_ACCESS_DENIED:		return EACCES;	/* 65	*/
-    case ERROR_BAD_NET_NAME:			return ENOENT;	/* 67	*/
-    case ERROR_FILE_EXISTS:			return EEXIST;	/* 80	*/
-    case ERROR_CANNOT_MAKE:			return EACCES;	/* 82	*/
-    case ERROR_FAIL_I24:			return EACCES;	/* 83	*/
-    case ERROR_INVALID_PARAMETER:		return EINVAL;	/* 87	*/
-    case ERROR_NO_PROC_SLOTS:			return EAGAIN;	/* 89	*/
-    case ERROR_DRIVE_LOCKED:			return EACCES;	/* 108	*/
-    case ERROR_BROKEN_PIPE:			return EPIPE;	/* 109	*/
-    case ERROR_DISK_FULL:			return ENOSPC;	/* 112	*/
-    case ERROR_INVALID_TARGET_HANDLE:		return EBADF;	/* 114	*/
-    case ERROR_WAIT_NO_CHILDREN:		return ECHILD;	/* 128	*/
-    case ERROR_CHILD_NOT_COMPLETE:		return ECHILD;	/* 129	*/
-    case ERROR_DIRECT_ACCESS_HANDLE:		return EBADF;	/* 130	*/
-    case ERROR_NEGATIVE_SEEK:			return EINVAL;	/* 131	*/
-    case ERROR_SEEK_ON_DEVICE:			return EACCES;	/* 132	*/
-    case ERROR_DIR_NOT_EMPTY:			return ENOTEMPTY;/* 145	*/
-    case ERROR_NOT_LOCKED:			return EACCES;	/* 158	*/
-    case ERROR_BAD_PATHNAME:			return ENOENT;	/* 161	*/
-    case ERROR_MAX_THRDS_REACHED:		return EAGAIN;	/* 164	*/
-    case ERROR_LOCK_FAILED:			return EACCES;	/* 167	*/
-    case ERROR_ALREADY_EXISTS:			return EEXIST;	/* 183	*/
-    case ERROR_INVALID_STARTING_CODESEG:	return ENOEXEC;	/* 188	*/
-    case ERROR_INVALID_STACKSEG:		return ENOEXEC;	/* 189	*/
-    case ERROR_INVALID_MODULETYPE:		return ENOEXEC;	/* 190	*/
-    case ERROR_INVALID_EXE_SIGNATURE:		return ENOEXEC;	/* 191	*/
-    case ERROR_EXE_MARKED_INVALID:		return ENOEXEC;	/* 192	*/
-    case ERROR_BAD_EXE_FORMAT:			return ENOEXEC;	/* 193	*/
-    case ERROR_ITERATED_DATA_EXCEEDS_64k:	return ENOEXEC;	/* 194	*/
-    case ERROR_INVALID_MINALLOCSIZE:		return ENOEXEC;	/* 195	*/
-    case ERROR_DYNLINK_FROM_INVALID_RING:	return ENOEXEC;	/* 196	*/
-    case ERROR_IOPL_NOT_ENABLED:		return ENOEXEC;	/* 197	*/
-    case ERROR_INVALID_SEGDPL:			return ENOEXEC;	/* 198	*/
-    case ERROR_AUTODATASEG_EXCEEDS_64k:		return ENOEXEC;	/* 199	*/
-    case ERROR_RING2SEG_MUST_BE_MOVABLE:	return ENOEXEC;	/* 200	*/
-    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:	return ENOEXEC;	/* 201	*/
-    case ERROR_INFLOOP_IN_RELOC_CHAIN:		return ENOEXEC;	/* 202	*/
-    case ERROR_FILENAME_EXCED_RANGE:		return ENOENT;	/* 206	*/
-    case ERROR_NESTING_NOT_ALLOWED:		return EAGAIN;	/* 215	*/
-    case ERROR_NOT_ENOUGH_QUOTA:		return ENOMEM;	/* 1816	*/
-    default:					return EINVAL;
-    }
-}
-
-static ETHR_INLINE thr_data_ *
-tid2thr(ethr_tid tid)
-{
-    ethr_tid ix;
-    thr_data_ *td;
-    
-    if (tid < 0)
-	return NULL;
-    ix = THR_IX(tid);
-    if (ix >= ETHR_MAX_THREADS)
-	return NULL;
-    td = thr_table[ix];
-    if (!td)
-	return NULL;
-    if (td->thr_id != tid)
-	return NULL;
-    return td;
-}
-
-static ETHR_INLINE void
-new_tid(ethr_tid *new_tid, ethr_tid *new_serial, ethr_tid *new_ix)
-{
-    ethr_tid tmp_serial = last_serial;
-    ethr_tid tmp_ix = last_ix + 1;
-    ethr_tid start_ix = tmp_ix;
-
-
-    do {
-	if (tmp_ix >= ETHR_MAX_THREADS) {
-	    tmp_serial++;
-	    if ((tmp_serial << serial_shift) < 0)
-		tmp_serial = 0;
-	    tmp_ix = 0;
-	}
-	if (!thr_table[tmp_ix]) {
-	    *new_tid	= (tmp_serial << serial_shift) | tmp_ix;
-	    *new_serial	= tmp_serial;
-	    *new_ix	= tmp_ix;
-	    return;
-	}
-	tmp_ix++;
-    } while (tmp_ix != start_ix);
-
-    *new_tid	= INVALID_TID;
-    *new_serial	= INVALID_TID;
-    *new_ix	= INVALID_TID;
-
-}
-
-
-static void thr_exit_cleanup(thr_data_ *td, void *res)
-{
-
-    ASSERT(td == OWN_THR_DATA);
-
-    run_exit_handlers();
-
-    EnterCriticalSection(&thr_table_cs);
-    CloseHandle(td->wait_event.handle);
-    if (td->thr_handle == INVALID_HANDLE_VALUE) {
-	/* We are detached; cleanup thread table */
-	ASSERT(td->joiner == INVALID_TID);
-	ASSERT(td == thr_table[THR_IX(td->thr_id)]);
-	thr_table[THR_IX(td->thr_id)] = NULL;
-	if (td != &main_thr_data)
-	    (*freep)((void *) td);
-    }
-    else {
-	/* Save result and let joining thread cleanup */
-	td->result = res;
-    }
-    LeaveCriticalSection(&thr_table_cs);
-}
-
-static unsigned __stdcall thr_wrapper(LPVOID args)
-{
-    void *(*func)(void*) = ((thr_wrap_data_ *) args)->func;
-    void *arg = ((thr_wrap_data_ *) args)->arg;
-    thr_data_ *td = ((thr_wrap_data_ *) args)->td;
-
-    td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-    if (td->wait_event.handle == INVALID_HANDLE_VALUE
-	|| !TlsSetValue(tls_own_thr_data, (LPVOID) td)) {
-	((thr_wrap_data_ *) args)->res = get_errno();
-	if (td->wait_event.handle != INVALID_HANDLE_VALUE)
-	    CloseHandle(td->wait_event.handle);
-	SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle);
-	_endthreadex((unsigned) 0);
-	ASSERT(0);
-    }
-
-    td->wait_event.prev = NULL;
-    td->wait_event.next = NULL;
-    td->wait_event.in_queue = 0;
-
-    if (thread_create_child_func)
-	(*thread_create_child_func)(((thr_wrap_data_ *) args)->prep_func_res);
-
-    ASSERT(td == OWN_THR_DATA);
-
-    ((thr_wrap_data_ *) args)->res = 0;
-    SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle);
-
-    thr_exit_cleanup(td, (*func)(arg));
-    return 0;
-}
-
-int
-ethr_fake_static_mutex_init(ethr_mutex *mtx)
-{
-    EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    /* Got here under race conditions; check again... */
-    if (!mtx->initialized) {
-	if (!InitializeCriticalSectionAndSpinCount(&mtx->cs,
-						   ETHR_MUTEX_SPIN_COUNT))
-	    return get_errno();
-	mtx->initialized = ETHR_MUTEX_INITIALIZED;
-    }
-    LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    return 0;
-}
-
-static int
-fake_static_cond_init(ethr_cond *cnd)
-{
-    EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    /* Got here under race conditions; check again... */
-    if (!cnd->initialized) {
-	if (!InitializeCriticalSectionAndSpinCount(&cnd->cs,
-						   ETHR_COND_SPIN_COUNT))
-	    return get_errno();
-	cnd->queue = NULL;
-	cnd->queue_end = NULL;
-	cnd->initialized = ETHR_COND_INITIALIZED;
-    }
-    LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    return 0;
-}
-
-#ifdef __GNUC__
-#define LL_LITERAL(X) X##LL
-#else
-#define LL_LITERAL(X) X##i64
-#endif
-
-#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600)
-
-static ETHR_INLINE void
-get_curr_time(long *sec, long *nsec)
-{
-    SYSTEMTIME t;
-    FILETIME ft;
-    LONGLONG lft;
-
-    GetSystemTime(&t);
-    SystemTimeToFileTime(&t, &ft);
-    memcpy(&lft, &ft, sizeof(lft));
-    *nsec = ((long) (lft % LL_LITERAL(10000000)))*100;
-    *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF);
-}
-
-static cnd_wait_event_ *cwe_freelist;
-static CRITICAL_SECTION cwe_cs;
-
-static int
-alloc_cwe(cnd_wait_event_ **cwe_res)
-{
-    cnd_wait_event_ *cwe;
-    EnterCriticalSection(&cwe_cs);
-    cwe = cwe_freelist;
-    if (cwe) {
-	cwe_freelist = cwe->next;
-	LeaveCriticalSection(&cwe_cs);
-    }
-    else {
-	LeaveCriticalSection(&cwe_cs);
-	cwe = (*allocp)(sizeof(cnd_wait_event_));
-	if (!cwe)
-	    return ENOMEM;
-	cwe->handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-	if (cwe->handle == INVALID_HANDLE_VALUE) {
-	    int res = get_errno();
-	    (*freep)(cwe);
-	    return res;
-	}
-    }
-    *cwe_res = cwe;
-    return 0;
-}
-
-static
-free_cwe(cnd_wait_event_ *cwe)
-{
-    EnterCriticalSection(&cwe_cs);
-    cwe->next = cwe_freelist;
-    cwe_freelist = cwe;
-    LeaveCriticalSection(&cwe_cs);
-}
-
-static ETHR_INLINE int
-condwait(ethr_cond *cnd,
-	 ethr_mutex *mtx,
-	 int with_timeout,
-	 ethr_timeval *timeout)
-{
-    int res;
-    thr_data_ *td;
-    cnd_wait_event_ *cwe;
-    DWORD code;
-    long time; /* time until timeout in milli seconds */
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-
-    if (!mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED
-	|| !cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)
-	|| (with_timeout && !timeout)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    td = OWN_THR_DATA;
-    if (td)
-	cwe = &td->wait_event;
-    else { /* A non-ethread thread */
-	res = alloc_cwe(&cwe);
-	if (res != 0)
-	    return res;
-    }
-
-    if (!cnd->initialized)
-	fake_static_cond_init(cnd);
-    EnterCriticalSection(&cnd->cs);
-
-    ASSERT(!cwe->in_queue);
-    if (cnd->queue_end) {
-	ASSERT(cnd->queue);
-	cwe->prev = cnd->queue_end;
-	cwe->next = NULL;
-	cnd->queue_end->next = cwe;
-	cnd->queue_end = cwe;
-    }
-    else {
-	ASSERT(!cnd->queue);
-	cwe->prev = NULL;
-	cwe->next = NULL;
-	cnd->queue = cwe;
-	cnd->queue_end = cwe;
-    }
-    cwe->in_queue = 1;
-
-    LeaveCriticalSection(&cnd->cs);
- 
-    LeaveCriticalSection(&mtx->cs);
-
-    if (!with_timeout)
-	time = INFINITE;
-    else {
-	long sec, nsec;
-	ASSERT(timeout);
-	get_curr_time(&sec, &nsec);
-	time = (timeout->tv_sec - sec)*1000;
-	time += (timeout->tv_nsec - nsec + 500)/1000000;
-	if (time < 0)
-	    time = 0;
-    }
-
-    /* wait for event to signal */
-    code = WaitForSingleObject(cwe->handle, time);
-
-    EnterCriticalSection(&mtx->cs);
-
-    if (code == WAIT_OBJECT_0) {
-	/* We were woken by a signal or a broadcast ... */
-	res = 0;
-
-	/* ... no need to remove event from wait queue since this was
-	   taken care of by the signal or broadcast */
-#ifdef DEBUG
-	EnterCriticalSection(&cnd->cs);
-	ASSERT(!cwe->in_queue);
-	LeaveCriticalSection(&cnd->cs);
-#endif
-
-    }
-    else {
-	/* We timed out... */
-	res = ETIMEDOUT;
-
-	/* ... probably have to remove event from wait queue ... */
-	EnterCriticalSection(&cnd->cs);
-
-	if (cwe->in_queue) { /* ... but we must check that we are in queue
-				since a signal or broadcast after timeout
-				may have removed us from the queue */
-	    if (cwe->prev) {
-		cwe->prev->next = cwe->next;
-	    }
-	    else {
-		ASSERT(cnd->queue == cwe);
-		cnd->queue = cwe->next;
-	    }
-
-	    if (cwe->next) {
-		cwe->next->prev = cwe->prev;
-	    }
-	    else {
-		ASSERT(cnd->queue_end == cwe);
-		cnd->queue_end = cwe->prev;
-	    }
-	    cwe->in_queue = 0;
-	}
-
-	LeaveCriticalSection(&cnd->cs);
-
-    }
-
-    if (!td)
-	free_cwe(cwe);
-
-    return res;
-
-}
-
-
-/*
- * ----------------------------------------------------------------------------
- * Exported functions
- * ----------------------------------------------------------------------------
- */
-
-int
-ethr_init(ethr_init_data *id)
-{
-#ifdef _WIN32_WINNT
-    DWORD major = (_WIN32_WINNT >> 8) & 0xff;
-    DWORD minor = _WIN32_WINNT & 0xff;
-    OSVERSIONINFO os_version;
-#endif
-    int err = 0;
-    thr_data_ *td = &main_thr_data;
-    unsigned long i;
-
-    if (!ethr_not_inited)
-	return EINVAL;
-
-#ifdef _WIN32_WINNT
-    os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
-    GetVersionEx(&os_version);
-    if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT
-	|| os_version.dwMajorVersion < major
-	|| (os_version.dwMajorVersion == major
-	    && os_version.dwMinorVersion < minor))
-	return ENOTSUP;
-#endif
-
-    ASSERT(ETHR_MAX_THREADS > 0);
-    for (i = ETHR_MAX_THREADS - 1, serial_shift = 0;
-	 i;
-	 serial_shift++, i >>= 1);
-    thr_ix_mask = ~(~((ethr_tid) 0) << serial_shift);
-
-    tls_own_thr_data = TlsAlloc();
-    if (tls_own_thr_data == TLS_OUT_OF_INDEXES)
-	goto error;
-
-    last_serial = 0;
-    last_ix = 0;
-
-    td->thr_id = 0;
-    td->thr_handle = GetCurrentThread();
-    td->joiner = INVALID_TID;
-    td->result = NULL;
-    td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-    if (td->wait_event.handle == INVALID_HANDLE_VALUE)
-	goto error;
-    td->wait_event.prev = NULL;
-    td->wait_event.next = NULL;
-    td->wait_event.in_queue = 0;
-    thr_table[0] = td;
-
-    if (!TlsSetValue(tls_own_thr_data, (LPVOID) td))
-	goto error;
-
-    ASSERT(td == OWN_THR_DATA);
-
-
-    cwe_freelist = NULL;
-    if (!InitializeCriticalSectionAndSpinCount(&cwe_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-
-    for (i = 1; i < ETHR_MAX_THREADS; i++)
-	thr_table[i] = NULL;
-
-    if (!InitializeCriticalSectionAndSpinCount(&thr_table_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-    if (!InitializeCriticalSectionAndSpinCount(&fake_static_init_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-    ethr_not_inited = 0;
-
-    err = init_common(id);
-    if (err)
-	goto error;
-
-    return 0;
-
- error:
-    ethr_not_inited = 1;
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-    if (td->thr_handle != INVALID_HANDLE_VALUE)
-	CloseHandle(td->thr_handle);
-    if (td->wait_event.handle != INVALID_HANDLE_VALUE)
-	CloseHandle(td->wait_event.handle);
-    return err;
-}
-
-/*
- * Thread functions.
- */
-
-int
-ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
-		ethr_thr_opts *opts)
-{
-    int err = 0;
-    thr_wrap_data_ twd;
-    thr_data_ *my_td, *child_td = NULL;
-    ethr_tid child_tid, child_serial, child_ix;
-    DWORD code;
-    unsigned ID;
-    unsigned stack_size = 0; /* 0 = system default */
-    int use_stack_size = (opts && opts->suggested_stack_size >= 0
-			  ? opts->suggested_stack_size
-			  : -1 /* Use system default */);
-
-#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
-    if (use_stack_size < 0)
-	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
-#endif
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!tid || !func) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    my_td = OWN_THR_DATA;
-    if (!my_td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    if (use_stack_size >= 0) {
-	size_t suggested_stack_size = (size_t) use_stack_size;
-#ifdef DEBUG
-	suggested_stack_size /= 2; /* Make sure we got margin */
-#endif
-	if (suggested_stack_size < min_stack_size)
-	    stack_size = (unsigned) ETHR_KW2B(min_stack_size);
-	else if (suggested_stack_size > max_stack_size)
-	    stack_size = (unsigned) ETHR_KW2B(max_stack_size);
-	else
-	    stack_size =
-		(unsigned) ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    /* Call prepare func if it exist */
-    if (thread_create_prepare_func)
-	twd.prep_func_res = (*thread_create_prepare_func)();
-    else
-	twd.prep_func_res = NULL;
-
-    /* Find a new thread id to use */
-    new_tid(&child_tid, &child_serial, &child_ix);
-    if (child_tid == INVALID_TID) {
-	err = EAGAIN;
-	goto error;
-    }
-
-    ASSERT(child_ix == THR_IX(child_tid));
-
-    *tid = child_tid;
-
-    ASSERT(!thr_table[child_ix]);
-
-    /* Alloc thread data */
-    thr_table[child_ix] = child_td = (thr_data_ *) (*allocp)(sizeof(thr_data_));
-    if (!child_td) {
-	err = ENOMEM;
-	goto error;
-    }
-
-    /* Init thread data */
-
-    child_td->thr_id		= child_tid;
-    child_td->thr_handle	= INVALID_HANDLE_VALUE;
-    child_td->joiner		= INVALID_TID;
-    child_td->result		= NULL;
-    /* 'child_td->wait_event' is initialized by child thread */
-
-
-    /* Init thread wrapper data */
-
-    twd.func			= func;
-    twd.arg			= arg;
-    twd.ptd			= my_td;
-    twd.td			= child_td;
-    twd.res			= 0;
-
-    ASSERT(!my_td->wait_event.in_queue);
-
-    /* spawn the thr_wrapper function */
-    child_td->thr_handle = (HANDLE) _beginthreadex(NULL,
-						   stack_size,
-						   thr_wrapper,
-						   (LPVOID) &twd,
-						   0,
-						   &ID);
-    if (child_td->thr_handle == (HANDLE) 0) {
-	child_td->thr_handle = INVALID_HANDLE_VALUE;
-	goto error;
-    }
-
-    ASSERT(child_td->thr_handle != INVALID_HANDLE_VALUE);
-
-    /* Wait for child to finish initialization */
-    code = WaitForSingleObject(my_td->wait_event.handle, INFINITE);
-    if (twd.res || code != WAIT_OBJECT_0) {
-	err = twd.res;
-	goto error;
-    }
-
-    if (opts && opts->detached) {
-	CloseHandle(child_td->thr_handle);
-	child_td->thr_handle = INVALID_HANDLE_VALUE;
-    }
-
-    last_serial	= child_serial;
-    last_ix	= child_ix;
-
-    ASSERT(thr_table[child_ix] == child_td);
-
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return 0;
-
- error:
-
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    if (child_ix != INVALID_TID) {
-
-	if (child_td) {
-	    ASSERT(thr_table[child_ix] == child_td);
-
-	    if (child_td->thr_handle != INVALID_HANDLE_VALUE) {
-		WaitForSingleObject(child_td->thr_handle, INFINITE);
-		CloseHandle(child_td->thr_handle);
-	    }
-
-	    (*freep)((void *) child_td);
-	    thr_table[child_ix] = NULL;
-	}
-    }
-
-    *tid = INVALID_TID;
-
-    LeaveCriticalSection(&thr_table_cs);
-    return err;
-}
-
-int ethr_thr_join(ethr_tid tid, void **res)
-{
-    int err = 0;
-    DWORD code;
-    thr_data_ *td;
-    thr_data_ *my_td;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    my_td = OWN_THR_DATA;
-
-    if (!my_td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    td = tid2thr(tid);
-    if (!td)
-	err = ESRCH;
-    else if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */
-	     || td->joiner != INVALID_TID) /* i.e. someone else is joining */
-	err = EINVAL;
-    else if (my_td == td)
-	err = EDEADLK;
-    else
-	td->joiner = my_td->thr_id;
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    if (err)
-	goto error;
-
-    /* Wait for thread to terminate */
-    code = WaitForSingleObject(td->thr_handle, INFINITE);
-    if (code != WAIT_OBJECT_0)
-	goto error;
-
-    EnterCriticalSection(&thr_table_cs);
-
-    ASSERT(td == tid2thr(tid));
-    ASSERT(td->thr_handle != INVALID_HANDLE_VALUE);
-    ASSERT(td->joiner == my_td->thr_id);
-
-    if (res)
-	*res = td->result;
-
-    CloseHandle(td->thr_handle);
-    ASSERT(td == thr_table[THR_IX(td->thr_id)]);
-    thr_table[THR_IX(td->thr_id)] = NULL;
-    if (td != &main_thr_data)
-	(*freep)((void *) td);
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return 0;
-
- error:
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-    return err;
-}
-
-
-int
-ethr_thr_detach(ethr_tid tid)
-{
-    int res;
-    DWORD code;
-    thr_data_ *td;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    if (!OWN_THR_DATA) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    td = tid2thr(tid);
-    if (!td)
-	res = ESRCH;
-    if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */
-	|| td->joiner != INVALID_TID) /* i.e. someone is joining */
-	res = EINVAL;
-    else {
-	res = 0;
-	CloseHandle(td->thr_handle);
-	td->thr_handle = INVALID_HANDLE_VALUE;
-    }
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return res;
-}
-
-
-void
-ethr_thr_exit(void *res)
-{
-    thr_data_ *td;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return;
-    }
-#endif
-    td = OWN_THR_DATA;
-    if (!td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return;
-    }
-    thr_exit_cleanup(td, res);
-    _endthreadex((unsigned) 0);
-}
-
-ethr_tid
-ethr_self(void)
-{
-    thr_data_ *td;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return INVALID_TID;
-    }
-#endif
-    /* It is okay for non-ethreads (i.e. native win32 threads) to call
-       ethr_self(). They will however be returned the INVALID_TID. */
-    td = OWN_THR_DATA;
-    if (!td)
-	return INVALID_TID;
-    return td->thr_id;
-}
-
-int
-ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
-{
-    /* INVALID_TID does not equal any tid, not even the INVALID_TID */
-    return tid1 == tid2 && tid1 != INVALID_TID;
-}
-
-/*
- * Mutex functions.
- */
-
-int
-ethr_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, ETHR_MUTEX_SPIN_COUNT))
-	return get_errno();
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#if ETHR_XCHK
-    mtx->is_rec_mtx = 0;
-#endif
-    return 0;
-}
-
-int
-ethr_rec_mutex_init(ethr_mutex *mtx)
-{
-    int res;
-    res = ethr_mutex_init(mtx);
-#if ETHR_XCHK
-    mtx->is_rec_mtx = 1;
-#endif
-    return res;
-}
-
-int
-ethr_mutex_destroy(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    DeleteCriticalSection(&mtx->cs);
-    mtx->initialized = 0;
-    return 0;
-}
-
-int ethr_mutex_set_forksafe(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return 0; /* No fork() */
-}
-
-int ethr_mutex_unset_forksafe(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return 0; /* No fork() */
-}
-
-int
-ethr_mutex_trylock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx
-	|| (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!mtx->initialized) {
-	int res = ethr_fake_static_mutex_init(mtx);
-	if (res != 0)
-	    return res;
-    }
-    return ethr_mutex_trylock__(mtx);
-}
-
-int
-ethr_mutex_lock(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx
-	|| (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_lock__(mtx);
-}
-
-int
-ethr_mutex_unlock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    int res;
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_unlock__(mtx);
-}
-
-/*
- * Condition variable functions.
- */
-
-int
-ethr_cond_init(ethr_cond *cnd)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, ETHR_COND_SPIN_COUNT))
-	return get_errno();
-    cnd->queue = NULL;
-    cnd->queue_end = NULL;
-    cnd->initialized = ETHR_COND_INITIALIZED;
-    return 0;
-}
-
-int
-ethr_cond_destroy(ethr_cond *cnd)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)
-	|| cnd->queue) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    DeleteCriticalSection(&cnd->cs);
-    cnd->initialized = 0;
-    return 0;
-}
-
-int
-ethr_cond_signal(ethr_cond *cnd)
-{
-    cnd_wait_event_ *cwe;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!cnd->initialized) {
-	int res = fake_static_cond_init(cnd);
-	if (res != 0)
-	    return res;
-    }
-    EnterCriticalSection(&cnd->cs);
-    cwe = cnd->queue;
-    if (cwe) {
-	ASSERT(cwe->in_queue);
-	SetEvent(cnd->queue->handle);
-	if (cwe->next)
-	    cwe->next->prev = NULL;
-	else {
-	    ASSERT(cnd->queue_end == cnd->queue);
-	    cnd->queue_end = NULL;
-	}
-	cnd->queue = cwe->next;
-	cwe->in_queue = 0;
-    }
-    LeaveCriticalSection(&cnd->cs);
-    return 0;
-}
-
-int
-ethr_cond_broadcast(ethr_cond *cnd)
-{
-    cnd_wait_event_ *cwe;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!cnd->initialized) {
-	int res = fake_static_cond_init(cnd);
-	if (res != 0)
-	    return res;
-    }
-    EnterCriticalSection(&cnd->cs);
-    for (cwe = cnd->queue; cwe; cwe = cwe->next) {
-	ASSERT(cwe->in_queue);
-	SetEvent(cwe->handle);
-	cwe->in_queue = 0;
-    }
-    cnd->queue = NULL;
-    cnd->queue_end = NULL;
-    LeaveCriticalSection(&cnd->cs);
-    return 0;
-
-}
-
-int
-ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
-{
-    return condwait(cnd, mtx, 0, NULL);
-}
-
-int
-ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout)
-{
-    return condwait(cnd, mtx, 1, timeout);
-}
-
-int
-ethr_time_now(ethr_timeval *time)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!time) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    get_curr_time(&time->tv_sec, &time->tv_nsec);
-    return 0;
-}
-
-/*
- * Thread specific data
- */
-
-int
-ethr_tsd_key_create(ethr_tsd_key *keyp)
-{
-    DWORD key;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!keyp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    key = TlsAlloc();
-    if (key == TLS_OUT_OF_INDEXES)
-	return get_errno();
-    *keyp = (ethr_tsd_key) key;
-    return 0;
-}
-
-int
-ethr_tsd_key_delete(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    if (!TlsFree((DWORD) key))
-	return get_errno();
-    return 0;
-}
-
-int
-ethr_tsd_set(ethr_tsd_key key, void *value)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    if (!TlsSetValue((DWORD) key, (LPVOID) value))
-	return get_errno();
-    return 0;
-}
-
-void *
-ethr_tsd_get(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return NULL;
-    }
-#endif
-    return (void *) TlsGetValue((DWORD) key);
-}
-
-/* Misc */
-
-#ifndef ETHR_HAVE_OPTIMIZED_LOCKS
-
-int
-ethr_do_spinlock_init(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX))
-	return 0;
-    else
-	return get_errno();
-}
-
-int
-ethr_do_rwlock_init(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    lock->counter = 0;
-    if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX))
-	return 0;
-    else
-	return get_errno();    
-}
-
-#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */
-
-#else
-#error "Missing thread implementation"
-#endif
-
-/* Atomics */
-
-int
-ethr_atomic_init(ethr_atomic_t *var, long i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_init__(var, i);
-}
-
-int
-ethr_atomic_set(ethr_atomic_t *var, long i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_set__(var, i);
-}
-
-int
-ethr_atomic_read(ethr_atomic_t *var, long *i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var || !i) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_read__(var, i);
-}
-
-
-int
-ethr_atomic_addtest(ethr_atomic_t *var, long incr, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var || !testp) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_addtest__(var, incr, testp);
-}   
-    
-int
-ethr_atomic_inctest(ethr_atomic_t *incp, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!incp || !testp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_inctest__(incp, testp);
-}
-
-int
-ethr_atomic_dectest(ethr_atomic_t *decp, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!decp || !testp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_dectest__(decp, testp);
-}
-
-int
-ethr_atomic_add(ethr_atomic_t *var, long incr)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_add__(var, incr);
-}   
-    
-int 
-ethr_atomic_inc(ethr_atomic_t *incp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!incp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_inc__(incp);
-}
-
-int
-ethr_atomic_dec(ethr_atomic_t *decp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!decp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_dec__(decp);
-}
-
-int
-ethr_atomic_and_old(ethr_atomic_t *var, long mask, long *old)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_and_old__(var, mask, old);
-}
-
-int
-ethr_atomic_or_old(ethr_atomic_t *var, long mask, long *old)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_or_old__(var, mask, old);
-}
-
-int
-ethr_atomic_xchg(ethr_atomic_t *var, long new, long *old)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }  
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }  
-#endif
-    return ethr_atomic_xchg__(var, new, old);
-}   
-
-int
-ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected, long *old)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }  
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }  
-#endif
-    return ethr_atomic_cmpxchg__(var, new, expected, old);
-}
-
-/* Spinlocks and rwspinlocks */
-
-int
-ethr_spinlock_init(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spinlock_init__(lock);
-}
-
-int
-ethr_spinlock_destroy(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spinlock_destroy__(lock);
-}
-
-
-int
-ethr_spin_unlock(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spin_unlock__(lock);
-}
-
-int
-ethr_spin_lock(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spin_lock__(lock);
-}
-
-int
-ethr_rwlock_init(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwlock_init__(lock);
-}
-
-int
-ethr_rwlock_destroy(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwlock_destroy__(lock);
-}
-
-int
-ethr_read_unlock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_read_unlock__(lock);
-}
-
-int
-ethr_read_lock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_read_lock__(lock);
-}
-
-int
-ethr_write_unlock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_write_unlock__(lock);
-}
-
-int
-ethr_write_lock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_write_lock__(lock);
-}
-
-
-int
-ethr_gate_init(ethr_gate *gp)
-{
-    int res;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_init(&gp->mtx);
-    if (res != 0)
-	return res;
-    res = ethr_cond_init(&gp->cnd);
-    if (res != 0) {
-	ethr_mutex_destroy(&gp->mtx);
-	return res;
-    }
-    gp->open = 0;
-    return 0;
-}
-
-int
-ethr_gate_destroy(ethr_gate *gp)
-{
-    int res, dres;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_destroy(&gp->mtx);
-    dres = ethr_cond_destroy(&gp->cnd);
-    if (res == 0)
-	res = dres;
-    gp->open = 0;
-    return res;
-}
-
-int
-ethr_gate_close(ethr_gate *gp)
-{
-    int res;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    gp->open = 0;
-    res = ethr_mutex_unlock__(&gp->mtx);
-    return res;
-}
-
-int
-ethr_gate_let_through(ethr_gate *gp, unsigned no)
-{
-    int res, ures;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    gp->open += no;
-    res = (gp->open == 1
-	   ? ethr_cond_signal(&gp->cnd)
-	   : ethr_cond_broadcast(&gp->cnd));
-    ures = ethr_mutex_unlock__(&gp->mtx);
-    if (res != 0)
-	res = ures;
-    return res;
-}
-
-int
-ethr_gate_swait(ethr_gate *gp, int spincount)
-{
-    int res, ures, n;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    n = spincount;
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    while (n >= 0 && !gp->open) {
-	res = ethr_mutex_unlock__(&gp->mtx);
-	if (res != 0)
-	    return res;
-	res = ethr_mutex_lock__(&gp->mtx);
-	if (res != 0)
-	    return res;
-	n--;
-    }
-    while (!gp->open) {
-	res = ethr_cond_wait(&gp->cnd, &gp->mtx);
-	if (res != 0 && res != EINTR)
-	    goto done;
-    }
-    gp->open--;
- done:
-    ures = ethr_mutex_unlock__(&gp->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-
-int
-ethr_gate_wait(ethr_gate *gp)
-{
-    return ethr_gate_swait(gp, 0);
-}
-
-
-/* rwmutex fallback */
-#ifdef ETHR_USE_RWMTX_FALLBACK
-
-int
-ethr_rwmutex_init(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_init(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    ethr_cond_init(&rwmtx->rcnd);
-    if (res != 0)
-	goto error_cleanup1;
-    res = ethr_cond_init(&rwmtx->wcnd);
-    if (res != 0)
-	goto error_cleanup2;
-    rwmtx->readers = 0;
-    rwmtx->waiting_readers = 0;
-    rwmtx->waiting_writers = 0;
-#if ETHR_XCHK
-    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
-#endif
-    return 0;
- error_cleanup2:
-    ethr_cond_destroy(&rwmtx->rcnd);
- error_cleanup1:
-    ethr_mutex_destroy(&rwmtx->mtx);
-    return res;
-}
-
-int
-ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
-{
-    int res, pres;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    rwmtx->initialized = 0;
-#endif
-    res = ethr_mutex_destroy(&rwmtx->mtx);
-    pres = ethr_cond_destroy(&rwmtx->rcnd);
-    if (res == 0)
-	res = pres;
-    pres = ethr_cond_destroy(&rwmtx->wcnd);
-    if (res == 0)
-	res = pres;
-    return res;
-}
-
-int
-ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_trylock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (!rwmtx->waiting_writers) {
-	res = ethr_mutex_unlock__(&rwmtx->mtx);
-	if (res == 0)
-	    return EBUSY;
-	return res;
-    }
-    rwmtx->readers++;
-    return ethr_mutex_unlock__(&rwmtx->mtx);
-}
-
-int
-ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    while (rwmtx->waiting_writers) {
-	rwmtx->waiting_readers++;
-	res = ethr_cond_wait(&rwmtx->rcnd, &rwmtx->mtx);
-	rwmtx->waiting_readers--;
-	if (res != 0 && res != EINTR) {
-	    (void) ethr_mutex_unlock__(&rwmtx->mtx);
-	    return res;
-	}
-    }
-    rwmtx->readers++;
-    return ethr_mutex_unlock__(&rwmtx->mtx);
-}
-
-int
-ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
-{
-    int res, ures;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    rwmtx->readers--;
-    if (!rwmtx->readers && rwmtx->waiting_writers)
-	res = ethr_cond_signal(&rwmtx->wcnd);
-    ures = ethr_mutex_unlock__(&rwmtx->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-int
-ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_trylock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (!rwmtx->readers && !rwmtx->waiting_writers)
-	return 0;
-    else {
-	res = ethr_mutex_unlock__(&rwmtx->mtx);
-	if (res == 0)
-	    return EBUSY;
-	return res;
-    }
-}
-
-int
-ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (!rwmtx->readers && !rwmtx->waiting_writers) 
-	return 0;
-    
-    while (rwmtx->readers) {
-	rwmtx->waiting_writers++;
-	res = ethr_cond_wait(&rwmtx->wcnd, &rwmtx->mtx);
-	rwmtx->waiting_writers--;
-	if (res != 0 && res != EINTR) {
-	    (void) ethr_rwmutex_rwunlock(rwmtx);
-	    return res;
-	}
-    }
-    return 0;
-}
-
-int
-ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
-{
-    int res, ures;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = 0;
-    if (rwmtx->waiting_writers)
-	res = ethr_cond_signal(&rwmtx->wcnd);
-    else if (rwmtx->waiting_readers)
-	res = ethr_cond_broadcast(&rwmtx->rcnd);
-    ures = ethr_mutex_unlock__(&rwmtx->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-#endif /* #ifdef ETHR_USE_RWMTX_FALLBACK */
-
-void
-ethr_compiler_barrier(void)
-{
-
-}
-
-#ifdef DEBUG
-
-#include <stdio.h>
-int ethr_assert_failed(char *f, int l, char *a)
-{
-    fprintf(stderr, "%s:%d: Assertion failed: %s\n", f, l, a);
-    abort();
-    return 0;
-}
-
-#endif
-
-
diff --git a/erts/lib_src/pthread/ethr_event.c b/erts/lib_src/pthread/ethr_event.c
new file mode 100644
index 0000000000..9434d60d0a
--- /dev/null
+++ b/erts/lib_src/pthread/ethr_event.c
@@ -0,0 +1,225 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Author: Rickard Green
+ */
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_EVENT_IMPL__
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ethread.h"
+
+#if defined(ETHR_LINUX_FUTEX_IMPL__)
+/* --- Linux futex implementation of ethread events ------------------------- */
+
+#include <sched.h>
+#include <errno.h>
+
+#define ETHR_YIELD_AFTER_BUSY_LOOPS 50
+
+int
+ethr_event_init(ethr_event *e)
+{
+    ethr_atomic32_init(&e->futex, ETHR_EVENT_OFF__);
+    return 0;
+}
+
+int
+ethr_event_destroy(ethr_event *e)
+{
+    return 0;
+}
+
+static ETHR_INLINE int
+wait__(ethr_event *e, int spincount)
+{
+    unsigned sc = spincount;
+    int res;
+    ethr_sint32_t val;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    if (spincount < 0)
+	ETHR_FATAL_ERROR__(EINVAL);
+
+    while (1) {
+	while (1) {
+	    val = ethr_atomic32_read(&e->futex);
+	    if (val == ETHR_EVENT_ON__)
+		return 0;
+	    if (sc == 0)
+		break;
+	    sc--;
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		res = ETHR_YIELD();
+		if (res != 0)
+		    ETHR_FATAL_ERROR__(res);
+	    }
+	}
+
+	if (val != ETHR_EVENT_OFF_WAITER__) {
+	    val = ethr_atomic32_cmpxchg(&e->futex,
+					ETHR_EVENT_OFF_WAITER__,
+					ETHR_EVENT_OFF__);
+
+	    if (val == ETHR_EVENT_ON__)
+		return 0;
+	    ETHR_ASSERT(val == ETHR_EVENT_OFF__);
+	}
+
+	res = ETHR_FUTEX__(&e->futex,
+			   ETHR_FUTEX_WAIT__,
+			   ETHR_EVENT_OFF_WAITER__);
+	if (res == EINTR)
+	    break;
+	if (res != 0 && res != EWOULDBLOCK)
+	    ETHR_FATAL_ERROR__(res);
+    }
+
+    return res;
+}
+
+#elif defined(ETHR_PTHREADS)
+/* --- Posix mutex/cond implementation of events ---------------------------- */
+
+int
+ethr_event_init(ethr_event *e)
+{
+    int res;
+    ethr_atomic32_init(&e->state, ETHR_EVENT_OFF__);
+    res = pthread_mutex_init(&e->mtx, NULL);
+    if (res != 0)
+	return res;
+    res = pthread_cond_init(&e->cnd, NULL);
+    if (res != 0) {
+	pthread_mutex_destroy(&e->mtx);
+	return res;
+    }
+    return 0;
+}
+
+int
+ethr_event_destroy(ethr_event *e)
+{
+    int res;
+    res = pthread_mutex_destroy(&e->mtx);
+    if (res != 0)
+	return res;
+    res = pthread_cond_destroy(&e->cnd);
+    if (res != 0)
+	return res;
+    return 0;
+}
+
+static ETHR_INLINE int
+wait__(ethr_event *e, int spincount)
+{
+    int sc = spincount;
+    ethr_sint32_t val;
+    int res, ulres;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    if (spincount < 0)
+	ETHR_FATAL_ERROR__(EINVAL);
+
+    while (1) {
+	val = ethr_atomic32_read(&e->state);
+	if (val == ETHR_EVENT_ON__)
+	    return 0;
+	if (sc == 0)
+	    break;
+	sc--;
+	ETHR_SPIN_BODY;
+	if (--until_yield == 0) {
+	    until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+	    res = ETHR_YIELD();
+	    if (res != 0)
+		ETHR_FATAL_ERROR__(res);
+	}
+    }
+
+    if (val != ETHR_EVENT_OFF_WAITER__) {
+	val = ethr_atomic32_cmpxchg(&e->state,
+				    ETHR_EVENT_OFF_WAITER__,
+				    ETHR_EVENT_OFF__);
+	if (val == ETHR_EVENT_ON__)
+	    return 0;
+	ETHR_ASSERT(val == ETHR_EVENT_OFF__);
+    }
+
+    ETHR_ASSERT(val == ETHR_EVENT_OFF_WAITER__
+		|| val == ETHR_EVENT_OFF__);
+
+    res = pthread_mutex_lock(&e->mtx);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+
+    while (1) {
+
+	val = ethr_atomic32_read(&e->state);
+	if (val == ETHR_EVENT_ON__)
+	    break;
+
+	res = pthread_cond_wait(&e->cnd, &e->mtx);
+	if (res == EINTR)
+	    break;
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+    }
+
+    ulres = pthread_mutex_unlock(&e->mtx);
+    if (ulres != 0)
+	ETHR_FATAL_ERROR__(ulres);
+
+    return res; /* 0 || EINTR */
+}
+
+#else
+#error No ethread event implementation
+#endif
+
+void
+ethr_event_reset(ethr_event *e)
+{
+    ethr_event_reset__(e);
+}
+
+void
+ethr_event_set(ethr_event *e)
+{
+    ethr_event_set__(e);
+}
+
+int
+ethr_event_wait(ethr_event *e)
+{
+    return wait__(e, 0);
+}
+
+int
+ethr_event_swait(ethr_event *e, int spincount)
+{
+    return wait__(e, spincount);
+}
diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c
new file mode 100644
index 0000000000..f047104103
--- /dev/null
+++ b/erts/lib_src/pthread/ethread.c
@@ -0,0 +1,451 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Pthread implementation of the ethread library
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_CHILD_WAIT_SPIN_COUNT 4000
+
+#include <stdio.h>
+#ifdef ETHR_TIME_WITH_SYS_TIME
+#  include <time.h>
+#  include <sys/time.h>
+#else
+#  ifdef ETHR_HAVE_SYS_TIME_H
+#    include <sys/time.h>
+#  else
+#    include <time.h>
+#  endif
+#endif
+#include <sys/types.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <limits.h>
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHREAD_IMPL__
+
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#ifndef ETHR_HAVE_ETHREAD_DEFINES
+#error Missing configure defines
+#endif
+
+pthread_key_t ethr_ts_event_key__;
+static int child_wait_spin_count;
+
+/*
+ * --------------------------------------------------------------------------
+ * Static functions
+ * --------------------------------------------------------------------------
+ */
+
+static void thr_exit_cleanup(void)
+{
+    ethr_run_exit_handlers__();
+}
+
+
+/* Argument passed to thr_wrapper() */
+typedef struct {
+    ethr_atomic32_t result;
+    ethr_ts_event *tse;
+    void *(*thr_func)(void *);
+    void *arg;
+    void *prep_func_res;
+} ethr_thr_wrap_data__;
+
+static void *thr_wrapper(void *vtwd)
+{
+    ethr_sint32_t result;
+    void *res;
+    ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd;
+    void *(*thr_func)(void *) = twd->thr_func;
+    void *arg = twd->arg;
+    ethr_ts_event *tsep = NULL;
+
+    result = (ethr_sint32_t) ethr_make_ts_event__(&tsep);
+
+    if (result == 0) {
+	tsep->iflgs |= ETHR_TS_EV_ETHREAD;
+	if (ethr_thr_child_func__)
+	    ethr_thr_child_func__(twd->prep_func_res);
+    }
+
+    tsep = twd->tse; /* We aren't allowed to follow twd after
+			result has been set! */
+
+    ethr_atomic32_set(&twd->result, result);
+
+    ethr_event_set(&tsep->event);
+
+    res = result == 0 ? (*thr_func)(arg) : NULL;
+
+    thr_exit_cleanup();
+    return res;
+}
+
+/* internal exports */
+
+int ethr_set_tse__(ethr_ts_event *tsep)
+{
+    return pthread_setspecific(ethr_ts_event_key__, (void *) tsep);
+}
+
+ethr_ts_event *ethr_get_tse__(void)
+{
+    return pthread_getspecific(ethr_ts_event_key__);
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * Exported functions
+ * --------------------------------------------------------------------------
+ */
+
+int
+ethr_init(ethr_init_data *id)
+{
+    int res;
+
+    if (!ethr_not_inited__)
+	return EINVAL;
+
+    ethr_not_inited__ = 0;
+
+    res = ethr_init_common__(id);
+    if (res != 0)
+	goto error;
+
+    child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT;
+    if (erts_get_cpu_configured(ethr_cpu_info__) == 1)
+	child_wait_spin_count = 0;
+
+    res = pthread_key_create(&ethr_ts_event_key__, ethr_ts_event_destructor__);
+
+    return 0;
+ error:
+    ethr_not_inited__ = 1;
+    return res;
+
+}
+
+int
+ethr_late_init(ethr_late_init_data *id)
+{
+    int res = ethr_late_init_common__(id);
+    if (res != 0)
+	return res;
+    ethr_not_completely_inited__ = 0;
+    return res;
+}
+
+int
+ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
+		ethr_thr_opts *opts)
+{
+    ethr_thr_wrap_data__ twd;
+    pthread_attr_t attr;
+    int res, dres;
+    int use_stack_size = (opts && opts->suggested_stack_size >= 0
+			  ? opts->suggested_stack_size
+			  : -1 /* Use system default */);
+
+#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
+    if (use_stack_size < 0)
+	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
+#endif
+
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!tid || !func) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+
+    ethr_atomic32_init(&twd.result, (ethr_sint32_t) -1);
+    twd.tse = ethr_get_ts_event();
+    twd.thr_func = func;
+    twd.arg = arg;
+
+    res = pthread_attr_init(&attr);
+    if (res != 0)
+	return res;
+
+    /* Error cleanup needed after this point */
+
+    /* Schedule child thread in system scope (if possible) ... */
+    res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
+    if (res != 0 && res != ENOTSUP)
+	goto error;
+
+    if (use_stack_size >= 0) {
+	size_t suggested_stack_size = (size_t) use_stack_size;
+	size_t stack_size;
+#ifdef ETHR_DEBUG
+	suggested_stack_size /= 2; /* Make sure we got margin */
+#endif
+#ifdef ETHR_STACK_GUARD_SIZE
+	/* The guard is at least on some platforms included in the stack size
+	   passed when creating threads */
+	suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE);
+#endif
+	if (suggested_stack_size < ethr_min_stack_size__)
+	    stack_size = ETHR_KW2B(ethr_min_stack_size__);
+	else if (suggested_stack_size > ethr_max_stack_size__)
+	    stack_size = ETHR_KW2B(ethr_max_stack_size__);
+	else
+	    stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
+	(void) pthread_attr_setstacksize(&attr, stack_size);
+    }
+
+#ifdef ETHR_STACK_GUARD_SIZE
+    (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE);
+#endif
+
+    /* Detached or joinable... */
+    res = pthread_attr_setdetachstate(&attr,
+				      (opts && opts->detached
+				       ? PTHREAD_CREATE_DETACHED
+				       : PTHREAD_CREATE_JOINABLE));
+    if (res != 0)
+	goto error;
+
+    /* Call prepare func if it exist */
+    if (ethr_thr_prepare_func__)
+	twd.prep_func_res = ethr_thr_prepare_func__();
+    else
+	twd.prep_func_res = NULL;
+
+    res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void*) &twd);
+
+    if (res == 0) {
+	int spin_count = child_wait_spin_count;
+
+	/* Wait for child to initialize... */
+	while (1) {
+	    ethr_sint32_t result;
+	    ethr_event_reset(&twd.tse->event);
+
+	    result = ethr_atomic32_read(&twd.result);
+	    if (result == 0)
+		break;
+
+	    if (result > 0) {
+		res = (int) result;
+		goto error;
+	    }
+
+	    res = ethr_event_swait(&twd.tse->event, spin_count);
+	    if (res != 0 && res != EINTR)
+		goto error;
+	    spin_count = 0;
+	}
+    }
+
+    /* Cleanup... */
+
+ error:
+    dres = pthread_attr_destroy(&attr);
+    if (res == 0)
+	res = dres;
+    if (ethr_thr_parent_func__)
+	ethr_thr_parent_func__(twd.prep_func_res);
+    return res;
+}
+
+int
+ethr_thr_join(ethr_tid tid, void **res)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_join((pthread_t) tid, res);
+}
+
+int
+ethr_thr_detach(ethr_tid tid)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_detach((pthread_t) tid);
+}
+
+void
+ethr_thr_exit(void *res)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return;
+    }
+#endif
+    thr_exit_cleanup();
+    pthread_exit(res);
+}
+
+ethr_tid
+ethr_self(void)
+{
+    return (ethr_tid) pthread_self();
+}
+
+int
+ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
+{
+    return pthread_equal((pthread_t) tid1, (pthread_t) tid2);
+}
+
+
+/*
+ * Thread specific events
+ */
+
+ethr_ts_event *
+ethr_get_ts_event(void)
+{
+    return ethr_get_ts_event__();
+}
+
+void
+ethr_leave_ts_event(ethr_ts_event *tsep)
+{
+    ethr_leave_ts_event__(tsep);
+}
+
+/*
+ * Thread specific data
+ */
+
+int
+ethr_tsd_key_create(ethr_tsd_key *keyp)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!keyp) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return pthread_key_create((pthread_key_t *) keyp, NULL);
+}
+
+int
+ethr_tsd_key_delete(ethr_tsd_key key)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_key_delete((pthread_key_t) key);
+}
+
+int
+ethr_tsd_set(ethr_tsd_key key, void *value)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_setspecific((pthread_key_t) key, value);
+}
+
+void *
+ethr_tsd_get(ethr_tsd_key key)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return NULL;
+    }
+#endif
+    return pthread_getspecific((pthread_key_t) key);
+}
+
+/*
+ * Signal functions
+ */
+
+#if ETHR_HAVE_ETHR_SIG_FUNCS
+
+int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!set && !oset) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+  return pthread_sigmask(how, set, oset);
+}
+
+int ethr_sigwait(const sigset_t *set, int *sig)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!set || !sig) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    if (sigwait(set, sig) < 0)
+	return errno;
+    return 0;
+}
+
+#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */
+
+ETHR_IMPL_NORETURN__
+ethr_abort__(void)
+{
+    abort();
+}
diff --git a/erts/lib_src/win/ethr_event.c b/erts/lib_src/win/ethr_event.c
new file mode 100644
index 0000000000..68f093f49c
--- /dev/null
+++ b/erts/lib_src/win/ethr_event.c
@@ -0,0 +1,123 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Author: Rickard Green
+ */
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_EVENT_IMPL__
+
+#include "ethread.h"
+
+/* --- Windows implementation of thread events ------------------------------ */
+
+#pragma intrinsic(_InterlockedExchangeAdd)
+#pragma intrinsic(_InterlockedCompareExchange)
+
+int
+ethr_event_init(ethr_event *e)
+{
+    e->state = ETHR_EVENT_OFF__;
+    e->handle = CreateEvent(NULL, FALSE, FALSE, NULL);
+    if (e->handle == INVALID_HANDLE_VALUE)
+	return ethr_win_get_errno__();
+    return 0;
+}
+
+int
+ethr_event_destroy(ethr_event *e)
+{
+    BOOL res = CloseHandle(e->handle);
+    return res == 0 ? ethr_win_get_errno__() : 0;
+}
+
+void
+ethr_event_set(ethr_event *e)
+{
+    ethr_event_set__(e);
+}
+
+void
+ethr_event_reset(ethr_event *e)
+{
+    ethr_event_reset__(e);
+}
+
+static ETHR_INLINE int
+wait(ethr_event *e, int spincount)
+{
+    LONG state;
+    DWORD code;
+    int sc, res, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    if (spincount < 0)
+	ETHR_FATAL_ERROR__(EINVAL);
+
+    sc = spincount;
+
+    while (1) {
+	long on;
+	while (1) {
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
+	    state = e->state;
+#else
+	    state = _InterlockedExchangeAdd(&e->state, (LONG) 0);
+#endif
+	    if (state == ETHR_EVENT_ON__)
+		return 0;
+	    if (sc == 0)
+		break;
+	    sc--;
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		res = ETHR_YIELD();
+		if (res != 0)
+		    ETHR_FATAL_ERROR__(res);
+	    }
+	}
+
+	if (state != ETHR_EVENT_OFF_WAITER__) {
+	    state = _InterlockedCompareExchange(&e->state,
+						ETHR_EVENT_OFF_WAITER__,
+						ETHR_EVENT_OFF__);
+	    if (state == ETHR_EVENT_ON__)
+		return 0;
+	    ETHR_ASSERT(state == ETHR_EVENT_OFF__);
+	}
+
+	code = WaitForSingleObject(e->handle, INFINITE);
+	if (code != WAIT_OBJECT_0)
+	    ETHR_FATAL_ERROR__(ethr_win_get_errno__());
+    }
+
+}
+
+int
+ethr_event_wait(ethr_event *e)
+{
+    return wait(e, 0);
+}
+
+int
+ethr_event_swait(ethr_event *e, int spincount)
+{
+    return wait(e, spincount);
+}
diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c
new file mode 100644
index 0000000000..789a360b11
--- /dev/null
+++ b/erts/lib_src/win/ethread.c
@@ -0,0 +1,586 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Windows native threads implementation of the ethread library
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_CHILD_WAIT_SPIN_COUNT 4000
+
+#undef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <process.h>
+#include <winerror.h>
+#include <stdio.h>
+#include <limits.h>
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHREAD_IMPL__
+
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#ifndef ETHR_HAVE_ETHREAD_DEFINES
+#error Missing configure defines
+#endif
+
+/* Argument passed to thr_wrapper() */
+typedef struct {
+    ethr_tid *tid;
+    ethr_atomic32_t result;
+    ethr_ts_event *tse;
+    void *(*thr_func)(void *);
+    void *arg;
+    void *prep_func_res;
+} ethr_thr_wrap_data__;
+
+#define ETHR_INVALID_TID_ID -1
+
+struct ethr_join_data_ {
+    HANDLE handle;
+    void *res;
+};
+
+static ethr_atomic_t thread_id_counter;
+static DWORD own_tid_key;
+static ethr_tid main_thr_tid;
+static int child_wait_spin_count;
+
+DWORD ethr_ts_event_key__;
+
+#define ETHR_GET_OWN_TID__	((ethr_tid *) TlsGetValue(own_tid_key))
+
+/*
+ * --------------------------------------------------------------------------
+ * Static functions
+ * --------------------------------------------------------------------------
+ */
+
+static void thr_exit_cleanup(ethr_tid *tid, void *res)
+{
+
+    ETHR_ASSERT(tid == ETHR_GET_OWN_TID__);
+
+    if (tid->jdata)
+	tid->jdata->res = res;
+
+    ethr_run_exit_handlers__();
+    ethr_ts_event_destructor__((void *) ethr_get_tse__());
+}
+
+static unsigned __stdcall thr_wrapper(LPVOID vtwd)
+{
+    ethr_tid my_tid;
+    ethr_sint32_t result;
+    void *res;
+    ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd;
+    void *(*thr_func)(void *) = twd->thr_func;
+    void *arg = twd->arg;
+    ethr_ts_event *tsep = NULL;
+
+    result = (ethr_sint32_t) ethr_make_ts_event__(&tsep);
+
+    if (result == 0) {
+	tsep->iflgs |= ETHR_TS_EV_ETHREAD;
+	my_tid = *twd->tid;
+	if (!TlsSetValue(own_tid_key, (LPVOID) &my_tid)) {
+	    result = (ethr_sint32_t) ethr_win_get_errno__();
+	    ethr_free_ts_event__(tsep);
+	}
+	else {
+	    if (ethr_thr_child_func__)
+		ethr_thr_child_func__(twd->prep_func_res);
+	}
+    }
+
+    tsep = twd->tse; /* We aren't allowed to follow twd after
+			result has been set! */
+
+    ethr_atomic32_set(&twd->result, result);
+
+    ethr_event_set(&tsep->event);
+
+    res = result == 0 ? (*thr_func)(arg) : NULL;
+
+    thr_exit_cleanup(&my_tid, res);
+    return 0;
+}
+
+/* internal exports */
+
+int
+ethr_win_get_errno__(void)
+{
+    return erts_get_last_win_errno();
+}
+
+int ethr_set_tse__(ethr_ts_event *tsep)
+{
+    return (TlsSetValue(ethr_ts_event_key__, (LPVOID) tsep)
+	    ? 0
+	    : ethr_win_get_errno__());
+}
+
+ethr_ts_event *ethr_get_tse__(void)
+{
+    return (ethr_ts_event *) TlsGetValue(ethr_ts_event_key__);
+}
+
+ETHR_IMPL_NORETURN__
+ethr_abort__(void)
+{
+#if 1
+    DebugBreak();
+#else
+    abort();
+#endif
+}
+
+/*
+ * ----------------------------------------------------------------------------
+ * Exported functions
+ * ----------------------------------------------------------------------------
+ */
+
+int
+ethr_init(ethr_init_data *id)
+{
+#ifdef _WIN32_WINNT
+    DWORD major = (_WIN32_WINNT >> 8) & 0xff;
+    DWORD minor = _WIN32_WINNT & 0xff;
+    OSVERSIONINFO os_version;
+#endif
+    int err = 0;
+    unsigned long i;
+
+    if (!ethr_not_inited__)
+	return EINVAL;
+
+#ifdef _WIN32_WINNT
+    os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+    GetVersionEx(&os_version);
+    if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT
+	|| os_version.dwMajorVersion < major
+	|| (os_version.dwMajorVersion == major
+	    && os_version.dwMinorVersion < minor))
+	return ENOTSUP;
+#endif
+    err = ethr_init_common__(id);
+    if (err)
+	goto error;
+
+    own_tid_key = TlsAlloc();
+    if (own_tid_key == TLS_OUT_OF_INDEXES)
+	goto error;
+
+    ethr_atomic_init(&thread_id_counter, 0);
+
+    main_thr_tid.id = 0;
+    main_thr_tid.jdata = NULL;
+
+    if (!TlsSetValue(own_tid_key, (LPVOID) &main_thr_tid))
+	goto error;
+
+    ETHR_ASSERT(&main_thr_tid == ETHR_GET_OWN_TID__);
+
+    ethr_ts_event_key__ = TlsAlloc();
+    if (ethr_ts_event_key__ == TLS_OUT_OF_INDEXES)
+	goto error;
+
+    child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT;
+    if (erts_get_cpu_configured(ethr_cpu_info__) == 1)
+	child_wait_spin_count = 0;
+
+    ethr_not_inited__ = 0;
+
+    return 0;
+
+ error:
+    ethr_not_inited__ = 1;
+    if (err == 0)
+	err = ethr_win_get_errno__();
+    ETHR_ASSERT(err != 0);
+    return err;
+}
+
+int
+ethr_late_init(ethr_late_init_data *id)
+{
+    int res = ethr_late_init_common__(id);
+    if (res != 0)
+	return res;
+    ethr_not_completely_inited__ = 0;
+    return res;
+}
+
+
+/*
+ * Thread functions.
+ */
+
+int
+ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
+		ethr_thr_opts *opts)
+{
+    HANDLE handle = INVALID_HANDLE_VALUE;
+    int err = 0;
+    ethr_thr_wrap_data__ twd;
+    DWORD code;
+    unsigned ID;
+    unsigned stack_size = 0; /* 0 = system default */
+    int use_stack_size = (opts && opts->suggested_stack_size >= 0
+			  ? opts->suggested_stack_size
+			  : -1 /* Use system default */);
+
+#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
+    if (use_stack_size < 0)
+	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
+#endif
+
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!tid || !func) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+
+    do {
+	tid->id = ethr_atomic_inc_read(&thread_id_counter);
+    } while (tid->id == ETHR_INVALID_TID_ID);
+
+    if (opts && opts->detached)
+	tid->jdata = NULL;
+    else {
+	tid->jdata = ethr_mem__.std.alloc(sizeof(struct ethr_join_data_));
+	if (!tid->jdata)
+	    return ENOMEM;
+	tid->jdata->handle = INVALID_HANDLE_VALUE;
+	tid->jdata->res = NULL;
+    }
+
+    if (use_stack_size >= 0) {
+	size_t suggested_stack_size = (size_t) use_stack_size;
+#ifdef ETHR_DEBUG
+	suggested_stack_size /= 2; /* Make sure we got margin */
+#endif
+	if (suggested_stack_size < ethr_min_stack_size__)
+	    stack_size = (unsigned) ETHR_KW2B(ethr_min_stack_size__);
+	else if (suggested_stack_size > ethr_max_stack_size__)
+	    stack_size = (unsigned) ETHR_KW2B(ethr_max_stack_size__);
+	else
+	    stack_size = (unsigned)
+	      ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
+    }
+
+    ethr_atomic32_init(&twd.result, -1);
+
+    twd.tid = tid;
+    twd.thr_func = func;
+    twd.arg = arg;
+    twd.tse = ethr_get_ts_event();
+
+    /* Call prepare func if it exist */
+    if (ethr_thr_prepare_func__)
+	twd.prep_func_res = ethr_thr_prepare_func__();
+    else
+	twd.prep_func_res = NULL;
+
+    /* spawn the thr_wrapper function */
+    handle = (HANDLE) _beginthreadex(NULL, stack_size, thr_wrapper,
+				     (LPVOID) &twd, 0, &ID);
+    if (handle == (HANDLE) 0) {
+	handle = INVALID_HANDLE_VALUE;
+	goto error;
+    }
+    else {
+	int spin_count = child_wait_spin_count;
+
+	ETHR_ASSERT(handle != INVALID_HANDLE_VALUE);
+
+	if (!tid->jdata)
+	    CloseHandle(handle);
+	else
+	    tid->jdata->handle = handle;
+
+	/* Wait for child to initialize... */
+	while (1) {
+	    ethr_sint32_t result;
+	    int err;
+	    ethr_event_reset(&twd.tse->event);
+
+	    result = ethr_atomic32_read(&twd.result);
+	    if (result == 0)
+		break;
+
+	    if (result > 0) {
+		err = (int) result;
+		goto error;
+	    }
+
+	    err = ethr_event_swait(&twd.tse->event, spin_count);
+	    if (err && err != EINTR)
+		goto error;
+	    spin_count = 0;
+	}
+    }
+
+    if (ethr_thr_parent_func__)
+	ethr_thr_parent_func__(twd.prep_func_res);
+
+    if (twd.tse)
+	ethr_leave_ts_event(twd.tse);
+
+    return 0;
+
+ error:
+
+    if (err == 0)
+	err = ethr_win_get_errno__();
+    ETHR_ASSERT(err != 0);
+
+    if (ethr_thr_parent_func__)
+	ethr_thr_parent_func__(twd.prep_func_res);
+
+    if (handle != INVALID_HANDLE_VALUE) {
+	WaitForSingleObject(handle, INFINITE);
+	CloseHandle(handle);
+    }
+
+    if (tid->jdata) {
+	ethr_mem__.std.free(tid->jdata);
+	tid->jdata = NULL;
+    }
+
+    tid->id = ETHR_INVALID_TID_ID;
+
+    if (twd.tse)
+	ethr_leave_ts_event(twd.tse);
+
+    return err;
+}
+
+int ethr_thr_join(ethr_tid tid, void **res)
+{
+    DWORD code;
+
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+
+    if (tid.id == ETHR_INVALID_TID_ID || !tid.jdata)
+	return EINVAL;
+
+    /* Wait for thread to terminate */
+    code = WaitForSingleObject(tid.jdata->handle, INFINITE);
+    if (code != WAIT_OBJECT_0)
+	return ethr_win_get_errno__();
+
+    CloseHandle(tid.jdata->handle);
+    tid.jdata->handle = INVALID_HANDLE_VALUE;
+
+    if (res)
+	*res = tid.jdata->res;
+
+    /*
+     * User better not try to join or detach again; or
+     * bad things will happen... (users responsibility)
+     */
+
+    ethr_mem__.std.free(tid.jdata);
+
+    return 0;
+}
+
+
+int
+ethr_thr_detach(ethr_tid tid)
+{
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+
+    if (tid.id == ETHR_INVALID_TID_ID || !tid.jdata)
+	return EINVAL;
+
+    CloseHandle(tid.jdata->handle);
+    tid.jdata->handle = INVALID_HANDLE_VALUE;
+
+    /*
+     * User better not try to join or detach again; or
+     * bad things will happen... (users responsibility)
+     */
+
+    ethr_mem__.std.free(tid.jdata);
+
+    return 0;
+}
+
+
+void
+ethr_thr_exit(void *res)
+{
+    ethr_tid *tid;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return;
+    }
+#endif
+    tid = ETHR_GET_OWN_TID__;
+    if (!tid) {
+	ETHR_ASSERT(0);
+	_endthreadex((unsigned) 0);
+    }
+    thr_exit_cleanup(tid, res);
+    _endthreadex((unsigned) 0);
+}
+
+ethr_tid
+ethr_self(void)
+{
+    ethr_tid *tid;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ethr_tid dummy_tid = {ETHR_INVALID_TID_ID, NULL};
+	ETHR_ASSERT(0);
+	return dummy_tid;
+    }
+#endif
+    /* It is okay for non-ethreads (i.e. native win32 threads) to call
+       ethr_self(). They will however be returned an invalid tid. */
+    tid = ETHR_GET_OWN_TID__;
+    if (!tid) {
+	ethr_tid dummy_tid = {ETHR_INVALID_TID_ID, NULL};
+	return dummy_tid;
+    }
+    return *tid;
+}
+
+int
+ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
+{
+    /* An invalid tid does not equal any tid, not even an invalid tid */
+    return tid1.id == tid2.id && tid1.id != ETHR_INVALID_TID_ID;
+}
+
+/*
+ * Thread specific data
+ */
+
+int
+ethr_tsd_key_create(ethr_tsd_key *keyp)
+{
+    DWORD key;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!keyp) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    key = TlsAlloc();
+    if (key == TLS_OUT_OF_INDEXES)
+	return ethr_win_get_errno__();
+    *keyp = (ethr_tsd_key) key;
+    return 0;
+}
+
+int
+ethr_tsd_key_delete(ethr_tsd_key key)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    if (!TlsFree((DWORD) key))
+	return ethr_win_get_errno__();
+    return 0;
+}
+
+int
+ethr_tsd_set(ethr_tsd_key key, void *value)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    if (!TlsSetValue((DWORD) key, (LPVOID) value))
+	return ethr_win_get_errno__();
+    return 0;
+}
+
+void *
+ethr_tsd_get(ethr_tsd_key key)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return NULL;
+    }
+#endif
+    return (void *) TlsGetValue((DWORD) key);
+}
+
+
+/*
+ * Thread specific events
+ */
+
+ethr_ts_event *
+ethr_get_ts_event(void)
+{
+    return ethr_get_ts_event__();
+}
+
+void
+ethr_leave_ts_event(ethr_ts_event *tsep)
+{
+    ethr_leave_ts_event__(tsep);
+}
+
+ethr_ts_event *
+ethr_create_ts_event__(void)
+{
+    ethr_ts_event *tsep;
+    ethr_make_ts_event__(&tsep);
+    return tsep;
+}
diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam
index 304ac41dce..dccb92a5af 100644
--- a/erts/preloaded/ebin/erl_prim_loader.beam
+++ b/erts/preloaded/ebin/erl_prim_loader.beam
diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam
index 21e5525b2f..b537546071 100644
--- a/erts/preloaded/ebin/erlang.beam
+++ b/erts/preloaded/ebin/erlang.beam
diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam
index 8ccc553c13..901def5eba 100644
--- a/erts/preloaded/ebin/init.beam
+++ b/erts/preloaded/ebin/init.beam
diff --git a/erts/preloaded/ebin/otp_ring0.beam b/erts/preloaded/ebin/otp_ring0.beam
index 8ae73ea9a7..62ebbb9ffe 100644
--- a/erts/preloaded/ebin/otp_ring0.beam
+++ b/erts/preloaded/ebin/otp_ring0.beam
diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam
index 3acda843fd..66fcb9e6a9 100644
--- a/erts/preloaded/ebin/prim_file.beam
+++ b/erts/preloaded/ebin/prim_file.beam
diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam
index 9457b6d360..9894050cba 100644
--- a/erts/preloaded/ebin/prim_inet.beam
+++ b/erts/preloaded/ebin/prim_inet.beam
diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam
index 6837cb4661..9ee70d59ec 100644
--- a/erts/preloaded/ebin/prim_zip.beam
+++ b/erts/preloaded/ebin/prim_zip.beam
diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam
index 4e4f85d312..542e266f88 100644
--- a/erts/preloaded/ebin/zlib.beam
+++ b/erts/preloaded/ebin/zlib.beam
diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile
index 785ad531f3..145638802f 100644
--- a/erts/preloaded/src/Makefile
+++ b/erts/preloaded/src/Makefile
@@ -1,7 +1,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2008-2009. All Rights Reserved.
+# Copyright Ericsson AB 2008-2010. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -20,8 +20,7 @@
 # be used when the preloaded modules actually are to be updated (i.e. the 
 # beam files are to be recompiled, which is normally not done). 
 # The beam files are placed in the current directory and should be copied 
-# to the ../ebin directory by using the commit target (only works in 
-# clearcase). 
+# to the ../ebin directory by using the copy target.
 
 include $(ERL_TOP)/make/target.mk
 include $(ERL_TOP)/make/$(TARGET)/otp.mk
@@ -62,26 +61,9 @@ debug opt: $(TARGET_FILES)
 clean:
 	rm -f $(TARGET_FILES)
 
-prepare:
-	cleartool co -nc $(STATIC_EBIN)/*
-	cleartool co -nc $(STATIC_EBIN)
-
 copy:
-	for x in *.beam; do\
-	  if test '!' -f $(STATIC_EBIN)/$$x; then\
-	    cleartool mkelem -nc $$x;\
-	  fi;\
-	done
 	cp *.beam $(STATIC_EBIN)
 
-commit:
-	cleartool ci -ident -nc $(STATIC_EBIN)/*.beam
-	cleartool ci -ident -nc $(STATIC_EBIN)
-
-cancel:
-	-cleartool unco -rm $(STATIC_EBIN)
-	-cleartool unco -rm $(STATIC_EBIN)/*.beam
-
 
 include $(ERL_TOP)/make/otp_release_targets.mk
 
diff --git a/erts/preloaded/src/erl_prim_loader.erl b/erts/preloaded/src/erl_prim_loader.erl
index a13292d5ab..024b20eadb 100644
--- a/erts/preloaded/src/erl_prim_loader.erl
+++ b/erts/preloaded/src/erl_prim_loader.erl
@@ -405,7 +405,7 @@ handle_timeout(State = #state{loader = inet}, Parent) ->
 efile_multi_get_file_from_port(State, ModFiles, Paths, Fun) ->
     Ref = make_ref(),
     %% More than 200 processes is no gain.
-    Max = min(200, erlang:system_info(thread_pool_size)),
+    Max = erlang:min(200, erlang:system_info(thread_pool_size)),
     efile_multi_get_file_from_port2(ModFiles, 0, Max, State, Paths, Fun, Ref, ok).
 
 efile_multi_get_file_from_port2([MF | MFs], Out, Max, State, Paths, Fun, Ref, Ret) when Out < Max ->
@@ -1189,9 +1189,6 @@ keyins(X, I, [Y | T]) when X < element(I,Y) -> [X,Y|T];
 keyins(X, I, [Y | T]) -> [Y | keyins(X, I, T)];
 keyins(X, _I, []) -> [X].
 
-min(X, Y) when X < Y -> X;
-min(_X, Y) -> Y.
-
 to_strs([P|Paths]) when is_atom(P) ->
     [atom_to_list(P)|to_strs(Paths)];
 to_strs([P|Paths]) when is_list(P) ->
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 1edb5e72db..4679a916c7 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -44,7 +44,14 @@
 -deprecated([hash/2]).
 -deprecated([concat_binary/1]).
 
--compile(nowarn_bif_clash).
+% Get rid of autoimports of spawn to avoid clashes with ourselves.
+-compile({no_auto_import,[spawn/1]}).
+-compile({no_auto_import,[spawn/4]}).
+-compile({no_auto_import,[spawn_link/1]}).
+-compile({no_auto_import,[spawn_link/4]}).
+-compile({no_auto_import,[spawn_opt/2]}).
+-compile({no_auto_import,[spawn_opt/4]}).
+-compile({no_auto_import,[spawn_opt/5]}).
 
 %%--------------------------------------------------------------------------
 
@@ -55,10 +62,10 @@
 %%--------------------------------------------------------------------------
 
 apply(Fun, Args) ->
-    apply(Fun, Args).
+    erlang:apply(Fun, Args).
 
 apply(Mod, Name, Args) ->
-    apply(Mod, Name, Args).
+    erlang:apply(Mod, Name, Args).
 
 %% Spawns with a fun
 
@@ -420,17 +427,11 @@ delay_trap(Result, Timeout) -> receive after Timeout -> Result end.
 %% have to reflect that, which we cannot forsee.
 %%
 set_cookie(Node, C) when Node =/= nonode@nohost, is_atom(Node) ->
-    Res = case C of
-	      _ when is_atom(C) ->
-		  auth:set_cookie(Node, C);
-	      {CI,CO} when is_atom(CI), is_atom(CO) ->
-		  auth:set_cookie(Node, {CI, CO});
-	      _ ->
-		  error
-	  end,
-    case Res of
-	error -> exit(badarg);
-	Other -> Other
+    case is_atom(C) of
+	true ->
+	    auth:set_cookie(Node, C);
+	false ->
+	    error(badarg)
     end.
 
 -spec get_cookie() -> atom().
diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl
index 3b98b9cddc..24430a3d40 100644
--- a/erts/preloaded/src/init.erl
+++ b/erts/preloaded/src/init.erl
@@ -51,6 +51,9 @@
 	 get_status/0,boot/1,get_arguments/0,get_plain_arguments/0,
 	 get_argument/1,script_id/0]).
 
+%% for the on_load functionality; not for general use
+-export([run_on_load_handlers/0]).
+
 %% internal exports
 -export([fetch_loaded/0,ensure_loaded/1,make_permanent/2,
 	 notify_when_started/1,wait_until_started/0, 
@@ -69,6 +72,7 @@
 		script_id = [],
 		loaded = [],
 		subscribed = []}).
+-type state() :: #state{}.
 
 -define(ON_LOAD_HANDLER, init__boot__on_load_handler).
 
@@ -143,10 +147,10 @@ restart() -> init ! {stop,restart}, ok.
 -spec reboot() -> 'ok'.
 reboot() -> init ! {stop,reboot}, ok.
 
--spec stop() -> no_return().
+-spec stop() -> 'ok'.
 stop() -> init ! {stop,stop}, ok.
 
--spec stop(non_neg_integer() | string()) -> no_return().
+-spec stop(non_neg_integer() | string()) -> 'ok'.
 stop(Status) -> init ! {stop,{stop,Status}}, ok.
 
 -spec boot([binary()]) -> no_return().
@@ -275,7 +279,7 @@ crash(String, List) ->
     halt(halt_string(String, List)).
 
 %% Status is {InternalStatus,ProvidedStatus}
--spec boot_loop(pid(), #state{}) -> no_return().
+-spec boot_loop(pid(), state()) -> no_return().
 boot_loop(BootPid, State) ->
     receive
 	{BootPid,loaded,ModLoaded} ->
@@ -308,24 +312,6 @@ boot_loop(BootPid, State) ->
 	{stop,Reason} ->
 	    stop(Reason,State);
 	{From,fetch_loaded} ->   %% Fetch and reset initially loaded modules.
-	    case whereis(?ON_LOAD_HANDLER) of
-		undefined ->
-		    %% There is no on_load handler process,
-		    %% probably because init:restart/0 has been
-		    %% called and it is not the first time we
-		    %% pass through here.
-		    ok;
-		Pid when is_pid(Pid) ->
-		    Pid ! run_on_load,
-		    receive
-			{'EXIT',Pid,on_load_done} ->
-			    ok;
-			{'EXIT',Pid,Res} ->
-			    %% Failure to run an on_load handler.
-			    %% This is fatal during start-up.
-			    exit(Res)
-		    end
-	    end,
 	    From ! {init,State#state.loaded},
 	    garb_boot_loop(BootPid,State#state{loaded = []});
 	{From,{ensure_loaded,Module}} ->
@@ -736,6 +722,7 @@ do_boot(Init,Flags,Start) ->
     BootList = get_boot(BootFile,Root),
     LoadMode = b2a(get_flag('-mode',Flags,false)),
     Deb = b2a(get_flag('-init_debug',Flags,false)),
+    catch ?ON_LOAD_HANDLER ! {init_debug_flag,Deb},
     BootVars = get_flag_args('-boot_var',Flags),
     ParallelLoad = 
 	(Pgm =:= "efile") and (erlang:system_info(thread_pool_size) > 0),
@@ -1335,23 +1322,44 @@ archive_extension() ->
 %%% Support for handling of on_load functions.
 %%%
 
+run_on_load_handlers() ->
+    Ref = monitor(process, ?ON_LOAD_HANDLER),
+    catch ?ON_LOAD_HANDLER ! run_on_load,
+    receive
+	{'DOWN',Ref,process,_,noproc} ->
+	    %% There is no on_load handler process,
+	    %% probably because init:restart/0 has been
+	    %% called and it is not the first time we
+	    %% pass through here.
+	    ok;
+	{'DOWN',Ref,process,_,on_load_done} ->
+	    ok;
+	{'DOWN',Ref,process,_,Res} ->
+	    %% Failure to run an on_load handler.
+	    %% This is fatal during start-up.
+	    exit(Res)
+    end.
+
 start_on_load_handler_process() ->
     register(?ON_LOAD_HANDLER,
-	     spawn_link(fun on_load_handler_init/0)).
+	     spawn(fun on_load_handler_init/0)).
 
 on_load_handler_init() ->
-    on_load_loop([]).
+    on_load_loop([], false).
 
-on_load_loop(Mods) ->
+on_load_loop(Mods, Debug0) ->
     receive
+	{init_debug_flag,Debug} ->
+	    on_load_loop(Mods, Debug);
 	{loaded,Mod} ->
-	    on_load_loop([Mod|Mods]);
+	    on_load_loop([Mod|Mods], Debug0);
 	run_on_load ->
-	    run_on_load_handlers(Mods),
+	    run_on_load_handlers(Mods, Debug0),
 	    exit(on_load_done)
     end.
 
-run_on_load_handlers([M|Ms]) ->
+run_on_load_handlers([M|Ms], Debug) ->
+    debug(Debug, {running_on_load_handler,M}),
     Fun = fun() ->
 		  Res = erlang:call_on_load_function(M),
 		  exit(Res)
@@ -1363,9 +1371,12 @@ run_on_load_handlers([M|Ms]) ->
 	    erlang:finish_after_on_load(M, Keep),
 	    case Keep of
 		false ->
-		    exit({on_load_function_failed,M});
+		    Error = {on_load_function_failed,M},
+		    debug(Debug, Error),
+		    exit(Error);
 		true ->
-		    run_on_load_handlers(Ms)
+		    debug(Debug, {on_load_handler_returned_ok,M}),
+		    run_on_load_handlers(Ms, Debug)
 	    end
     end;
-run_on_load_handlers([]) -> ok.
+run_on_load_handlers([], _) -> ok.
diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl
index 43e6f6cd88..8e0790e74a 100644
--- a/erts/preloaded/src/prim_file.erl
+++ b/erts/preloaded/src/prim_file.erl
@@ -1,19 +1,19 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2000-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 2000-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 -module(prim_file).
@@ -25,7 +25,7 @@
 %%% Interface towards a single file's contents. Uses ?FD_DRV.
 
 %% Generic file contents operations
--export([open/2, close/1, sync/1, position/2, truncate/1,
+-export([open/2, close/1, datasync/1, sync/1, advise/4, position/2, truncate/1,
 	 write/2, pwrite/2, pwrite/3, read/2, read_line/1, pread/2, pread/3, copy/3]).
 
 %% Specialized file operations
@@ -64,7 +64,7 @@
 -include("file.hrl").
 
 -define(DRV,    efile).
--define(FD_DRV, efile).
+-define(FD_DRV, "efile").
 
 -define(LARGEFILESIZE, (1 bsl 63)).
 
@@ -96,6 +96,8 @@
 -define(FILE_IPREAD,           27).
 -define(FILE_ALTNAME,          28).
 -define(FILE_READ_LINE,        29).
+-define(FILE_FDATASYNC,        30).
+-define(FILE_ADVISE,           31).
 
 %% Driver responses
 -define(FILE_RESP_OK,          0).
@@ -107,6 +109,8 @@
 -define(FILE_RESP_LDATA,       6).
 -define(FILE_RESP_N2DATA,      7).
 -define(FILE_RESP_EOF,         8).
+-define(FILE_RESP_FNAME,       9).
+-define(FILE_RESP_ALL_DATA,   10).
 
 %% Open modes for the driver's open function.
 -define(EFILE_MODE_READ,       1).
@@ -114,9 +118,10 @@
 -define(EFILE_MODE_READ_WRITE, 3).  
 -define(EFILE_MODE_APPEND,     4).
 -define(EFILE_COMPRESSED,      8).
+-define(EFILE_MODE_EXCL,       16).
 
 %% Use this mask to get just the mode bits to be passed to the driver.
--define(EFILE_MODE_MASK, 15).
+-define(EFILE_MODE_MASK, 31).
 
 %% Seek modes for the driver's seek function.
 -define(EFILE_SEEK_SET, 0).
@@ -130,6 +135,13 @@
 %% IPREAD variants
 -define(IPREAD_S32BU_P32BU, 0).
 
+%% POSIX file advises
+-define(POSIX_FADV_NORMAL,     0).
+-define(POSIX_FADV_RANDOM,     1).
+-define(POSIX_FADV_SEQUENTIAL, 2).
+-define(POSIX_FADV_WILLNEED,   3).
+-define(POSIX_FADV_DONTNEED,   4).
+-define(POSIX_FADV_NOREUSE,    5).
 
 
 %%%-----------------------------------------------------------------
@@ -143,7 +155,7 @@
 %% Opens a file using the driver port Port. Returns {error, Reason}
 %% | {ok, FileDescriptor}
 open(Port, File, ModeList) when is_port(Port), 
-                                is_list(File), 
+                                (is_list(File) orelse is_binary(File)), 
                                 is_list(ModeList) ->
     case open_mode(ModeList) of
 	{Mode, _Portopts, _Setopts} ->
@@ -155,10 +167,11 @@ open(_,_,_) ->
     {error, badarg}.
 
 %% Opens a file. Returns {error, Reason} | {ok, FileDescriptor}.
-open(File, ModeList) when is_list(File), is_list(ModeList) ->
+open(File, ModeList) when (is_list(File) orelse is_binary(File)), 
+			  is_list(ModeList) ->
     case open_mode(ModeList) of
 	{Mode, Portopts, Setopts} ->
-	    open_int({?FD_DRV, Portopts}, File, Mode, Setopts);
+	    open_int({?FD_DRV, Portopts},File, Mode, Setopts);
 	Reason ->
 	    {error, Reason}
     end;
@@ -186,7 +199,7 @@ open_int({Driver, Portopts}, File, Mode, Setopts) ->
     end;
 open_int(Port, File, Mode, Setopts) ->
     M = Mode band ?EFILE_MODE_MASK,
-    case drv_command(Port, [<<?FILE_OPEN, M:32>>, File, 0]) of
+    case drv_command(Port, [<<?FILE_OPEN, M:32>>, pathname(File)]) of
 	{ok, Number} ->
 	    open_int_setopts(Port, Number, Setopts);
 	Error ->
@@ -220,7 +233,35 @@ close(#file_descriptor{module = ?MODULE, data = {Port, _}}) ->
 close(Port) when is_port(Port) ->
     drv_close(Port).
 
+-define(ADVISE(Offs, Len, Adv),
+	<<?FILE_ADVISE, Offs:64/signed, Len:64/signed,
+	  Adv:32/signed>>).
 
+%% Returns {error, Reason} | ok.
+advise(#file_descriptor{module = ?MODULE, data = {Port, _}},
+       Offset, Length, Advise) ->
+    case Advise of
+	normal ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_NORMAL),
+	    drv_command(Port, Cmd);
+	random ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_RANDOM),
+	    drv_command(Port, Cmd);
+	sequential ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_SEQUENTIAL),
+	    drv_command(Port, Cmd);
+	will_need ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_WILLNEED),
+	    drv_command(Port, Cmd);
+	dont_need ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_DONTNEED),
+	    drv_command(Port, Cmd);
+	no_reuse ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_NOREUSE),
+	    drv_command(Port, Cmd);
+	_ ->
+	    {error, einval}
+    end.
 
 %% Returns {error, Reason} | ok.
 write(#file_descriptor{module = ?MODULE, data = {Port, _}}, Bytes) ->
@@ -292,6 +333,9 @@ pwrite(#file_descriptor{module = ?MODULE}, _, _) ->
     {error, badarg}.
 
 
+%% Returns {error, Reason} | ok.
+datasync(#file_descriptor{module = ?MODULE, data = {Port, _}}) ->
+    drv_command(Port, [?FILE_FDATASYNC]).
 
 %% Returns {error, Reason} | ok.
 sync(#file_descriptor{module = ?MODULE, data = {Port, _}}) ->
@@ -448,7 +492,7 @@ ipread_s32bu_p32bu(#file_descriptor{module = ?MODULE, data = {_, _}},
 
 
 %% Returns {ok, Contents} | {error, Reason}
-read_file(File) ->
+read_file(File) when (is_list(File) orelse is_binary(File)) ->
     case drv_open(?FD_DRV, [binary]) of
 	{ok, Port} ->
 	    Result = read_file(Port, File),
@@ -456,11 +500,14 @@ read_file(File) ->
 	    Result;
 	{error, _} = Error ->
 	    Error
-    end.
+    end;
+read_file(_) ->
+    {error, badarg}.
 
 %% Takes a Port opened with open/1.
-read_file(Port, File) when is_port(Port) ->
-    Cmd = [?FILE_READ_FILE | File],
+read_file(Port, File) when is_port(Port),
+			   (is_list(File) orelse is_binary(File)) ->
+    Cmd = [?FILE_READ_FILE | pathname(File)],
     case drv_command(Port, Cmd) of
 	{error, enomem} ->
 	    %% It could possibly help to do a 
@@ -471,12 +518,14 @@ read_file(Port, File) when is_port(Port) ->
 	    drv_command(Port, Cmd);
 	Result ->
 	    Result
-    end.
+    end;
+read_file(_,_) ->
+    {error, badarg}.
 
     
 
 %% Returns {error, Reason} | ok.
-write_file(File, Bin) ->
+write_file(File, Bin) when (is_list(File) orelse is_binary(File)) ->
     case open(File, [binary, write]) of
 	{ok, Handle} ->
 	    Result = write(Handle, Bin),
@@ -484,8 +533,10 @@ write_file(File, Bin) ->
 	    Result;
 	Error ->
 	    Error
-    end.
-
+    end;
+write_file(_, _) -> 
+    {error, badarg}.
+    
 
 
 %%%-----------------------------------------------------------------
@@ -498,7 +549,7 @@ write_file(File, Bin) ->
 %% Returns {ok, Port}, the Port should be used as first argument in all
 %% the following functions. Returns {error, Reason} upon failure.
 start() ->
-    try erlang:open_port({spawn, atom_to_list(?DRV)}, []) of
+    try erlang:open_port({spawn, atom_to_list(?DRV)}, [binary]) of
 	Port ->
 	    {ok, Port}
     catch
@@ -555,7 +606,7 @@ get_cwd(_, _) ->
     {error, badarg}.
 
 get_cwd_int(Drive) ->
-    get_cwd_int({?DRV, []}, Drive).
+    get_cwd_int({?DRV, [binary]}, Drive).
 
 get_cwd_int(Port, Drive) ->
     drv_command(Port, <<?FILE_PWD, Drive>>).
@@ -565,7 +616,7 @@ get_cwd_int(Port, Drive) ->
 %% set_cwd/{1,2}
 
 set_cwd(Dir) ->
-    set_cwd_int({?DRV, []}, Dir).
+    set_cwd_int({?DRV, [binary]}, Dir).
 
 set_cwd(Port, Dir) when is_port(Port) ->
     set_cwd_int(Port, Dir).
@@ -591,89 +642,88 @@ set_cwd_int(Port, Dir0) ->
 	 end),
     %% Dir is now either a string or an EXIT tuple.
     %% An EXIT tuple will fail in the following catch.
-    drv_command(Port, [?FILE_CHDIR, Dir, 0]).
+    drv_command(Port, [?FILE_CHDIR, pathname(Dir)]).
 
 
 
 %% delete/{1,2}
 
 delete(File) ->
-    delete_int({?DRV, []}, File).
+    delete_int({?DRV, [binary]}, File).
 
 delete(Port, File) when is_port(Port) ->
     delete_int(Port, File).
 
 delete_int(Port, File) ->
-    drv_command(Port, [?FILE_DELETE, File, 0]).
+    drv_command(Port, [?FILE_DELETE, pathname(File)]).
 
 
 
 %% rename/{2,3}
 
 rename(From, To) ->
-    rename_int({?DRV, []}, From, To).
+    rename_int({?DRV, [binary]}, From, To).
 
 rename(Port, From, To) when is_port(Port) ->
     rename_int(Port, From, To).
 
 rename_int(Port, From, To) ->
-    drv_command(Port, [?FILE_RENAME, From, 0, To, 0]).
+    drv_command(Port, [?FILE_RENAME, pathname(From), pathname(To)]).
 
 
 
 %% make_dir/{1,2}
 
 make_dir(Dir) ->
-    make_dir_int({?DRV, []}, Dir).
+    make_dir_int({?DRV, [binary]}, Dir).
 
 make_dir(Port, Dir) when is_port(Port) ->
     make_dir_int(Port, Dir).
 
 make_dir_int(Port, Dir) ->
-    drv_command(Port, [?FILE_MKDIR, Dir, 0]).
+    drv_command(Port, [?FILE_MKDIR, pathname(Dir)]).
 
 
 
 %% del_dir/{1,2}
 
 del_dir(Dir) ->
-    del_dir_int({?DRV, []}, Dir).
+    del_dir_int({?DRV, [binary]}, Dir).
 
 del_dir(Port, Dir) when is_port(Port) ->
     del_dir_int(Port, Dir).
 
 del_dir_int(Port, Dir) ->
-    drv_command(Port, [?FILE_RMDIR, Dir, 0]).
+    drv_command(Port, [?FILE_RMDIR, pathname(Dir)]).
 
 
 
 %% read_file_info/{1,2}
 
 read_file_info(File) ->
-    read_file_info_int({?DRV, []}, File).
+    read_file_info_int({?DRV, [binary]}, File).
 
 read_file_info(Port, File) when is_port(Port) ->
     read_file_info_int(Port, File).
 
 read_file_info_int(Port, File) ->
-    drv_command(Port, [?FILE_FSTAT, File, 0]).
+    drv_command(Port, [?FILE_FSTAT, pathname(File)]).
 
 %% altname/{1,2}
 
 altname(File) ->
-    altname_int({?DRV, []}, File).
+    altname_int({?DRV, [binary]}, File).
 
 altname(Port, File) when is_port(Port) ->
     altname_int(Port, File).
 
 altname_int(Port, File) ->
-    drv_command(Port, [?FILE_ALTNAME, File, 0]).
-
+    drv_command(Port, [?FILE_ALTNAME, pathname(File)]).
 
 %% write_file_info/{2,3}
 
 write_file_info(File, Info) ->
-    write_file_info_int({?DRV, []}, File, Info).
+    write_file_info_int({?DRV, [binary]}, File, Info).
 
 write_file_info(Port, File, Info) when is_port(Port) ->
     write_file_info_int(Port, File, Info).
@@ -699,72 +749,72 @@ write_file_info_int(Port,
 			date_to_bytes(Atime), 
 			date_to_bytes(Mtime), 
 			date_to_bytes(Ctime),
-			File, 0]).
+			pathname(File)]).
 
 
 
 %% make_link/{2,3}
 
 make_link(Old, New) ->
-    make_link_int({?DRV, []}, Old, New).
+    make_link_int({?DRV, [binary]}, Old, New).
 
 make_link(Port, Old, New) when is_port(Port) ->
     make_link_int(Port, Old, New).
 
 make_link_int(Port, Old, New) ->
-    drv_command(Port, [?FILE_LINK, Old, 0, New, 0]).
+    drv_command(Port, [?FILE_LINK, pathname(Old), pathname(New)]).
 
 
 
 %% make_symlink/{2,3}
 
 make_symlink(Old, New) ->
-    make_symlink_int({?DRV, []}, Old, New).
+    make_symlink_int({?DRV, [binary]}, Old, New).
 
 make_symlink(Port, Old, New) when is_port(Port) ->
     make_symlink_int(Port, Old, New).
 
 make_symlink_int(Port, Old, New) ->
-    drv_command(Port, [?FILE_SYMLINK, Old, 0, New, 0]).
+    drv_command(Port, [?FILE_SYMLINK, pathname(Old), pathname(New)]).
 
 
 
 %% read_link/{2,3}
 
 read_link(Link) ->
-    read_link_int({?DRV, []}, Link).
+    read_link_int({?DRV, [binary]}, Link).
 
 read_link(Port, Link) when is_port(Port) ->
     read_link_int(Port, Link).
 
 read_link_int(Port, Link) ->
-    drv_command(Port, [?FILE_READLINK, Link, 0]).
+    drv_command(Port, [?FILE_READLINK, pathname(Link)]).
 
 
 
 %% read_link_info/{2,3}
 
 read_link_info(Link) ->
-    read_link_info_int({?DRV, []}, Link).
+    read_link_info_int({?DRV, [binary]}, Link).
 
 read_link_info(Port, Link) when is_port(Port) ->
     read_link_info_int(Port, Link).
 
 read_link_info_int(Port, Link) ->
-    drv_command(Port, [?FILE_LSTAT, Link, 0]).
+    drv_command(Port, [?FILE_LSTAT, pathname(Link)]).
 
 
 
 %% list_dir/{1,2}
 
 list_dir(Dir) ->
-    list_dir_int({?DRV, []}, Dir).
+    list_dir_int({?DRV, [binary]}, Dir).
 
 list_dir(Port, Dir) when is_port(Port) ->
     list_dir_int(Port, Dir).
 
 list_dir_int(Port, Dir) ->
-    drv_command(Port, [?FILE_READDIR, Dir, 0], []).
+    drv_command(Port, [?FILE_READDIR, pathname(Dir)], []).
 
 
 
@@ -782,7 +832,7 @@ drv_open(Driver, Portopts) ->
 	    {ok, Port}
     catch
 	error:Reason ->
-	    {error,Reason}
+	    {error, Reason}
     end.
 
 
@@ -814,9 +864,9 @@ drv_command(Port, Command) ->
 drv_command(Port, Command, R) when is_binary(Command) ->
     drv_command(Port, Command, true, R);
 drv_command(Port, Command, R) ->
-    try erlang:iolist_to_binary(Command) of
-	Bin ->
-	    drv_command(Port, Bin, true, R)
+    try erlang:iolist_size(Command) of
+	_ ->
+	    drv_command(Port, Command, true, R)
     catch
 	error:Reason ->
 	    {error, Reason}
@@ -918,6 +968,8 @@ open_mode([compressed|Rest], Mode, Portopts, Setopts) ->
 open_mode([append|Rest], Mode, Portopts, Setopts) ->
     open_mode(Rest, Mode bor ?EFILE_MODE_APPEND bor ?EFILE_MODE_WRITE, 
 	      Portopts, Setopts);
+open_mode([exclusive|Rest], Mode, Portopts, Setopts) ->
+    open_mode(Rest, Mode bor ?EFILE_MODE_EXCL, Portopts, Setopts);
 open_mode([delayed_write|Rest], Mode, Portopts, Setopts) ->
     open_mode([{delayed_write, 64*1024, 2000}|Rest], Mode,
 	      Portopts, Setopts);
@@ -983,16 +1035,14 @@ lseek_position(_) ->
 
 translate_response(?FILE_RESP_OK, []) ->
     ok;
-translate_response(?FILE_RESP_OK, Data) ->
-    {ok, Data};
 translate_response(?FILE_RESP_ERROR, List) when is_list(List) ->
     {error, list_to_atom(List)};
 translate_response(?FILE_RESP_NUMBER, List) ->
     {N, []} = get_uint64(List),
     {ok, N};
 translate_response(?FILE_RESP_DATA, List) ->
-    {N, Data} = get_uint64(List),
-    {ok, {N, Data}};
+    {_N, _Data} = ND = get_uint64(List),
+    {ok, ND};
 translate_response(?FILE_RESP_INFO, List) when is_list(List) ->
     {ok, transform_info_ints(get_uint32s(List))};
 translate_response(?FILE_RESP_NUMERR, L0) ->
@@ -1031,6 +1081,14 @@ translate_response(?FILE_RESP_N2DATA = X, L0) when is_list(L0) ->
     end;
 translate_response(?FILE_RESP_EOF, []) ->
     eof;
+translate_response(?FILE_RESP_FNAME, []) ->
+    ok;
+translate_response(?FILE_RESP_FNAME, Data) when is_binary(Data) ->
+    {ok, prim_file:internal_native2name(Data)};
+translate_response(?FILE_RESP_FNAME, Data) ->
+    {ok, Data};
+translate_response(?FILE_RESP_ALL_DATA, Data) ->
+    {ok, Data};
 translate_response(X, Data) ->
     {error, {bad_response_from_port, [X | Data]}}.
 
@@ -1077,14 +1135,14 @@ date_to_bytes(undefined) ->
 date_to_bytes({{Y, Mon, D}, {H, Min, S}}) ->
     <<Y:32, Mon:32, D:32, H:32, Min:32, S:32>>.
 
-% uint64([[X1, X2, X3, X4] = Y1 | [X5, X6, X7, X8] = Y2]) ->
-%     (uint32(Y1) bsl 32) bor uint32(Y2).
+%% uint64([[X1, X2, X3, X4] = Y1 | [X5, X6, X7, X8] = Y2]) ->
+%%     (uint32(Y1) bsl 32) bor uint32(Y2).
 
-% uint64(X1, X2, X3, X4, X5, X6, X7, X8) ->
-%     (uint32(X1, X2, X3, X4) bsl 32) bor uint32(X5, X6, X7, X8).
+%% uint64(X1, X2, X3, X4, X5, X6, X7, X8) ->
+%%     (uint32(X1, X2, X3, X4) bsl 32) bor uint32(X5, X6, X7, X8).
 
-% uint32([X1,X2,X3,X4]) ->
-%     (X1 bsl 24) bor (X2 bsl 16) bor (X3 bsl 8) bor X4.
+%% uint32([X1,X2,X3,X4]) ->
+%%     (X1 bsl 24) bor (X2 bsl 16) bor (X3 bsl 8) bor X4.
 
 uint32(X1,X2,X3,X4) ->
     (X1 bsl 24) bor (X2 bsl 16) bor (X3 bsl 8) bor X4.
@@ -1166,3 +1224,9 @@ lists_split([Hd | Tl], N, Rev) ->
 
 reverse(X) -> lists:reverse(X, []).
 reverse(L, T) -> lists:reverse(L, T).
+
+% Will add zero termination too
+% The 'EXIT' tuple from a bad argument will eventually generate an error
+% in list_to_binary, which is caught and generates the {error,badarg} return
+pathname(File) ->
+    (catch prim_file:internal_name2native(File)).
diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl
index 0feb591efb..8f2e845b4f 100644
--- a/erts/preloaded/src/prim_inet.erl
+++ b/erts/preloaded/src/prim_inet.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2000-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -37,7 +37,7 @@
 -export([setopt/3, setopts/2, getopt/2, getopts/2, is_sockopt_val/2]).
 -export([chgopt/3, chgopts/2]).
 -export([getstat/2, getfd/1, getindex/1, getstatus/1, gettype/1, 
-	 getiflist/1, ifget/3, ifset/3,
+	 getifaddrs/1, getiflist/1, ifget/3, ifset/3,
 	 gethostname/1]).
 -export([getservbyname/3, getservbyport/3]).
 -export([peername/1, setpeername/2]).
@@ -216,9 +216,10 @@ bindx(S, AddFlag, Addrs) ->
 	sctp ->
 	    %% Really multi-homed "bindx". Stringified args:
 	    %% [AddFlag, (Port, IP)+]:
-	    Args = ?int8(AddFlag) ++
-		lists:concat([?int16(Port)++ip_to_bytes(IP) ||
-				 {IP, Port} <- Addrs]),
+	    Args =
+		[?int8(AddFlag)|
+		 [[?int16(Port)|ip_to_bytes(IP)] ||
+		     {IP, Port} <- Addrs]],
 	    case ctl_cmd(S, ?SCTP_REQ_BINDX, Args) of
 		{ok,_} -> {ok, S};
 		Error  -> Error
@@ -623,7 +624,7 @@ chgopt(S, Opt, Value) when is_port(S) ->
     chgopts(S, [{Opt,Value}]).
 
 chgopts(S, Opts) when is_port(S), is_list(Opts) ->
-    case inet:getopts(S, need_template(Opts)) of
+    case getopts(S, need_template(Opts)) of
 	{ok,Templates} ->
 	    try merge_options(Opts, Templates) of
 		NewOpts ->
@@ -636,7 +637,94 @@ chgopts(S, Opts) when is_port(S), is_list(Opts) ->
     
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%
-%% IFLIST(insock()) -> {ok,IfNameList} | {error, Reason}
+%% getifaddrs(insock()) -> {ok,IfAddrsList} | {error, Reason}
+%%
+%%   IfAddrsList = [{Name,[Opts]}]
+%%   Name = string()
+%%   Opts = {flags,[Flag]} | {addr,Addr} | {netmask,Addr} | {broadaddr,Addr}
+%%        | {dstaddr,Addr} | {hwaddr,HwAddr} | {mtu,integer()}
+%%   Flag = up | broadcast | loopback | running | multicast
+%%   Addr = ipv4addr() | ipv6addr()
+%%   HwAddr = ethernet_addr()
+%%
+%% get interface name and addresses list
+%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+getifaddrs(S) when is_port(S) ->
+    case ctl_cmd(S, ?INET_REQ_GETIFADDRS, []) of
+	{ok, Data} ->
+	    {ok, comp_ifaddrs(build_ifaddrs(Data), ktree_empty())};
+	{error,enotsup} ->
+	    case getiflist(S) of
+		{ok, IFs} ->
+		    {ok, getifaddrs_ifget(S, IFs)};
+		Err1 -> Err1
+	    end;
+	Err2 -> Err2
+    end.
+
+%% Restructure interface properties per interface and remove duplicates
+
+comp_ifaddrs([{If,Opts}|IfOpts], T) ->
+    case ktree_is_defined(If, T) of
+	true ->
+	    OptSet = comp_ifaddrs_add(ktree_get(If, T), Opts),
+	    comp_ifaddrs(IfOpts, ktree_update(If, OptSet, T));
+	false ->
+	    OptSet = comp_ifaddrs_add(ktree_empty(), Opts),
+	    comp_ifaddrs(IfOpts, ktree_insert(If, OptSet, T))
+     end;
+comp_ifaddrs([], T) ->
+    [{If,ktree_keys(ktree_get(If, T))} || If <- ktree_keys(T)].
+
+comp_ifaddrs_add(OptSet, [Opt|Opts]) ->
+    case ktree_is_defined(Opt, OptSet) of
+	true
+	  when element(1, Opt) =:= flags;
+	       element(1, Opt) =:= hwaddr ->
+	    comp_ifaddrs_add(OptSet, Opts);
+	_ ->
+	    comp_ifaddrs_add(ktree_insert(Opt, undefined, OptSet), Opts)
+    end;
+comp_ifaddrs_add(OptSet, []) -> OptSet.
+
+%% Legacy emulation of getifaddrs
+
+getifaddrs_ifget(_, []) -> [];
+getifaddrs_ifget(S, [IF|IFs]) ->
+    case ifget(S, IF, [flags]) of
+	{ok,[{flags,Flags}]=FlagsVals} ->
+	    BroadOpts =
+		case member(broadcast, Flags) of
+		    true ->
+			[broadaddr,hwaddr];
+		    false ->
+			[hwaddr]
+		end,
+	    P2POpts =
+		case member(pointtopoint, Flags) of
+		    true ->
+			[dstaddr|BroadOpts];
+		    false ->
+			BroadOpts
+		end,
+	    getifaddrs_ifget(S, IFs, IF, FlagsVals, [addr,netmask|P2POpts]);
+	_ ->
+	    getifaddrs_ifget(S, IFs, IF, [], [addr,netmask,hwaddr])
+    end.
+
+getifaddrs_ifget(S, IFs, IF, FlagsVals, Opts) ->
+    OptVals =
+	case ifget(S, IF, Opts) of
+	    {ok,OVs} -> OVs;
+	    _ -> []
+	end,
+    [{IF,FlagsVals++OptVals}|getifaddrs_ifget(S, IFs)].
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%
+%% getiflist(insock()) -> {ok,IfNameList} | {error, Reason}
 %%
 %% get interface name list
 %%
@@ -1206,7 +1294,7 @@ type_opt_1(sctp_default_send_param) ->
 	timetolive        = [uint32,0],
 	tsn               = [],
 	cumtsn            = [],
-	assoc_id          = [sctp_assoc_id,0]}}];
+	assoc_id          = [[sctp_assoc_id,0]]}}];
 %% for SCTP_OPT_EVENTS
 type_opt_1(sctp_events) ->
     [{record,#sctp_event_subscribe{
@@ -1325,6 +1413,19 @@ type_value_2({enum,List}, Enum) ->
 	{value,_} 				    -> true;
 	false                                       -> false
     end;
+type_value_2(sockaddr, Addr) ->
+    case Addr of
+	any                                         -> true;
+	loopback                                    -> true;
+	{A,B,C,D} when ?ip(A,B,C,D)                 -> true;
+	{A,B,C,D,E,F,G,H} when ?ip6(A,B,C,D,E,F,G,H) -> true;
+	_                                           -> false
+    end;
+type_value_2(linkaddr, Addr) when is_list(Addr) ->
+    case len(Addr, 32768) of
+	undefined                                   -> false;
+	_                                           -> true
+    end;
 type_value_2({bitenumlist,List}, EnumList) -> 
     case enum_vals(EnumList, List) of
 	Ls when is_list(Ls)                         -> true;
@@ -1413,14 +1514,21 @@ enc_value_2(addr, {any,Port}) ->
     [?INET_AF_ANY|?int16(Port)];
 enc_value_2(addr, {loopback,Port}) ->
     [?INET_AF_LOOPBACK|?int16(Port)];
-enc_value_2(addr, {IP,Port}) ->
-    case tuple_size(IP) of
-	4 ->
-	    [?INET_AF_INET,?int16(Port)|ip4_to_bytes(IP)];
-	8 ->
-	    [?INET_AF_INET6,?int16(Port)|ip6_to_bytes(IP)]
-    end;
-enc_value_2(ether, [X1,X2,X3,X4,X5,X6]) -> [X1,X2,X3,X4,X5,X6];
+enc_value_2(addr, {IP,Port}) when tuple_size(IP) =:= 4 ->
+    [?INET_AF_INET,?int16(Port)|ip4_to_bytes(IP)];
+enc_value_2(addr, {IP,Port}) when tuple_size(IP) =:= 8 ->
+    [?INET_AF_INET6,?int16(Port)|ip6_to_bytes(IP)];
+enc_value_2(ether, [_,_,_,_,_,_]=Xs) -> Xs;
+enc_value_2(sockaddr, any) ->
+    [?INET_AF_ANY];
+enc_value_2(sockaddr, loopback) ->
+    [?INET_AF_LOOPBACK];
+enc_value_2(sockaddr, IP) when tuple_size(IP) =:= 4 ->
+    [?INET_AF_INET|ip4_to_bytes(IP)];
+enc_value_2(sockaddr, IP) when tuple_size(IP) =:= 8 ->
+    [?INET_AF_INET6|ip6_to_bytes(IP)];
+enc_value_2(linkaddr, Linkaddr) ->
+    [?int16(length(Linkaddr)),Linkaddr];
 enc_value_2(sctp_assoc_id, Val) -> ?int32(Val);
 %% enc_value_2(sctp_assoc_id, Bin) -> [byte_size(Bin),Bin];
 enc_value_2({enum,List}, Enum) ->
@@ -1464,7 +1572,11 @@ dec_value(time, [X3,X2,X1,X0|T]) ->
 	Val -> {Val, T}
     end;
 dec_value(ip, [A,B,C,D|T])             -> {{A,B,C,D}, T};
-dec_value(ether,[X1,X2,X3,X4,X5,X6|T]) -> {[X1,X2,X3,X4,X5,X6],T};
+%% dec_value(ether, [X1,X2,X3,X4,X5,X6|T]) -> {[X1,X2,X3,X4,X5,X6],T};
+dec_value(sockaddr, [X|T]) ->
+    get_ip(X, T);
+dec_value(linkaddr, [X1,X0|T]) ->
+    split(?i16(X1,X0), T);
 dec_value({enum,List}, [X3,X2,X1,X0|T]) ->
     Val = ?i32(X3,X2,X1,X0),
     case enum_name(Val, List) of
@@ -1480,7 +1592,7 @@ dec_value({bitenumlist,List}, [X3,X2,X1,X0|T]) ->
 %%     {enum_names(Val, List), T};
 dec_value(binary,[L0,L1,L2,L3|List]) ->
     Len = ?i32(L0,L1,L2,L3),
-    {X,T}=lists:split(Len,List),
+    {X,T}=split(Len,List),
     {list_to_binary(X),T};
 dec_value(Types, List) when is_tuple(Types) ->
     {L,T} = dec_value_tuple(Types, List, 1, []),
@@ -1495,7 +1607,7 @@ dec_value_tuple(Types, List, N, Acc)
     {Term,Tail} = dec_value(element(N, Types), List),
     dec_value_tuple(Types, Tail, N+1, [Term|Acc]);
 dec_value_tuple(_, List, _, Acc) ->
-    {lists:reverse(Acc),List}.
+    {rev(Acc),List}.
 
 borlist([V|Vs], Value) ->
     borlist(Vs, V bor Value);
@@ -1702,11 +1814,11 @@ merge_fields(_, _, _) -> [].
 %%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-type_ifopt(addr)      -> ip;
-type_ifopt(broadaddr) -> ip;
-type_ifopt(dstaddr)   -> ip;
+type_ifopt(addr)      -> sockaddr;
+type_ifopt(broadaddr) -> sockaddr;
+type_ifopt(dstaddr)   -> sockaddr;
 type_ifopt(mtu)       -> int;
-type_ifopt(netmask)   -> ip;
+type_ifopt(netmask)   -> sockaddr;
 type_ifopt(flags)     ->
     {bitenumlist,
      [{up, ?INET_IFF_UP},
@@ -1718,7 +1830,7 @@ type_ifopt(flags)     ->
       {no_pointtopoint, ?INET_IFF_NPOINTTOPOINT},
       {running, ?INET_IFF_RUNNING},
       {multicast, ?INET_IFF_MULTICAST}]};
-type_ifopt(hwaddr)    -> ether;
+type_ifopt(hwaddr)    -> linkaddr;
 type_ifopt(Opt) when is_atom(Opt) -> undefined.
 
 enc_ifopt(addr)      -> ?INET_IFOPT_ADDR;
@@ -1903,6 +2015,30 @@ encode_ifname(Name) ->
     if N > 255 -> {error, einval};
        true -> {ok,[N | Name]}
     end.
+
+build_ifaddrs(Cs) ->
+    build_ifaddrs(Cs, []).
+%%
+build_ifaddrs([], []) ->
+    [];
+build_ifaddrs([0|Cs], Acc) ->
+    Name = utf8_to_characters(rev(Acc)),
+    {Opts,Rest} = build_ifaddrs_opts(Cs, []),
+    [{Name,Opts}|build_ifaddrs(Rest)];
+build_ifaddrs([C|Cs], Acc) ->
+    build_ifaddrs(Cs, [C|Acc]).
+
+build_ifaddrs_opts([0|Cs], Acc) ->
+    {rev(Acc),Cs};
+build_ifaddrs_opts([C|Cs]=CCs, Acc) ->
+    case dec_ifopt(C) of
+	undefined ->
+	    erlang:error(badarg, [CCs,Acc]);
+	Opt ->
+	    Type = type_ifopt(Opt),
+	    {Val,Rest} = dec_value(Type, Cs),
+	    build_ifaddrs_opts(Rest, [{Opt,Val}|Acc])
+    end.
     
 build_iflist(Cs) ->
     build_iflist(Cs, [], []).
@@ -1927,6 +2063,80 @@ rev(L) -> rev(L,[]).
 rev([C|L],Acc) -> rev(L,[C|Acc]);
 rev([],Acc) -> Acc.
 
+split(N, L) -> split(N, L, []).
+split(0, L, R) when is_list(L) -> {rev(R),L};
+split(N, [H|T], R) when is_integer(N), N > 0 -> split(N-1, T, [H|R]).
+
+len(L, N) -> len(L, N, 0).
+len([], N, C) when is_integer(N), N >= 0 -> C;
+len(L, 0, _) when is_list(L) -> undefined;
+len([_|L], N, C) when is_integer(N), N >= 0 -> len(L, N-1, C+1).
+
+member(X, [X|_]) -> true;
+member(X, [_|Xs]) -> member(X, Xs);
+member(_, []) -> false.
+
+
+
+%% Lookup tree that keeps key insert order
+
+ktree_empty() -> {[],tree()}.
+ktree_is_defined(Key, {_,T}) -> tree(T, Key, is_defined).
+ktree_get(Key, {_,T}) -> tree(T, Key, get).
+ktree_insert(Key, V, {Keys,T}) -> {[Key|Keys],tree(T, Key, {insert,V})}.
+ktree_update(Key, V, {Keys,T}) -> {Keys,tree(T, Key, {update,V})}.
+ktree_keys({Keys,_}) -> rev(Keys).
+
+%% Simple lookup tree. Hash the key to get statistical balance.
+%% Key is matched equal, not compared equal.
+
+tree() -> nil.
+tree(T, Key, Op) -> tree(T, Key, Op, erlang:phash2(Key)).
+
+tree(nil, _, is_defined, _) -> false;
+tree(nil, K, {insert,V}, _) -> {K,V,nil,nil};
+tree({K,_,_,_}, K, is_defined, _) -> true;
+tree({K,V,_,_}, K, get, _) -> V;
+tree({K,_,L,R}, K, {update,V}, _) -> {K,V,L,R};
+tree({K0,V0,L,R}, K, Op, H) ->
+    H0 = erlang:phash2(K0),
+    if  H0 < H;  H0 =:= H, K0 < K ->
+	    if  is_tuple(Op) ->
+		    {K0,V0,tree(L, K, Op, H),R};
+		true ->
+		    tree(L, K, Op, H)
+	    end;
+	true ->
+	    if  is_tuple(Op) ->
+		    {K0,V0,L,tree(R, K, Op, H)};
+		true ->
+		    tree(R, K, Op, H)
+	    end
+    end.
+
+
+
+utf8_to_characters([]) -> [];
+utf8_to_characters([B|Bs]=Arg) when (B band 16#FF) =:= B ->
+    if  16#F8 =< B ->
+	    erlang:error(badarg, [Arg]);
+	16#F0 =< B ->
+	    utf8_to_characters(Bs, B band 16#07, 3);
+	16#E0 =< B ->
+	    utf8_to_characters(Bs, B band 16#0F, 2);
+	16#C0 =< B ->
+	    utf8_to_characters(Bs, B band 16#1F, 1);
+	16#80 =< B ->
+	    erlang:error(badarg, [Arg]);
+	true ->
+	    [B|utf8_to_characters(Bs)]
+    end.
+%%
+utf8_to_characters(Bs, U, 0) ->
+    [U|utf8_to_characters(Bs)];
+utf8_to_characters([B|Bs], U, N) when ((B band 16#3F) bor 16#80) =:= B ->
+    utf8_to_characters(Bs, (U bsl 6) bor (B band 16#3F), N-1).
+
 ip_to_bytes(IP) when tuple_size(IP) =:= 4 -> ip4_to_bytes(IP);
 ip_to_bytes(IP) when tuple_size(IP) =:= 8 -> ip6_to_bytes(IP).
 
diff --git a/erts/preloaded/src/prim_zip.erl b/erts/preloaded/src/prim_zip.erl
index 3f5a5b9721..6a9856fdad 100644
--- a/erts/preloaded/src/prim_zip.erl
+++ b/erts/preloaded/src/prim_zip.erl
@@ -65,34 +65,55 @@ filter_fun() ->
 open(F) ->
     open(filter_fun(), undefined, F).
 
-open(FilterFun, FilterAcc, F) ->
-    case ?CATCH do_open(FilterFun, FilterAcc, F) of
-	{ok, PrimZip, Acc} ->
-	    {ok, PrimZip, Acc};
-	Error ->
-	    {error, Error}
-    end.
+open(FilterFun, FilterAcc, F) when is_function(FilterFun, 2) ->
+    try
+	do_open(FilterFun, FilterAcc, F)
+    catch
+	throw:{filter_fun_throw, Reason} ->
+	    throw(Reason);
+	throw:InternalReason ->
+	    {error, InternalReason};
+	Class:Reason ->
+	    erlang:error(erlang:raise(Class, Reason, erlang:get_stacktrace()))
+    end;
+open(_, _, _) ->
+    {error, einval}.
 
 do_open(FilterFun, FilterAcc, F) ->
     Input = get_zip_input(F),
     In0 = Input({open, F, [read, binary, raw]}, []),
     Z = zlib:open(),
     PrimZip = #primzip{files = [], zlib = Z, in = In0, input = Input},
-    {PrimZip2, FilterAcc2} = get_central_dir(PrimZip, FilterFun, FilterAcc),
-    {ok, PrimZip2, FilterAcc2}.
+    try
+	{PrimZip2, FilterAcc2} = get_central_dir(PrimZip, FilterFun, FilterAcc),
+	{ok, PrimZip2, FilterAcc2}
+    catch
+	Class:Reason ->
+	    close(PrimZip),
+	    erlang:error(erlang:raise(Class, Reason, erlang:get_stacktrace()))
+    end.
 
 %% iterate over all files in a zip archive
-foldl(FilterFun, FilterAcc, #primzip{files = Files} = PrimZip) ->
-    case ?CATCH do_foldl(FilterFun, FilterAcc, Files, [], PrimZip, PrimZip) of
-	{ok, FilterAcc2, PrimZip2} -> {ok, PrimZip2, FilterAcc2};
-	Error                      -> {error, Error}
+foldl(FilterFun, FilterAcc, #primzip{files = Files} = PrimZip)
+  when is_function(FilterFun, 2) ->
+    try
+	{ok, FilterAcc2, PrimZip2} =
+	    do_foldl(FilterFun, FilterAcc, Files, [], PrimZip, PrimZip),
+	{ok, PrimZip2, FilterAcc2}
+    catch
+	throw:{filter_fun_throw, Reason} ->
+	    throw(Reason);
+	throw:InternalReason ->
+	    {error, InternalReason};
+	Class:Reason ->
+	    erlang:error(erlang:raise(Class, Reason, erlang:get_stacktrace()))
     end;
 foldl(_, _, _) ->
     {error, einval}.
 
 do_foldl(FilterFun, FilterAcc, [PF | Tail], Acc0, PrimZip, PrimZipOrig) ->
     #primzip_file{name = F, get_info = GetInfo, get_bin = GetBin} = PF,
-    case FilterFun({F, GetInfo, GetBin}, FilterAcc) of
+    try FilterFun({F, GetInfo, GetBin}, FilterAcc) of
 	{Continue, Include, FilterAcc2} ->
 	    Acc1 = include_acc(Include, PF, Acc0),
 	    case Continue of
@@ -103,6 +124,9 @@ do_foldl(FilterFun, FilterAcc, [PF | Tail], Acc0, PrimZip, PrimZipOrig) ->
 	    end;
 	FilterRes ->
 	    throw({illegal_filter, FilterRes})
+    catch
+	throw:Reason ->
+	    throw({filter_fun_throw, Reason})
     end;
 do_foldl(_FilterFun, FilterAcc, [], Acc, PrimZip, _PrimZipOrig) ->
     {ok, FilterAcc, PrimZip#primzip{files = reverse(Acc)}}.
@@ -121,12 +145,14 @@ include_acc(Include, PF, Acc) ->
 	List when is_list(List) ->
 	    %% Add new entries
 	    Fun = fun(I, A) -> include_acc(I, PF, A) end,
-	    lists_foldl(Fun, Acc, List)
+	    lists_foldl(Fun, Acc, List);
+	Bad ->
+	    throw({illegal_filter, Bad})
     end.
 
 lists_foldl(F, Accu, [Hd|Tail]) ->
     lists_foldl(F, F(Hd, Accu), Tail);
-lists_foldl(F, Accu, []) when is_function(F, 2) -> 
+lists_foldl(F, Accu, []) when is_function(F, 2) ->
     Accu.
 
 %% close a zip archive
@@ -139,7 +165,9 @@ close(_) ->
 get_zip_input({F, B}) when is_binary(B), is_list(F) ->
     fun binary_io/2;
 get_zip_input(F) when is_list(F) ->
-    fun prim_file_io/2.
+    fun prim_file_io/2;
+get_zip_input(_) ->
+    throw(einval).
 
 %% get a file from the archive
 get_z_file(F, Offset, ChunkSize, #primzip{zlib = Z, in = In0, input = Input}) ->
@@ -218,15 +246,15 @@ get_cd_loop(N, BCD, Acc0, PrimZip, FileName, Offset, CFH, EndOffset, FilterFun,
     GetInfo = fun() -> cd_file_header_to_file_info(FileName, CFH, <<>>) end,
     GetBin = fun() -> get_z_file(FileName, Offset, Size, PrimZip) end,
     PF = #primzip_file{name = FileName, get_info = GetInfo, get_bin = GetBin},
-    case FilterFun({FileName, GetInfo, GetBin}, FilterAcc) of
+    try FilterFun({FileName, GetInfo, GetBin}, FilterAcc) of
 	{Continue, Include, FilterAcc2} ->
 	    Acc1 =
 		case Include of
-		    false -> 
+		    false ->
 			Acc0;
 		    true ->
 			[PF | Acc0];
-		    {true, Nick} -> 
+		    {true, Nick} ->
 			[PF#primzip_file{name = Nick} | Acc0];
 		    {true, Nick, GI, GB} ->
 			PF2 = #primzip_file{name = Nick, get_info = GI, get_bin = GB},
@@ -247,13 +275,16 @@ get_cd_loop(N, BCD, Acc0, PrimZip, FileName, Offset, CFH, EndOffset, FilterFun,
 	    end;
 	FilterRes ->
 	    throw({illegal_filter, FilterRes})
+    catch
+	throw:Reason ->
+	    throw({filter_fun_throw, Reason})
     end.
 
 get_file_header(BCD) ->
     BCFH =
 	case BCD of
 	    <<?CENTRAL_FILE_MAGIC:32/little,
-	     B:(?CENTRAL_FILE_HEADER_SZ-4)/binary, 
+	     B:(?CENTRAL_FILE_HEADER_SZ-4)/binary,
 	     _/binary>> ->
 		B;
 	    _ ->
@@ -266,11 +297,11 @@ get_file_header(BCD) ->
     ToGet = FileNameLen + ExtraLen + CommentLen,
     {B2, BCDRest} =
 	case BCD of
-	    <<_:?CENTRAL_FILE_HEADER_SZ/binary, 
+	    <<_:?CENTRAL_FILE_HEADER_SZ/binary,
 	     G:ToGet/binary,
-	     Rest/binary>> -> 
+	     Rest/binary>> ->
 		{G, Rest};
-	    _ -> 
+	    _ ->
 		throw(bad_central_directory)
 	end,
     FileName = get_filename_from_b2(B2, FileNameLen, ExtraLen, CommentLen),
@@ -319,9 +350,9 @@ prim_file_io({file_info, F}, _) ->
     end;
 prim_file_io({open, FN, Opts}, _) ->
     case ?CATCH prim_file:open(FN, Opts++[binary]) of
-	{ok, H} -> 
+	{ok, H} ->
 	    H;
-	{error, E} -> 
+	{error, E} ->
 	    throw(E)
     end;
 prim_file_io({read, N}, H) ->
@@ -476,7 +507,7 @@ cd_file_header_to_file_info(FileName,
 %%     FI;     % not yet supported, and not widely used
 add_extra_info(FI, _) ->
     FI.
-%% 
+%%
 %% unix_extra_field_and_var_from_bin(<<TSize:16/little,
 %% 				   ATime:32/little,
 %% 				   MTime:32/little,
@@ -500,7 +531,7 @@ dos_date_time_to_datetime(DosDate, DosTime) ->
     <<Hour:5, Min:6, Sec:5>> = <<DosTime:16>>,
     <<YearFrom1980:7, Month:4, Day:5>> = <<DosDate:16>>,
     {{YearFrom1980+1980, Month, Day},
-     {Hour, Min, Sec}}. 
+     {Hour, Min, Sec}}.
 
 cd_file_header_from_bin(<<VersionMadeBy:16/little,
 			 VersionNeeded:16/little,
@@ -622,7 +653,7 @@ reverse(X) ->
 
 reverse([H|T], Y) ->
     reverse(T, [H|Y]);
-reverse([], X) -> 
+reverse([], X) ->
     X.
 
 last([E|Es]) -> last(E, Es).
diff --git a/erts/test/Makefile b/erts/test/Makefile
index 796403e182..68be3f2178 100644
--- a/erts/test/Makefile
+++ b/erts/test/Makefile
@@ -28,6 +28,7 @@ EBIN = .
 # ----------------------------------------------------
 
 MODULES= \
+	autoimport_SUITE \
 	erlc_SUITE \
 	install_SUITE \
 	nt_SUITE \
@@ -38,12 +39,14 @@ MODULES= \
 	erlexec_SUITE \
 	z_SUITE
 
+ERL_XML = $(ERL_TOP)/erts/doc/src/erlang.xml
+ERL_XML_TARGET = autoimport_SUITE_data/erlang.xml
 
 ERL_FILES= $(MODULES:%=%.erl)
 
 TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR))
 
-EXTRA_FILES = install_SUITE_data/install_bin
+EXTRA_FILES = install_SUITE_data/install_bin $(ERL_XML_TARGET)
 
 # ----------------------------------------------------
 # Release directory specification
@@ -65,6 +68,10 @@ install_SUITE_data/install_bin: ../../make/install_bin
 	rm -f $@
 	cp -p $< $@
 
+$(ERL_XML_TARGET): $(ERL_XML)
+	rm -f $@
+	cp -p $< $@
+
 clean:
 	rm -f $(TARGET_FILES) $(EXTRA_FILES)
 	rm -f core *~
@@ -82,7 +89,7 @@ release_tests_spec: opt
 	$(INSTALL_DIR) $(RELSYSDIR)
 	$(INSTALL_DATA) system.spec system.dynspec system.spec.vxworks \
 		$(ERL_FILES) $(TARGET_FILES) $(RELSYSDIR)
-	chmod -f -R u+w $(RELSYSDIR)
+	chmod -R u+w $(RELSYSDIR)
 	tar cf - *_SUITE_data utils | (cd $(RELSYSDIR); tar xf -)
 
 release_docs_spec:
diff --git a/erts/test/autoimport_SUITE.erl b/erts/test/autoimport_SUITE.erl
new file mode 100644
index 0000000000..0e4708e046
--- /dev/null
+++ b/erts/test/autoimport_SUITE.erl
@@ -0,0 +1,185 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%% Purpose: Test erlang.xml re autoimports
+-module(autoimport_SUITE).
+
+-include_lib("test_server/include/test_server.hrl").
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 autoimports/1]).
+-define(TEST_TIMEOUT, ?t:seconds(180)).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [autoimports].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+init_per_testcase(_Func, Config) ->
+    Dog = test_server:timetrap(?TEST_TIMEOUT),
+    [{watchdog, Dog} | Config].
+
+end_per_testcase(_Func, Config) ->
+    Dog = ?config(watchdog, Config),
+    catch test_server:timetrap_cancel(Dog),
+    ok.
+
+
+autoimports(suite) ->
+    [];
+autoimports(doc) ->
+    ["Check that erlang.xml documents autoimports correctly"];
+autoimports(Config) when is_list(Config) ->
+    ?line XML = filename:join([?config(data_dir,Config),"erlang.xml"]),
+    ?line case xml(XML) of
+	      [] ->
+		  ok;
+	      List ->
+		  lists:foreach(fun({[],F,A}) ->
+					io:format("erlang:~s/~p is wrongly "
+						  "documented as ~s/~p~n",
+						  [F,A,F,A]);
+				   ({"erlang",F,A}) ->
+					io:format("~s/~p is wrongly "
+						  "documented as "
+						  "erlang:~s/~p~n",
+						  [F,A,F,A])
+				end,List),
+		  ?t:fail({wrong_autoimports,List})
+	  end.
+
+%%
+%% Ugly chunk of code to heuristically analyze the erlang.xml
+%% documentation file. Don't view this as an example...
+%%
+
+xml(XMLFile) ->
+    {ok,File} = file:open(XMLFile,[read]),
+    DocData = xloop(file:read_line(File),File),
+    file:close(File),
+    analyze(DocData).
+
+xloop({ok,Line},File) ->
+    case re:run(Line,"\\<name\\>",[{capture,none}]) of
+	nomatch ->
+	    xloop(file:read_line(File),File);
+	match ->
+	    X = re:replace(Line,"\\).*",")",[{return,list}]),
+	    Y = re:replace(X,".*\\>","",[{return,list}]),
+	    Mod = get_module(Y),
+	    Rest1 = fstrip(Mod++":",Y),
+	    Func = get_function(Rest1),
+	    Rest2 = fstrip(Func++"(", Rest1),
+	    Argc = count_args(Rest2,1,0,false),
+	    [{Mod,Func,Argc} |
+	     xloop(file:read_line(File),File)]
+    end;
+xloop(_,_) ->
+    [].
+
+analyze([{[],Func,Arity}|T]) ->
+    case erl_internal:bif(list_to_atom(Func),Arity) of
+	true ->
+	    analyze(T);
+	false ->
+	    [{[],Func,Arity} |
+	     analyze(T)]
+    end;
+analyze([{"erlang",Func,Arity}|T]) ->
+    case erl_internal:bif(list_to_atom(Func),Arity) of
+	true ->
+	    [{"erlang",Func,Arity}|analyze(T)];
+	false ->
+	    analyze(T)
+    end;
+analyze([_|T]) ->
+   analyze(T);
+analyze([]) ->
+   [].
+
+
+count_args([],_,N,false) ->
+    N;
+count_args([],_,N,true) ->
+    N+1;
+count_args(_,0,N,true) ->
+    N+1;
+count_args(_,0,N,false) ->
+    N;
+count_args([Par|T],Level,N,Got) when (Par =:= 40) or
+				 (Par =:= 123) or (Par =:= 91) ->
+    count_args(T,Level+1,N,(Level =:= 1) or Got);
+count_args([41|T],1,N,true) ->
+    count_args(T,0,N+1,false);
+count_args([Par|T],Level,N, Got) when (Par =:= 41) or
+				 (Par =:= 125) or (Par =:= 93) ->
+    count_args(T,Level-1,N,Got);
+count_args([$,|T],1,N,true) ->
+    count_args(T,1,N+1,false);
+count_args([$ |T],A,B,C) ->
+    count_args(T,A,B,C);
+count_args([_|T],1,N,_) ->
+    count_args(T,1,N,true);
+count_args([_|T],A,B,C) ->
+    count_args(T,A,B,C).
+
+fstrip([],X) ->
+    X;
+fstrip(_,[]) ->
+    [];
+fstrip([H|T1],[H|T2]) ->
+    fstrip(T1,T2);
+fstrip(_,L) ->
+    L.
+
+get_module(X) ->
+    get_module(X,[]).
+get_module([],_) ->
+    [];
+get_module([$:|_],Acc) ->
+    lists:reverse(Acc);
+get_module([40|_],_) -> %(
+    [];
+get_module([H|T],Acc) ->
+    get_module(T,[H|Acc]).
+
+get_function(X) ->
+    get_function(X,[]).
+get_function([],_) ->
+    [];
+get_function([40|_],Acc) -> %(
+    lists:reverse(Acc);
+get_function([H|T],Acc) ->
+    get_function(T,[H|Acc]).
diff --git a/erts/test/autoimport_SUITE_data/dummy.txt b/erts/test/autoimport_SUITE_data/dummy.txt
new file mode 100644
index 0000000000..972643e527
--- /dev/null
+++ b/erts/test/autoimport_SUITE_data/dummy.txt
@@ -0,0 +1,19 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1998-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%% Purpouse: Dummy
diff --git a/erts/test/erl_print_SUITE.erl b/erts/test/erl_print_SUITE.erl
index 3bb7d4d016..ee1a200530 100644
--- a/erts/test/erl_print_SUITE.erl
+++ b/erts/test/erl_print_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -32,14 +32,35 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(10)).
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, fin_per_testcase/2]).
 
--export([erlang_display/1, integer/1, float/1, string/1, character/1, snprintf/1, quote/1]).
+-export([erlang_display/1, integer/1, float/1, 
+	 string/1, character/1, snprintf/1, quote/1]).
 
 -include_lib("test_server/include/test_server.hrl").
-    
-all(doc) -> [];
-all(suite) -> test_cases().
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    test_cases().
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 %%
 %%
@@ -47,14 +68,9 @@ all(suite) -> test_cases().
 %%
 %%
 
-test_cases() ->
-    [erlang_display,
-     integer,
-     float,
-     string,
-     character,
-     snprintf,
-     quote].
+test_cases() -> 
+    [erlang_display, integer, float, string, character,
+     snprintf, quote].
 
 erlang_display(doc) -> [];
 erlang_display(suite) -> [];
diff --git a/erts/test/erlc_SUITE.erl b/erts/test/erlc_SUITE.erl
index 437f020f99..62e0e6813d 100644
--- a/erts/test/erlc_SUITE.erl
+++ b/erts/test/erlc_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,14 +20,33 @@
 
 %% Tests the erlc command by compiling various types of files.
 
--export([all/1, compile_erl/1, compile_yecc/1, compile_script/1,
-	 compile_mib/1, good_citizen/1, deep_cwd/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, compile_erl/1,
+	 compile_yecc/1, compile_script/1,
+	 compile_mib/1, good_citizen/1, deep_cwd/1, arg_overflow/1]).
 
 -include_lib("test_server/include/test_server.hrl").
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [compile_erl, compile_yecc, compile_script, compile_mib,
-     good_citizen, deep_cwd].
+     good_citizen, deep_cwd, arg_overflow].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 %% Copy from erlc_SUITE_data/include/erl_test.hrl.
 
@@ -189,6 +208,18 @@ deep_cwd_1(PrivDir) ->
     ?line true = filelib:is_file("test.beam"),
     ok.
 
+%% Test that a large number of command line switches does not
+%% overflow the argument buffer
+arg_overflow(Config) when is_list(Config) ->
+    ?line {SrcDir, _OutDir, Cmd} = get_cmd(Config),
+    ?line FileName = filename:join(SrcDir, "erl_test_ok.erl"),
+    ?line Args = lists:flatten([ ["-D", integer_to_list(N), "=1 "] ||
+            N <- lists:seq(1,10000) ]),
+    ?line run(Config, Cmd, FileName, Args,
+	      ["Warning: function foo/0 is unused\$",
+	       "_OK_"]),
+    ok.
+
 erlc() ->
     case os:find_executable("erlc") of
 	false ->
diff --git a/erts/test/erlexec_SUITE.erl b/erts/test/erlexec_SUITE.erl
index 164ce9faaf..0dfe6c2e5f 100644
--- a/erts/test/erlexec_SUITE.erl
+++ b/erts/test/erlexec_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2007-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -31,9 +31,11 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(1)).
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
--export([args_file/1, evil_args_file/1, env/1, args_file_env/1, otp_7461/1, otp_7461_remote/1, otp_8209/1]).
+-export([args_file/1, evil_args_file/1, env/1, args_file_env/1, otp_7461/1, otp_7461_remote/1, otp_8209/1, zdbbl_dist_buf_busy_limit/1]).
 
 -include_lib("test_server/include/test_server.hrl").
     
@@ -43,7 +45,7 @@ init_per_testcase(Case, Config) ->
     SavedEnv = save_env(),
     [{testcase, Case}, {watchdog, Dog}, {erl_flags_env, SavedEnv} |Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     SavedEnv = ?config(erl_flags_env, Config),
     restore_env(SavedEnv),
@@ -51,10 +53,26 @@ fin_per_testcase(_Case, Config) ->
     ?t:timetrap_cancel(Dog),
     ok.
 
-all(doc) -> [];
-all(suite) ->
-    [args_file, evil_args_file, env, args_file_env, otp_7461, otp_8209].
+suite() -> [{ct_hooks,[ts_install_cth]}].
 
+all() -> 
+    [args_file, evil_args_file, env, args_file_env,
+     otp_7461, otp_8209, zdbbl_dist_buf_busy_limit].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 otp_8209(doc) ->
     ["Test that plain first argument does not "
@@ -330,6 +348,25 @@ otp_7461_remote([halt, Pid]) ->
     io:format("halt order from ~p to node ~p\n",[Pid,node()]),
     halt().
 
+zdbbl_dist_buf_busy_limit(doc) ->    
+    ["Check +zdbbl flag"];
+zdbbl_dist_buf_busy_limit(suite) ->
+    [];
+zdbbl_dist_buf_busy_limit(Config) when is_list(Config) ->
+    LimKB = 1122233,
+    LimB = LimKB*1024,
+    ?line {ok,[[PName]]} = init:get_argument(progname),
+    ?line SNameS = "erlexec_test_02",
+    ?line SName = list_to_atom(SNameS++"@"++
+                         hd(tl(string:tokens(atom_to_list(node()),"@")))),
+    ?line Cmd = PName ++ " -sname "++SNameS++" -setcookie "++
+        atom_to_list(erlang:get_cookie()) ++
+	" +zdbbl " ++ integer_to_list(LimKB),
+    ?line open_port({spawn,Cmd},[]),
+    ?line pong = loop_ping(SName,40),
+    ?line LimB = rpc:call(SName,erlang,system_info,[dist_buf_busy_limit]),
+    ?line ok = cleanup_node(SNameS, 10),
+    ok.
     
 
 %%
diff --git a/erts/test/erlexec_SUITE_data/erlexec_tests.c b/erts/test/erlexec_SUITE_data/erlexec_tests.c
index a49a0b21a8..1d1ca881d9 100644
--- a/erts/test/erlexec_SUITE_data/erlexec_tests.c
+++ b/erts/test/erlexec_SUITE_data/erlexec_tests.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /* Used by test case otp_7461 to spawn a child process with a given
@@ -22,7 +22,7 @@
  * Author: Sverker Eriksson
  */
 
-#if defined (__WIN32__) || defined(VXWORKS) || defined(_OSE_)
+#if defined (__WIN32__) || defined(VXWORKS)
 int main() {return 0;}
 
 #else /* UNIX only */
diff --git a/erts/test/ethread_SUITE.erl b/erts/test/ethread_SUITE.erl
index bbc79e9381..71d8c1c679 100644
--- a/erts/test/ethread_SUITE.erl
+++ b/erts/test/ethread_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2004-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2004-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -31,54 +31,50 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(10)).
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, fin_per_testcase/2]).
 
 -export([create_join_thread/1,
 	 equal_tids/1,
 	 mutex/1,
 	 try_lock_mutex/1,
-	 recursive_mutex/1,
-	 time_now/1,
 	 cond_wait/1,
-	 cond_timedwait/1,
 	 broadcast/1,
 	 detached_thread/1,
 	 max_threads/1,
-	 forksafety/1,
-	 vfork/1,
 	 tsd/1,
 	 spinlock/1,
 	 rwspinlock/1,
 	 rwmutex/1,
-	 atomic/1,
-	 gate/1]).
+	 atomic/1]).
 
 -include_lib("test_server/include/test_server.hrl").
 
-tests() ->
-    [create_join_thread,
-     equal_tids,
-     mutex,
-     try_lock_mutex,
-     recursive_mutex,
-     time_now,
-     cond_wait,
-     cond_timedwait,
-     broadcast,
-     detached_thread,
-     max_threads,
-     forksafety,
-     vfork,
-     tsd,
-     spinlock,
-     rwspinlock,
-     rwmutex,
-     atomic,
-     gate].
-
-all(doc) -> [];
-all(suite) -> tests().
+tests() -> 
+    [create_join_thread, equal_tids, mutex, try_lock_mutex,
+     cond_wait, broadcast, detached_thread,
+     max_threads, tsd, spinlock, rwspinlock, rwmutex, atomic].
 
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    tests().
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 %%
 %%
@@ -114,24 +110,6 @@ try_lock_mutex(suite) ->
 try_lock_mutex(Config) ->
     run_case(Config, "try_lock_mutex", "").
 
-recursive_mutex(doc) ->
-    ["Tests recursive mutexes."];
-recursive_mutex(suite) ->
-    [];
-recursive_mutex(Config) ->
-    run_case(Config, "recursive_mutex", "").
-
-time_now(doc) ->
-    ["Tests ethr_time_now by comparing time values with Erlang."];
-time_now(suite) ->
-    [];
-time_now(Config) ->
-    run_case(Config, "time_now", "", fun (P) ->
-					     spawn_link(fun () ->
-								watchdog(P)
-							end)
-				     end).
-
 wd_dispatch(P) ->
     receive
 	bye ->
@@ -169,13 +147,6 @@ cond_wait(suite) ->
 cond_wait(Config) ->
     run_case(Config, "cond_wait", "").
 
-cond_timedwait(doc) ->
-    ["Tests ethr_cond_timedwait with ethr_cond_signal and ethr_cond_broadcast."];
-cond_timedwait(suite) ->
-    [];
-cond_timedwait(Config) ->
-    run_case(Config, "cond_timedwait", "").
-
 broadcast(doc) ->
     ["Tests that a ethr_cond_broadcast really wakes up all waiting threads"];
 broadcast(suite) ->
@@ -195,26 +166,15 @@ max_threads(doc) ->
 max_threads(suite) ->
     [];
 max_threads(Config) ->
-    run_case(Config, "max_threads", "").
-
-forksafety(doc) ->
-    ["Tests forksafety."];
-forksafety(suite) ->
-    [];
-forksafety(Config) ->
-    run_case(Config, "forksafety", "").
-
-vfork(doc) ->
-    ["Tests vfork with threads."];
-vfork(suite) ->
-    case ?t:os_type() of
-	{unix, osf1} ->
-	    {skip, "vfork() known to hang multi-threaded applications on osf1"};
+    case {os:type(), os:version()} of
+	{{unix,darwin}, {9, _, _}} ->
+	    %% For some reason pthread_create() crashes when more
+	    %% threads cannot be created, instead of returning an
+	    %% error code on our MacOS X Leopard machine...
+	    {skipped, "MacOS X Leopard cannot cope with this test..."};
 	_ ->
-	    []
-    end;
-vfork(Config) ->
-    run_case(Config, "vfork", "").
+	    run_case(Config, "max_threads", "")
+    end.
 
 tsd(doc) ->
     ["Tests thread specific data."];
@@ -251,13 +211,6 @@ atomic(suite) ->
 atomic(Config) ->
     run_case(Config, "atomic", "").
 
-gate(doc) ->
-    ["Tests gates."];
-gate(suite) ->
-    [];
-gate(Config) ->
-    run_case(Config, "gate", "").
-
 %%
 %%
 %% Auxiliary functions
diff --git a/erts/test/ethread_SUITE_data/ethread_tests.c b/erts/test/ethread_SUITE_data/ethread_tests.c
index f779f13c51..0b59ff5aa6 100644
--- a/erts/test/ethread_SUITE_data/ethread_tests.c
+++ b/erts/test/ethread_SUITE_data/ethread_tests.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -90,10 +90,12 @@ static void print_line(char *frmt,...)
     print_eol();
 }
 
+#if 0 /* Currently not used; silence annoying warning... */
 static void print(char *frmt,...)
 {
     PRINT_VA_LIST(frmt);
 }
+#endif
 
 static void fail(char *frmt,...)
 {
@@ -197,8 +199,8 @@ create_join_thread_test(void)
     }
 
     for (i = 1; i <= CJTT_NO_THREADS; i++) {
-	int *tres;
-	res = ethr_thr_join(cjtt_tids[i], (void **) &tres);
+	void *tres;
+	res = ethr_thr_join(cjtt_tids[i], &tres);
 	ASSERT(res == 0);
 	ASSERT(tres == &cjtt_res[i]);
 	ASSERT(cjtt_res[i] == i);
@@ -216,8 +218,8 @@ create_join_thread_test(void)
 #define ETT_THREADS 100000
 
 static ethr_tid ett_tids[3];
-static ethr_mutex ett_mutex = ETHR_MUTEX_INITER;
-static ethr_cond ett_cond = ETHR_COND_INITER;
+static ethr_mutex ett_mutex;
+static ethr_cond ett_cond;
 static int ett_terminate;
 
 static void *
@@ -234,14 +236,12 @@ static void *
 ett_thread2(void *unused)
 {
     int res;
-    res = ethr_mutex_lock(&ett_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&ett_mutex);
     while (!ett_terminate) {
 	res = ethr_cond_wait(&ett_cond, &ett_mutex);
 	ASSERT(res == 0);
     }
-    res = ethr_mutex_unlock(&ett_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&ett_mutex);
     return NULL;
 }
 
@@ -250,6 +250,10 @@ equal_tids_test(void)
 {
     int res, i;
 
+    res = ethr_mutex_init(&ett_mutex);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&ett_cond);
+    ASSERT(res == 0);
     ett_tids[0] = ethr_self();
     
     res = ethr_thr_create(&ett_tids[1], ett_thread, (void *) &ett_tids[1], NULL);
@@ -298,13 +302,10 @@ equal_tids_test(void)
 	ASSERT(res == 0);
     }
 
-    res = ethr_mutex_lock(&ett_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&ett_mutex);
     ett_terminate = 1;
-    res = ethr_cond_signal(&ett_cond);
-    ASSERT(res == 0);
-    res = ethr_mutex_unlock(&ett_mutex);
-    ASSERT(res == 0);
+    ethr_cond_signal(&ett_cond);
+    ethr_mutex_unlock(&ett_mutex);
     res = ethr_thr_join(ett_tids[1], NULL);
     ASSERT(res == 0);
 
@@ -321,17 +322,14 @@ equal_tids_test(void)
  * Tests mutexes.
  */
 
-static ethr_mutex mt_mutex = ETHR_MUTEX_INITER;
+static ethr_mutex mt_mutex;
 static int mt_data;
 
 void *
 mt_thread(void *unused)
 {
-    int res;
-
     print_line("Aux thread tries to lock mutex");
-    res = ethr_mutex_lock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&mt_mutex);
     print_line("Aux thread locked mutex");
 
     ASSERT(mt_data == 0);
@@ -345,8 +343,7 @@ mt_thread(void *unused)
 
     ASSERT(mt_data == 1);
 
-    res = ethr_mutex_unlock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&mt_mutex);
     print_line("Aux thread unlocked mutex");
 
     return NULL;
@@ -356,18 +353,18 @@ mt_thread(void *unused)
 static void
 mutex_test(void)
 {
-    int do_restart = 1;
     int res;
     ethr_tid tid;
 
-    print_line("Running test with statically initialized mutex");
+    print_line("Trying to initialize mutex");
+    res = ethr_mutex_init(&mt_mutex);
+    ASSERT(res == 0);
+    print_line("Initialized mutex");
 
- restart:
     mt_data = 0;
 
     print_line("Main thread tries to lock mutex");
-    res = ethr_mutex_lock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&mt_mutex);
     print_line("Main thread locked mutex");
 
     ASSERT(mt_data == 0);
@@ -383,8 +380,7 @@ mutex_test(void)
 
     ASSERT(mt_data == 0);
 
-    res = ethr_mutex_unlock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&mt_mutex);
     print_line("Main thread unlocked mutex");
 
     print_line("Main thread goes to sleep for 1 second");
@@ -392,8 +388,7 @@ mutex_test(void)
     print_line("Main thread woke up");
 
     print_line("Main thread tries to lock mutex");
-    res = ethr_mutex_lock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&mt_mutex);
     print_line("Main thread locked mutex");
 
     ASSERT(mt_data == 1);
@@ -404,8 +399,7 @@ mutex_test(void)
 
     ASSERT(mt_data == 1);
 
-    res = ethr_mutex_unlock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&mt_mutex);
     print_line("Main thread unlocked mutex");
 
     res = ethr_thr_join(tid, NULL);
@@ -416,20 +410,6 @@ mutex_test(void)
     ASSERT(res == 0);
     print_line("Main thread destroyed mutex");
 
-    if (do_restart) {
-	do_restart = 0;
-
-	print_line("Running test with dynamically initialized mutex");
-
-	print_line("Trying to initialize mutex");
-	res = ethr_mutex_init(&mt_mutex);
-	ASSERT(res == 0);
-	print_line("Initialized mutex");
-
-	goto restart;
-
-    }
-
 }
 
 /*
@@ -438,9 +418,9 @@ mutex_test(void)
  * Tests try lock mutex operation.
  */
 
-static ethr_mutex tlmt_mtx1 = ETHR_MUTEX_INITER;
-static ethr_mutex tlmt_mtx2 = ETHR_MUTEX_INITER;
-static ethr_cond tlmt_cnd2 = ETHR_COND_INITER;
+static ethr_mutex tlmt_mtx1;
+static ethr_mutex tlmt_mtx2;
+static ethr_cond tlmt_cnd2;
 
 static int tlmt_mtx1_locked;
 static int tlmt_mtx1_do_unlock;
@@ -450,32 +430,24 @@ tlmt_thread(void *unused)
 {
     int res;
 
-    res = ethr_mutex_lock(&tlmt_mtx1);
-    ASSERT(res == 0);
-    res = ethr_mutex_lock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&tlmt_mtx1);
+    ethr_mutex_lock(&tlmt_mtx2);
 
     tlmt_mtx1_locked = 1;
-    res = ethr_cond_signal(&tlmt_cnd2);
-    ASSERT(res == 0);
+    ethr_cond_signal(&tlmt_cnd2);
 
     while (!tlmt_mtx1_do_unlock) {
 	res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2);
 	ASSERT(res == 0 || res == EINTR);
     }
 
-    res = ethr_mutex_unlock(&tlmt_mtx2);
-    ASSERT(res == 0);
-    res = ethr_mutex_unlock(&tlmt_mtx1);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&tlmt_mtx2);
+    ethr_mutex_unlock(&tlmt_mtx1);
 
-    res = ethr_mutex_lock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&tlmt_mtx2);
     tlmt_mtx1_locked = 0;
-    res = ethr_cond_signal(&tlmt_cnd2);
-    ASSERT(res == 0);
-    res = ethr_mutex_unlock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_cond_signal(&tlmt_cnd2);
+    ethr_mutex_unlock(&tlmt_mtx2);
 
     return NULL;
 }
@@ -486,48 +458,49 @@ try_lock_mutex_test(void)
     int i, res;
     ethr_tid tid;
 
+    res = ethr_mutex_init(&tlmt_mtx1);
+    ASSERT(res == 0);
+    res = ethr_mutex_init(&tlmt_mtx2);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&tlmt_cnd2);
+    ASSERT(res == 0);
+
     tlmt_mtx1_locked = 0;
     tlmt_mtx1_do_unlock = 0;
 
     res = ethr_thr_create(&tid, tlmt_thread, NULL, NULL);
     ASSERT(res == 0);
 
-    res = ethr_mutex_lock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&tlmt_mtx2);
 
     while (!tlmt_mtx1_locked) {
 	res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2);
 	ASSERT(res == 0 || res == EINTR);
     }
 
-    res = ethr_mutex_unlock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&tlmt_mtx2);
 
     for (i = 0; i < 10; i++) {
 	res = ethr_mutex_trylock(&tlmt_mtx1);
 	ASSERT(res == EBUSY);
     }
 
-    res = ethr_mutex_lock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&tlmt_mtx2);
 
     tlmt_mtx1_do_unlock = 1;
-    res = ethr_cond_signal(&tlmt_cnd2);
-    ASSERT(res == 0);
+    ethr_cond_signal(&tlmt_cnd2);
 
     while (tlmt_mtx1_locked) {
 	res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2);
 	ASSERT(res == 0 || res == EINTR);
     }
 
-    res = ethr_mutex_unlock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&tlmt_mtx2);
 
     res = ethr_mutex_trylock(&tlmt_mtx1);
     ASSERT(res == 0);
 
-    res = ethr_mutex_unlock(&tlmt_mtx1);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&tlmt_mtx1);
 
     res = ethr_thr_join(tid, NULL);
     ASSERT(res == 0);
@@ -541,328 +514,88 @@ try_lock_mutex_test(void)
 }
 
 /*
- * The recursive mutex test case.
- *
- * Tests recursive mutexes.
- */
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-static ethr_mutex rmt_mutex
-#ifdef ETHR_REC_MUTEX_INITER 
-                            = ETHR_REC_MUTEX_INITER
-#endif
-                                                   ;
-static int rmt_data;
-
-void *
-rmt_thread(void *unused)
-{
-    int res;
-
-    print_line("Aux thread tries to lock mutex");
-    res = ethr_mutex_lock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Aux thread locked mutex");
-
-    ASSERT(rmt_data == 0);
-
-    rmt_data = 1;
-    print_line("Aux thread wrote");
-
-    print_line("Aux thread goes to sleep for 1 second");
-    do_sleep(1);
-    print_line("Aux thread woke up");
-
-    ASSERT(rmt_data == 1);
-
-    res = ethr_mutex_unlock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Aux thread unlocked mutex");
-
-    return NULL;
-}
-
-#endif
-
-static void
-recursive_mutex_test(void)
-{
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-    int do_restart = 1;
-    int res;
-    ethr_tid tid;
-
-#ifdef ETHR_REC_MUTEX_INITER
-    print_line("Running test with statically initialized mutex");
-#else
-    goto dynamic_init;
-#endif
-
- restart:
-    rmt_data = 0;
-
-    print_line("Main thread tries to lock mutex");
-    res = ethr_mutex_lock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread locked mutex");
-
-    print_line("Main thread tries to lock mutex again");
-    res = ethr_mutex_lock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread locked mutex again");
-
-    ASSERT(rmt_data == 0);
-
-    print_line("Main thread about to create aux thread");
-    res = ethr_thr_create(&tid, rmt_thread, NULL, NULL);
-    ASSERT(res == 0);
-    print_line("Main thread created aux thread");
-
-    print_line("Main thread goes to sleep for 1 second");
-    do_sleep(1);
-    print_line("Main thread woke up");
-
-    ASSERT(rmt_data == 0);
-
-    res = ethr_mutex_unlock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread unlocked mutex");
-
-    print_line("Main thread goes to sleep for 1 second");
-    do_sleep(1);
-    print_line("Main thread woke up");
-
-    ASSERT(rmt_data == 0);
-
-    res = ethr_mutex_unlock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread unlocked mutex again");
-
-    print_line("Main thread goes to sleep for 1 second");
-    do_sleep(1);
-    print_line("Main thread woke up");
-
-    print_line("Main thread tries to lock mutex");
-    res = ethr_mutex_lock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread locked mutex");
-
-    ASSERT(rmt_data == 1);
-
-    print_line("Main thread goes to sleep for 1 second");
-    do_sleep(1);
-    print_line("Main thread woke up");
-
-    ASSERT(rmt_data == 1);
-
-    res = ethr_mutex_unlock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread unlocked mutex");
-
-    res = ethr_thr_join(tid, NULL);
-    ASSERT(res == 0);
-    print_line("Main thread joined aux thread");
-
-    res = ethr_mutex_destroy(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread destroyed mutex");
-
-    if (do_restart) {
-#ifndef ETHR_REC_MUTEX_INITER
-    dynamic_init:
-#endif
-	do_restart = 0;
-
-	print_line("Running test with dynamically initialized mutex");
-
-	print_line("Trying to initialize mutex");
-	res = ethr_rec_mutex_init(&rmt_mutex);
-	ASSERT(res == 0);
-	print_line("Initialized mutex");
-
-	goto restart;
-    }
-
-#ifndef ETHR_REC_MUTEX_INITER
-    succeed("Static initializer for recursive mutexes not supported");
-#endif
-
-#else /* #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-    skip("Recursive mutexes not supported");
-#endif /* #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-}
-
-/*
- * The time now test.
- *
- * Tests ethr_time_now by comparing time values with Erlang.
- */
-#define TNT_MAX_TIME_DIFF 200000
-#define TNT_MAX_TIME_VALUES 52
-
-static void
-time_now_test(void)
-{
-    int scanf_res, time_now_res, i, no_values, max_abs_diff;
-    static ethr_timeval tv[TNT_MAX_TIME_VALUES];
-    static int ms[TNT_MAX_TIME_VALUES];
-
-    i = 0;
-    do {
-	ASSERT(i < TNT_MAX_TIME_VALUES);
-	scanf_res = scanf("%d", &ms[i]);
-	time_now_res = ethr_time_now(&tv[i]);
-	ASSERT(scanf_res == 1);
-	ASSERT(time_now_res == 0);
-#if 0
-	print_line("Got %d; %ld:%ld", ms[i], tv[i].tv_sec, tv[i].tv_nsec);
-#endif
-	i++;
-    } while (ms[i-1] >= 0);
-
-    no_values = i-1;
-
-    ASSERT(ms[0] == 0);
-
-    print_line("TNT_MAX_TIME_DIFF = %d (us)", TNT_MAX_TIME_DIFF);
-
-    max_abs_diff = 0;
-
-    for (i = 1; i < no_values; i++) {
-	long diff;
-	long tn_us;
-	long e_us;
-
-	tn_us = (tv[i].tv_sec - tv[0].tv_sec) * 1000000;
-	tn_us += (tv[i].tv_nsec - tv[0].tv_nsec)/1000;
-
-	e_us = ms[i]*1000;
-	
-	diff = e_us - tn_us;
-
-	print_line("Erlang time = %ld us; ethr_time_now = %ld us; diff %ld us",
-		   e_us, tn_us, diff);
-
-	if (max_abs_diff < abs((int) diff)) {
-	    max_abs_diff = abs((int) diff);
-	}
-
-	ASSERT(e_us - TNT_MAX_TIME_DIFF <= tn_us);
-	ASSERT(tn_us <= e_us + TNT_MAX_TIME_DIFF);
-    }
-
-    print_line("Max absolute diff = %d us", max_abs_diff);
-    succeed("Max absolute diff = %d us", max_abs_diff);
-}
-
-/*
  * The cond wait test case.
  *
  * Tests ethr_cond_wait with ethr_cond_signal and ethr_cond_broadcast.
  */
 
 
-static ethr_mutex cwt_mutex = ETHR_MUTEX_INITER;
-static ethr_cond cwt_cond = ETHR_COND_INITER;
+static ethr_mutex cwt_mutex;
+static ethr_cond cwt_cond;
 static int cwt_counter;
 
 void *
-cwt_thread(void *is_timedwait_test_ptr)
+cwt_thread(void *unused)
 {
-    int use_timedwait = *((int *) is_timedwait_test_ptr);
     int res;
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
-    if (use_timedwait) {
-	ethr_timeval tv;
-	res = ethr_time_now(&tv);
-	ASSERT(res == 0);
-	tv.tv_sec += 3600; /* Make sure we won't time out */
-
-	do {
-	    res = ethr_cond_timedwait(&cwt_cond, &cwt_mutex, &tv);
-	} while (res == EINTR);
-	ASSERT(res == 0);
-    }
-    else {
-	do {
-	    res = ethr_cond_wait(&cwt_cond, &cwt_mutex);
-	} while (res == EINTR);
-	ASSERT(res == 0);
-    }
+    do {
+	res = ethr_cond_wait(&cwt_cond, &cwt_mutex);
+    } while (res == EINTR);
+    ASSERT(res == 0);
 
     cwt_counter++;
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     return NULL;
 }
 
 static void
-cond_wait_test(int is_timedwait_test)
+cond_wait_test(void)
 {
-    int do_restart = !is_timedwait_test;
     ethr_tid tid1, tid2;
     int res;
 
-    if (!is_timedwait_test)
-	print_line("Running test with statically initialized mutex and cond");
+    res = ethr_mutex_init(&cwt_mutex);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&cwt_cond);
+    ASSERT(res == 0);
 
- restart:
     /* Wake with signal */
 
     cwt_counter = 0;
 
-    res = ethr_thr_create(&tid1, cwt_thread, (void *) &is_timedwait_test, NULL);
+    res = ethr_thr_create(&tid1, cwt_thread, NULL, NULL);
     ASSERT(res == 0);
-    res = ethr_thr_create(&tid2, cwt_thread, (void *) &is_timedwait_test, NULL);
+    res = ethr_thr_create(&tid2, cwt_thread, NULL, NULL);
     ASSERT(res == 0);
 
     do_sleep(1); /* Make sure threads waits on cond var */
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
-    res = ethr_cond_signal(&cwt_cond); /* Wake one thread */
-    ASSERT(res == 0);
+    ethr_cond_signal(&cwt_cond); /* Wake one thread */
 
     do_sleep(1); /* Make sure awakened thread waits on mutex */
 
     ASSERT(cwt_counter == 0);
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     do_sleep(1);  /* Let awakened thread proceed */
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
     ASSERT(cwt_counter == 1);
 
-    res = ethr_cond_signal(&cwt_cond); /* Wake the other thread */
-    ASSERT(res == 0);
+    ethr_cond_signal(&cwt_cond); /* Wake the other thread */
 
     do_sleep(1); /* Make sure awakened thread waits on mutex */
 
     ASSERT(cwt_counter == 1);
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     do_sleep(1);  /* Let awakened thread proceed */
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
     ASSERT(cwt_counter == 2);
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     res = ethr_thr_join(tid1, NULL);
     ASSERT(res == 0);
@@ -875,35 +608,30 @@ cond_wait_test(int is_timedwait_test)
 
     cwt_counter = 0;
 
-    res = ethr_thr_create(&tid1, cwt_thread, (void *) &is_timedwait_test, NULL);
+    res = ethr_thr_create(&tid1, cwt_thread, NULL, NULL);
     ASSERT(res == 0);
-    res = ethr_thr_create(&tid2, cwt_thread, (void *) &is_timedwait_test, NULL);
+    res = ethr_thr_create(&tid2, cwt_thread, NULL, NULL);
     ASSERT(res == 0);
 
     do_sleep(1); /* Make sure threads waits on cond var */
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
-    res = ethr_cond_broadcast(&cwt_cond); /* Wake the threads */
-    ASSERT(res == 0);
+    ethr_cond_broadcast(&cwt_cond); /* Wake the threads */
 
     do_sleep(1); /* Make sure awakened threads wait on mutex */
 
     ASSERT(cwt_counter == 0);
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     do_sleep(1);  /* Let awakened threads proceed */
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
     ASSERT(cwt_counter == 2);
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     res = ethr_thr_join(tid1, NULL);
     ASSERT(res == 0);
@@ -916,113 +644,6 @@ cond_wait_test(int is_timedwait_test)
     res = ethr_cond_destroy(&cwt_cond);
     ASSERT(res == 0);
 
-    if (do_restart) {
-	do_restart = 0;
-	res = ethr_mutex_init(&cwt_mutex);
-	ASSERT(res == 0);
-	res = ethr_cond_init(&cwt_cond);
-	ASSERT(res == 0);
-	print_line("Running test with dynamically initialized mutex and cond");
-	goto restart;
-    }
-}
-
-/*
- * The cond timedwait test case.
- *
- * Tests ethr_cond_timedwait with ethr_cond_signal and ethr_cond_broadcast.
- */
-
-#define CTWT_MAX_TIME_DIFF 100000
-
-static long
-ctwt_check_timeout(long to)
-{
-    int res;
-    ethr_timeval tva, tvb;
-    long diff, abs_diff;
-
-    res = ethr_time_now(&tva);
-    ASSERT(res == 0);
-
-    tva.tv_sec += to / 1000;
-    tva.tv_nsec += (to % 1000) * 1000000;
-    if (tva.tv_nsec >= 1000000000) {
-	tva.tv_sec++;
-	tva.tv_nsec -= 1000000000;
-	ASSERT(tva.tv_nsec < 1000000000);
-    }
-
-    do {
-	res = ethr_cond_timedwait(&cwt_cond, &cwt_mutex, &tva);
-    } while (res == EINTR);
-    ASSERT(res == ETIMEDOUT);
-
-    res = ethr_time_now(&tvb);
-    ASSERT(res == 0);
-
-    diff = (tvb.tv_sec - tva.tv_sec) * 1000000;
-    diff += (tvb.tv_nsec - tva.tv_nsec)/1000;
-
-    print("Timeout=%ld; ", to);
-    print("tva.tv_sec=%ld tva.tv_nsec=%ld; ", tva.tv_sec, tva.tv_nsec);
-    print("tvb.tv_sec=%ld tvb.tv_nsec=%ld; ", tvb.tv_sec, tvb.tv_nsec);
-    print_line("diff (tvb - tva) = %ld us", diff);
-
-    abs_diff = (long) abs((int) diff);
-
-    ASSERT(CTWT_MAX_TIME_DIFF >= abs_diff);
-    return abs_diff;
-}
-
-static void
-cond_timedwait_test(void)
-{
-    int do_restart = 1;
-    long abs_diff, max_abs_diff = 0;
-    int res;
-
-#define CTWT_UPD_MAX_DIFF if (abs_diff > max_abs_diff) max_abs_diff = abs_diff;
-
-    print_line("Running test with statically initialized mutex and cond");
-
-    print_line("CTWT_MAX_TIME_DIFF=%d", CTWT_MAX_TIME_DIFF);
-
- restart:
-
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
-
-    abs_diff = ctwt_check_timeout(300);
-    CTWT_UPD_MAX_DIFF;
-    abs_diff = ctwt_check_timeout(700);
-    CTWT_UPD_MAX_DIFF;
-    abs_diff = ctwt_check_timeout(1000);
-    CTWT_UPD_MAX_DIFF;
-    abs_diff = ctwt_check_timeout(2300);
-    CTWT_UPD_MAX_DIFF;
-    abs_diff = ctwt_check_timeout(5100);
-    CTWT_UPD_MAX_DIFF;
-
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
-
-    cond_wait_test(1);
-
-    if (do_restart) {
-	do_restart = 0;
-	res = ethr_mutex_init(&cwt_mutex);
-	ASSERT(res == 0);
-	res = ethr_cond_init(&cwt_cond);
-	ASSERT(res == 0);
-	print_line("Running test with dynamically initialized mutex and cond");
-	goto restart;
-    }
-
-    print_line("Max absolute diff = %d us", max_abs_diff);
-    succeed("Max absolute diff = %d us", max_abs_diff);
-
-#undef CTWT_UPD_MAX_DIFF
 }
 
 /*
@@ -1037,9 +658,9 @@ cond_timedwait_test(void)
 static int bct_woken = 0;
 static int bct_waiting = 0;
 static int bct_done = 0;
-static ethr_mutex bct_mutex = ETHR_MUTEX_INITER;
-static ethr_cond bct_cond = ETHR_COND_INITER;
-static ethr_cond bct_cntrl_cond = ETHR_COND_INITER;
+static ethr_mutex bct_mutex;
+static ethr_cond bct_cond;
+static ethr_cond bct_cntrl_cond;
 
 
 static void *
@@ -1047,30 +668,24 @@ bct_thread(void *unused)
 {
     int res;
 
-    res = ethr_mutex_lock(&bct_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&bct_mutex);
 
     while (!bct_done) {
 
 	bct_waiting++;
-	if (bct_waiting == BCT_THREADS) {
-	    res = ethr_cond_signal(&bct_cntrl_cond);
-	    ASSERT(res == 0);
-	}
+	if (bct_waiting == BCT_THREADS)
+	    ethr_cond_signal(&bct_cntrl_cond);
 	do {
 	    res = ethr_cond_wait(&bct_cond, &bct_mutex);
 	} while (res == EINTR);
 	ASSERT(res == 0);
 	bct_woken++;
-	if (bct_woken == BCT_THREADS) {
-	    res = ethr_cond_signal(&bct_cntrl_cond);
-	    ASSERT(res == 0);
-	}
+	if (bct_woken == BCT_THREADS)
+	    ethr_cond_signal(&bct_cntrl_cond);
 
     }
 
-    res = ethr_mutex_unlock(&bct_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&bct_mutex);
 
     return NULL;
 }
@@ -1081,14 +696,20 @@ broadcast_test(void)
     int res, i;
     ethr_tid tid[BCT_THREADS];
 
+    res = ethr_mutex_init(&bct_mutex);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&bct_cntrl_cond);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&bct_cond);
+    ASSERT(res == 0);
+
     for (i = 0; i < BCT_THREADS; i++) {
 	res = ethr_thr_create(&tid[i], bct_thread, NULL, NULL);
 	ASSERT(res == 0);
 
     }
 
-    res = ethr_mutex_lock(&bct_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&bct_mutex);
 
     for (i = 0; i < BCT_NO_OF_WAITS; i++) {
 
@@ -1101,8 +722,7 @@ broadcast_test(void)
 	bct_woken = 0;
 
 	/* Wake all threads */
-	res = ethr_cond_broadcast(&bct_cond);
-	ASSERT(res == 0);
+	ethr_cond_broadcast(&bct_cond);
 
 	while (bct_woken != BCT_THREADS) {
 	    res = ethr_cond_wait(&bct_cntrl_cond, &bct_mutex);
@@ -1114,13 +734,11 @@ broadcast_test(void)
     bct_done = 1;
 
     /* Wake all threads */
-    res = ethr_cond_broadcast(&bct_cond);
-    ASSERT(res == 0);
+    ethr_cond_broadcast(&bct_cond);
 
-    res = ethr_mutex_unlock(&bct_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&bct_mutex);
 
-    for (i = 0; i < BCT_THREADS - 1; i++) {
+    for (i = 0; i < BCT_THREADS; i++) {
 	res = ethr_thr_join(tid[i], NULL);
 	ASSERT(res == 0);
     }
@@ -1143,26 +761,22 @@ broadcast_test(void)
 #define DT_THREADS (50*1024)
 #define DT_BATCH_SIZE 64
 
-static ethr_mutex dt_mutex = ETHR_MUTEX_INITER;
-static ethr_cond dt_cond = ETHR_COND_INITER;
+static ethr_mutex dt_mutex;
+static ethr_cond dt_cond;
 static int dt_count;
 static int dt_limit;
 
 static void *
 dt_thread(void *unused)
 {
-    int res;
-
-    res = ethr_mutex_lock(&dt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&dt_mutex);
 
     dt_count++;
 
     if (dt_count >= dt_limit)
 	ethr_cond_signal(&dt_cond);
 
-    res = ethr_mutex_unlock(&dt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&dt_mutex);
 
     return NULL;
 }
@@ -1174,6 +788,11 @@ detached_thread_test(void)
     ethr_tid tid[DT_BATCH_SIZE];
     int i, j, res;
 
+    res = ethr_mutex_init(&dt_mutex);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&dt_cond);
+    ASSERT(res == 0);
+
     thr_opts.detached = 1;
     dt_count = 0;
     dt_limit = 0;
@@ -1187,14 +806,12 @@ detached_thread_test(void)
 	    ASSERT(res == 0);
 	}
 
-	res = ethr_mutex_lock(&dt_mutex);
-	ASSERT(res == 0);
+	ethr_mutex_lock(&dt_mutex);
 	while (dt_count < dt_limit) {
 	    res = ethr_cond_wait(&dt_cond, &dt_mutex);
 	    ASSERT(res == 0 || res == EINTR);
 	}
-	res = ethr_mutex_unlock(&dt_mutex);
-	ASSERT(res == 0);
+	ethr_mutex_unlock(&dt_mutex);
 
 	print_line("dt_count = %d", dt_count);
     }
@@ -1211,8 +828,8 @@ detached_thread_test(void)
 #define MTT_TIMES 10
 
 static int mtt_terminate;
-static ethr_mutex mtt_mutex = ETHR_MUTEX_INITER;
-static ethr_cond mtt_cond = ETHR_COND_INITER;
+static ethr_mutex mtt_mutex;
+static ethr_cond mtt_cond;
 static char mtt_string[22*MTT_TIMES]; /* 22 is enough for ", %d" */
 
 
@@ -1220,16 +837,14 @@ void *mtt_thread(void *unused)
 {
     int res;
 
-    res = ethr_mutex_lock(&mtt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&mtt_mutex);
 
     while (!mtt_terminate) {
 	res = ethr_cond_wait(&mtt_cond, &mtt_mutex);
 	ASSERT(res == 0 || res == EINTR);
     }
 
-    res = ethr_mutex_unlock(&mtt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&mtt_mutex);
 
     return NULL; 
 }
@@ -1265,16 +880,13 @@ mtt_create_join_threads(void)
     print_line("%d = ethr_thr_create()", res);
     print_line("Number of created threads: %d", no_threads);
 
-    res = ethr_mutex_lock(&mtt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&mtt_mutex);
 
     mtt_terminate = 1;
 
-    res = ethr_cond_broadcast(&mtt_cond);
-    ASSERT(res == 0);
+    ethr_cond_broadcast(&mtt_cond);
 
-    res = ethr_mutex_unlock(&mtt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&mtt_mutex);
 
     while (ix) {
 	res = ethr_thr_join(tids[--ix], NULL);
@@ -1292,9 +904,14 @@ mtt_create_join_threads(void)
 static void
 max_threads_test(void)
 {
-    int no_threads[MTT_TIMES], i, up, down, eq;
+    int no_threads[MTT_TIMES], i, up, down, eq, res;
     char *str;
 
+    res = ethr_mutex_init(&mtt_mutex);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&mtt_cond);
+    ASSERT(res == 0);
+
     for (i = 0; i < MTT_TIMES; i++) {
 	no_threads[i] = mtt_create_join_threads();
     }
@@ -1326,272 +943,6 @@ max_threads_test(void)
 
 }
 
-
-/*
- * The forksafety test case.
- *
- * Tests forksafety.
- */
-#ifdef __WIN32__
-#define NO_FORK_PRESENT
-#endif
-
-#ifndef NO_FORK_PRESENT
-
-static ethr_mutex ft_test_inner_mutex = ETHR_MUTEX_INITER;
-static ethr_mutex ft_test_outer_mutex = ETHR_MUTEX_INITER;
-static ethr_mutex ft_go_mutex = ETHR_MUTEX_INITER;
-static ethr_cond ft_go_cond = ETHR_COND_INITER;
-static int ft_go;
-static int ft_have_forked;
-
-static void *
-ft_thread(void *unused)
-{
-    int res;
-
-    res = ethr_mutex_lock(&ft_test_outer_mutex);
-    ASSERT(res == 0);
-
-    res = ethr_mutex_lock(&ft_go_mutex);
-    ASSERT(res == 0);
-
-    ft_go = 1;
-    res = ethr_cond_signal(&ft_go_cond);
-    ASSERT(res == 0);
-    res = ethr_mutex_unlock(&ft_go_mutex);
-    ASSERT(res == 0);
-
-    do_sleep(1);
-    ASSERT(!ft_have_forked);
-
-    res = ethr_mutex_lock(&ft_test_inner_mutex);
-    ASSERT(res == 0);
-
-    res = ethr_mutex_unlock(&ft_test_inner_mutex);
-    ASSERT(res == 0);
-
-    do_sleep(1);
-    ASSERT(!ft_have_forked);
-
-    res = ethr_mutex_unlock(&ft_test_outer_mutex);
-    ASSERT(res == 0);
-
-    do_sleep(1);
-    ASSERT(ft_have_forked);
-    
-
-    return NULL;
-}
-
-#endif /* #ifndef NO_FORK_PRESENT */
-
-static void
-forksafety_test(void)
-{
-#ifdef NO_FORK_PRESENT
-    skip("No fork() present; nothing to test");
-#elif defined(DEBUG)
-    skip("Doesn't work in debug build");
-#else
-    char snd_msg[] = "ok, bye!";
-    char rec_msg[sizeof(snd_msg)*2];
-    ethr_tid tid;
-    int res;
-    int fds[2];
-
-
-    res = ethr_mutex_set_forksafe(&ft_test_inner_mutex);
-    if (res == ENOTSUP) {
-	skip("Forksafety not supported on this platform!");
-    }
-    ASSERT(res == 0);
-    res = ethr_mutex_set_forksafe(&ft_test_outer_mutex);
-    ASSERT(res == 0);
-
-
-    res = pipe(fds);
-    ASSERT(res == 0);
-
-    ft_go = 0;
-    ft_have_forked = 0;
-
-    res = ethr_mutex_lock(&ft_go_mutex);
-    ASSERT(res == 0);
-    
-    res = ethr_thr_create(&tid, ft_thread, NULL, NULL);
-    ASSERT(res == 0);
-
-    do {
-	res = ethr_cond_wait(&ft_go_cond, &ft_go_mutex);
-    } while (res == EINTR || !ft_go);
-    ASSERT(res == 0);
-
-    res = ethr_mutex_unlock(&ft_go_mutex);
-    ASSERT(res == 0);
-
-    res = fork();
-    ft_have_forked = 1;
-    if (res == 0) {
-	close(fds[0]);
-	res = ethr_mutex_lock(&ft_test_outer_mutex);
-	if (res != 0)
-	    _exit(1);
-	res = ethr_mutex_lock(&ft_test_inner_mutex);
-	if (res != 0)
-	    _exit(1);
-	res = ethr_mutex_unlock(&ft_test_inner_mutex);
-	if (res != 0)
-	    _exit(1);
-	res = ethr_mutex_unlock(&ft_test_outer_mutex);
-	if (res != 0)
-	    _exit(1);
-
-	res = ethr_mutex_destroy(&ft_test_inner_mutex);
-	if (res != 0)
-	    _exit(1);
-	res = ethr_mutex_destroy(&ft_test_outer_mutex);
-	if (res != 0)
-	    _exit(1);
-
-	res = (int) write(fds[1], (void *) snd_msg, sizeof(snd_msg));
-	if (res != sizeof(snd_msg))
-	    _exit(1);
-	close(fds[1]);
-	_exit(0);
-    }
-    ASSERT(res > 0);
-    close(fds[1]);
-
-    res = (int) read(fds[0], (void *) rec_msg, sizeof(rec_msg));
-    ASSERT(res == (int) sizeof(snd_msg));
-    ASSERT(strcmp(snd_msg, rec_msg) == 0);
-
-    close(fds[0]);
-#endif
-}
-
-
-/*
- * The vfork test case.
- *
- * Tests vfork with threads.
- */
-
-#ifdef __WIN32__
-#define NO_VFORK_PRESENT
-#endif
-
-#ifndef NO_VFORK_PRESENT
-
-#undef vfork
-
-static ethr_mutex vt_mutex = ETHR_MUTEX_INITER;
-
-static void *
-vt_thread(void *vprog)
-{
-    char *prog = (char *) vprog;
-    int res;
-    char snd_msg[] = "ok, bye!";
-    char rec_msg[sizeof(snd_msg)*2];
-    int fds[2];
-    char closefd[20];
-    char writefd[20];
-
-    res = pipe(fds);
-    ASSERT(res == 0);
-
-    res = sprintf(closefd, "%d", fds[0]);
-    ASSERT(res <= 20);
-    res = sprintf(writefd, "%d", fds[1]);
-    ASSERT(res <= 20);
-
-    print("parent: About to vfork and execute ");
-    print("execlp(\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", NULL)",
-	  prog, prog, "vfork", "exec", snd_msg, closefd, writefd);
-    print_line(" in child");
-    res = vfork();
-    if (res == 0) {
-	execlp(prog, prog, "vfork", "exec", snd_msg, closefd, writefd, NULL);
-	_exit(1);
-    }
-    ASSERT(res > 0);
-
-    print_line("parent: I'm back");
-
-    close(fds[1]);
-
-    res = (int) read(fds[0], (void *) rec_msg, sizeof(rec_msg));
-    print_line("parent: %d = read()", res);
-    print_line("parent: rec_msg=\"%s\"", rec_msg);
-    ASSERT(res == (int) sizeof(snd_msg));
-    ASSERT(strcmp(snd_msg, rec_msg) == 0);
-
-    close(fds[0]);
-
-    return NULL;
-}
-
-#endif /* #ifndef NO_VFORK_PRESENT */
-
-static void
-vfork_test(int argc, char *argv[])
-{
-#ifdef NO_VFORK_PRESENT
-    skip("No vfork() present; nothing to test");
-#else
-    int res;
-    ethr_tid tid;
-
-    if (argc == 6 && strcmp("exec", argv[2]) == 0) {
-	/* We are child after vfork() and execlp() ... */
-
-	char *snd_msg;
-	int closefd;
-	int writefd;
-
-	snd_msg = argv[3];
-	closefd = atoi(argv[4]);
-	writefd = atoi(argv[5]);
-
-	print_line("child: snd_msg=\"%s\"; closefd=%d writefd=%d",
-		   snd_msg, closefd, writefd);
-
-	close(closefd);
-
-	res = (int) write(writefd, (void *) snd_msg, strlen(snd_msg)+1);
-	print_line("child: %d = write()", res);
-	if (res != strlen(snd_msg)+1)
-	    exit(1);
-	close(writefd);
-	print_line("child: bye");
-	exit(0);
-    }
-    ASSERT(argc == 2);
-
-    res = ethr_mutex_set_forksafe(&vt_mutex);
-    ASSERT(res == 0 || res == ENOTSUP);
-    res = ethr_mutex_lock(&vt_mutex);
-    ASSERT(res == 0);
-
-    res = ethr_thr_create(&tid, vt_thread, (void *) argv[0], NULL);
-    ASSERT(res == 0);
-
-    do_sleep(1);
-
-    res = ethr_mutex_unlock(&vt_mutex);
-    ASSERT(res == 0);
-
-    res = ethr_thr_join(tid, NULL);
-    ASSERT(res == 0);
-
-    res = ethr_mutex_destroy(&vt_mutex);
-    ASSERT(res == 0);
-#endif
-}
-
-
 /*
  * The tsd test case.
  *
@@ -1651,11 +1002,8 @@ static int st_data;
 void *
 st_thread(void *unused)
 {
-    int res;
-
     print_line("Aux thread tries to lock spinlock");
-    res = ethr_spin_lock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_lock(&st_spinlock);
     print_line("Aux thread locked spinlock");
 
     ASSERT(st_data == 0);
@@ -1669,8 +1017,7 @@ st_thread(void *unused)
 
     ASSERT(st_data == 1);
 
-    res = ethr_spin_unlock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_unlock(&st_spinlock);
     print_line("Aux thread unlocked spinlock");
 
     return NULL;
@@ -1691,8 +1038,7 @@ spinlock_test(void)
     st_data = 0;
 
     print_line("Main thread tries to lock spinlock");
-    res = ethr_spin_lock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_lock(&st_spinlock);
     print_line("Main thread locked spinlock");
 
     ASSERT(st_data == 0);
@@ -1708,8 +1054,7 @@ spinlock_test(void)
 
     ASSERT(st_data == 0);
 
-    res = ethr_spin_unlock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_unlock(&st_spinlock);
     print_line("Main thread unlocked spinlock");
 
     print_line("Main thread goes to sleep for 1 second");
@@ -1717,8 +1062,7 @@ spinlock_test(void)
     print_line("Main thread woke up");
 
     print_line("Main thread tries to lock spinlock");
-    res = ethr_spin_lock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_lock(&st_spinlock);
     print_line("Main thread locked spinlock");
 
     ASSERT(st_data == 1);
@@ -1729,8 +1073,7 @@ spinlock_test(void)
 
     ASSERT(st_data == 1);
 
-    res = ethr_spin_unlock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_unlock(&st_spinlock);
     print_line("Main thread unlocked spinlock");
 
     res = ethr_thr_join(tid, NULL);
@@ -1757,23 +1100,19 @@ void *
 rwst_thread(void *unused)
 {
     int data;
-    int res;
 
     print_line("Aux thread tries to read lock rwspinlock");
-    res = ethr_read_lock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_read_lock(&rwst_rwspinlock);
     print_line("Aux thread read locked rwspinlock");
 
     ASSERT(rwst_data == 4711);
 
     print_line("Aux thread tries to read unlock rwspinlock");
-    res = ethr_read_unlock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_read_unlock(&rwst_rwspinlock);
     print_line("Aux thread read unlocked rwspinlock");
 
     print_line("Aux thread tries to write lock rwspinlock");
-    res = ethr_write_lock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_write_lock(&rwst_rwspinlock);
     print_line("Aux thread write locked rwspinlock");
 
     data = ++rwst_data;
@@ -1787,8 +1126,7 @@ rwst_thread(void *unused)
     ++rwst_data;
 
     print_line("Aux thread tries to write unlock rwspinlock");
-    res = ethr_write_unlock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_write_unlock(&rwst_rwspinlock);
     print_line("Aux thread write unlocked rwspinlock");
 
     return NULL;
@@ -1810,8 +1148,7 @@ rwspinlock_test(void)
     rwst_data = 4711;
 
     print_line("Main thread tries to read lock rwspinlock");
-    res = ethr_read_lock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_read_lock(&rwst_rwspinlock);
     print_line("Main thread read locked rwspinlock");
 
     ASSERT(rwst_data == 4711);
@@ -1828,13 +1165,11 @@ rwspinlock_test(void)
     ASSERT(rwst_data == 4711);
 
     print_line("Main thread tries to read unlock rwspinlock");
-    res = ethr_read_unlock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_read_unlock(&rwst_rwspinlock);
     print_line("Main thread read unlocked rwspinlock");
 
     print_line("Main thread tries to write lock rwspinlock");
-    res = ethr_write_lock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_write_lock(&rwst_rwspinlock);
     print_line("Main thread write locked rwspinlock");
 
     data = ++rwst_data;
@@ -1847,8 +1182,7 @@ rwspinlock_test(void)
     ++rwst_data;
 
     print_line("Main thread tries to write unlock rwspinlock");
-    res = ethr_write_unlock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_write_unlock(&rwst_rwspinlock);
     print_line("Main thread write unlocked rwspinlock");
 
     res = ethr_thr_join(tid, NULL);
@@ -1875,23 +1209,19 @@ void *
 rwmt_thread(void *unused)
 {
     int data;
-    int res;
 
     print_line("Aux thread tries to read lock rwmutex");
-    res = ethr_rwmutex_rlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rlock(&rwmt_rwmutex);
     print_line("Aux thread read locked rwmutex");
 
     ASSERT(rwmt_data == 4711);
 
     print_line("Aux thread tries to read unlock rwmutex");
-    res = ethr_rwmutex_runlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_runlock(&rwmt_rwmutex);
     print_line("Aux thread read unlocked rwmutex");
 
     print_line("Aux thread tries to write lock rwmutex");
-    res = ethr_rwmutex_rwlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rwlock(&rwmt_rwmutex);
     print_line("Aux thread write locked rwmutex");
 
     data = ++rwmt_data;
@@ -1905,8 +1235,7 @@ rwmt_thread(void *unused)
     ++rwmt_data;
 
     print_line("Aux thread tries to write unlock rwmutex");
-    res = ethr_rwmutex_rwunlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rwunlock(&rwmt_rwmutex);
     print_line("Aux thread write unlocked rwmutex");
 
     return NULL;
@@ -1928,8 +1257,7 @@ rwmutex_test(void)
     rwmt_data = 4711;
 
     print_line("Main thread tries to read lock rwmutex");
-    res = ethr_rwmutex_rlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rlock(&rwmt_rwmutex);
     print_line("Main thread read locked rwmutex");
 
     ASSERT(rwmt_data == 4711);
@@ -1946,13 +1274,11 @@ rwmutex_test(void)
     ASSERT(rwmt_data == 4711);
 
     print_line("Main thread tries to read unlock rwmutex");
-    res = ethr_rwmutex_runlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_runlock(&rwmt_rwmutex);
     print_line("Main thread read unlocked rwmutex");
 
     print_line("Main thread tries to write lock rwmutex");
-    res = ethr_rwmutex_rwlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rwlock(&rwmt_rwmutex);
     print_line("Main thread write locked rwmutex");
 
     data = ++rwmt_data;
@@ -1965,8 +1291,7 @@ rwmutex_test(void)
     ++rwmt_data;
 
     print_line("Main thread tries to write unlock rwmutex");
-    res = ethr_rwmutex_rwunlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rwunlock(&rwmt_rwmutex);
     print_line("Main thread write unlocked rwmutex");
 
     res = ethr_thr_join(tid, NULL);
@@ -1998,65 +1323,53 @@ static ethr_atomic_t at_data;
 void *
 at_thread(void *unused)
 {
-    int res, i;
+    int i;
     long val, go;
     
 
-    res = ethr_atomic_inctest(&at_ready, &val);
-    ASSERT(res == 0);
+    val = ethr_atomic_inc_read(&at_ready);
     ASSERT(val > 0);
     ASSERT(val <= AT_THREADS);
 
     do {
-	res = ethr_atomic_read(&at_go, &go);
-	ASSERT(res == 0);
+	go = ethr_atomic_read(&at_go);
     } while (!go);
 
     for (i = 0; i < AT_ITER; i++) {
-	res = ethr_atomic_or_old(&at_data, at_set_val, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_read_bor(&at_data, at_set_val);
 	ASSERT(val >= (i == 0 ? 0 : at_set_val) + (long) 4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_and_old(&at_data, ~at_rm_val, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_read_band(&at_data, ~at_rm_val);
 	ASSERT(val >= at_set_val + (long) 4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_read(&at_data, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_read(&at_data);
 	ASSERT(val >= at_set_val + (long) 4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_inctest(&at_data, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_inc_read(&at_data);
 	ASSERT(val > at_set_val + (long) 4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_dectest(&at_data, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_dec_read(&at_data);
 	ASSERT(val >= at_set_val + (long) 4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_inc(&at_data);
-	ASSERT(res == 0);
+	ethr_atomic_inc(&at_data);
 
-	res = ethr_atomic_dec(&at_data);
-	ASSERT(res == 0);
+	ethr_atomic_dec(&at_data);
 
-	res = ethr_atomic_addtest(&at_data, (long) 4711, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_add_read(&at_data, (long) 4711);
 	ASSERT(val >= at_set_val + (long) 2*4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_add(&at_data, (long) -4711);
-	ASSERT(res == 0);
+	ethr_atomic_add(&at_data, (long) -4711);
 	ASSERT(val >= at_set_val + (long) 4711);
 	ASSERT(val <= at_max_val);
     }
 
-    res = ethr_atomic_inc(&at_done);
-    ASSERT(res == 0);
+    ethr_atomic_inc(&at_done);
     return NULL;
 }
 
@@ -2069,14 +1382,13 @@ atomic_test(void)
     ethr_tid tid[AT_THREADS];
     ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
 
-    if (sizeof(long) > 4) {
+#if ETHR_SIZEOF_PTR > 4
 	at_rm_val = ((long) 1) << 57;
 	at_set_val = ((long) 1) << 60;
-    }
-    else {
+#else
 	at_rm_val = ((long) 1) << 27;
 	at_set_val = ((long) 1) << 30;
-    }
+#endif
 
     at_max_val = at_set_val + at_rm_val + ((long) AT_THREADS + 1) * 4711;
     data_init = at_rm_val + (long) 4711;
@@ -2085,22 +1397,15 @@ atomic_test(void)
     thr_opts.detached = 1;
 
     print_line("Initializing");
-    res = ethr_atomic_init(&at_ready, 0);
-    ASSERT(res == 0);
-    res = ethr_atomic_init(&at_go, 0);
-    ASSERT(res == 0);
-    res = ethr_atomic_init(&at_done, data_init);
-    ASSERT(res == 0);
-    res = ethr_atomic_init(&at_data, data_init);
-    ASSERT(res == 0);
+    ethr_atomic_init(&at_ready, 0);
+    ethr_atomic_init(&at_go, 0);
+    ethr_atomic_init(&at_done, data_init);
+    ethr_atomic_init(&at_data, data_init);
 
-    res = ethr_atomic_read(&at_data, &val);
-    ASSERT(res == 0);
+    val = ethr_atomic_read(&at_data);
     ASSERT(val == data_init);
-    res = ethr_atomic_set(&at_done, 0);
-    ASSERT(res == 0);
-    res = ethr_atomic_read(&at_done, &val);
-    ASSERT(res == 0);
+    ethr_atomic_set(&at_done, 0);
+    val = ethr_atomic_read(&at_done);
     ASSERT(val == 0);
 
     print_line("Creating threads");
@@ -2111,31 +1416,27 @@ atomic_test(void)
 
     print_line("Waiting for threads to ready up");
     do {
-	res = ethr_atomic_read(&at_ready, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_read(&at_ready);
 	ASSERT(val >= 0);
 	ASSERT(val <= AT_THREADS);
     } while (val != AT_THREADS);
 
     print_line("Letting threads loose");
-    res = ethr_atomic_xchg(&at_go, 17, &val);
-    ASSERT(res == 0);
+    val = ethr_atomic_xchg(&at_go, 17);
     ASSERT(val == 0);
-    res = ethr_atomic_read(&at_go, &val);
-    ASSERT(res == 0);
+    val = ethr_atomic_read(&at_go);
     ASSERT(val == 17);
 
 
     print_line("Waiting for threads to finish");
     do {
-	res = ethr_atomic_read(&at_done, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_read(&at_done);
 	ASSERT(val >= 0);
 	ASSERT(val <= AT_THREADS);
     } while (val != AT_THREADS);
 
     print_line("Checking result");
-    res = ethr_atomic_read(&at_data, &val);
+    val = ethr_atomic_read(&at_data);
     ASSERT(res == 0);
     ASSERT(val == data_final);
     print_line("Result ok");
@@ -2143,190 +1444,6 @@ atomic_test(void)
 }
 
 
-/*
- * The gate test case.
- *
- * Tests gates.
- */
-
-#define GT_THREADS 10
-
-static ethr_atomic_t gt_wait1;
-static ethr_atomic_t gt_wait2;
-static ethr_atomic_t gt_done;
-
-static ethr_gate gt_gate1;
-static ethr_gate gt_gate2;
-
-void *
-gt_thread(void *thr_no)
-{
-    int no = (int)(long) thr_no;
-    int swait = no % 2 == 0;
-    int res;
-    long done;
-
-
-    do {
-
-	res = ethr_atomic_inc(&gt_wait1);
-	ASSERT(res == 0);
-
-	if (swait)
-	    res = ethr_gate_swait(&gt_gate1, INT_MAX);
-	else
-	    res = ethr_gate_wait(&gt_gate1);
-	ASSERT(res == 0);
-
-	res = ethr_atomic_dec(&gt_wait1);
-	ASSERT(res == 0);
-
-	res = ethr_atomic_inc(&gt_wait2);
-	ASSERT(res == 0);
-
-	if (swait)
-	    res = ethr_gate_swait(&gt_gate2, INT_MAX);
-	else
-	    res = ethr_gate_wait(&gt_gate2);
-	ASSERT(res == 0);
-
-	res = ethr_atomic_dec(&gt_wait2);
-	ASSERT(res == 0);
-
-	res = ethr_atomic_read(&gt_done, &done);
-	ASSERT(res == 0);
-    } while (!done);
-    return NULL;
-}
-
-
-static void
-gate_test(void)
-{
-    long val;
-    int res, i;
-    ethr_tid tid[GT_THREADS];
-
-    print_line("Initializing");
-    res = ethr_atomic_init(&gt_wait1, 0);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_atomic_init(&gt_wait2, 0);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_atomic_init(&gt_done, 0);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_gate_init(&gt_gate1);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_gate_init(&gt_gate2);
-    ASSERT_EQ(res, 0, "%d");
-
-    print_line("Creating threads");
-    for (i = 0; i < GT_THREADS; i++) {
-	res = ethr_thr_create(&tid[i], gt_thread, (void *) i, NULL);
-	ASSERT_EQ(res, 0, "%d");
-    }
-
-    print_line("Waiting for threads to ready up");
-    do {
-	res = ethr_atomic_read(&gt_wait1, &val);
-	ASSERT_EQ(res, 0, "%d");
-	ASSERT(0 <= val && val <= GT_THREADS);
-    } while (val != GT_THREADS);
-
-    print_line("Testing");
-
-    res = ethr_gate_let_through(&gt_gate1, 8);
-    ASSERT_EQ(res, 0, "%d");
-
-    WAIT_UNTIL_LIM((res = ethr_atomic_read(&gt_wait2, &val),
-		    (res != 0 || val == 8)),
-		   60);
-
-    res = ethr_atomic_read(&gt_wait1, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, GT_THREADS - 8, "%ld");
-
-    res = ethr_atomic_read(&gt_wait2, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 8, "%ld");
-    
-    res = ethr_gate_let_through(&gt_gate2, 4);
-    ASSERT_EQ(res, 0, "%d");
-
-    WAIT_UNTIL_LIM((res = ethr_atomic_read(&gt_wait2, &val),
-		    (res != 0 || val == 4)),
-		   60);
-
-    res = ethr_atomic_read(&gt_wait1, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, GT_THREADS - 4, "%ld");
-
-    res = ethr_atomic_read(&gt_wait2, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 4, "%ld");
-    
-    res = ethr_gate_let_through(&gt_gate1, GT_THREADS);
-    ASSERT_EQ(res, 0, "%d");
-
-    WAIT_UNTIL_LIM((res = ethr_atomic_read(&gt_wait2, &val),
-		    (res != 0 || val == GT_THREADS)),
-		   60);
-    res = ethr_atomic_read(&gt_wait1, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 0, "%ld");
-
-    res = ethr_atomic_read(&gt_wait2, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, GT_THREADS, "%ld");
-    
-    res = ethr_gate_let_through(&gt_gate2, GT_THREADS);
-    ASSERT_EQ(res, 0, "%d");
-
-    WAIT_UNTIL_LIM((res = ethr_atomic_read(&gt_wait2, &val),
-		    (res != 0 || val == 4)),
-		   60);
-    res = ethr_atomic_read(&gt_wait1, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, GT_THREADS - 4, "%ld");
-
-    res = ethr_atomic_read(&gt_wait2, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 4, "%ld");
-    
-    res = ethr_atomic_set(&gt_done, 1);
-    ASSERT_EQ(res, 0, "%d");
-
-    res = ethr_gate_let_through(&gt_gate2, GT_THREADS);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_gate_let_through(&gt_gate1, GT_THREADS - 4);
-    ASSERT_EQ(res, 0, "%d");
-
-    WAIT_UNTIL_LIM(((res = ethr_atomic_read(&gt_wait1, &val)) != 0
-		    || (val == 0
-			&& ((res = ethr_atomic_read(&gt_wait2, &val)) != 0
-			    || val == 0))),
-		   60);
-
-    res = ethr_atomic_read(&gt_wait1, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 0, "%ld");
-
-    res = ethr_atomic_read(&gt_wait2, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 0, "%ld");
-    
-    print_line("Joining threads");
-    for (i = 0; i < GT_THREADS; i++) {
-	res = ethr_thr_join(tid[i], NULL);
-	ASSERT_EQ(res, 0, "%d");
-    }
-
-    res = ethr_gate_destroy(&gt_gate1);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_gate_destroy(&gt_gate2);
-    ASSERT_EQ(res, 0, "%d");
-
-}
-
 #endif /* #ifndef ETHR_NO_THREAD_LIB */
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
@@ -2342,14 +1459,12 @@ main(int argc, char *argv[])
 #ifndef ETHR_NO_THREAD_LIB
     {
 	char *testcase;
-	int res;
 
 	send_my_pid();
 
 	testcase = argv[1];
-	res = ethr_init(NULL);
 
-	if (res != 0)
+	if (ethr_init(NULL) != 0 || ethr_late_init(NULL) != 0)
 	    fail("Failed to initialize the ethread library");
 
 	if (strcmp(testcase, "create_join_thread") == 0)
@@ -2360,24 +1475,14 @@ main(int argc, char *argv[])
 	    mutex_test();
 	else if (strcmp(testcase, "try_lock_mutex") == 0)
 	    try_lock_mutex_test();
-	else if (strcmp(testcase, "recursive_mutex") == 0)
-	    recursive_mutex_test();
-	else if (strcmp(testcase, "time_now") == 0)
-	    time_now_test();
 	else if (strcmp(testcase, "cond_wait") == 0)
-	    cond_wait_test(0);
-	else if (strcmp(testcase, "cond_timedwait") == 0)
-	    cond_timedwait_test();
+	    cond_wait_test();
 	else if (strcmp(testcase, "broadcast") == 0)
 	    broadcast_test();
 	else if (strcmp(testcase, "detached_thread") == 0)
 	    detached_thread_test();
 	else if (strcmp(testcase, "max_threads") == 0)
 	    max_threads_test();
-	else if (strcmp(testcase, "forksafety") == 0)
-	    forksafety_test();
-	else if (strcmp(testcase, "vfork") == 0)
-	    vfork_test(argc, argv);
 	else if (strcmp(testcase, "tsd") == 0)
 	    tsd_test();
 	else if (strcmp(testcase, "spinlock") == 0)
@@ -2388,8 +1493,6 @@ main(int argc, char *argv[])
 	    rwmutex_test();
 	else if (strcmp(testcase, "atomic") == 0)
 	    atomic_test();
-	else if (strcmp(testcase, "gate") == 0)
-	    gate_test();
 	else
 	    skip("Test case \"%s\" not implemented yet", testcase);
 
diff --git a/erts/test/install_SUITE.erl b/erts/test/install_SUITE.erl
index e14790bc1b..214031a6fe 100644
--- a/erts/test/install_SUITE.erl
+++ b/erts/test/install_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -29,8 +29,9 @@
 
 %-define(line_trace, 1).
 
--export([all/1, init_per_suite/1, end_per_suite/1,
-	 init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, 
+	 init_per_suite/1, end_per_suite/1,
+	 init_per_testcase/2, end_per_testcase/2]).
 
 -export([bin_default/1,
 	 bin_default_dirty/1,
@@ -64,27 +65,32 @@
 	       erlang_bindir = "",
 	       bindir_symlinks = ""}).
 
-need_symlink_cases() ->
-    [bin_unreachable_absolute,
-     bin_unreachable_relative,
-     bin_same_dir,
-     bin_ok_symlink,
-     bin_dirname_fail,
+need_symlink_cases() -> 
+    [bin_unreachable_absolute, bin_unreachable_relative,
+     bin_same_dir, bin_ok_symlink, bin_dirname_fail,
      bin_no_use_dirname_fail].
 
-dont_need_symlink_cases() ->
-    [bin_default,
-     bin_default_dirty,
-     bin_outside_eprfx,
-     bin_outside_eprfx_dirty,
-     bin_not_abs,
-     bin_unreasonable_path,
-     'bin white space',
+dont_need_symlink_cases() -> 
+    [bin_default, bin_default_dirty, bin_outside_eprfx,
+     bin_outside_eprfx_dirty, bin_not_abs,
+     bin_unreasonable_path, 'bin white space',
      bin_no_srcfile].
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     dont_need_symlink_cases() ++ need_symlink_cases().
 
+groups() -> 
+    [].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 %%
 %% The test cases
 %%
@@ -585,7 +591,7 @@ init_per_testcase_aux(true, _OsType, Case, Config) ->
      {test_dir, make_dirs(?config(priv_dir, Config), atom_to_list(Case))}
      | Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
diff --git a/erts/test/nt_SUITE.erl b/erts/test/nt_SUITE.erl
index 530fb55270..7d6da28ad6 100644
--- a/erts/test/nt_SUITE.erl
+++ b/erts/test/nt_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1998-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,7 +22,9 @@
 -include_lib("test_server/include/test_server.hrl").
 -include_lib("kernel/include/file.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,nt/1,handle_eventlog/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,init_per_testcase/2,
+	 end_per_testcase/2,nt/1,handle_eventlog/2,
 	 middleman/1,service_basic/1, service_env/1, user_env/1, synced/1, 
 	 service_prio/1, 
 	 logout/1, debug/1, restart/1, restart_always/1,stopaction/1,
@@ -31,20 +33,38 @@
 
 -define(TEST_SERVICES, [1,2,3,4,5,6,7,8,9,10,11]).
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     case os:type() of
-	{win32,nt} ->
-	    [nt, service_basic, service_env, user_env, synced, service_prio, 
-	     logout, debug,
-	     restart, restart_always, stopaction];
-	_ -> [nt] %%% Just to give a little hint why they are skipped...
+	{win32, nt} ->
+	    [nt, service_basic, service_env, user_env, synced,
+	     service_prio, logout, debug, restart, restart_always,
+	     stopaction];
+	_ -> [nt]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 init_per_testcase(_Func, Config) ->
     Dog = test_server:timetrap(?TEST_TIMEOUT),
     [{watchdog, Dog} | Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     lists:foreach(fun(X) -> 
 			  catch remove_service("test_service_" ++ 
 					 integer_to_list(X)) end,
diff --git a/erts/test/otp_SUITE.erl b/erts/test/otp_SUITE.erl
index 425ad31782..d61fbbddcf 100644
--- a/erts/test/otp_SUITE.erl
+++ b/erts/test/otp_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2000-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,8 @@
 
 -module(otp_SUITE).
 
--export([all/1,init_per_suite/1,end_per_suite/1]).
+-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2,
+	 init_per_suite/1,end_per_suite/1]).
 -export([undefined_functions/1,deprecated_not_in_obsolete/1,
 	 obsolete_but_not_deprecated/1,call_to_deprecated/1,
          call_to_size_1/1,strong_components/1]).
@@ -28,10 +29,22 @@
 
 -import(lists, [filter/2,foldl/3,foreach/2]).
 
-all(suite) ->
-    [undefined_functions,deprecated_not_in_obsolete,
-     obsolete_but_not_deprecated,call_to_deprecated,
-     call_to_size_1,strong_components].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [undefined_functions, deprecated_not_in_obsolete,
+     obsolete_but_not_deprecated, call_to_deprecated,
+     call_to_size_1, strong_components].
+
+groups() -> 
+    [].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_suite(Config) ->
     Dog = test_server:timetrap(?t:minutes(10)),
diff --git a/erts/test/run_erl_SUITE.erl b/erts/test/run_erl_SUITE.erl
index efeafbad8c..6350dc47dd 100644
--- a/erts/test/run_erl_SUITE.erl
+++ b/erts/test/run_erl_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,9 @@
 
 -module(run_erl_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 basic/1,heavy/1,heavier/1,defunct/1]).
 -export([ping_me_back/1]).
 
@@ -29,13 +31,31 @@ init_per_testcase(_Case, Config) ->
     Dog = ?t:timetrap(?t:minutes(2)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
 
-all(suite) ->
-    [basic,heavy,heavier,defunct].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [basic, heavy, heavier, defunct].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 basic(Config) when is_list(Config) ->
     case os:type() of
diff --git a/erts/test/system.spec b/erts/test/system.spec
index 9bfe2dbcf8..e0561ba0b2 100644
--- a/erts/test/system.spec
+++ b/erts/test/system.spec
@@ -1 +1 @@
-{topcase, {dir, "../system_test"}}.
+{suites,"../system_test",all}.
diff --git a/erts/test/z_SUITE.erl b/erts/test/z_SUITE.erl
index 8faddeb0d3..8fceab32a6 100644
--- a/erts/test/z_SUITE.erl
+++ b/erts/test/z_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -35,26 +35,45 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(5)).
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
 -export([search_for_core_files/1, core_files/1]).
 
--include_lib("test_server/include/test_server.hrl").
+-include_lib("common_test/include/ct.hrl").
     
 
 init_per_testcase(Case, Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
     [{testcase, Case}, {watchdog, Dog} |Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
 
-all(doc) -> [];
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [core_files].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 
 core_files(doc) ->
     [];
@@ -253,6 +272,8 @@ core_file_search(#core_search_conf{search_dir = Base,
 				     core_cand(Conf, Core, Cores);
 				 "core." ++ _ ->
 				     core_cand(Conf, Core, Cores);
+				 Bin when is_binary(Bin) -> %Icky filename; ignore
+				     Cores;
 				 BName ->
 				     case lists:suffix(".core", BName) of
 					 true -> core_cand(Conf, Core, Cores);
diff --git a/erts/vsn.mk b/erts/vsn.mk
index 14c6a3e4dd..193a914a70 100644
--- a/erts/vsn.mk
+++ b/erts/vsn.mk
@@ -1,7 +1,7 @@
 # 
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1997-2010. All Rights Reserved.
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -17,8 +17,8 @@
 # %CopyrightEnd%
 # 
 
-VSN = 5.7.5.1
-SYSTEM_VSN = R13B04
+VSN = 5.8.4
+SYSTEM_VSN = R14B03
 
 # Port number 4365 in 4.2
 # Port number 4366 in 4.3