Implement platform specific aligned sys_alloc and use when supported

erts_sys_aligned_alloc() is currently implemented using posix_memalign if
it exist, or using _aligned_malloc on Windows.

If erts_sys_aligned_alloc() exist allocators will create sys_alloc
carriers similar to how this was done pre-R16.

5 files changed, 80 insertions, 4 deletions

diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index 3cab9e18da..127576b850 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -32,13 +32,13 @@
 
 #if HAVE_MMAP
 #  define HAVE_ERTS_MSEG 1
-#  define HAVE_SUPER_ALIGNED_MB_CARRIERS 1
+#  define ERTS_HAVE_MSEG_SUPER_ALIGNED 1
 #else
 #  define HAVE_ERTS_MSEG 0
-#  define HAVE_SUPER_ALIGNED_MB_CARRIERS 0
+#  define ERTS_HAVE_MSEG_SUPER_ALIGNED 0
 #endif
 
-#if HAVE_SUPER_ALIGNED_MB_CARRIERS
+#if ERTS_HAVE_MSEG_SUPER_ALIGNED
 #  define MSEG_ALIGN_BITS (18)
    /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */
 #else
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index c8fcec8547..2c47aa06c2 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -107,6 +107,10 @@
 #endif
 #include <netdb.h>
 
+#ifdef HAVE_POSIX_MEMALIGN
+#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 1
+#endif
+
 /*
  * Make sure that MAXPATHLEN is defined.
  */
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index a7ea4b2490..fdc3167c62 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -2524,6 +2524,52 @@ void erts_sys_alloc_init(void)
 {
 }
 
+#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+void *erts_sys_aligned_alloc(UWord alignment, UWord size)
+{
+#ifdef HAVE_POSIX_MEMALIGN
+    void *ptr = NULL;
+    int error;
+    ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */
+    error = posix_memalign(&ptr, (size_t) alignment, (size_t) size);
+#if HAVE_ERTS_MSEG
+    if (error || !ptr) {
+	erts_mseg_clear_cache();
+	error = posix_memalign(&ptr, (size_t) alignment, (size_t) size);
+    }
+#endif
+    if (error) {
+	errno = error;
+	return NULL;
+    }
+    if (!ptr)
+	errno = ENOMEM;
+    ASSERT(!ptr || (((UWord) ptr) & (alignment - 1)) == 0);
+    return ptr;
+#else
+#  error "Missing erts_sys_aligned_alloc() implementation"
+#endif
+}
+
+void erts_sys_aligned_free(UWord alignment, void *ptr)
+{
+    ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */
+    free(ptr);
+}
+
+void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size)
+{
+    void *new_ptr = erts_sys_aligned_alloc(alignment, size);
+    if (new_ptr) {
+	UWord copy_size = old_size < size ? old_size : size;
+	sys_memcpy(new_ptr, ptr, (size_t) copy_size);
+	erts_sys_aligned_free(alignment, ptr);
+    }
+    return new_ptr;
+}
+
+#endif
+
 void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz)
 {
     void *res = malloc((size_t) sz);
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index 5ce1a61303..c44c5d9492 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -95,6 +95,8 @@
 #  define ERTS_I64_LITERAL(X) X##i64
 #endif
 
+#define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 1
+
 /*
  * Practial Windows specific macros.
  */
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index e0422de026..8244a43583 100755
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -32,7 +32,7 @@
 #include "erl_threads.h"
 #include "../../drivers/win32/win_con.h"
 #include "erl_cpu_topology.h"
-
+#include <malloc.h>
 
 void erts_sys_init_float(void);
 
@@ -2906,6 +2906,30 @@ void erts_sys_free(ErtsAlcType_t t, void *x, void *p)
     free(p);
 }
 
+void *erts_sys_aligned_alloc(UWord alignment, UWord size)
+{
+    void *ptr;
+    ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */
+    ptr = _aligned_malloc((size_t) size, (size_t) alignment);
+    ASSERT(!ptr || (((UWord) ptr) & (alignment - 1)) == 0);
+    return ptr;
+}
+
+void erts_sys_aligned_free(UWord alignment, void *ptr)
+{
+    ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */
+    _aligned_free(ptr);
+}
+
+void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size)
+{
+    void *new_ptr;
+    ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */
+    new_ptr = _aligned_realloc(ptr, (size_t) size, (size_t) alignment);
+    ASSERT(!new_ptr || (((UWord) new_ptr) & (alignment - 1)) == 0);
+    return new_ptr;
+}
+
 static Preload* preloaded = NULL;
 static unsigned* res_name = NULL;
 static int num_preloaded = 0;