erts: Add support for fast erts_is_literal()

4 files changed, 32 insertions, 5 deletions

diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index e8ff2bf752..03ca080c14 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -349,6 +349,12 @@ struct ErtsMemMapper_ {
 
 ErtsMemMapper erts_dflt_mmapper;
 
+#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+ErtsMemMapper erts_literal_mmapper;
+char* erts_literals_start;
+UWord erts_literals_size;
+#endif
+
 #define ERTS_MMAP_SIZE_SC_SA_INC(SZ) 						\
     do {									\
 	mm->size.supercarrier.used.total += (SZ);			\
@@ -2108,6 +2114,7 @@ static void hard_dbg_mseg_init(void);
 void
 erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
 {
+    static int is_first_call = 1;
     int virtual_map = 0;
     char *start = NULL, *end = NULL;
     UWord pagesize;
@@ -2145,7 +2152,9 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
 #endif
 
     erts_smp_mtx_init(&mm->mtx, "erts_mmap");
-    erts_mtx_init(&am.init_mutex, "mmap_init_atoms");
+    if (is_first_call) {
+        erts_mtx_init(&am.init_mutex, "mmap_init_atoms");
+    }
 
 #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
     if (init->virtual_range.start) {
@@ -2302,6 +2311,14 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
 #ifdef HARD_DEBUG_MSEG
     hard_dbg_mseg_init();
 #endif
+
+#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+   if (mm == &erts_literal_mmapper) {
+       erts_literals_start = erts_literal_mmapper.sa.bot;
+       erts_literals_size  = erts_literal_mmapper.sua.top - erts_literals_start;
+    }
+#endif
+    is_first_call = 0;
 }
 
 
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index 8707c23527..61d912fd28 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -50,8 +50,11 @@ typedef struct {
 #define ERTS_MMAP_INIT_DEFAULT_INITER \
     {{NULL, NULL}, {NULL, NULL}, 0, 1, (1 << 16), 1}
 
+#define ERTS_MMAP_INIT_LITERAL_INITER \
+    {{NULL, NULL}, {NULL, NULL}, 1024*1024*1024, 1, (1 << 16), 0}
+
 typedef struct ErtsMemMapper_ ErtsMemMapper;
-extern ErtsMemMapper erts_dflt_mmapper;
+
 void *erts_mmap(ErtsMemMapper*, Uint32 flags, UWord *sizep);
 void erts_munmap(ErtsMemMapper*, Uint32 flags, void *ptr, UWord size);
 void *erts_mremap(ErtsMemMapper*, Uint32 flags, void *ptr, UWord old_size, UWord *sizep);
@@ -121,6 +124,11 @@ Eterm erts_mmap_debug_info(ErtsMemMapper*, struct process*);
 #  define ERTS_HAVE_OS_MMAP 1
 #endif
 
+extern ErtsMemMapper erts_dflt_mmapper;
+#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+extern ErtsMemMapper erts_literal_mmapper;
+#endif
+
 /*#define HARD_DEBUG_MSEG*/
 #ifdef HARD_DEBUG_MSEG
 #  define HARD_DBG_INSERT_MSEG hard_dbg_insert_mseg
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index aadcc755c1..20695899eb 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1402,7 +1402,7 @@ erts_mseg_init(ErtsMsegInit_t *init)
 
     erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
 
-    erts_mmap_init(&erts_dflt_mmapper, &init->mmap);
+    erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap);
 
     if (!IS_2POW(GET_PAGE_SIZE))
 	erl_exit(ERTS_ABORT_EXIT, "erts_mseg: Unexpected page_size %beu\n", GET_PAGE_SIZE);
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index 677f8ea4ed..2acd8f8505 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -59,7 +59,8 @@ typedef struct {
     Uint rmcbf;
     Uint mcs;
     Uint nos;
-    ErtsMMapInit mmap;
+    ErtsMMapInit dflt_mmap;
+    ErtsMMapInit literal_mmap;
 } ErtsMsegInit_t;
 
 #define ERTS_MSEG_INIT_DEFAULT_INITIALIZER				\
@@ -68,7 +69,8 @@ typedef struct {
     20,			/* rmcbf: Relative max cache bad fit	*/	\
     10,			/* mcs:   Max cache size		*/	\
     1000,		/* cci:   Cache check interval		*/	\
-    ERTS_MMAP_INIT_DEFAULT_INITER					\
+    ERTS_MMAP_INIT_DEFAULT_INITER,					\
+    ERTS_MMAP_INIT_LITERAL_INITER                                       \
 }
 
 typedef struct {