aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/beam_bif_load.c8
-rw-r--r--erts/emulator/beam/bif.tab2
-rw-r--r--erts/emulator/beam/erl_alloc.c88
-rw-r--r--erts/emulator/beam/erl_alloc.types17
-rw-r--r--erts/emulator/beam/erl_alloc_util.c97
-rw-r--r--erts/emulator/beam/erl_alloc_util.h24
-rw-r--r--erts/emulator/beam/erl_bif_info.c1
-rw-r--r--erts/emulator/beam/erl_gc.c20
-rw-r--r--erts/emulator/beam/erl_map.c75
-rw-r--r--erts/emulator/beam/erl_map.h1
-rw-r--r--erts/emulator/beam/erl_nif.c5
-rw-r--r--erts/emulator/beam/erl_process.c5
-rw-r--r--erts/emulator/beam/external.c18
13 files changed, 273 insertions, 88 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index b10250dc49..87508dcf5f 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -832,7 +832,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
/*
* Message queue can contains funs, but (at least currently) no
- * constants. If we got references to this module from the message
+ * literals. If we got references to this module from the message
* queue, a GC cannot remove these...
*/
@@ -853,7 +853,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
for (; hfrag; hfrag = hfrag->next) {
if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size))
return am_true;
- /* Should not contain any constants... */
+ /* Should not contain any literals... */
ASSERT(!any_heap_refs(&hfrag->mem[0],
&hfrag->mem[hfrag->used_size],
literals,
@@ -908,7 +908,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
#ifdef DEBUG
/*
* Message buffer fragments should not have any references
- * to constants, and off heap lists should already have
+ * to literals, and off heap lists should already have
* been moved into process off heap structure.
*/
for (msgp = rp->msg_frag; msgp; msgp = msgp->next) {
@@ -945,7 +945,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
need_gc &= ~done_gc;
/*
- * Try to get rid of constants by by garbage collecting.
+ * Try to get rid of literals by by garbage collecting.
* Clear both fvalue and ftrace.
*/
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 58cd31cee9..872f0f9b2a 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -652,6 +652,8 @@ bif erts_debug:size_shared/1
bif erts_debug:copy_shared/1
bif erlang:has_prepared_code_on_load/1
+bif maps:take/2
+
#
# Obsolete
#
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index a6519bd9e4..d04977b9ae 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -30,6 +30,7 @@
#endif
#define ERTS_ALLOC_C__
#define ERTS_ALC_INTERNAL__
+#define ERTS_WANT_MEM_MAPPERS
#include "sys.h"
#define ERL_THREADS_EMU_INTERNAL__
#include "erl_threads.h"
@@ -131,6 +132,9 @@ static ErtsAllocatorState_t ets_alloc_state;
static ErtsAllocatorState_t driver_alloc_state;
static ErtsAllocatorState_t fix_alloc_state;
static ErtsAllocatorState_t literal_alloc_state;
+#ifdef ERTS_ALC_A_EXEC
+static ErtsAllocatorState_t exec_alloc_state;
+#endif
static ErtsAllocatorState_t test_alloc_state;
typedef struct {
@@ -216,6 +220,7 @@ typedef struct {
struct au_init driver_alloc;
struct au_init fix_alloc;
struct au_init literal_alloc;
+ struct au_init exec_alloc;
struct au_init test_alloc;
} erts_alc_hndl_args_init_t;
@@ -307,9 +312,9 @@ set_default_literal_alloc_opts(struct au_init *ip)
ip->init.util.name_prefix = "literal_";
ip->init.util.alloc_no = ERTS_ALC_A_LITERAL;
#ifndef SMALL_MEMORY
- ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */
+ ip->init.util.mmbcs = 1024*1024; /* Main carrier size */
#else
- ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */
+ ip->init.util.mmbcs = 256*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_LITERAL;
ip->init.util.asbcst = 0;
@@ -329,15 +334,48 @@ set_default_literal_alloc_opts(struct au_init *ip)
ip->init.util.sys_dealloc = &erts_alcu_literal_32_sys_dealloc;
#elif defined(ARCH_64)
# ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
- ip->init.util.mseg_alloc = &erts_alcu_literal_64_mseg_alloc;
- ip->init.util.mseg_realloc = &erts_alcu_literal_64_mseg_realloc;
- ip->init.util.mseg_dealloc = &erts_alcu_literal_64_mseg_dealloc;
+ ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc;
+ ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc;
+ ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc;
+ ip->init.util.mseg_mmapper = &erts_literal_mmapper;
# endif
#else
# error Unknown architecture
#endif
}
+#ifdef ERTS_ALC_A_EXEC
+static void
+set_default_exec_alloc_opts(struct au_init *ip)
+{
+ SET_DEFAULT_ALLOC_OPTS(ip);
+ ip->enable = 1;
+ ip->thr_spec = 0;
+ ip->disable_allowed = 0;
+ ip->thr_spec_allowed = 0;
+ ip->carrier_migration_allowed = 0;
+ ip->atype = BESTFIT;
+ ip->init.bf.ao = 1;
+ ip->init.util.ramv = 0;
+ ip->init.util.mmsbc = 0;
+ ip->init.util.sbct = ~((UWord) 0);
+ ip->init.util.name_prefix = "exec_";
+ ip->init.util.alloc_no = ERTS_ALC_A_EXEC;
+ ip->init.util.mmbcs = 0; /* No main carrier */
+ ip->init.util.ts = ERTS_ALC_MTA_EXEC;
+ ip->init.util.asbcst = 0;
+ ip->init.util.rsbcst = 0;
+ ip->init.util.rsbcmt = 0;
+ ip->init.util.rmbcmt = 0;
+ ip->init.util.acul = 0;
+
+ ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc;
+ ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc;
+ ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc;
+ ip->init.util.mseg_mmapper = &erts_exec_mmapper;
+}
+#endif /* ERTS_ALC_A_EXEC */
+
static void
set_default_temp_alloc_opts(struct au_init *ip)
{
@@ -659,6 +697,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
set_default_fix_alloc_opts(&init.fix_alloc,
fix_type_sizes);
set_default_literal_alloc_opts(&init.literal_alloc);
+#ifdef ERTS_ALC_A_EXEC
+ set_default_exec_alloc_opts(&init.exec_alloc);
+#endif
set_default_test_alloc_opts(&init.test_alloc);
if (argc && argv)
@@ -688,6 +729,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.driver_alloc.thr_spec = 0;
init.fix_alloc.thr_spec = 0;
init.literal_alloc.thr_spec = 0;
+#ifdef ERTS_ALC_A_EXEC
+ init.exec_alloc.thr_spec = 0;
+#endif
#endif
/* Make adjustments for carrier migration support */
@@ -701,6 +745,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
adjust_carrier_migration_support(&init.driver_alloc);
adjust_carrier_migration_support(&init.fix_alloc);
adjust_carrier_migration_support(&init.literal_alloc);
+#ifdef ERTS_ALC_A_EXEC
+ adjust_carrier_migration_support(&init.exec_alloc);
+#endif
if (init.erts_alloc_config) {
/* Adjust flags that erts_alloc_config won't like */
@@ -716,6 +763,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.driver_alloc.thr_spec = 0;
init.fix_alloc.thr_spec = 0;
init.literal_alloc.thr_spec = 0;
+#ifdef ERTS_ALC_A_EXEC
+ init.exec_alloc.thr_spec = 0;
+#endif
/* No carrier migration */
init.temp_alloc.init.util.acul = 0;
@@ -728,6 +778,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.driver_alloc.init.util.acul = 0;
init.fix_alloc.init.util.acul = 0;
init.literal_alloc.init.util.acul = 0;
+#ifdef ERTS_ALC_A_EXEC
+ init.exec_alloc.init.util.acul = 0;
+#endif
}
#ifdef ERTS_SMP
@@ -745,6 +798,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
adjust_tpref(&init.driver_alloc, erts_no_schedulers);
adjust_tpref(&init.fix_alloc, erts_no_schedulers);
adjust_tpref(&init.literal_alloc, erts_no_schedulers);
+#ifdef ERTS_ALC_A_EXEC
+ adjust_tpref(&init.exec_alloc, erts_no_schedulers);
+#endif
#else
/* No thread specific if not smp */
@@ -764,6 +820,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
refuse_af_strategy(&init.driver_alloc);
refuse_af_strategy(&init.fix_alloc);
refuse_af_strategy(&init.literal_alloc);
+#ifdef ERTS_ALC_A_EXEC
+ refuse_af_strategy(&init.exec_alloc);
+#endif
#ifdef ERTS_SMP
if (!init.temp_alloc.thr_spec)
@@ -808,6 +867,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu);
+#ifdef ERTS_ALC_A_EXEC
+ set_au_allocator(ERTS_ALC_A_EXEC, &init.exec_alloc, ncpu);
+#endif
set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu);
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
@@ -864,7 +926,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
start_au_allocator(ERTS_ALC_A_LITERAL,
&init.literal_alloc,
&literal_alloc_state);
-
+#ifdef ERTS_ALC_A_EXEC
+ start_au_allocator(ERTS_ALC_A_EXEC,
+ &init.exec_alloc,
+ &exec_alloc_state);
+#endif
start_au_allocator(ERTS_ALC_A_TEST,
&init.test_alloc,
&test_alloc_state);
@@ -1499,6 +1565,16 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
else
handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0);
break;
+ case 'X':
+ if (has_prefix("scs", argv[i]+3)) {
+#ifdef ERTS_ALC_A_EXEC
+ init->mseg.exec_mmap.scs =
+#endif
+ get_mb_value(argv[i]+6, argv, &i);
+ }
+ else
+ handle_au_arg(&init->exec_alloc, &argv[i][3], argv, &i, 0);
+ break;
case 'D':
handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0);
break;
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index d2fe440d47..ba216c7eb4 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -87,6 +87,9 @@ allocator EHEAP true eheap_alloc
allocator ETS true ets_alloc
allocator FIXED_SIZE true fix_alloc
allocator LITERAL true literal_alloc
++if exec_alloc
+allocator EXEC true exec_alloc
++endif
+else # Non smp build
@@ -98,6 +101,9 @@ allocator EHEAP false eheap_alloc
allocator ETS false ets_alloc
allocator FIXED_SIZE false fix_alloc
allocator LITERAL false literal_alloc
++if exec_alloc
+allocator EXEC false exec_alloc
++endif
+endif
@@ -258,7 +264,6 @@ type PRTSD STANDARD SYSTEM port_specific_data
type CPUDATA LONG_LIVED SYSTEM cpu_data
type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids
type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data
-type ZLIB STANDARD SYSTEM zlib
type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map
type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts
type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q
@@ -291,8 +296,10 @@ type THR_Q_LL LONG_LIVED SYSTEM long_lived_thr_queue
+if smp
type ASYNC SHORT_LIVED SYSTEM async
+type ZLIB STANDARD SYSTEM zlib
+else
-# sl_alloc is not thread safe in non smp build; therefore, we use driver_alloc
+# sl/std_alloc is not thread safe in non smp build; therefore, we use driver_alloc
+type ZLIB DRIVER SYSTEM zlib
type ASYNC DRIVER SYSTEM async
+endif
@@ -338,8 +345,14 @@ type SL_MPATHS SHORT_LIVED SYSTEM sl_migration_paths
# Currently most hipe code use this type.
type HIPE SYSTEM SYSTEM hipe_data
++if exec_alloc
+type HIPE_EXEC EXEC CODE hipe_code
+endif
++endif
+
+
+
+if heap_frag_elim_test
type SSB SHORT_LIVED PROCESSES ssb
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index b913886d86..2995f2f822 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -859,28 +859,36 @@ void*
erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
{
void* res;
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ Uint sz = ERTS_SUPERALIGNED_CEILING(*size_p);
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
- res = erts_alcu_mseg_alloc(allctr, size_p, flags);
- if (res)
- set_literal_range(res, *size_p);
+ res = erts_alcu_mseg_alloc(allctr, &sz, flags);
+ if (res) {
+ set_literal_range(res, sz);
+ *size_p = sz;
+ }
return res;
}
void*
erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg,
- Uint old_size, Uint *new_size_p)
+ Uint old_size, Uint *new_size_p)
{
void* res;
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ Uint new_sz = ERTS_SUPERALIGNED_CEILING(*new_size_p);
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
if (seg && old_size)
clear_literal_range(seg, old_size);
- res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p);
- if (res)
- set_literal_range(res, *new_size_p);
+ res = erts_alcu_mseg_realloc(allctr, seg, old_size, &new_sz);
+ if (res) {
+ set_literal_range(res, new_sz);
+ *new_size_p = new_sz;
+ }
return res;
}
@@ -888,8 +896,9 @@ void
erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
Uint flags)
{
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
erts_alcu_mseg_dealloc(allctr, seg, size, flags);
@@ -898,8 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
#elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+/* Used by literal allocator that has its own mmapper (super carrier) */
void*
-erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
+erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
{
void* res;
UWord size = (UWord) *size_p;
@@ -907,33 +917,33 @@ erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
if (flags & ERTS_MSEG_FLG_2POW)
mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
- res = erts_mmap(&erts_literal_mmapper, mmap_flags, &size);
+ res = erts_mmap(allctr->mseg_mmapper, mmap_flags, &size);
*size_p = (Uint)size;
INC_CC(allctr->calls.mseg_alloc);
return res;
}
void*
-erts_alcu_literal_64_mseg_realloc(Allctr_t *allctr, void *seg,
+erts_alcu_mmapper_mseg_realloc(Allctr_t *allctr, void *seg,
Uint old_size, Uint *new_size_p)
{
void *res;
UWord new_size = (UWord) *new_size_p;
- res = erts_mremap(&erts_literal_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size);
+ res = erts_mremap(allctr->mseg_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size);
*new_size_p = (Uint) new_size;
INC_CC(allctr->calls.mseg_realloc);
return res;
}
void
-erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
+erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
Uint flags)
{
Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY;
if (flags & ERTS_MSEG_FLG_2POW)
mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
- erts_munmap(&erts_literal_mmapper, mmap_flags, seg, (UWord)size);
+ erts_munmap(allctr->mseg_mmapper, mmap_flags, seg, (UWord)size);
INC_CC(allctr->calls.mseg_dealloc);
}
#endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */
@@ -941,9 +951,10 @@ erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
#endif /* HAVE_ERTS_MSEG */
void*
-erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
+erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
{
void *res;
+ const Uint size = *size_p;
#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
if (superalign)
res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size);
@@ -957,9 +968,10 @@ erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
}
void*
-erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
+erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign)
{
void *res;
+ const Uint size = *size_p;
#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
if (superalign)
@@ -994,38 +1006,48 @@ erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
#ifdef ARCH_32
void*
-erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
+erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
{
void* res;
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ Uint size = ERTS_SUPERALIGNED_CEILING(*size_p);
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
- res = erts_alcu_sys_alloc(allctr, size, 1);
- if (res)
+ res = erts_alcu_sys_alloc(allctr, &size, 1);
+ if (res) {
set_literal_range(res, size);
+ *size_p = size;
+ }
return res;
}
void*
-erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
+erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint old_size, int superalign)
{
void* res;
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ Uint size = ERTS_SUPERALIGNED_CEILING(*size_p);
+
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
if (ptr && old_size)
clear_literal_range(ptr, old_size);
- res = erts_alcu_sys_realloc(allctr, ptr, size, old_size, 1);
- if (res)
+ res = erts_alcu_sys_realloc(allctr, ptr, &size, old_size, 1);
+ if (res) {
set_literal_range(res, size);
+ *size_p = size;
+ }
return res;
}
void
erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
{
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
erts_alcu_sys_dealloc(allctr, ptr, size, 1);
@@ -3864,12 +3886,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
? UNIT_CEILING(bcrr_sz)
: SYS_ALLOC_CARRIER_CEILING(bcrr_sz));
- crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
+ crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC);
if (!crr) {
if (crr_sz > UNIT_CEILING(bcrr_sz)) {
crr_sz = UNIT_CEILING(bcrr_sz);
- crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
+ crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC);
}
if (!crr) {
#if HAVE_ERTS_MSEG
@@ -4027,7 +4049,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
new_crr = (Carrier_t *) allctr->sys_realloc(allctr,
(void *) old_crr,
- new_crr_sz,
+ &new_crr_sz,
old_crr_sz,
0);
if (new_crr) {
@@ -4048,7 +4070,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
new_crr_sz = UNIT_CEILING(new_crr_sz);
new_crr = (Carrier_t *) allctr->sys_realloc(allctr,
(void *) old_crr,
- new_crr_sz,
+ &new_crr_sz,
old_crr_sz,
0);
if (new_crr)
@@ -6127,6 +6149,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->mseg_alloc = init->mseg_alloc;
allctr->mseg_realloc = init->mseg_realloc;
allctr->mseg_dealloc = init->mseg_dealloc;
+ allctr->mseg_mmapper = init->mseg_mmapper;
}
else {
ASSERT(!init->mseg_realloc && !init->mseg_dealloc);
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 527a0cd91d..f50f09907a 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -72,9 +72,10 @@ typedef struct {
void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags);
void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags);
+ ErtsMemMapper *mseg_mmapper;
#endif
- void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign);
- void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign);
+ void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign);
+ void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign);
void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign);
} AllctrInit_t;
@@ -205,18 +206,18 @@ void* erts_alcu_literal_32_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uin
void erts_alcu_literal_32_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
# elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
-void* erts_alcu_literal_64_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
-void* erts_alcu_literal_64_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
-void erts_alcu_literal_64_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
+void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
+void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
# endif
#endif /* HAVE_ERTS_MSEG */
-void* erts_alcu_sys_alloc(Allctr_t*, Uint size, int superalign);
-void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign);
+void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign);
+void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign);
void erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
#ifdef ARCH_32
-void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint size, int superalign);
-void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign);
+void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint *size_p, int superalign);
+void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign);
void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
#endif
@@ -601,9 +602,10 @@ struct Allctr_t_ {
void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags);
void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags);
+ ErtsMemMapper *mseg_mmapper;
#endif
- void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign);
- void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign);
+ void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign);
+ void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign);
void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign);
void (*init_atoms) (void);
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 7a72b0d8cc..c9741b361f 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -22,6 +22,7 @@
# include "config.h"
#endif
+#define ERTS_WANT_MEM_MAPPERS
#include "sys.h"
#include "erl_vm.h"
#include "global.h"
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index afb2ec1cc2..df5d0f4918 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2056,8 +2056,26 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
*hp++ = val;
break;
case TAG_PRIMARY_LIST:
+#ifdef SHCOPY_SEND
+ if (erts_is_literal(val,list_val(val))) {
+ *hp++ = val;
+ } else {
+ *hp++ = offset_ptr(val, offs);
+ }
+#else
+ *hp++ = offset_ptr(val, offs);
+#endif
+ break;
case TAG_PRIMARY_BOXED:
- *hp++ = offset_ptr(val, offs);
+#ifdef SHCOPY_SEND
+ if (erts_is_literal(val,boxed_val(val))) {
+ *hp++ = val;
+ } else {
+ *hp++ = offset_ptr(val, offs);
+ }
+#else
+ *hp++ = offset_ptr(val, offs);
+#endif
break;
case TAG_PRIMARY_HEADER:
*hp++ = val;
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 03a96cb00a..8efc983f04 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -54,6 +54,7 @@
* - maps:new/0
* - maps:put/3
* - maps:remove/2
+ * - maps:take/2
* - maps:to_list/1
* - maps:update/3
* - maps:values/1
@@ -93,7 +94,7 @@ static Uint hashmap_subtree_size(Eterm node);
static Eterm hashmap_to_list(Process *p, Eterm map);
static Eterm hashmap_keys(Process *p, Eterm map);
static Eterm hashmap_values(Process *p, Eterm map);
-static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
+static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node, Eterm *value);
static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size);
static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size);
static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys);
@@ -1521,10 +1522,45 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
BIF_ERROR(BIF_P, BADMAP);
}
-/* maps:remove/3 */
+/* maps:take/2 */
-int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
+BIF_RETTYPE maps_take_2(BIF_ALIST_2) {
+ if (is_map(BIF_ARG_2)) {
+ Eterm res, map, val;
+ if (erts_maps_take(BIF_P, BIF_ARG_1, BIF_ARG_2, &map, &val)) {
+ Eterm *hp = HAlloc(BIF_P, 3);
+ res = make_tuple(hp);
+ *hp++ = make_arityval(2);
+ *hp++ = val;
+ *hp++ = map;
+ BIF_RET(res);
+ }
+ BIF_RET(am_error);
+ }
+ BIF_P->fvalue = BIF_ARG_2;
+ BIF_ERROR(BIF_P, BADMAP);
+}
+
+/* maps:remove/2 */
+
+BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
+ if (is_map(BIF_ARG_2)) {
+ Eterm res;
+ (void) erts_maps_take(BIF_P, BIF_ARG_1, BIF_ARG_2, &res, NULL);
+ BIF_RET(res);
+ }
+ BIF_P->fvalue = BIF_ARG_2;
+ BIF_ERROR(BIF_P, BADMAP);
+}
+
+/* erts_maps_take
+ * return 1 if key is found, otherwise 0
+ * If the key is not found res (output map) will be map (input map)
+ */
+int erts_maps_take(Process *p, Eterm key, Eterm map,
+ Eterm *res, Eterm *value) {
Uint32 hx;
+ Eterm ret;
if (is_flatmap(map)) {
Sint n;
Uint need;
@@ -1537,7 +1573,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
if (n == 0) {
*res = map;
- return 1;
+ return 0;
}
ks = flatmap_get_keys(mp);
@@ -1564,6 +1600,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
if (is_immed(key)) {
while (1) {
if (*ks == key) {
+ if (value) *value = *vs;
goto found_key;
} else if (--n) {
*mhp++ = *vs++;
@@ -1574,6 +1611,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
} else {
while(1) {
if (EQ(*ks, key)) {
+ if (value) *value = *vs;
goto found_key;
} else if (--n) {
*mhp++ = *vs++;
@@ -1589,7 +1627,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
HRelease(p, hp_start + need, hp_start);
*res = map;
- return 1;
+ return 0;
found_key:
/* Copy rest of keys and values */
@@ -1601,19 +1639,13 @@ found_key:
}
ASSERT(is_hashmap(map));
hx = hashmap_make_hash(key);
- *res = hashmap_delete(p, hx, key, map);
- return 1;
-}
-
-BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
- if (is_map(BIF_ARG_2)) {
- Eterm res;
- if (erts_maps_remove(BIF_P, BIF_ARG_1, BIF_ARG_2, &res)) {
- BIF_RET(res);
- }
+ ret = hashmap_delete(p, hx, key, map, value);
+ if (is_value(ret)) {
+ *res = ret;
+ return 1;
}
- BIF_P->fvalue = BIF_ARG_2;
- BIF_ERROR(BIF_P, BADMAP);
+ *res = map;
+ return 0;
}
int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) {
@@ -2322,7 +2354,8 @@ static Eterm hashmap_values(Process* p, Eterm node) {
return res;
}
-static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
+static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key,
+ Eterm map, Eterm *value) {
Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL;
Eterm *ptr;
Eterm hdr, res = map, node = map;
@@ -2337,8 +2370,12 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
switch(primary_tag(node)) {
case TAG_PRIMARY_LIST:
if (EQ(CAR(list_val(node)), key)) {
+ if (value) {
+ *value = CDR(list_val(node));
+ }
goto unroll;
}
+ res = THE_NON_VALUE;
goto not_found;
case TAG_PRIMARY_BOXED:
ptr = boxed_val(node);
@@ -2365,6 +2402,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
n = hashmap_bitcount(hval);
} else {
/* not occupied */
+ res = THE_NON_VALUE;
goto not_found;
}
@@ -2394,6 +2432,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
break;
}
/* not occupied */
+ res = THE_NON_VALUE;
goto not_found;
default:
erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 7af9100906..8b5c9582ba 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -82,6 +82,7 @@ struct ErtsEStack_;
Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map);
int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res);
int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
+int erts_maps_take(Process *p, Eterm key, Eterm map, Eterm *res, Eterm *value);
Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
Eterm node, int is_update);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 9a0eea42f0..941f44b9ec 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2443,14 +2443,13 @@ int enif_make_map_remove(ErlNifEnv* env,
Eterm key,
Eterm *map_out)
{
- int res;
if (!is_map(map_in)) {
return 0;
}
flush_env(env);
- res = erts_maps_remove(env->proc, key, map_in, map_out);
+ (void) erts_maps_take(env->proc, key, map_in, map_out, NULL);
cache_env(env);
- return res;
+ return 1;
}
int enif_map_iterator_create(ErlNifEnv *env,
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1a66044627..d485affa3b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10932,9 +10932,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
INITIALIZE_SHCOPY(info);
#endif
-#ifdef ERTS_SMP
erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
-#endif
/*
* Check for errors.
@@ -11235,6 +11233,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
* Schedule process for execution.
*/
+ erts_smp_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR);
+
schedule_process(p, state, 0);
VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id));
@@ -11248,6 +11248,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
DTRACE2(process_spawn, process_name, mfa);
}
#endif
+ return res;
error:
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 723c25ff77..3c002d43a7 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -4429,22 +4429,32 @@ init_done:
SKIP(1+atom_extra_skip);
atom_extra_skip = 0;
break;
- case PID_EXT:
case NEW_PID_EXT:
+ atom_extra_skip = 12;
+ goto case_PID;
+ case PID_EXT:
atom_extra_skip = 9;
+ case_PID:
/* In case it is an external pid */
heap_size += EXTERNAL_THING_HEAD_SIZE + 1;
terms++;
break;
- case PORT_EXT:
case NEW_PORT_EXT:
+ atom_extra_skip = 8;
+ goto case_PORT;
+ case PORT_EXT:
atom_extra_skip = 5;
+ case_PORT:
/* In case it is an external port */
heap_size += EXTERNAL_THING_HEAD_SIZE + 1;
terms++;
break;
- case NEW_REFERENCE_EXT:
case NEWER_REFERENCE_EXT:
+ atom_extra_skip = 4;
+ goto case_NEW_REFERENCE;
+ case NEW_REFERENCE_EXT:
+ atom_extra_skip = 1;
+ case_NEW_REFERENCE:
{
int id_words;
@@ -4455,7 +4465,7 @@ init_done:
goto error;
ep += 2;
- atom_extra_skip = 1 + 4*id_words;
+ atom_extra_skip += 4*id_words;
/* In case it is an external ref */
#if defined(ARCH_64)
heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1;
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-25 12:56:36 +0000