diff options
Diffstat (limited to 'erts/emulator/beam/erl_alloc.c')
| -rw-r--r-- | erts/emulator/beam/erl_alloc.c | 193 |
1 files changed, 166 insertions, 27 deletions
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index d68f22d573..5544712e8d 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -130,6 +130,8 @@ static ErtsAllocatorState_t binary_alloc_state; static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; static ErtsAllocatorState_t fix_alloc_state; +static ErtsAllocatorState_t literal_alloc_state; +static ErtsAllocatorState_t test_alloc_state; typedef struct { erts_smp_atomic32_t refc; @@ -211,6 +213,8 @@ typedef struct { struct au_init ets_alloc; struct au_init driver_alloc; struct au_init fix_alloc; + struct au_init literal_alloc; + struct au_init test_alloc; } erts_alc_hndl_args_init_t; #define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} @@ -272,9 +276,9 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.util.name_prefix = "ll_"; ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED; #ifndef SMALL_MEMORY - ip->init.util.mmbcs = 2*1024*1024 - 40; /* Main carrier size */ + ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ #else - ip->init.util.mmbcs = 1*1024*1024 - 40; /* Main carrier size */ + ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_LONG_LIVED; ip->init.util.asbcst = 0; @@ -285,6 +289,51 @@ set_default_ll_alloc_opts(struct au_init *ip) } static void +set_default_literal_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 1; + ip->thr_spec = 0; + ip->atype = BESTFIT; + ip->init.bf.ao = 1; + ip->init.util.ramv = 0; + ip->init.util.mmsbc = 0; + ip->init.util.sbct = ~((UWord) 0); + ip->init.util.name_prefix = "literal_"; + ip->init.util.alloc_no = ERTS_ALC_A_LITERAL; +#ifndef SMALL_MEMORY + ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ +#else + ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ +#endif + ip->init.util.ts = ERTS_ALC_MTA_LITERAL; + ip->init.util.asbcst = 0; + ip->init.util.rsbcst = 0; + ip->init.util.rsbcmt = 0; + ip->init.util.rmbcmt = 0; + ip->init.util.acul = 0; + +#if defined(ARCH_32) +# if HAVE_ERTS_MSEG + ip->init.util.mseg_alloc = &erts_alcu_literal_32_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_literal_32_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_literal_32_mseg_dealloc; +# endif + ip->init.util.sys_alloc = &erts_alcu_literal_32_sys_alloc; + ip->init.util.sys_realloc = &erts_alcu_literal_32_sys_realloc; + ip->init.util.sys_dealloc = &erts_alcu_literal_32_sys_dealloc; +#elif defined(ARCH_64) +# ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION + ip->init.util.mseg_alloc = &erts_alcu_literal_64_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_literal_64_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_literal_64_mseg_dealloc; +# endif +#else +# error Unknown architecture +#endif +} + +static void set_default_temp_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); @@ -398,6 +447,33 @@ set_default_fix_alloc_opts(struct au_init *ip, ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } +static void +set_default_test_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 0; /* Disabled by default */ + ip->thr_spec = -1 * erts_no_schedulers; + ip->atype = AOFIRSTFIT; + ip->init.aoff.flavor = AOFF_BF; + ip->init.util.name_prefix = "test_"; + ip->init.util.alloc_no = ERTS_ALC_A_TEST; + ip->init.util.mmbcs = 0; /* Main carrier size */ + ip->init.util.ts = ERTS_ALC_MTA_TEST; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; + + /* Use a constant minimal MBC size */ +#if ERTS_SA_MB_CARRIERS + ip->init.util.smbcs = ERTS_SACRR_UNIT_SZ; + ip->init.util.lmbcs = ERTS_SACRR_UNIT_SZ; + ip->init.util.sbct = ERTS_SACRR_UNIT_SZ; +#else + ip->init.util.smbcs = 1 << 12; + ip->init.util.lmbcs = 1 << 12; + ip->init.util.sbct = 1 << 12; +#endif +} + + #ifdef ERTS_SMP static void @@ -535,7 +611,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)] = sizeof(ErtsDrvSelectDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)] - = sizeof(ErlMessage); + = sizeof(ErtsMessageRef); #ifdef ERTS_SMP fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)] = sizeof(ErtsThrQElement_t); @@ -577,6 +653,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_default_driver_alloc_opts(&init.driver_alloc); set_default_fix_alloc_opts(&init.fix_alloc, fix_type_sizes); + set_default_literal_alloc_opts(&init.literal_alloc); + set_default_test_alloc_opts(&init.test_alloc); if (argc && argv) handle_args(argc, argv, &init); @@ -604,6 +682,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; init.fix_alloc.thr_spec = 0; + init.literal_alloc.thr_spec = 0; #endif /* Make adjustments for carrier migration support */ @@ -616,6 +695,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_carrier_migration_support(&init.ets_alloc); adjust_carrier_migration_support(&init.driver_alloc); adjust_carrier_migration_support(&init.fix_alloc); + adjust_carrier_migration_support(&init.literal_alloc); if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ @@ -630,6 +710,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; init.fix_alloc.thr_spec = 0; + init.literal_alloc.thr_spec = 0; /* No carrier migration */ init.temp_alloc.init.util.acul = 0; @@ -641,6 +722,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ets_alloc.init.util.acul = 0; init.driver_alloc.init.util.acul = 0; init.fix_alloc.init.util.acul = 0; + init.literal_alloc.init.util.acul = 0; } #ifdef ERTS_SMP @@ -657,6 +739,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_tpref(&init.ets_alloc, erts_no_schedulers); adjust_tpref(&init.driver_alloc, erts_no_schedulers); adjust_tpref(&init.fix_alloc, erts_no_schedulers); + adjust_tpref(&init.literal_alloc, erts_no_schedulers); #else /* No thread specific if not smp */ @@ -675,6 +758,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) refuse_af_strategy(&init.ets_alloc); refuse_af_strategy(&init.driver_alloc); refuse_af_strategy(&init.fix_alloc); + refuse_af_strategy(&init.literal_alloc); #ifdef ERTS_SMP if (!init.temp_alloc.thr_spec) @@ -685,6 +769,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #if HAVE_ERTS_MSEG init.mseg.nos = erts_no_schedulers; erts_mseg_init(&init.mseg); +# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + erts_mmap_init(&erts_literal_mmapper, &init.mseg.literal_mmap); +# endif #endif erts_alcu_init(&init.alloc_util); @@ -718,6 +805,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu); set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu); set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (!erts_allctrs[i].alloc) @@ -770,6 +859,13 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, &fix_alloc_state); + start_au_allocator(ERTS_ALC_A_LITERAL, + &init.literal_alloc, + &literal_alloc_state); + + start_au_allocator(ERTS_ALC_A_TEST, + &init.test_alloc, + &test_alloc_state); erts_mtrace_install_wrapper_functions(); extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); @@ -952,7 +1048,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, } for (i = 0; i < size; i++) { - void *as; + Allctr_t *as; atype = init->atype; if (!init->thr_spec) @@ -989,22 +1085,22 @@ start_au_allocator(ErtsAlcType_t alctr_n, switch (atype) { case GOODFIT: - as = (void *) erts_gfalc_start((GFAllctr_t *) as0, + as = erts_gfalc_start((GFAllctr_t *) as0, &init->init.gf, &init->init.util); break; case BESTFIT: - as = (void *) erts_bfalc_start((BFAllctr_t *) as0, + as = erts_bfalc_start((BFAllctr_t *) as0, &init->init.bf, &init->init.util); break; case AFIT: - as = (void *) erts_afalc_start((AFAllctr_t *) as0, + as = erts_afalc_start((AFAllctr_t *) as0, &init->init.af, &init->init.util); break; case AOFIRSTFIT: - as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0, + as = erts_aoffalc_start((AOFFAllctr_t *) as0, &init->init.aoff, &init->init.util); break; @@ -1351,6 +1447,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) &init->fix_alloc, &init->sl_alloc, &init->temp_alloc + /* test_alloc not affected by +Mea??? or +Mu??? */ }; int aui_sz = (int) sizeof(aui)/sizeof(aui[0]); char *arg; @@ -1406,25 +1503,25 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) } else if (has_prefix("scs", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.scs = + init->mseg.dflt_mmap.scs = #endif get_mb_value(argv[i]+6, argv, &i); } else if (has_prefix("sco", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.sco = + init->mseg.dflt_mmap.sco = #endif get_bool_value(argv[i]+6, argv, &i); } else if (has_prefix("scrpm", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.scrpm = + init->mseg.dflt_mmap.scrpm = #endif get_bool_value(argv[i]+8, argv, &i); } else if (has_prefix("scrfsd", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.scrfsd = + init->mseg.dflt_mmap.scrfsd = #endif get_amount_value(argv[i]+9, argv, &i); } @@ -1441,6 +1538,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'T': handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0); break; + case 'Z': + handle_au_arg(&init->test_alloc, &argv[i][3], argv, &i, 0); + break; case 'Y': { /* sys_alloc */ if (has_prefix("tt", param+2)) { /* set trim threshold */ @@ -2093,11 +2193,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } - /* All alloc_util allocators *have* to be enabled */ + /* All alloc_util allocators *have* to be enabled, except test_alloc */ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_A_TEST: break; default: if (!erts_allctrs_info[ai].enabled @@ -2138,6 +2239,8 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) * contain any allocated memory. */ continue; + case ERTS_ALC_A_TEST: + continue; case ERTS_ALC_A_EHEAP: save = &size.processes; break; @@ -2539,14 +2642,17 @@ erts_alloc_util_allocators(void *proc) /* * Currently all allocators except sys_alloc are * alloc_util allocators. + * Also hide test_alloc which is disabled by default + * and only intended for our own testing. */ - sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2; + sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2; ASSERT(sz > 0); hp = HAlloc((Process *) proc, sz); res = NIL; for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) { switch (i) { case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_A_TEST: break; default: { char *alc_str = (char *) ERTS_ALC_A2AD(i); @@ -2629,7 +2735,7 @@ erts_allocator_info(int to, void *arg) erts_mseg_info(i, &to, arg, 0, NULL, NULL); } erts_print(to, arg, "=allocator:mseg_alloc.erts_mmap\n"); - erts_mmap_info(&to, arg, NULL, NULL, &emis); + erts_mmap_info(&erts_dflt_mmapper, &to, arg, NULL, NULL, &emis); } #endif @@ -2855,12 +2961,12 @@ reply_alloc_info(void *vair) int global_instances = air->req_sched == sched_id; ErtsProcLocks rp_locks; Process *rp = air->proc; - Eterm ref_copy = NIL, ai_list, msg; - Eterm *hp = NULL, *hp_end = NULL, *hp_start = NULL; + Eterm ref_copy = NIL, ai_list, msg = NIL; + Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL; Eterm **hpp; Uint sz, *szp; ErlOffHeap *ohp = NULL; - ErlHeapFragment *bp = NULL; + ErtsMessage *mp = NULL; struct erts_mmap_info_struct emis; int i; Eterm (*info_func)(Allctr_t *, @@ -2981,7 +3087,8 @@ reply_alloc_info(void *vair) ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); - ainfo = (air->only_sz ? NIL : erts_mmap_info(NULL, NULL, hpp, szp, &emis)); + ainfo = (air->only_sz ? NIL : + erts_mmap_info(&erts_dflt_mmapper, NULL, NULL, hpp, szp, &emis)); ainfo = erts_bld_tuple3(hpp, szp, alloc_atom, erts_bld_atom(hpp,szp,"erts_mmap"), @@ -3061,20 +3168,17 @@ reply_alloc_info(void *vair) if (hpp) break; - hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); hp_start = hp; hp_end = hp + sz; szp = NULL; hpp = &hp; } - if (bp) - bp = erts_resize_message_buffer(bp, hp - hp_start, &msg, 1); - else { - ASSERT(hp); - HRelease(rp, hp_end, hp); - } - erts_queue_message(rp, &rp_locks, bp, msg, NIL); + if (hp != hp_end) + erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1); + + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (air->req_sched == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -3430,6 +3534,41 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) #else case 0xf13: return (UWord) 0; #endif + case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1); + + case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0; + + case 0xf16: { + Uint extra_hdr_sz = UNIT_CEILING((Uint)a1); + ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; + Uint offset = ts->allctr[0]->mbc_header_size; + void* orig_creating_mbc = ts->allctr[0]->creating_mbc; + void* orig_destroying_mbc = ts->allctr[0]->destroying_mbc; + void* new_creating_mbc = *(void**)a2; /* inout arg */ + void* new_destroying_mbc = *(void**)a3; /* inout arg */ + int i; + + for (i=0; i < ts->size; i++) { + Allctr_t* ap = ts->allctr[i]; + if (ap->mbc_header_size != offset + || ap->creating_mbc != orig_creating_mbc + || ap->destroying_mbc != orig_destroying_mbc + || ap->mbc_list.first != NULL) + return -1; + } + for (i=0; i < ts->size; i++) { + ts->allctr[i]->mbc_header_size += extra_hdr_sz; + ts->allctr[i]->creating_mbc = new_creating_mbc; + ts->allctr[i]->destroying_mbc = new_destroying_mbc; + } + *(void**)a2 = orig_creating_mbc; + *(void**)a3 = orig_destroying_mbc; + return offset; + } + case 0xf17: { + ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; + return ts->allctr[0]->largest_mbc_size; + } default: break; } |