erts: Add new allocator LITERAL

6 files changed, 62 insertions, 14 deletions

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 11508a1b39..68689b8e7f 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -684,6 +684,9 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	erts_cleanup_funs_on_purge(code, end);
 	beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length,
 			    erts_active_code_ix());
+        if (code[MI_LITERALS_START]) {
+            erts_free(ERTS_ALC_T_LITERAL, (void *) code[MI_LITERALS_START]);
+        }
 	erts_free(ERTS_ALC_T_CODE, (void *) code);
 	modp->curr.code = NULL;
 	modp->curr.code_length = 0;
@@ -1019,6 +1022,9 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1)
 	    beam_catches_delmod(modp->old.catches, code, modp->old.code_length,
 				code_ix);
 	    decrement_refc(code);
+            if (code[MI_LITERALS_START]) {
+                erts_free(ERTS_ALC_T_LITERAL, (void *) code[MI_LITERALS_START]);
+            }
 	    erts_free(ERTS_ALC_T_CODE, (void *) code);
 	    modp->old.code = NULL;
 	    modp->old.code_length = 0;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index a9d47eb7b0..f0b4be4c3d 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -654,6 +654,7 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader,
     stp->code[MI_COMPILE_PTR] = 0;
     stp->code[MI_COMPILE_SIZE] = 0;
     stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0;
+    stp->code[MI_LITERALS_START] = 0;
     stp->code[MI_MD5_PTR] = 0;
 
     /*
@@ -921,6 +922,9 @@ loader_state_dtor(Binary* magic)
 	stp->bin = 0;
     }
     if (stp->code != 0) {
+        if (stp->code[MI_LITERALS_START]) {
+            erts_free(ERTS_ALC_T_LITERAL, (void*) stp->code[MI_LITERALS_START]);
+        }
 	erts_free(ERTS_ALC_T_CODE, stp->code);
 	stp->code = 0;
     }
@@ -4348,7 +4352,6 @@ static int
 freeze_code(LoaderState* stp)
 {
     BeamInstr* code = stp->code;
-    Uint *literal_end = NULL;
     int i;
     byte* str_table;
     unsigned strtab_size = stp->chunks[STR_CHUNK].size;
@@ -4378,7 +4381,6 @@ freeze_code(LoaderState* stp)
 	    sizeof(Eterm) + (stp->current_li+1) * stp->loc_size;
     }
     size = (stp->ci * sizeof(BeamInstr)) +
-	(stp->total_literal_size * sizeof(Eterm)) +
 	strtab_size + attr_size + compile_size + MD5_SIZE + line_size;
 
     /*
@@ -4406,10 +4408,9 @@ freeze_code(LoaderState* stp)
     }
     CHKBLK(ERTS_ALC_T_CODE,code);
 
-    literal_end = (Uint *) (code+stp->ci);
     /*
-     * Place the literal heap directly after the code and fix up all
-     * instructions that refer to it.
+     * Place the literals in their own allocated heap (for fast range check)
+     * and fix up all instructions that refer to it.
      */
     {
 	Uint* ptr;
@@ -4420,7 +4421,8 @@ freeze_code(LoaderState* stp)
 
         ERTS_INIT_OFF_HEAP(&code_off_heap);
 
-	low = (Uint *) (code+stp->ci);
+        low = erts_alloc(ERTS_ALC_T_LITERAL,
+                         stp->total_literal_size*sizeof(Eterm));
 	high = low + stp->total_literal_size;
 	code[MI_LITERALS_START] = (BeamInstr) low;
 	code[MI_LITERALS_END] = (BeamInstr) high;
@@ -4443,7 +4445,6 @@ freeze_code(LoaderState* stp)
 	    op_ptr[0] = lit->term;
 	    lp = lp->next;
 	}
-	literal_end += stp->total_literal_size;
     }
     CHKBLK(ERTS_ALC_T_CODE,code);
 
@@ -4452,9 +4453,9 @@ freeze_code(LoaderState* stp)
      */
     if (stp->line_instr == 0) {
 	code[MI_LINE_TABLE] = (BeamInstr) 0;
-	str_table = (byte *) literal_end;
+	str_table = (byte *) (code+stp->ci);
     } else {
-	Eterm* line_tab = (Eterm *) literal_end;
+	Eterm* line_tab = (Eterm *) (code+stp->ci);
 	Eterm* p;
 	int ftab_size = stp->num_functions;
 	int num_instrs = stp->current_li;
@@ -4486,7 +4487,7 @@ freeze_code(LoaderState* stp)
 		*locp++ = (Uint16) stp->line_instr[i].loc;
 	    }
 	    *locp++ = LINE_INVALID_LOCATION;
-	    str_table = (byte *) locp;
+            str_table = (byte *) locp;
 	} else {
 	    Uint32* locp = (Uint32 *) p;
 	    ASSERT(stp->loc_size == 4);
@@ -4494,9 +4495,8 @@ freeze_code(LoaderState* stp)
 		*locp++ = stp->line_instr[i].loc;
 	    }
 	    *locp++ = LINE_INVALID_LOCATION;
-	    str_table = (byte *) locp;
+            str_table = (byte *) locp;
 	}
-
 	CHKBLK(ERTS_ALC_T_CODE,code);
     }
 
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 70d5357e27..d83a2930d6 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -130,6 +130,7 @@ static ErtsAllocatorState_t binary_alloc_state;
 static ErtsAllocatorState_t ets_alloc_state;
 static ErtsAllocatorState_t driver_alloc_state;
 static ErtsAllocatorState_t fix_alloc_state;
+static ErtsAllocatorState_t literal_alloc_state;
 
 typedef struct {
     erts_smp_atomic32_t refc;
@@ -211,6 +212,7 @@ typedef struct {
     struct au_init ets_alloc;
     struct au_init driver_alloc;
     struct au_init fix_alloc;
+    struct au_init literal_alloc;
 } erts_alc_hndl_args_init_t;
 
 #define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
@@ -285,6 +287,32 @@ set_default_ll_alloc_opts(struct au_init *ip)
 }
 
 static void
+set_default_literal_alloc_opts(struct au_init *ip)
+{
+    SET_DEFAULT_ALLOC_OPTS(ip);
+    ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
+    ip->thr_spec		= 0;
+    ip->atype			= BESTFIT;
+    ip->init.bf.ao		= 1;
+    ip->init.util.ramv		= 0;
+    ip->init.util.mmsbc		= 0;
+    ip->init.util.sbct		= ~((UWord) 0);
+    ip->init.util.name_prefix	= "literal_";
+    ip->init.util.alloc_no	= ERTS_ALC_A_LITERAL;
+#ifndef SMALL_MEMORY
+    ip->init.util.mmbcs 	= 2*1024*1024; /* Main carrier size */
+#else
+    ip->init.util.mmbcs 	= 1*1024*1024; /* Main carrier size */
+#endif
+    ip->init.util.ts 		= ERTS_ALC_MTA_LITERAL;
+    ip->init.util.asbcst	= 0;
+    ip->init.util.rsbcst	= 0;
+    ip->init.util.rsbcmt	= 0;
+    ip->init.util.rmbcmt	= 0;
+    ip->init.util.acul		= 0;
+}
+
+static void
 set_default_temp_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
@@ -577,6 +605,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     set_default_driver_alloc_opts(&init.driver_alloc);
     set_default_fix_alloc_opts(&init.fix_alloc,
 			       fix_type_sizes);
+    set_default_literal_alloc_opts(&init.literal_alloc);
 
     if (argc && argv)
 	handle_args(argc, argv, &init);
@@ -604,6 +633,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     init.ets_alloc.thr_spec = 0;
     init.driver_alloc.thr_spec = 0;
     init.fix_alloc.thr_spec = 0;
+    init.literal_alloc.thr_spec = 0;
 #endif
 
     /* Make adjustments for carrier migration support */
@@ -616,6 +646,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     adjust_carrier_migration_support(&init.ets_alloc);
     adjust_carrier_migration_support(&init.driver_alloc);
     adjust_carrier_migration_support(&init.fix_alloc);
+    adjust_carrier_migration_support(&init.literal_alloc);
 
     if (init.erts_alloc_config) {
 	/* Adjust flags that erts_alloc_config won't like */
@@ -630,6 +661,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.ets_alloc.thr_spec = 0;
 	init.driver_alloc.thr_spec = 0;
 	init.fix_alloc.thr_spec = 0;
+        init.literal_alloc.thr_spec = 0;
 
 	/* No carrier migration */
 	init.temp_alloc.init.util.acul = 0;
@@ -641,6 +673,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.ets_alloc.init.util.acul = 0;
 	init.driver_alloc.init.util.acul = 0;
 	init.fix_alloc.init.util.acul = 0;
+        init.literal_alloc.init.util.acul = 0;
     }
 
 #ifdef ERTS_SMP
@@ -657,6 +690,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     adjust_tpref(&init.ets_alloc, erts_no_schedulers);
     adjust_tpref(&init.driver_alloc, erts_no_schedulers);
     adjust_tpref(&init.fix_alloc, erts_no_schedulers);
+    adjust_tpref(&init.literal_alloc, erts_no_schedulers);
 
 #else
     /* No thread specific if not smp */
@@ -675,6 +709,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     refuse_af_strategy(&init.ets_alloc);
     refuse_af_strategy(&init.driver_alloc);
     refuse_af_strategy(&init.fix_alloc);
+    refuse_af_strategy(&init.literal_alloc);
 
 #ifdef ERTS_SMP 
     if (!init.temp_alloc.thr_spec)
@@ -718,6 +753,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu);
     set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
     set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu);
 
     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
 	if (!erts_allctrs[i].alloc)
@@ -770,6 +806,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     start_au_allocator(ERTS_ALC_A_FIXED_SIZE,
 		       &init.fix_alloc,
 		       &fix_alloc_state);
+    start_au_allocator(ERTS_ALC_A_LITERAL,
+                       &init.literal_alloc,
+                       &literal_alloc_state);
 
     erts_mtrace_install_wrapper_functions();
     extra_block_size += erts_instr_init(init.instr.stat, init.instr.map);
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 4804fb407d..7d519c1be4 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -86,6 +86,7 @@ allocator	LONG_LIVED	true		ll_alloc
 allocator	EHEAP		true		eheap_alloc
 allocator	ETS		true		ets_alloc
 allocator	FIXED_SIZE	true		fix_alloc
+allocator	LITERAL 	true		literal_alloc
 
 +else # Non smp build
 
@@ -96,6 +97,7 @@ allocator	LONG_LIVED	false		ll_alloc
 allocator	EHEAP		false		eheap_alloc
 allocator	ETS		false		ets_alloc
 allocator	FIXED_SIZE	false		fix_alloc
+allocator	LITERAL 	false		literal_alloc
 
 +endif
 
@@ -343,6 +345,7 @@ type	DDLL_PROCESS	STANDARD	SYSTEM		ddll_processes
 type	MONITOR_LH	STANDARD	PROCESSES	monitor_lh
 type	NLINK_LH	STANDARD	PROCESSES	nlink_lh
 type	CODE		LONG_LIVED	CODE		code
+type	LITERAL		LITERAL 	CODE		literal
 type	DB_HEIR_DATA	STANDARD	ETS		db_heir_data
 type	DB_MS_PSDO_PROC	LONG_LIVED	ETS		db_match_pseudo_proc
 type	SCHDLR_DATA	LONG_LIVED	SYSTEM		scheduler_data
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 9a132ee007..79ccf8db64 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4321,7 +4321,7 @@ static void os_info_init(void)
     os_flavor(buf, 1024);
     flav = erts_atom_put((byte *) buf, strlen(buf), ERTS_ATOM_ENC_LATIN1, 1);
     erts_free(ERTS_ALC_T_TMP, (void *) buf);
-    hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM, (3+4)*sizeof(Eterm));
+    hp = erts_alloc(ERTS_ALC_T_LITERAL, (3+4)*sizeof(Eterm));
     os_type_tuple = TUPLE2(hp, type, flav);
     hp += 3;
     os_version(&major, &minor, &build);
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index cc68e1f74d..7193c3d301 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -488,7 +488,7 @@ static void *const_term_alloc(void *tmpl)
     alloc_size = size + (offsetof(struct const_term, mem)/sizeof(Eterm));
     hipe_constants_size += alloc_size;
 
-    p = (struct const_term*)erts_alloc(ERTS_ALC_T_HIPE, alloc_size * sizeof(Eterm));
+    p = (struct const_term*)erts_alloc(ERTS_ALC_T_LITERAL, alloc_size * sizeof(Eterm));
 
     /* I have absolutely no idea if having a private 'off_heap'
        works or not. _Some_ off_heap object is required for