Merge remote branch 'upstream/dev' into dev

1 files changed, 156 insertions, 28 deletions

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index cda404af5e..bbc8a445a7 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -50,6 +50,9 @@
 #include "erl_bestfit_alloc.h"
 #define GET_ERL_AF_ALLOC_IMPL
 #include "erl_afit_alloc.h"
+#define GET_ERL_AOFF_ALLOC_IMPL
+#include "erl_ao_firstfit_alloc.h"
+
 
 #define ERTS_ALC_DEFAULT_MAX_THR_PREF 16
 
@@ -85,15 +88,19 @@ typedef union {
     char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))];
     AFAllctr_t afa;
     char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))];
+    AOFFAllctr_t aoffa;
+    char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))];
 } ErtsAllocatorState_t;
 
-static ErtsAllocatorState_t sl_alloc_state;
+static ErtsAllocatorState_t sbmbc_alloc_state;
 static ErtsAllocatorState_t std_alloc_state;
 static ErtsAllocatorState_t ll_alloc_state;
 #if HALFWORD_HEAP
-static ErtsAllocatorState_t std_alloc_low_state;
-static ErtsAllocatorState_t ll_alloc_low_state;
+static ErtsAllocatorState_t sbmbc_low_alloc_state;
+static ErtsAllocatorState_t std_low_alloc_state;
+static ErtsAllocatorState_t ll_low_alloc_state;
 #endif
+static ErtsAllocatorState_t sl_alloc_state;
 static ErtsAllocatorState_t temp_alloc_state;
 static ErtsAllocatorState_t eheap_alloc_state;
 static ErtsAllocatorState_t binary_alloc_state;
@@ -120,7 +127,8 @@ static void *fix_core_alloc(Uint size)
 enum allctr_type {
     GOODFIT,
     BESTFIT,
-    AFIT
+    AFIT,
+    AOFIRSTFIT
 };
 
 struct au_init {
@@ -132,6 +140,7 @@ struct au_init {
 	GFAllctrInit_t	gf;
 	BFAllctrInit_t	bf;
 	AFAllctrInit_t	af;
+	AOFFAllctrInit_t aoff;
     } init;
     struct {
 	int mmbcs;
@@ -145,7 +154,8 @@ struct au_init {
     ERTS_DEFAULT_ALLCTR_INIT,		\
     ERTS_DEFAULT_GF_ALLCTR_INIT,	\
     ERTS_DEFAULT_BF_ALLCTR_INIT,	\
-    ERTS_DEFAULT_AF_ALLCTR_INIT		\
+    ERTS_DEFAULT_AF_ALLCTR_INIT,	\
+    ERTS_DEFAULT_AOFF_ALLCTR_INIT       \
 }
 
 typedef struct {
@@ -162,6 +172,7 @@ typedef struct {
 	char *mtrace;
 	char *nodename;
     } instr;
+    struct au_init sbmbc_alloc;
     struct au_init sl_alloc;
     struct au_init std_alloc;
     struct au_init ll_alloc;
@@ -171,8 +182,9 @@ typedef struct {
     struct au_init ets_alloc;
     struct au_init driver_alloc;
 #if HALFWORD_HEAP
-    struct au_init std_alloc_low;
-    struct au_init ll_alloc_low;
+    struct au_init sbmbc_low_alloc;
+    struct au_init std_low_alloc;
+    struct au_init ll_low_alloc;
 #endif
 } erts_alc_hndl_args_init_t;
 
@@ -185,6 +197,34 @@ do {									\
 } while (0)
 
 static void
+set_default_sbmbc_alloc_opts(struct au_init *ip)
+{
+    SET_DEFAULT_ALLOC_OPTS(ip);
+    ip->enable			= 0;
+    ip->thr_spec		= 0;
+    ip->atype			= BESTFIT;
+    ip->init.bf.ao		= 1;
+    ip->init.util.ramv		= 0;
+    ip->init.util.mmsbc		= 0;
+    ip->init.util.mmmbc		= 500;
+    ip->init.util.sbct		= ~((UWord) 0);
+    ip->init.util.name_prefix	= "sbmbc_";
+    ip->init.util.alloc_no	= ERTS_ALC_A_SBMBC;
+#ifndef SMALL_MEMORY
+    ip->init.util.mmbcs 	= 2*1024*1024; /* Main carrier size */
+#else
+    ip->init.util.mmbcs 	= 1*1024*1024; /* Main carrier size */
+#endif
+    ip->init.util.ts 		= ERTS_ALC_MTA_SBMBC;
+    ip->init.util.asbcst	= 0;
+    ip->init.util.rsbcst	= 0;
+    ip->init.util.rsbcmt	= 0;
+    ip->init.util.rmbcmt	= 0;
+    ip->init.util.sbmbct	= 0;
+    ip->init.util.sbmbcs	= 0;
+}
+
+static void
 set_default_sl_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
@@ -202,6 +242,7 @@ set_default_sl_alloc_opts(struct au_init *ip)
     ip->init.util.ts 		= ERTS_ALC_MTA_SHORT_LIVED;
     ip->init.util.rsbcst	= 80;
 #if HALFWORD_HEAP
+    ip->init.util.force         = 1;
     ip->init.util.low_mem       = 1;
 #endif
 
@@ -249,6 +290,8 @@ set_default_ll_alloc_opts(struct au_init *ip)
     ip->init.util.rsbcst	= 0;
     ip->init.util.rsbcmt	= 0;
     ip->init.util.rmbcmt	= 0;
+    ip->init.util.sbmbct	= 0;
+    ip->init.util.sbmbcs	= 0;
 }
 
 static void
@@ -269,6 +312,7 @@ set_default_temp_alloc_opts(struct au_init *ip)
     ip->init.util.rsbcst	= 90;
     ip->init.util.rmbcmt	= 100;
 #if HALFWORD_HEAP
+    ip->init.util.force         = 1;
     ip->init.util.low_mem       = 1;
 #endif
 }
@@ -291,6 +335,7 @@ set_default_eheap_alloc_opts(struct au_init *ip)
     ip->init.util.ts 		= ERTS_ALC_MTA_EHEAP;
     ip->init.util.rsbcst	= 50;
 #if HALFWORD_HEAP
+    ip->init.util.force         = 1;
     ip->init.util.low_mem       = 1;
 #endif
 }
@@ -436,10 +481,13 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     hdbg_init();
 #endif
 
+    erts_have_sbmbc_alloc = 0;
+
     erts_sys_alloc_init();
     init_thr_ix(erts_no_schedulers);
     erts_init_utils_mem();
 
+    set_default_sbmbc_alloc_opts(&init.sbmbc_alloc);
     set_default_sl_alloc_opts(&init.sl_alloc);
     set_default_std_alloc_opts(&init.std_alloc);
     set_default_ll_alloc_opts(&init.ll_alloc);
@@ -453,6 +501,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	handle_args(argc, argv, &init);
 
     if (erts_no_schedulers <= 1) {
+	init.sbmbc_alloc.thr_spec = 0;
 	init.sl_alloc.thr_spec = 0;
 	init.std_alloc.thr_spec = 0;
 	init.ll_alloc.thr_spec = 0;
@@ -464,6 +513,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 
     if (init.erts_alloc_config) {
 	/* Adjust flags that erts_alloc_config won't like */
+	init.sbmbc_alloc.thr_spec = 0;
 	init.temp_alloc.thr_spec = 0;
 	init.sl_alloc.thr_spec = 0;
 	init.std_alloc.thr_spec = 0;
@@ -480,6 +530,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.temp_alloc.thr_spec = erts_no_schedulers;
 
     /* Others must use thread preferred interface */
+    adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers);
     adjust_tpref(&init.sl_alloc, erts_no_schedulers);
     adjust_tpref(&init.std_alloc, erts_no_schedulers);
     adjust_tpref(&init.ll_alloc, erts_no_schedulers);
@@ -497,6 +548,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
      * The following allocators cannot be run with afit strategy.
      * Make sure they don't...
      */
+    refuse_af_strategy(&init.sbmbc_alloc);
     refuse_af_strategy(&init.sl_alloc);
     refuse_af_strategy(&init.std_alloc);
     refuse_af_strategy(&init.ll_alloc);
@@ -518,6 +570,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     erts_afalc_init();
     erts_bfalc_init();
     erts_gfalc_init();
+    erts_aoffalc_init();
 
     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
 	erts_allctrs[i].alloc		= NULL;
@@ -551,19 +604,30 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 
 #if HALFWORD_HEAP
     /* Init low memory variants by cloning */
-    init.std_alloc_low = init.std_alloc;
-    init.std_alloc_low.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW;
-    init.std_alloc_low.init.util.low_mem  = 1;
-
-    init.ll_alloc_low = init.ll_alloc;
-    init.ll_alloc_low.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW;
-    init.ll_alloc_low.init.util.low_mem = 1;
-
-    set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_alloc_low);
-    set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_alloc_low);
+    init.sbmbc_low_alloc = init.sbmbc_alloc;
+    init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_";
+    init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW;
+    init.sbmbc_low_alloc.init.util.low_mem = 1;
+
+    init.std_low_alloc = init.std_alloc;
+    init.std_low_alloc.init.util.name_prefix	= "std_low_";
+    init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW;
+    init.std_low_alloc.init.util.force = 1;
+    init.std_low_alloc.init.util.low_mem = 1;
+
+    init.ll_low_alloc = init.ll_alloc;
+    init.ll_low_alloc.init.util.name_prefix	= "ll_low_";
+    init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW;
+    init.ll_low_alloc.init.util.force = 1;
+    init.ll_low_alloc.init.util.low_mem = 1;
+
+    set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc);
+    set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc);
+    set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc);
 #endif /* HALFWORD */
 
     set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc);
+    set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc);
     set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc);
     set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc);
     set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc);
@@ -593,6 +657,20 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 
     erts_mtrace_init(init.instr.mtrace, init.instr.nodename);
 
+    /* sbmbc_alloc() needs to be started first */
+    start_au_allocator(ERTS_ALC_A_SBMBC,
+		       &init.sbmbc_alloc,
+		       &sbmbc_alloc_state);
+#if HALFWORD_HEAP
+    start_au_allocator(ERTS_ALC_A_SBMBC_LOW,
+		       &init.sbmbc_low_alloc,
+		       &sbmbc_low_alloc_state);
+    erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable
+			     && init.sbmbc_low_alloc.enable);
+#else
+    erts_have_sbmbc_alloc = init.sbmbc_alloc.enable;
+#endif
+
     start_au_allocator(ERTS_ALC_A_TEMPORARY,
 		       &init.temp_alloc,
 		       &temp_alloc_state);
@@ -610,11 +688,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 		       &ll_alloc_state);
 #if HALFWORD_HEAP
     start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW,
-		       &init.ll_alloc_low,
-		       &ll_alloc_low_state);
+		       &init.ll_low_alloc,
+		       &ll_low_alloc_state);
     start_au_allocator(ERTS_ALC_A_STANDARD_LOW,
-		       &init.std_alloc_low,
-		       &std_alloc_low_state);
+		       &init.std_low_alloc,
+		       &std_low_alloc_state);
 #endif
     start_au_allocator(ERTS_ALC_A_EHEAP,
 		       &init.eheap_alloc,
@@ -680,14 +758,11 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
     ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
     ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
 
-#if HALFWORD_HEAP
-    /* If halfword heap, silently ignore any disabling of internal
-     * allocators for low memory
+    /*
+     * Some allocators are forced on if halfword heap is used.
      */
-    if (init->init.util.low_mem) {
+    if (init->init.util.force)
 	init->enable = 1;
-    }
-#endif
 
     if (!init->enable) {
 	af->alloc = erts_sys_alloc;
@@ -837,6 +912,12 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 					   &init->init.af,
 					   &init->init.util);
 	    break;
+    	case AOFIRSTFIT:
+	    as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0,
+					     &init->init.aoff,
+					     &init->init.util);
+	    break;
+
 	default:
 	    as = NULL;
 	    ASSERT(0);
@@ -947,6 +1028,20 @@ get_kb_value(char *param_end, char** argv, int* ip)
 }
 
 static Uint
+get_byte_value(char *param_end, char** argv, int* ip)
+{
+    Sint tmp;
+    char *rest;
+    char *param = argv[*ip]+1;
+    char *value = get_value(param_end, argv, ip);
+    errno = 0;
+    tmp = (Sint) strtol(value, &rest, 10);
+    if (errno != 0 || rest == value || tmp < 0)
+	bad_value(param, param_end, value);
+    return (Uint) tmp;
+}
+
+static Uint
 get_amount_value(char *param_end, char** argv, int* ip)
 {
     Sint tmp;
@@ -1017,6 +1112,9 @@ handle_au_arg(struct au_init *auip,
 	    else if (strcmp("af", alg) == 0) {
 		auip->atype = AFIT;
 	    }
+	    else if (strcmp("aoff", alg) == 0) {
+		auip->atype = AOFIRSTFIT;
+	    }
 	    else {
 		bad_value(param, sub_param + 1, alg);
 	    }
@@ -1085,6 +1183,12 @@ handle_au_arg(struct au_init *auip,
 	if(has_prefix("sbct", sub_param)) {
 	    auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip);
 	}
+	else if (has_prefix("sbmbcs", sub_param)) {
+	    auip->init.util.sbmbcs = get_byte_value(sub_param + 6, argv, ip);
+	}
+	else if (has_prefix("sbmbct", sub_param)) {
+	    auip->init.util.sbmbct = get_byte_value(sub_param + 6, argv, ip);
+	}
 	else if (has_prefix("smbcs", sub_param)) {
 	    auip->default_.smbcs = 0;
 	    auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip);
@@ -1123,6 +1227,7 @@ static void
 handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 {
     struct au_init *aui[] = {
+	&init->sbmbc_alloc,
 	&init->binary_alloc,
 	&init->std_alloc,
 	&init->ets_alloc,
@@ -1150,6 +1255,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		case 'B':
 		    handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i);
 		    break;
+		case 'C':
+		    handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i);
+		    break;
 		case 'D':
 		    handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i);
 		    break;
@@ -1856,12 +1964,16 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	    return am_badarg;
     }
 
-    /* All alloc_util allocators *have* to be enabled */
+    /* All alloc_util allocators except sbmbc_alloc *have* to be enabled */
     
     for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
 	switch (ai) {
 	case ERTS_ALC_A_SYSTEM:
 	case ERTS_ALC_A_FIXED_SIZE:
+	case ERTS_ALC_A_SBMBC:
+#if HALFWORD_HEAP
+	case ERTS_ALC_A_SBMBC_LOW:
+#endif
 	    break;
 	default:
 	    if (!erts_allctrs_info[ai].enabled
@@ -1901,6 +2013,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		      * Often not thread safe and usually never
 		      * contain any allocated memory.
 		      */
+		case ERTS_ALC_A_SBMBC:
+		    /* Included in other allocators */
+#if HALFWORD_HEAP
+		case ERTS_ALC_A_SBMBC_LOW:
+		    /* Included in other allocators */
+#endif
 		    continue;
 		case ERTS_ALC_A_EHEAP:
 		    save = &size.processes;
@@ -2882,6 +3000,7 @@ unsigned long erts_alc_test(unsigned long op,
     case 0x2:	return erts_bfalc_test(op, a1, a2);
     case 0x3:	return erts_afalc_test(op, a1, a2);
     case 0x4:	return erts_mseg_test(op,  a1, a2, a3);
+    case 0x5:	return erts_aoffalc_test(op, a1, a2);
     case 0xf:
 	switch (op) {
 	case 0xf00:
@@ -2925,6 +3044,7 @@ unsigned long erts_alc_test(unsigned long op,
 	    init.atype = GOODFIT;
 	    init.init.util.name_prefix = (char *) a1;
 	    init.init.util.ts = a2 ? 1 : 0;
+	    init.init.util.sbmbct = 0;
 
 	    if ((char **) a3) {
 		char **argv = (char **) a3;
@@ -2960,6 +3080,14 @@ unsigned long erts_alc_test(unsigned long op,
 					  &init.init.af,
 					  &init.init.util);
 		break;
+	    case AOFIRSTFIT:
+		allctr = erts_aoffalc_start((AOFFAllctr_t *)
+					  erts_alloc(ERTS_ALC_T_UNDEF,
+						     sizeof(AOFFAllctr_t)),
+					  &init.init.aoff,
+					  &init.init.util);
+		break;
+
 	    default:
 		ASSERT(0);
 		allctr = NULL;