diff options
Diffstat (limited to 'erts/emulator/beam')
| -rw-r--r-- | erts/emulator/beam/beam_emu.c | 9 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_alloc.c | 16 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_ao_firstfit_alloc.c | 190 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_ao_firstfit_alloc.h | 11 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_bestfit_alloc.c | 6 | ||||
| -rwxr-xr-x | erts/emulator/beam/erl_bif_info.c | 76 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_bits.c | 34 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_init.c | 2 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_utils.h | 5 | ||||
| -rw-r--r-- | erts/emulator/beam/external.c | 4 | ||||
| -rwxr-xr-x | erts/emulator/beam/global.h | 2 | ||||
| -rw-r--r-- | erts/emulator/beam/utils.c | 287 |
12 files changed, 509 insertions, 133 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 5781009f58..da36c4437e 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6256,3 +6256,12 @@ erts_current_reductions(Process *current, Process *p) } } +int +erts_beam_jump_table(void) +{ +#if defined(NO_JUMP_TABLE) + return 0; +#else + return 1; +#endif +} diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 86575ccb9b..5eacff8829 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -249,7 +249,7 @@ set_default_acul(struct au_init *ip, int acul) { ip->thr_spec = 1; ip->atype = AOFIRSTFIT; - ip->init.aoff.bf_within_carrier = 1; + ip->init.aoff.flavor = AOFF_BF; ip->init.util.acul = acul; } @@ -564,7 +564,7 @@ static ERTS_INLINE int strategy_support_carrier_migration(struct au_init *auip) { /* - * Currently only aoff and aoffcaobf support carrier + * Currently only aoff, aoffcbf and aoffcaobf support carrier * migration, i.e, type AOFIRSTFIT. */ return auip->atype == AOFIRSTFIT; @@ -587,9 +587,9 @@ ensure_carrier_migration_support(struct au_init *auip) * default to a strategy that can... */ if (!strategy_support_carrier_migration(auip)) { - /* Default to aoffcaobf */ + /* Default to aoffcbf */ auip->atype = AOFIRSTFIT; - auip->init.aoff.bf_within_carrier = 1; + auip->init.aoff.flavor = AOFF_BF; } } @@ -1290,11 +1290,15 @@ handle_au_arg(struct au_init *auip, } else if (strcmp("aoff", alg) == 0) { auip->atype = AOFIRSTFIT; - auip->init.aoff.bf_within_carrier = 0; + auip->init.aoff.flavor = AOFF_AOFF; + } + else if (strcmp("aoffcbf", alg) == 0) { + auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_BF; } else if (strcmp("aoffcaobf", alg) == 0) { auip->atype = AOFIRSTFIT; - auip->init.aoff.bf_within_carrier = 1; + auip->init.aoff.flavor = AOFF_AOBF; } else { bad_value(param, sub_param + 1, alg); diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index f73ac2eb6e..4e6c8b317e 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -34,10 +34,11 @@ * sub-tree ('max_sz'). By that we can start from root and keep * left (for low addresses) while dismissing entire sub-trees with * too small blocks. - * AOFFCBF: + * Bestfit within carrier: * The only difference for "bestfit within carrier" is the tree * sorting order. Blocks within the same carrier are sorted - * wrt size instead of address. + * wrt size instead of address. The 'max_sz' field is maintained + * in order to dismiss entire carriers with too small blocks. * * Authors: Rickard Green/Sverker Eriksson */ @@ -67,6 +68,15 @@ # define LEFT_VISITED_FLG (((Uint) 1) << 2) # define RIGHT_VISITED_FLG (((Uint) 1) << 3) #endif +#ifdef DEBUG +# define IS_BF_FLG (((Uint) 1) << 4) +#endif + +#define IS_TREE_NODE(N) (((AOFF_RBTree_t *) (N))->flags & TREE_NODE_FLG) +#define IS_LIST_ELEM(N) (!IS_TREE_NODE(((AOFF_RBTree_t *) (N)))) + +#define SET_TREE_NODE(N) (((AOFF_RBTree_t *) (N))->flags |= TREE_NODE_FLG) +#define SET_LIST_ELEM(N) (((AOFF_RBTree_t *) (N))->flags &= ~TREE_NODE_FLG) #define IS_RED(N) (((AOFF_RBTree_t *) (N)) \ && ((AOFF_RBTree_t *) (N))->flags & RED_FLG) @@ -98,6 +108,16 @@ struct AOFF_RBTree_t_ { }; #define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) +/* BF block nodes keeps list of all with equal size + */ +typedef struct { + AOFF_RBTree_t t; + AOFF_RBTree_t *next; +}AOFF_RBTreeList_t; + +#define LIST_NEXT(N) (((AOFF_RBTreeList_t*) (N))->next) +#define LIST_PREV(N) (((AOFF_RBTreeList_t*) (N))->t.parent) + typedef struct AOFF_Carrier_t_ AOFF_Carrier_t; struct AOFF_Carrier_t_ { @@ -119,11 +139,11 @@ struct AOFF_Carrier_t_ { #ifdef HARD_DEBUG # define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) -# define HARD_CHECK_TREE(CRR,BF,ROOT,SZ) check_tree(CRR, BF, ROOT, SZ) -static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint); +# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ) +static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint); #else # define HARD_CHECK_IS_MEMBER(ROOT,NODE) -# define HARD_CHECK_TREE(CRR,BF,ROOT,SZ) +# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) #endif @@ -161,25 +181,25 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, else ASSERT(new_max == old_max); } -static ERTS_INLINE SWord cmp_blocks(int bestfit, +static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor, AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs) { ASSERT(lhs != rhs); - ASSERT(!bestfit || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); - if (bestfit) { + ASSERT(flavor == AOFF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); + if (flavor != AOFF_AOFF) { SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); - if (diff) return diff; + if (diff || flavor == AOFF_BF) return diff; } return (char*)lhs - (char*)rhs; } -static ERTS_INLINE SWord cmp_cand_blk(int bestfit, +static ERTS_INLINE SWord cmp_cand_blk(enum AOFF_Flavor flavor, Block_t* cand_blk, AOFF_RBTree_t* rhs) { - if (bestfit) { + if (flavor != AOFF_AOFF) { if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) { SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr); - if (diff) return diff; + if (diff || flavor == AOFF_BF) return diff; } } return (char*)cand_blk - (char*)rhs; @@ -198,7 +218,7 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); /* Generic tree functions used by both carrier and block trees. */ static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del); -static void rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); +static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size); #ifdef HARD_DEBUG static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node); @@ -234,21 +254,21 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); - alc->bf_within_carrier = aoffinit->bf_within_carrier; + alc->flavor = aoffinit->flavor; allctr->mbc_header_size = sizeof(AOFF_Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; - allctr->min_block_size = sizeof(AOFF_RBTree_t); + allctr->min_block_size = (aoffinit->flavor == AOFF_BF ? + sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t)); - allctr->vsn_str = aoffinit->bf_within_carrier ? - ERTS_ALC_AOFF_CBF_ALLOC_VSN_STR : ERTS_ALC_AOFF_ALLOC_VSN_STR; + allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; /* Callback functions */ allctr->get_free_block = aoff_get_free_block; allctr->link_free_block = aoff_link_free_block; - allctr->unlink_free_block = aoff_unlink_free_block; + allctr->unlink_free_block = aoff_unlink_free_block; allctr->info_options = info_options; allctr->get_next_mbc_size = NULL; @@ -359,9 +379,7 @@ replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y) y->parent = x->parent; y->right = x->right; y->left = x->left; - - y->max_sz = x->max_sz; - lower_max_size(y, NULL); + y->max_sz = x->max_sz; } static void @@ -458,23 +476,47 @@ tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk) } static void -aoff_unlink_free_block(Allctr_t *allctr, Block_t *del) +aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) { -#ifdef HARD_DEBUG AOFFAllctr_t* alc = (AOFFAllctr_t*)allctr; -#endif - AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(del); + AOFF_RBTree_t* del = (AOFF_RBTree_t*)blk; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr); ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz); - HARD_CHECK_IS_MEMBER(alc->mbc_root, &crr->rbt_node); - HARD_CHECK_TREE(&crr->crr, alc->bf_within_carrier, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + + if (alc->flavor == AOFF_BF) { + ASSERT(del->flags & IS_BF_FLG); + if (IS_LIST_ELEM(del)) { + /* Remove from list */ + ASSERT(LIST_PREV(del)); + ASSERT(LIST_PREV(del)->flags & IS_BF_FLG); + LIST_NEXT(LIST_PREV(del)) = LIST_NEXT(del); + if (LIST_NEXT(del)) { + ASSERT(LIST_NEXT(del)->flags & IS_BF_FLG); + LIST_PREV(LIST_NEXT(del)) = LIST_PREV(del); + } + return; + } + else if (LIST_NEXT(del)) { + /* Replace tree node by next element in list... */ + + ASSERT(AOFF_BLK_SZ(LIST_NEXT(del)) == AOFF_BLK_SZ(del)); + ASSERT(IS_LIST_ELEM(LIST_NEXT(del))); + + replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del)); + + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + return; + } + } rbt_delete(&crr->root, (AOFF_RBTree_t*)del); - HARD_CHECK_TREE(&crr->crr, alc->bf_within_carrier, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); - /* Update the carrier tree with a potentially new (lower) max_sz - */ + /* Update the carrier tree with a potentially new (lower) max_sz + */ if (crr->root) { if (crr->rbt_node.hdr.bhdr == crr->root->max_sz) { return; @@ -488,6 +530,7 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *del) lower_max_size(&crr->rbt_node, NULL); } + static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del) { @@ -542,7 +585,9 @@ rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del) } if (y != z) { /* We spliced out the successor of z; replace z by the successor */ + ASSERT(z != &null_x); replace(root, z, y); + lower_max_size(y, NULL); } if (spliced_is_black) { @@ -666,12 +711,11 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block) ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr)); ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0)); - HARD_CHECK_IS_MEMBER(alc->mbc_root, &blk_crr->rbt_node); - HARD_CHECK_TREE(&blk_crr->crr, alc->bf_within_carrier, blk_crr->root, 0); + HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); - rbt_insert(alc->bf_within_carrier, &blk_crr->root, blk); + rbt_insert(alc->flavor, &blk_crr->root, blk); - /* Update the carrier tree with a potential new (larger) max_sz + /* Update the carrier tree with a potentially new (larger) max_sz */ crr_node = &blk_crr->rbt_node; if (blk_sz > crr_node->hdr.bhdr) { @@ -683,15 +727,19 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block) if (!crr_node) break; } } - HARD_CHECK_TREE(&blk_crr->crr, alc->bf_within_carrier, blk_crr->root, 0); + HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); } static void -rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) +rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) { Uint blk_sz = AOFF_BLK_SZ(blk); - blk->flags = 0; +#ifdef DEBUG + blk->flags = (flavor == AOFF_BF) ? IS_BF_FLG : 0; +#else + blk->flags = 0; +#endif blk->left = NULL; blk->right = NULL; blk->max_sz = blk_sz; @@ -704,10 +752,12 @@ rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) else { AOFF_RBTree_t *x = *root; while (1) { + SWord diff; if (x->max_sz < blk_sz) { x->max_sz = blk_sz; } - if (cmp_blocks(bestfit, blk, x) < 0) { + diff = cmp_blocks(flavor, blk, x); + if (diff < 0) { if (!x->left) { blk->parent = x; x->left = blk; @@ -715,7 +765,7 @@ rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) } x = x->left; } - else { + else if (diff > 0) { if (!x->right) { blk->parent = x; x->right = blk; @@ -723,6 +773,18 @@ rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) } x = x->right; } + else { + ASSERT(flavor == AOFF_BF); + ASSERT(blk->flags & IS_BF_FLG); + ASSERT(x->flags & IS_BF_FLG); + SET_LIST_ELEM(blk); + LIST_NEXT(blk) = LIST_NEXT(x); + LIST_PREV(blk) = x; + if (LIST_NEXT(x)) + LIST_PREV(LIST_NEXT(x)) = blk; + LIST_NEXT(x) = blk; + return; + } } /* Insert block into size tree */ @@ -732,6 +794,10 @@ rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) if (IS_RED(blk->parent)) tree_insert_fixup(root, blk); } + if (flavor == AOFF_BF) { + SET_TREE_NODE(blk); + LIST_NEXT(blk) = NULL; + } } static AOFF_RBTree_t* @@ -780,7 +846,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, /* Get block within carrier tree */ #ifdef HARD_DEBUG - dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->bf_within_carrier, crr->root, size); + dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, size); #endif blk = rbt_search(crr->root, size); @@ -793,7 +859,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (!blk) return NULL; - if (cand_blk && cmp_cand_blk(alc->bf_within_carrier, cand_blk, blk) < 0) { + if (cand_blk && cmp_cand_blk(alc->flavor, cand_blk, blk) < 0) { return NULL; /* cand_blk was better */ } @@ -813,7 +879,7 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) /* Link carrier in address order tree */ crr->rbt_node.hdr.bhdr = 0; - rbt_insert(0, root, &crr->rbt_node); + rbt_insert(AOFF_AOFF, root, &crr->rbt_node); /* aoff_link_free_block will add free block later */ crr->root = NULL; @@ -843,7 +909,7 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) /* Link carrier in address order tree */ - rbt_insert(0, root, &crr->rbt_node); + rbt_insert(AOFF_AOFF, root, &crr->rbt_node); HARD_CHECK_TREE(NULL, 0, *root, 0); } @@ -883,6 +949,7 @@ static struct { Eterm as; Eterm aoff; Eterm aoffcaobf; + Eterm aoffcbf; #ifdef DEBUG Eterm end_of_atoms; #endif @@ -912,6 +979,7 @@ init_atoms(void) AM_INIT(as); AM_INIT(aoff); AM_INIT(aoffcaobf); + AM_INIT(aoffcbf); #ifdef DEBUG for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { @@ -943,13 +1011,15 @@ info_options(Allctr_t *allctr, { AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; Eterm res = THE_NON_VALUE; + const char* flavor_str[3] = {"aoff", "aoffcaobf", "aoffcbf"}; + Eterm flavor_atom[3] = {am.aoff, am.aoffcaobf, am.aoffcbf}; if (print_to_p) { erts_print(*print_to_p, print_to_arg, "%sas: %s\n", prefix, - alc->bf_within_carrier ? "aoffcaobf" : "aoff"); + flavor_str[alc->flavor]); } if (hpp || szp) { @@ -959,8 +1029,7 @@ info_options(Allctr_t *allctr, __FILE__, __LINE__);; res = NIL; - add_2tup(hpp, szp, &res, am.as, - alc->bf_within_carrier ? am.aoffcaobf : am.aoff); + add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->flavor]); } return res; @@ -978,6 +1047,7 @@ UWord erts_aoffalc_test(UWord op, UWord a1, UWord a2) { switch (op) { + case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_AOBF; case 0x501: { AOFF_RBTree_t *node = ((AOFFAllctr_t *) a1)->mbc_root; Uint size = (Uint) a2; @@ -987,10 +1057,13 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2) case 0x502: return (UWord) ((AOFF_RBTree_t *) a1)->parent; case 0x503: return (UWord) ((AOFF_RBTree_t *) a1)->left; case 0x504: return (UWord) ((AOFF_RBTree_t *) a1)->right; + case 0x505: return (UWord) LIST_NEXT(a1); case 0x506: return (UWord) IS_BLACK((AOFF_RBTree_t *) a1); + case 0x507: return (UWord) IS_TREE_NODE((AOFF_RBTree_t *) a1); case 0x508: return (UWord) 0; /* IS_BF_ALGO */ case 0x509: return (UWord) ((AOFF_RBTree_t *) a1)->max_sz; - case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->bf_within_carrier; + case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_BF; + case 0x50b: return (UWord) LIST_PREV(a1); default: ASSERT(0); return ~((UWord) 0); } } @@ -1049,7 +1122,7 @@ static void print_tree(AOFF_RBTree_t*); */ static AOFF_RBTree_t * -check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint size) +check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint size) { AOFF_RBTree_t *res = NULL; Sint blacks; @@ -1061,6 +1134,7 @@ check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint size) #ifdef PRINT_TREE print_tree(root); #endif + ASSERT(within_crr || flavor == AOFF_AOFF); if (!root) return res; @@ -1116,6 +1190,20 @@ check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint size) ASSERT(crr == within_crr); ASSERT((char*)x > (char*)crr); ASSERT(((char*)x + AOFF_BLK_SZ(x)) <= ((char*)crr + CARRIER_SZ(crr))); + + } + if (flavor == AOFF_BF) { + AOFF_RBTree_t* y = x; + AOFF_RBTree_t* nxt = LIST_NEXT(y); + ASSERT(IS_TREE_NODE(x)); + while (nxt) { + ASSERT(IS_LIST_ELEM(nxt)); + ASSERT(AOFF_BLK_SZ(nxt) == AOFF_BLK_SZ(x)); + ASSERT(FBLK_TO_MBC(&nxt->hdr) == within_crr); + ASSERT(LIST_PREV(nxt) == y); + y = nxt; + nxt = LIST_NEXT(nxt); + } } if (IS_RED(x)) { @@ -1127,13 +1215,13 @@ check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint size) if (x->left) { ASSERT(x->left->parent == x); - ASSERT(cmp_blocks(bestfit, x->left, x) < 0); + ASSERT(cmp_blocks(flavor, x->left, x) < 0); ASSERT(x->left->max_sz <= x->max_sz); } if (x->right) { ASSERT(x->right->parent == x); - ASSERT(cmp_blocks(bestfit, x->right, x) > 0); + ASSERT(cmp_blocks(flavor, x->right, x) > 0); ASSERT(x->right->max_sz <= x->max_sz); } ASSERT(x->max_sz >= AOFF_BLK_SZ(x)); @@ -1142,7 +1230,7 @@ check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint size) || x->max_sz == (x->right ? x->right->max_sz : 0)); if (size && AOFF_BLK_SZ(x) >= size) { - if (!res || cmp_blocks(bestfit, x, res) < 0) { + if (!res || cmp_blocks(flavor, x, res) < 0) { res = x; } } diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index e92cb5abb6..25b344c6a8 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -24,12 +24,17 @@ #include "erl_alloc_util.h" #define ERTS_ALC_AOFF_ALLOC_VSN_STR "0.9" -#define ERTS_ALC_AOFF_CBF_ALLOC_VSN_STR "0.9" typedef struct AOFFAllctr_t_ AOFFAllctr_t; +enum AOFF_Flavor { + AOFF_AOFF = 0, + AOFF_AOBF = 1, + AOFF_BF = 2 +}; + typedef struct { - int bf_within_carrier; + enum AOFF_Flavor flavor; } AOFFAllctrInit_t; #define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/} @@ -52,7 +57,7 @@ struct AOFFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ struct AOFF_RBTree_t_* mbc_root; - int bf_within_carrier; + enum AOFF_Flavor flavor; }; UWord erts_aoffalc_test(UWord, UWord, UWord); diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 58e53c3d00..41f449bb28 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -966,15 +966,17 @@ UWord erts_bfalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x200: return (UWord) ((BFAllctr_t *) a1)->address_order; + case 0x200: return (UWord) ((BFAllctr_t *) a1)->address_order; /* IS_AOBF */ case 0x201: return (UWord) ((BFAllctr_t *) a1)->mbc_root; case 0x202: return (UWord) ((RBTree_t *) a1)->parent; case 0x203: return (UWord) ((RBTree_t *) a1)->left; case 0x204: return (UWord) ((RBTree_t *) a1)->right; - case 0x205: return (UWord) ((RBTreeList_t *) a1)->next; + case 0x205: return (UWord) LIST_NEXT(a1); case 0x206: return (UWord) IS_BLACK((RBTree_t *) a1); case 0x207: return (UWord) IS_TREE_NODE((RBTree_t *) a1); case 0x208: return (UWord) 1; /* IS_BF_ALGO */ + case 0x20a: return (UWord) !((BFAllctr_t *) a1)->address_order; /* IS_BF */ + case 0x20b: return (UWord) LIST_PREV(a1); default: ASSERT(0); return ~((UWord) 0); } } diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 74a37f374d..e4fbd21c0e 100755 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2577,74 +2577,14 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = hsz ? HAlloc(BIF_P, hsz) : NULL; res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); BIF_RET(res); - } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) { -#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ - || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ - || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) - int i; - char **str; -#endif -#ifdef ETHR_NATIVE_ATOMIC32_IMPL - erts_printf("32-bit native atomics: %s\n", - ETHR_NATIVE_ATOMIC32_IMPL); - str = ethr_native_atomic32_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_atomic32_%s()\n", str[i]); -#endif -#ifdef ETHR_NATIVE_ATOMIC64_IMPL - erts_printf("64-bit native atomics: %s\n", - ETHR_NATIVE_ATOMIC64_IMPL); - str = ethr_native_atomic64_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_atomic64_%s()\n", str[i]); -#endif -#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL - if (ethr_have_native_dw_atomic()) { - erts_printf("Double word native atomics: %s\n", - ETHR_NATIVE_DW_ATOMIC_IMPL); - str = ethr_native_dw_atomic_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_dw_atomic_%s()\n", str[i]); - str = ethr_native_su_dw_atomic_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_su_dw_atomic_%s()\n", str[i]); - } -#endif -#ifdef ETHR_NATIVE_SPINLOCK_IMPL - erts_printf("Native spin-locks: %s\n", ETHR_NATIVE_SPINLOCK_IMPL); -#endif -#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL - erts_printf("Native rwspin-locks: %s\n", ETHR_NATIVE_RWSPINLOCK_IMPL); -#endif -#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ - erts_printf("SSE2 support: %s\n", (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ - ? "yes" : "no")); -#endif -#ifdef ETHR_X86_OUT_OF_ORDER - erts_printf("x86" -#ifdef ARCH_64 - "_64" -#endif - " out of order\n"); -#endif -#ifdef ETHR_SPARC_TSO - erts_printf("Sparc TSO\n"); -#endif -#ifdef ETHR_SPARC_PSO - erts_printf("Sparc PSO\n"); -#endif -#ifdef ETHR_SPARC_RMO - erts_printf("Sparc RMO\n"); -#endif -#if defined(ETHR_PPC_HAVE_LWSYNC) - erts_printf("Have lwsync instruction: yes\n"); -#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) - erts_printf("Have lwsync instruction: no\n"); -#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) - erts_printf("Have lwsync instruction: %s (runtime test)\n", - ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no"); -#endif - BIF_RET(am_true); + } else if (ERTS_IS_ATOM_STR("ethread_info", BIF_ARG_1)) { + BIF_RET(erts_get_ethread_info(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("emu_args", BIF_ARG_1)) { + BIF_RET(erts_get_emu_args(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("beam_jump_table", BIF_ARG_1)) { + BIF_RET(erts_beam_jump_table() ? am_true : am_false); } else if (ERTS_IS_ATOM_STR("dynamic_trace", BIF_ARG_1)) { #if defined(USE_DTRACE) diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 3753b618e1..43eb691338 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -484,8 +484,16 @@ erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer ERTS_FP_ERROR_THOROUGH(p, f32, return THE_NON_VALUE); f.fd = f32; } else { +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef ftmp; + ftmp.fd = f64; + f.fw[0] = ftmp.fw[1]; + f.fw[1] = ftmp.fw[0]; + ERTS_FP_ERROR_THOROUGH(p, f.fd, return THE_NON_VALUE); +#else ERTS_FP_ERROR_THOROUGH(p, f64, return THE_NON_VALUE); f.fd = f64; +#endif } mb->offset += num_bits; hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT); @@ -1014,8 +1022,13 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) #endif } else if (is_small(arg)) { u.f64 = (double) signed_val(arg); +#ifdef DOUBLE_MIDDLE_ENDIAN + a = u.i32[1]; + b = u.i32[0]; +#else a = u.i32[0]; b = u.i32[1]; +#endif } else if (is_big(arg)) { if (big_to_double(arg, &u.f64) < 0) { return 0; @@ -1118,21 +1131,42 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) byte *bptr; double f64; float f32; +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef fbuf, ftmp; +#endif if (num_bits == 64) { if (is_float(arg)) { +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef *fdp = (FloatDef*)(float_val(arg) + 1); + ftmp = *fdp; +#else bptr = (byte *) (float_val(arg) + 1); +#endif } else if (is_small(arg)) { f64 = (double) signed_val(arg); +#ifdef DOUBLE_MIDDLE_ENDIAN + ftmp.fd = f64; +#else bptr = (byte *) &f64; +#endif } else if (is_big(arg)) { if (big_to_double(arg, &f64) < 0) { return 0; } +#ifdef DOUBLE_MIDDLE_ENDIAN + ftmp.fd = f64; +#else bptr = (byte *) &f64; +#endif } else { return 0; } +#ifdef DOUBLE_MIDDLE_ENDIAN + fbuf.fw[0] = ftmp.fw[1]; + fbuf.fw[1] = ftmp.fw[0]; + bptr = fbuf.fb; +#endif } else if (num_bits == 32) { if (is_float(arg)) { FloatDef f; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 4bae3dfeb4..8d137df7ae 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -637,6 +637,8 @@ early_init(int *argc, char **argv) /* char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; + erts_save_emu_args(*argc, argv); + erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index a2064bd8a3..80d29d554a 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -24,6 +24,8 @@ #include "erl_smp.h" #include "erl_printf.h" +struct process; + typedef struct { #ifdef DEBUG int smp_api; @@ -155,6 +157,9 @@ Uint32 block_hash(byte *, unsigned, Uint32); Uint32 make_hash2(Eterm); Uint32 make_hash(Eterm); +void erts_save_emu_args(int argc, char **argv); +Eterm erts_get_emu_args(struct process *c_p); +Eterm erts_get_ethread_info(struct process * c_p); Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str); Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui); diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 249b1e0923..1c88765381 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2325,7 +2325,7 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla GET_DOUBLE(obj, f); if (dflags & DFLAG_NEW_FLOATS) { *ep++ = NEW_FLOAT_EXT; -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) put_int32(f.fw[0], ep); ep += 4; put_int32(f.fw[1], ep); @@ -2804,7 +2804,7 @@ dec_term_atom_common: volatile unsigned long *fpexnp = erts_get_current_fp_exception(); #endif -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) ff.fw[0] = get_int32(ep); ep += 4; ff.fw[1] = get_int32(ep); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index c0d66d6cc1..2eec15867d 100755 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1042,6 +1042,8 @@ extern erts_driver_t vanilla_driver; extern erts_driver_t spawn_driver; extern erts_driver_t fd_driver; +int erts_beam_jump_table(void); + /* Should maybe be placed in erl_message.h, but then we get an include mess. */ ERTS_GLB_INLINE Eterm * erts_alloc_message_heap_state(Uint size, diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 62caa67ce1..84deaecb87 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1319,7 +1319,7 @@ make_hash2(Eterm term) { FloatDef ff; GET_DOUBLE(term, ff); -#if defined(WORDS_BIGENDIAN) +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); #else UINT32_HASH_2(ff.fw[1], ff.fw[0], HCONST_12); @@ -3466,6 +3466,291 @@ erts_free_read_env(void *value) erts_free(ERTS_ALC_T_TMP, value); } + +typedef struct { + size_t sz; + char *ptr; +} ErtsEmuArg; + +typedef struct { + int argc; + ErtsEmuArg *arg; + size_t no_bytes; +} ErtsEmuArgs; + +ErtsEmuArgs saved_emu_args = {0}; + +void +erts_save_emu_args(int argc, char **argv) +{ +#ifdef DEBUG + char *end_ptr; +#endif + char *ptr; + int i; + size_t arg_sz[100]; + size_t size; + + ASSERT(!saved_emu_args.argc); + + size = sizeof(ErtsEmuArg)*argc; + for (i = 0; i < argc; i++) { + size_t sz = sys_strlen(argv[i]); + if (i < sizeof(arg_sz)/sizeof(arg_sz[0])) + arg_sz[i] = sz; + size += sz+1; + } + ptr = (char *) malloc(size); +#ifdef DEBUG + end_ptr = ptr + size; +#endif + saved_emu_args.arg = (ErtsEmuArg *) ptr; + ptr += sizeof(ErtsEmuArg)*argc; + saved_emu_args.argc = argc; + saved_emu_args.no_bytes = 0; + for (i = 0; i < argc; i++) { + size_t sz; + if (i < sizeof(arg_sz)/sizeof(arg_sz[0])) + sz = arg_sz[i]; + else + sz = sys_strlen(argv[i]); + saved_emu_args.arg[i].ptr = ptr; + saved_emu_args.arg[i].sz = sz; + saved_emu_args.no_bytes += sz; + ptr += sz+1; + sys_strcpy(saved_emu_args.arg[i].ptr, argv[i]); + } + ASSERT(ptr == end_ptr); +} + +Eterm +erts_get_emu_args(Process *c_p) +{ +#ifdef DEBUG + Eterm *end_hp; +#endif + int i; + Uint hsz; + Eterm *hp, res; + + hsz = saved_emu_args.no_bytes*2; + hsz += saved_emu_args.argc*2; + + hp = HAlloc(c_p, hsz); +#ifdef DEBUG + end_hp = hp + hsz; +#endif + res = NIL; + + for (i = saved_emu_args.argc-1; i >= 0; i--) { + Eterm arg = buf_to_intlist(&hp, + saved_emu_args.arg[i].ptr, + saved_emu_args.arg[i].sz, + NIL); + res = CONS(hp, arg, res); + hp += 2; + } + + ASSERT(hp == end_hp); + + return res; +} + + +Eterm +erts_get_ethread_info(Process *c_p) +{ + Uint sz, *szp; + Eterm res, *hp, **hpp, *end_hp = NULL; + + sz = 0; + szp = &sz; + hpp = NULL; + + while (1) { + Eterm tup, list, name; +#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ + || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ + || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) + char buf[1024]; + int i; + char **str; +#endif + + res = NIL; + +#ifdef ETHR_X86_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "sse2"), +#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + erts_bld_string(hpp, szp, + (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + ? "yes" : "no")) +#else + erts_bld_string(hpp, szp, "yes") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, + "x86" +#ifdef ARCH_64 + "_64" +#endif + " OOO"), + erts_bld_string(hpp, szp, +#ifdef ETHR_X86_OUT_OF_ORDER + "yes" +#else + "no" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); +#endif + +#ifdef ETHR_SPARC_V9_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Sparc V9"), + erts_bld_string(hpp, szp, +#if defined(ETHR_SPARC_TSO) + "TSO" +#elif defined(ETHR_SPARC_PSO) + "PSO" +#elif defined(ETHR_SPARC_RMO) + "RMO" +#else + "undefined" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); + +#endif + +#ifdef ETHR_PPC_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "lwsync"), + erts_bld_string(hpp, szp, +#if defined(ETHR_PPC_HAVE_LWSYNC) + "yes" +#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) + "no" +#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) + ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no" +#else + "undefined" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); + +#endif + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Native rw-spinlocks"), +#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL + erts_bld_string(hpp, szp, ETHR_NATIVE_RWSPINLOCK_IMPL) +#else + erts_bld_string(hpp, szp, "no") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Native spinlocks"), +#ifdef ETHR_NATIVE_SPINLOCK_IMPL + erts_bld_string(hpp, szp, ETHR_NATIVE_SPINLOCK_IMPL) +#else + erts_bld_string(hpp, szp, "no") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + + list = NIL; +#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL + if (ethr_have_native_dw_atomic()) { + name = erts_bld_string(hpp, szp, ETHR_NATIVE_DW_ATOMIC_IMPL); + str = ethr_native_dw_atomic_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_dw_atomic_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } + str = ethr_native_su_dw_atomic_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_su_dw_atomic_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } + } + else +#endif + name = erts_bld_string(hpp, szp, "no"); + + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "Double word native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + list = NIL; +#ifdef ETHR_NATIVE_ATOMIC64_IMPL + name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC64_IMPL); + str = ethr_native_atomic64_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_atomic64_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } +#else + name = erts_bld_string(hpp, szp, "no"); +#endif + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "64-bit native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + list = NIL; +#ifdef ETHR_NATIVE_ATOMIC32_IMPL + name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC32_IMPL); + str = ethr_native_atomic32_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_atomic32_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } +#else + name = erts_bld_string(hpp, szp, "no"); +#endif + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "32-bit native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + if (hpp) { + HRelease(c_p, end_hp, *hpp) + return res; + } + + hp = HAlloc(c_p, sz); + end_hp = hp + sz; + hpp = &hp; + szp = NULL; + } +} + /* * To be used to silence unused result warnings, but do not abuse it. */ |