diff options
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r-- | erts/emulator/beam/erl_init.c | 63 | ||||
-rw-r--r-- | erts/emulator/beam/erl_node_container_utils.h | 40 | ||||
-rw-r--r-- | erts/emulator/beam/erl_port.h | 4 | ||||
-rw-r--r-- | erts/emulator/beam/erl_process.h | 8 | ||||
-rw-r--r-- | erts/emulator/beam/erl_process_lock.h | 2 | ||||
-rw-r--r-- | erts/emulator/beam/erl_ptab.c | 103 | ||||
-rw-r--r-- | erts/emulator/beam/erl_ptab.h | 150 | ||||
-rw-r--r-- | erts/emulator/beam/erl_term.c | 2 | ||||
-rw-r--r-- | erts/emulator/beam/erl_term.h | 19 | ||||
-rw-r--r-- | erts/emulator/beam/global.h | 2 | ||||
-rw-r--r-- | erts/emulator/beam/io.c | 20 | ||||
-rw-r--r-- | erts/emulator/beam/sys.h | 25 |
12 files changed, 303 insertions, 135 deletions
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 9ef66456ac..f83e380b02 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -111,6 +111,11 @@ const int etp_lock_check = 1; #else const int etp_lock_check = 0; #endif +#ifdef WORDS_BIGENDIAN +const int etp_big_endian = 1; +#else +const int etp_big_endian = 0; +#endif /* * Note about VxWorks: All variables must be initialized by executable code, * not by an initializer. Otherwise a new instance of the emulator will @@ -123,7 +128,10 @@ extern void ConNormalExit(void); extern void ConWaitForExit(void); #endif -static void erl_init(int ncpu, int proc_tab_sz); +static void erl_init(int ncpu, + int proc_tab_sz, + int port_tab_sz, + int port_tab_sz_ignore_files); static erts_atomic_t exiting; @@ -291,12 +299,18 @@ void erts_short_init(void) { int ncpu = early_init(NULL, NULL); - erl_init(ncpu, ERTS_DEFAULT_MAX_PROCESSES); + erl_init(ncpu, + ERTS_DEFAULT_MAX_PROCESSES, + ERTS_DEFAULT_MAX_PORTS, + 0); erts_initialized = 1; } static void -erl_init(int ncpu, int proc_tab_sz) +erl_init(int ncpu, + int proc_tab_sz, + int port_tab_sz, + int port_tab_sz_ignore_files) { init_benchmarking(); @@ -333,7 +347,7 @@ erl_init(int ncpu, int proc_tab_sz) init_dist(); erl_drv_thr_init(); erts_init_async(); - init_io(); + erts_init_io(port_tab_sz, port_tab_sz_ignore_files); init_copy(); init_load(); erts_init_bif(); @@ -556,7 +570,10 @@ void erts_usage(void) erts_fprintf(stderr, "-P number set maximum number of processes on this node,\n"); erts_fprintf(stderr, " valid range is [%d-%d]\n", - ERTS_MIN_PROCESSES, ERTS_MAX_PROCESSES); + ERTS_MIN_PROCESSES, ERTS_MAX_PROCESSES); + erts_fprintf(stderr, "-Q number set maximum number of ports on this node,\n"); + erts_fprintf(stderr, " valid range is [%d-%d]\n", + ERTS_MIN_PORTS, ERTS_MAX_PORTS); erts_fprintf(stderr, "-R number set compatibility release number,\n"); erts_fprintf(stderr, " valid range [%d-%d]\n", this_rel-2, this_rel); @@ -946,12 +963,13 @@ erl_start(int argc, char **argv) { int i = 1; char* arg=NULL; - char* Parg = NULL; int have_break_handler = 1; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; int ncpu = early_init(&argc, argv); int proc_tab_sz = ERTS_DEFAULT_MAX_PROCESSES; + int port_tab_sz = ERTS_DEFAULT_MAX_PORTS; + int port_tab_sz_ignore_files = 0; envbufsz = sizeof(envbuf); if (erts_sys_getenv(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) @@ -1210,15 +1228,29 @@ erl_start(int argc, char **argv) arg); break; - case 'P': - /* set maximum number of processes */ - Parg = get_arg(argv[i]+2, argv[i+1], &i); - proc_tab_sz = atoi(Parg); - if (proc_tab_sz < ERTS_MIN_PROCESSES - || proc_tab_sz > ERTS_MAX_PROCESSES) { - erts_fprintf(stderr, "bad number of processes %s\n", Parg); + case 'P': /* set maximum number of processes */ + arg = get_arg(argv[i]+2, argv[i+1], &i); + errno = 0; + proc_tab_sz = strtol(arg, NULL, 10); + if (errno != 0 + || proc_tab_sz < ERTS_MIN_PROCESSES + || ERTS_MAX_PROCESSES < proc_tab_sz) { + erts_fprintf(stderr, "bad number of processes %s\n", arg); + erts_usage(); + } + break; + + case 'Q': /* set maximum number of ports */ + arg = get_arg(argv[i]+2, argv[i+1], &i); + errno = 0; + port_tab_sz = strtol(arg, NULL, 10); + if (errno != 0 + || port_tab_sz < ERTS_MIN_PROCESSES + || ERTS_MAX_PROCESSES < port_tab_sz) { + erts_fprintf(stderr, "bad number of ports %s\n", arg); erts_usage(); } + port_tab_sz_ignore_files = 1; break; case 'S' : /* Was handled in early_init() just read past it */ @@ -1511,7 +1543,10 @@ erl_start(int argc, char **argv) boot_argc = argc - i; /* Number of arguments to init */ boot_argv = &argv[i]; - erl_init(ncpu, proc_tab_sz); + erl_init(ncpu, + proc_tab_sz, + port_tab_sz, + port_tab_sz_ignore_files); init_shared_memory(boot_argc, boot_argv); load_preloaded(); diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 56f658993b..667bda255b 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -118,8 +118,18 @@ extern ErtsPTab erts_proc; -#define internal_pid_index(x) erts_ptab_data2ix(&erts_proc, \ - internal_pid_data((x))) +#define make_internal_pid(D) erts_ptab_make_id(&erts_proc, \ + (D), \ + _TAG_IMMED1_PID) + +#define internal_pid_index(PID) (ASSERT_EXPR(is_internal_pid((PID))), \ + erts_ptab_id2pix(&erts_proc, (PID))) + +#define internal_pid_data(PID) (ASSERT_EXPR(is_internal_pid((PID))), \ + erts_ptab_id2data(&erts_proc, (PID))) + +#define internal_pid_number(x) _GET_PID_NUM(internal_pid_data((x))) +#define internal_pid_serial(x) _GET_PID_SER(internal_pid_data((x))) #define internal_pid_node_name(x) (internal_pid_node((x))->sysname) #define external_pid_node_name(x) (external_pid_node((x))->sysname) @@ -164,18 +174,14 @@ extern ErtsPTab erts_proc; * 32-bit CPU. */ -#define ERTS_MAX_PROCESSES ((SWORD_CONSTANT(1) << 27)-1) -#if (ERTS_MAX_PROCESSES > MAX_SMALL) -# error "The maximum number of processes must fit in a SMALL." -#endif - +#define ERTS_MAX_PROCESSES (ERTS_PTAB_MAX_SIZE-1) #define ERTS_MAX_PID_DATA ((1 << _PID_DATA_SIZE) - 1) #define ERTS_MAX_PID_NUMBER ((1 << _PID_NUM_SIZE) - 1) #define ERTS_MAX_PID_SERIAL ((1 << _PID_SER_SIZE) - 1) #define ERTS_PROC_BITS (_PID_SER_SIZE + _PID_NUM_SIZE) -#define ERTS_INVALID_PID make_internal_pid(ERTS_MAX_PID_DATA) +#define ERTS_INVALID_PID ERTS_PTAB_INVALID_ID(_TAG_IMMED1_PID) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Ports * @@ -183,8 +189,17 @@ extern ErtsPTab erts_proc; extern ErtsPTab erts_port; -#define internal_port_index(x) erts_ptab_data2ix(&erts_port, \ - internal_port_data((x))) +#define make_internal_port(D) erts_ptab_make_id(&erts_port, \ + (D), \ + _TAG_IMMED1_PORT) + +#define internal_port_index(PRT) (ASSERT_EXPR(is_internal_port((PRT))), \ + erts_ptab_id2pix(&erts_port, (PRT))) + +#define internal_port_data(PRT) (ASSERT_EXPR(is_internal_port((PRT))), \ + erts_ptab_id2data(&erts_port, (PRT))) + +#define internal_port_number(x) _GET_PORT_NUM(internal_port_data((x))) #define internal_port_node_name(x) (internal_port_node((x))->sysname) #define external_port_node_name(x) (external_port_node((x))->sysname) @@ -227,14 +242,13 @@ extern ErtsPTab erts_port; which defines the maximum number of simultaneous Ports in the Erlang node. ERTS_MAX_PORTS is a hard upper limit. */ -#define ERTS_MAX_PORTS (1 << ERTS_PORTS_BITS) - +#define ERTS_MAX_PORTS (ERTS_PTAB_MAX_SIZE-1) #define ERTS_MAX_PORT_DATA ((1 << _PORT_DATA_SIZE) - 1) #define ERTS_MAX_PORT_NUMBER ((1 << _PORT_NUM_SIZE) - 1) #define ERTS_PORTS_BITS (_PORT_NUM_SIZE) -#define ERTS_INVALID_PORT make_internal_port(ERTS_MAX_PORT_DATA) +#define ERTS_INVALID_PORT ERTS_PTAB_INVALID_ID(_TAG_IMMED1_PORT) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Refs * diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index fb34a6da2d..3de0eed9ea 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -28,6 +28,10 @@ typedef struct _erl_drv_port Port; #include "erl_port_task.h" #include "erl_ptab.h" #include "erl_thr_progress.h" +#include "erl_trace.h" + +#define ERTS_DEFAULT_MAX_PORTS (1 << 16) +#define ERTS_MIN_PORTS 1024 typedef struct erts_driver_t_ erts_driver_t; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index f5b7921820..1feb5b96db 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -72,7 +72,7 @@ struct ErtsNodesMonitor_; #define ERTS_MAX_NO_OF_SCHEDULERS 1024 -#define ERTS_DEFAULT_MAX_PROCESSES (1 << 15) +#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18) #define ERTS_HEAP_ALLOC(Type, Size) \ erts_alloc((Type), (Size)) @@ -1727,10 +1727,10 @@ extern int erts_disable_proc_not_running_opt; /* Minimum NUMBER of processes for a small system to start */ -#ifdef ERTS_SMP +#define ERTS_MIN_PROCESSES 1024 +#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS +#undef ERTS_MIN_PROCESSES #define ERTS_MIN_PROCESSES ERTS_NO_OF_PIX_LOCKS -#else -#define ERTS_MIN_PROCESSES 16 #endif void erts_smp_notify_inc_runq(ErtsRunQueue *runq); diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 7ca52303da..537cd3f24f 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -205,7 +205,7 @@ typedef struct erts_proc_lock_t_ { & ~ERTS_PROC_LOCK_MAIN) -#define ERTS_PIX_LOCKS_BITS 8 +#define ERTS_PIX_LOCKS_BITS 10 #define ERTS_NO_OF_PIX_LOCKS (1 << ERTS_PIX_LOCKS_BITS) diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index eba0c286cc..7ab756ae8d 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -429,7 +429,7 @@ erts_ptab_init_table(ErtsPTab *ptab, char *name) { size_t tab_sz; - int max_data_bits; + int bits; char *tab_end; erts_smp_atomic_t *tab_entry; erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; @@ -440,8 +440,13 @@ erts_ptab_init_table(ErtsPTab *ptab, erts_smp_atomic32_init_nob(&ptab->vola.tile.count, 0); last_data_init_nob(ptab, ~((Uint64) 0)); - if (size > (1 << ERTS_PROC_BITS)) - size = 1 << ERTS_PROC_BITS; + /* A size that is a power of 2 is to prefer performance wise */ + bits = erts_fit_in_bits_int32(size-1); + size = 1 << bits; + if (size > ERTS_PTAB_MAX_SIZE) { + size = ERTS_PTAB_MAX_SIZE; + bits = erts_fit_in_bits_int32((Sint32) size - 1); + } ptab->r.o.max = size; @@ -454,34 +459,31 @@ erts_ptab_init_table(ErtsPTab *ptab, tab_entry++; } - max_data_bits = erts_fit_in_bits_int32((Sint32) ptab->r.o.max - 1); - ptab->r.o.tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; ptab->r.o.pix_per_cache_line = (ERTS_CACHE_LINE_SIZE / sizeof(erts_smp_atomic_t)); - if ((ptab->r.o.max & (ptab->r.o.max - 1)) - | (ptab->r.o.pix_per_cache_line & (ptab->r.o.pix_per_cache_line - 1))) { - /* ptab->r.o.max or ptab->r.o.pix_per_cache_line not a power of 2 :-( */ - ptab->r.o.pix_cl_mask = 0; - ptab->r.o.pix_cl_shift = 0; - ptab->r.o.pix_cli_mask = 0; - ptab->r.o.pix_cli_shift = 0; - } - else { - ASSERT((ptab->r.o.tab_cache_lines - & (ptab->r.o.tab_cache_lines - 1)) == 0); - ptab->r.o.pix_cl_mask - = ptab->r.o.tab_cache_lines-1; - ptab->r.o.pix_cl_shift - = erts_fit_in_bits_int32(ptab->r.o.pix_per_cache_line-1); - ptab->r.o.pix_cli_shift - = erts_fit_in_bits_int32(ptab->r.o.pix_cl_mask); - ptab->r.o.pix_cli_mask - = (1 << (max_data_bits - ptab->r.o.pix_cli_shift)) - 1; - } + ASSERT((ptab->r.o.max & (ptab->r.o.max - 1)) == 0); /* power of 2 */ + ASSERT((ptab->r.o.pix_per_cache_line + & (ptab->r.o.pix_per_cache_line - 1)) == 0); /* power of 2 */ + ASSERT((ptab->r.o.tab_cache_lines + & (ptab->r.o.tab_cache_lines - 1)) == 0); /* power of 2 */ + + ptab->r.o.pix_mask + = (1 << bits) - 1; + ptab->r.o.pix_cl_mask + = ptab->r.o.tab_cache_lines-1; + ptab->r.o.pix_cl_shift + = erts_fit_in_bits_int32(ptab->r.o.pix_per_cache_line-1); + ptab->r.o.pix_cli_shift + = erts_fit_in_bits_int32(ptab->r.o.pix_cl_mask); + ptab->r.o.pix_cli_mask + = (1 << (bits - ptab->r.o.pix_cli_shift)) - 1; + + ASSERT(ptab->r.o.pix_cl_shift + ptab->r.o.pix_cli_shift == bits); ptab->r.o.invalid_element = invalid_element; - ptab->r.o.invalid_data = ERTS_PTAB_ID2DATA(invalid_element->id); + ptab->r.o.invalid_data = erts_ptab_id2data(ptab, invalid_element->id); + ptab->r.o.release_element = release_element; if (release_element) ptab->r.o.release_element = release_element; else @@ -493,6 +495,26 @@ erts_ptab_init_table(ErtsPTab *ptab, ptab->list.data.chunks = (((ptab->r.o.max - 1) / ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE) + 1); + + if (size == ERTS_PTAB_MAX_SIZE) { + int pix; + /* + * We want a table size of a power of 2 which ERTS_PTAB_MAX_SIZE + * is. We only have ERTS_PTAB_MAX_SIZE-1 unique identifiers and + * we don't want to shrink the size to ERTS_PTAB_MAX_SIZE/2. + * + * In order to fix this, we insert a pointer from the table + * to the invalid_element, wich will be interpreted as a + * slot currently being modified. This way we will be able to + * have ERTS_PTAB_MAX_SIZE-1 valid elements in the table while + * still having a table size of the power of 2. + */ + erts_smp_atomic32_inc_nob(&ptab->vola.tile.count); + pix = erts_ptab_data2pix(ptab, ptab->r.o.invalid_data); + erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], + (erts_aint_t) ptab->r.o.invalid_element); + } + } int @@ -541,7 +563,7 @@ erts_ptab_new_element(ErtsPTab *ptab, /* Reserve slot */ while (1) { ld++; - pix = erts_ptab_data2ix(ptab, ERTS_PTAB_LastData2EtermData(ld)); + pix = erts_ptab_data2pix(ptab, ERTS_PTAB_LastData2EtermData(ld)); if (erts_smp_atomic_read_nob(&ptab->r.o.tab[pix]) == ERTS_AINT_NULL) { erts_aint_t val; val = erts_smp_atomic_cmpxchg_relb(&ptab->r.o.tab[pix], @@ -558,8 +580,8 @@ erts_ptab_new_element(ErtsPTab *ptab, if (data == ptab->r.o.invalid_data) { /* Do not use invalid data; fix it... */ ld += ptab->r.o.max; - ASSERT(pix == erts_ptab_data2ix(ptab, - ERTS_PTAB_LastData2EtermData(ld))); + ASSERT(pix == erts_ptab_data2pix(ptab, + ERTS_PTAB_LastData2EtermData(ld))); data = ERTS_PTAB_LastData2EtermData(ld); ASSERT(data != ptab->r.o.invalid_data); } @@ -609,7 +631,7 @@ save_deleted_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el) ptdep->prev = ptab->list.data.deleted.end; ptdep->next = NULL; - ptdep->ix = erts_ptab_data2ix(ptab, ERTS_PTAB_ID2DATA(ptab_el->id)); + ptdep->ix = erts_ptab_id2pix(ptab, ptab_el->id); ptdep->u.element.id = ptab_el->id; ptdep->u.element.inserted = ptab_el->u.alive.started_interval; ptdep->u.element.deleted = @@ -632,7 +654,7 @@ erts_ptab_delete_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el) { int maybe_save; - int pix = erts_ptab_data2ix(ptab, ERTS_PTAB_ID2DATA(ptab_el->id)); + int pix = erts_ptab_id2pix(ptab, ptab_el->id); ASSERT(erts_get_scheduler_id()); /* *Need* to be a scheduler */ @@ -1274,9 +1296,9 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next) data = ERTS_PTAB_LastData2EtermData(ld); if (ptab->r.o.invalid_data == data) { ld += ptab->r.o.max; - ASSERT(erts_ptab_data2ix(ptab, data) - == erts_ptab_data2ix(ptab, - ERTS_PTAB_LastData2EtermData(ld))); + ASSERT(erts_ptab_data2pix(ptab, data) + == erts_ptab_data2pix(ptab, + ERTS_PTAB_LastData2EtermData(ld))); } last_data_set_relb(ptab, ld); } @@ -1295,9 +1317,9 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next) data = ERTS_PTAB_LastData2EtermData(ld); if (ptab->r.o.invalid_data == data) { ld += ptab->r.o.max; - ASSERT(erts_ptab_data2ix(ptab, data) - == erts_ptab_data2ix(ptab, - ERTS_PTAB_LastData2EtermData(ld))); + ASSERT(erts_ptab_data2pix(ptab, data) + == erts_ptab_data2pix(ptab, + ERTS_PTAB_LastData2EtermData(ld))); data = ERTS_PTAB_LastData2EtermData(ld); } res = data; @@ -1510,10 +1532,9 @@ debug_ptab_list_check_del_list(ErtsPTab *ptab) prev_x_interval_p = &ptdep->u.element.deleted; ERTS_PTAB_LIST_ASSERT( erts_ptab_is_valid_id(ptdep->u.element.id)); - ERTS_PTAB_LIST_ASSERT( - erts_ptab_data2ix(ptab, - ERTS_PTAB_ID2DATA(ptdep->u.element.id)) - == ptdep->ix); + ERTS_PTAB_LIST_ASSERT(erts_ptab_id2pix(ptab, + ptdep->u.element.id) + == ptdep->ix); } } diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h index b2ae829d62..8a130f42a3 100644 --- a/erts/emulator/beam/erl_ptab.h +++ b/erts/emulator/beam/erl_ptab.h @@ -98,13 +98,14 @@ typedef struct { typedef struct { erts_smp_atomic_t *tab; - int max; - int tab_cache_lines; - int pix_per_cache_line; - int pix_cl_mask; - int pix_cl_shift; - int pix_cli_mask; - int pix_cli_shift; + Uint32 max; + Uint32 tab_cache_lines; + Uint32 pix_per_cache_line; + Uint32 pix_mask; + Uint32 pix_cl_mask; + Uint32 pix_cl_shift; + Uint32 pix_cli_mask; + Uint32 pix_cli_shift; ErtsPTabElementCommon *invalid_element; Eterm invalid_data; void (*release_element)(void *); @@ -139,6 +140,12 @@ typedef struct { #define ERTS_PTAB_ID_DATA_SIZE 28 #define ERTS_PTAB_ID_DATA_SHIFT (_TAG_IMMED1_SIZE) +/* ERTS_PTAB_MAX_SIZE must be a power of 2 */ +#define ERTS_PTAB_MAX_SIZE (SWORD_CONSTANT(1) << 27) +#if (ERTS_PTAB_MAX_SIZE-1) > MAX_SMALL +# error "The maximum number of processes/ports must fit in a SMALL." +#endif + /* * Currently pids and ports are allowed. @@ -156,14 +163,14 @@ typedef struct { # error "Unexpected port tag size" #endif +#define ERTS_PTAB_INVALID_ID(TAG) \ + ((Eterm) \ + ((((1 << ERTS_PTAB_ID_DATA_SIZE) - 1) << ERTS_PTAB_ID_DATA_SHIFT) \ + | (TAG))) + #define erts_ptab_is_valid_id(ID) \ (is_internal_pid((ID)) || is_internal_port((ID))) -#define ERTS_PTAB_ID2DATA(ID) \ - (ASSERT_EXPR(erts_ptab_is_valid_id((ID))), \ - (((ID) >> ERTS_PTAB_ID_DATA_SHIFT) \ - & ~(~((Uint) 0) << ERTS_PTAB_ID_DATA_SIZE))) - void erts_ptab_init(void); void erts_ptab_init_table(ErtsPTab *ptab, ErtsAlcType_t atype, @@ -182,7 +189,13 @@ int erts_ptab_initialized(ErtsPTab *ptab); ERTS_GLB_INLINE erts_interval_t *erts_ptab_interval(ErtsPTab *ptab); ERTS_GLB_INLINE int erts_ptab_max(ErtsPTab *ptab); ERTS_GLB_INLINE int erts_ptab_count(ErtsPTab *ptab); -ERTS_GLB_INLINE int erts_ptab_data2ix(ErtsPTab *ptab, Eterm data); +ERTS_GLB_INLINE Uint erts_ptab_pixdata2data(ErtsPTab *ptab, Eterm pixdata); +ERTS_GLB_INLINE Uint32 erts_ptab_pixdata2pix(ErtsPTab *ptab, Eterm pixdata); +ERTS_GLB_INLINE Uint32 erts_ptab_data2pix(ErtsPTab *ptab, Eterm data); +ERTS_GLB_INLINE Uint erts_ptab_data2pixdata(ErtsPTab *ptab, Eterm data); +ERTS_GLB_INLINE Eterm erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag); +ERTS_GLB_INLINE int erts_ptab_id2pix(ErtsPTab *ptab, Eterm id); +ERTS_GLB_INLINE Uint erts_ptab_id2data(ErtsPTab *ptab, Eterm id); ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_nob(ErtsPTab *ptab, int ix); ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix); ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix); @@ -211,39 +224,118 @@ erts_ptab_interval(ErtsPTab *ptab) ERTS_GLB_INLINE int erts_ptab_max(ErtsPTab *ptab) { - return ptab->r.o.max; + int max = ptab->r.o.max; + return max == ERTS_PTAB_MAX_SIZE ? max - 1 : max; } ERTS_GLB_INLINE int erts_ptab_count(ErtsPTab *ptab) { + int max = ptab->r.o.max; erts_aint32_t res = erts_smp_atomic32_read_nob(&ptab->vola.tile.count); - if (res > ptab->r.o.max) - return ptab->r.o.max; + if (max == ERTS_PTAB_MAX_SIZE) { + max--; + res--; + } + if (res > max) + return max; ASSERT(res >= 0); return (int) res; } -ERTS_GLB_INLINE int erts_ptab_data2ix(ErtsPTab *ptab, Eterm data) +ERTS_GLB_INLINE Uint erts_ptab_pixdata2data(ErtsPTab *ptab, Eterm pixdata) { - int n, pix; + Uint32 data = ((Uint32) pixdata) & ~ptab->r.o.pix_mask; + data |= (pixdata >> ptab->r.o.pix_cl_shift) & ptab->r.o.pix_cl_mask; + data |= (pixdata & ptab->r.o.pix_cli_mask) << ptab->r.o.pix_cli_shift; + return data; +} - n = (int) data; - if (ptab->r.o.pix_cl_mask) { - pix = ((n & ptab->r.o.pix_cl_mask) << ptab->r.o.pix_cl_shift); - pix += ((n >> ptab->r.o.pix_cli_shift) & ptab->r.o.pix_cli_mask); - } - else { - n %= ptab->r.o.max; - pix = n % ptab->r.o.tab_cache_lines; - pix *= ptab->r.o.pix_per_cache_line; - pix += n / ptab->r.o.tab_cache_lines; - } +ERTS_GLB_INLINE Uint32 erts_ptab_pixdata2pix(ErtsPTab *ptab, Eterm pixdata) +{ + return ((Uint32) pixdata) & ptab->r.o.pix_mask; +} + +ERTS_GLB_INLINE Uint32 erts_ptab_data2pix(ErtsPTab *ptab, Eterm data) +{ + Uint32 n, pix; + n = (Uint32) data; + pix = ((n & ptab->r.o.pix_cl_mask) << ptab->r.o.pix_cl_shift); + pix += ((n >> ptab->r.o.pix_cli_shift) & ptab->r.o.pix_cli_mask); ASSERT(0 <= pix && pix < ptab->r.o.max); return pix; } +ERTS_GLB_INLINE Uint erts_ptab_data2pixdata(ErtsPTab *ptab, Eterm data) +{ + Uint pixdata = data & ~((Uint) ptab->r.o.pix_mask); + pixdata |= (Uint) erts_ptab_data2pix(ptab, data); + ASSERT(data == erts_ptab_pixdata2data(ptab, pixdata)); + return pixdata; +} + +#if ERTS_SIZEOF_TERM == 8 + +ERTS_GLB_INLINE Eterm +erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag) +{ + HUint huint; + Uint32 low_data = (Uint32) data; + low_data &= (1 << ERTS_PTAB_ID_DATA_SIZE) - 1; + low_data <<= ERTS_PTAB_ID_DATA_SHIFT; + huint.hval[ERTS_HUINT_HVAL_HIGH] = erts_ptab_data2pix(ptab, data); + huint.hval[ERTS_HUINT_HVAL_LOW] = low_data | ((Uint32) tag); + return (Eterm) huint.val; +} + +ERTS_GLB_INLINE int +erts_ptab_id2pix(ErtsPTab *ptab, Eterm id) +{ + HUint huint; + huint.val = id; + return (int) huint.hval[ERTS_HUINT_HVAL_HIGH]; +} + +ERTS_GLB_INLINE Uint +erts_ptab_id2data(ErtsPTab *ptab, Eterm id) +{ + HUint huint; + huint.val = id; + return (Uint) (huint.hval[ERTS_HUINT_HVAL_LOW] >> ERTS_PTAB_ID_DATA_SHIFT); +} + +#elif ERTS_SIZEOF_TERM == 4 + +ERTS_GLB_INLINE Eterm +erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag) +{ + Eterm id; + data &= ((1 << ERTS_PTAB_ID_DATA_SIZE) - 1); + id = (Eterm) erts_ptab_data2pixdata(ptab, data); + return (id << ERTS_PTAB_ID_DATA_SHIFT) | tag; +} + +ERTS_GLB_INLINE int +erts_ptab_id2pix(ErtsPTab *ptab, Eterm id) +{ + Uint pixdata = (Uint) id; + pixdata >>= ERTS_PTAB_ID_DATA_SHIFT; + return (int) erts_ptab_pixdata2pix(ptab, pixdata); +} + +ERTS_GLB_INLINE Uint +erts_ptab_id2data(ErtsPTab *ptab, Eterm id) +{ + Uint pixdata = (Uint) id; + pixdata >>= ERTS_PTAB_ID_DATA_SHIFT; + return erts_ptab_pixdata2data(ptab, pixdata); +} + +#else +#error "Unsupported size of term" +#endif + ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_nob(ErtsPTab *ptab, int ix) { ASSERT(0 <= ix && ix < ptab->r.o.max); diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index f77e8b798f..b86e3e16ca 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -144,9 +144,7 @@ ET_DEFINE_CHECKED(Uint,bignum_header_arity,Eterm,_is_bignum_header); ET_DEFINE_CHECKED(Eterm*,big_val,Wterm,is_big); ET_DEFINE_CHECKED(Eterm*,float_val,Wterm,is_float); ET_DEFINE_CHECKED(Eterm*,tuple_val,Wterm,is_tuple); -ET_DEFINE_CHECKED(Uint,internal_pid_data,Eterm,is_internal_pid); ET_DEFINE_CHECKED(struct erl_node_*,internal_pid_node,Eterm,is_internal_pid); -ET_DEFINE_CHECKED(Uint,internal_port_data,Eterm,is_internal_port); ET_DEFINE_CHECKED(struct erl_node_*,internal_port_node,Eterm,is_internal_port); ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Wterm,is_internal_ref); ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Wterm,is_internal_ref); diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 7349d3e0c7..bc4c3a09a0 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -591,23 +591,13 @@ _ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm) #define make_pid_data(Ser, Num) \ ((Uint) ((Ser) << _PID_NUM_SIZE | (Num))) -#define make_internal_pid(X) \ - ((Eterm) (((X) << _PID_DATA_SHIFT) | _TAG_IMMED1_PID)) - #define is_internal_pid(x) (((x) & _TAG_IMMED1_MASK) == _TAG_IMMED1_PID) #define is_not_internal_pid(x) (!is_internal_pid((x))) -#define _unchecked_internal_pid_data(x) _GET_PID_DATA((x)) -_ET_DECLARE_CHECKED(Uint,internal_pid_data,Eterm) -#define internal_pid_data(x) _ET_APPLY(internal_pid_data,(x)) - #define _unchecked_internal_pid_node(x) erts_this_node _ET_DECLARE_CHECKED(struct erl_node_*,internal_pid_node,Eterm) #define internal_pid_node(x) _ET_APPLY(internal_pid_node,(x)) -#define internal_pid_number(x) _GET_PID_NUM(internal_pid_data((x))) -#define internal_pid_serial(x) _GET_PID_SER(internal_pid_data((x))) - #define internal_pid_data_words(x) (1) /* @@ -646,18 +636,9 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_pid_node,Eterm) #define _GET_PORT_NUM(X) _GETBITS((X), 0, _PORT_NUM_SIZE) -#define make_internal_port(X) \ - ((Eterm) (((X) << _PORT_DATA_SHIFT) | _TAG_IMMED1_PORT)) - #define is_internal_port(x) (((x) & _TAG_IMMED1_MASK) == _TAG_IMMED1_PORT) #define is_not_internal_port(x) (!is_internal_port(x)) -#define _unchecked_internal_port_data(x) _GET_PORT_DATA((x)) -_ET_DECLARE_CHECKED(Uint,internal_port_data,Eterm) -#define internal_port_data(x) _ET_APPLY(internal_port_data,(x)) - -#define internal_port_number(x) _GET_PORT_NUM(internal_port_data((x))) - #define _unchecked_internal_port_node(x) erts_this_node _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm) #define internal_port_node(x) _ET_APPLY(internal_port_node,(x)) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index ae31ab54b5..0de11afee9 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -899,7 +899,7 @@ void erts_wake_process_later(Port*, Process*); Port *erts_open_driver(erts_driver_t*, Eterm, char*, SysDriverOpts*, int *, int *); int erts_is_port_ioq_empty(Port *); void erts_terminate_port(Port *); -void init_io(void); +void erts_init_io(int, int); void erts_do_exit_port(Port *, Eterm, Eterm); void erts_port_command(Process *, Eterm, Port *, Eterm); Eterm erts_port_control(Process*, Port*, Uint, Eterm); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 21fc76b8db..1177e7e691 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1238,13 +1238,10 @@ release_port(void *vport) } #endif -/* initialize the port array */ -void init_io(void) +void erts_init_io(int port_tab_size, + int port_tab_size_ignore_files) { ErlDrvEntry** dp; - char maxports[21]; /* enough for any 64-bit integer */ - size_t maxportssize = sizeof(maxports); - int port_tab_size; #ifdef ERTS_SMP init_xports_list_alloc(); @@ -1252,15 +1249,16 @@ void init_io(void) pdl_init(); - if (erts_sys_getenv("ERL_MAX_PORTS", maxports, &maxportssize) == 0) - port_tab_size = atoi(maxports); - else - port_tab_size = sys_max_files(); + if (!port_tab_size_ignore_files) { + int max_files = sys_max_files(); + if (port_tab_size < max_files) + port_tab_size = max_files; + } if (port_tab_size > ERTS_MAX_PORTS) port_tab_size = ERTS_MAX_PORTS; - if (port_tab_size < 1024) - port_tab_size = 1024; + else if (port_tab_size < ERTS_MIN_PORTS) + port_tab_size = ERTS_MIN_PORTS; erts_smp_mtx_init(&erts_driver_list_lock,"driver_list"); driver_list = NULL; diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 0d3b910278..12313f0984 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -244,9 +244,11 @@ int real_printf(const char *fmt, ...); #if SIZEOF_VOID_P == 8 #undef ARCH_32 #define ARCH_64 +#define ERTS_SIZEOF_TERM 8 #elif SIZEOF_VOID_P == 4 #define ARCH_32 #undef ARCH_64 +#define ERTS_SIZEOF_TERM 4 #else #error Neither 32 nor 64 bit architecture #endif @@ -254,6 +256,8 @@ int real_printf(const char *fmt, ...); # define HALFWORD_HEAP 1 # define HALFWORD_ASSERT 0 # define ASSERT_HALFWORD(COND) ASSERT(COND) +# undef ERTS_SIZEOF_TERM +# define ERTS_SIZEOF_TERM 4 #else # define HALFWORD_HEAP 0 # define HALFWORD_ASSERT 0 @@ -380,6 +384,27 @@ typedef unsigned char byte; #error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found #endif +#ifdef WORDS_BIGENDIAN +# define ERTS_HUINT_HVAL_HIGH 0 +# define ERTS_HUINT_HVAL_LOW 1 +#else +# define ERTS_HUINT_HVAL_HIGH 1 +# define ERTS_HUINT_HVAL_LOW 0 +#endif +#if ERTS_SIZEOF_TERM == 8 +typedef union { + Uint val; + Uint32 hval[2]; +} HUint; +#elif ERTS_SIZEOF_TERM == 4 +typedef union { + Uint val; + Uint16 hval[2]; +} HUint; +#else +#error "Unsupported size of term" +#endif + # define ERTS_EXTRA_DATA_ALIGN_SZ(X) \ (((size_t) 8) - (((size_t) (X)) & ((size_t) 7))) |