diff options
Diffstat (limited to 'erts/emulator/beam/erl_init.c')
| -rw-r--r-- | erts/emulator/beam/erl_init.c | 391 |
1 files changed, 169 insertions, 222 deletions
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 8a297cded0..1eb3dba240 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2011. All Rights Reserved. + * Copyright Ericsson AB 1997-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -34,7 +34,6 @@ #include "erl_binary.h" #include "dist.h" #include "erl_mseg.h" -#include "erl_nmgc.h" #include "erl_threads.h" #include "erl_bif_timer.h" #include "erl_instrument.h" @@ -42,6 +41,9 @@ #include "erl_misc_utils.h" #include "packet_parser.h" #include "erl_cpu_topology.h" +#include "erl_thr_progress.h" +#include "erl_thr_queue.h" +#include "erl_async.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -68,6 +70,8 @@ static void erl_init(int ncpu); #define ERTS_MIN_COMPAT_REL 7 +static erts_atomic_t exiting; + #ifdef ERTS_SMP erts_smp_atomic32_t erts_writing_erl_crash_dump; erts_tsd_key_t erts_is_crash_dumping_key; @@ -87,7 +91,6 @@ int erts_use_sender_punish; */ Uint display_items; /* no of items to display in traces etc */ -Uint display_loads; /* print info about loaded modules */ int H_MIN_SIZE; /* The minimum heap grain */ int BIN_VH_MIN_SIZE; /* The minimum binary virtual*/ @@ -99,8 +102,6 @@ int erts_backtrace_depth; /* How many functions to show in a backtrace * in error codes. */ -int erts_async_max_threads; /* number of threads for async support */ -int erts_async_thread_suggested_stack_size; erts_smp_atomic32_t erts_max_gen_gcs; Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, @@ -109,7 +110,6 @@ Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, int erts_compat_rel; -static int use_multi_run_queue; static int no_schedulers; static int no_schedulers_online; @@ -153,28 +153,6 @@ Export *erts_delay_trap = NULL; int erts_use_r9_pids_ports; -#ifdef HYBRID -Eterm *global_heap; -Eterm *global_hend; -Eterm *global_htop; -Eterm *global_saved_htop; -Eterm *global_old_heap; -Eterm *global_old_hend; -ErlOffHeap erts_global_offheap; -Uint global_heap_sz = SH_DEFAULT_SIZE; - -#ifndef INCREMENTAL -Eterm *global_high_water; -Eterm *global_old_htop; -#endif - -Uint16 global_gen_gcs; -Uint16 global_max_gen_gcs; -Uint global_gc_flags; - -Uint global_heap_min_sz = SH_DEFAULT_SIZE; -#endif - int ignore_break; int replace_intr; @@ -247,19 +225,16 @@ erl_init(int ncpu) { init_benchmarking(); -#ifdef ERTS_SMP - erts_system_block_init(); -#endif - erts_init_monitors(); erts_init_gc(); erts_init_time(); erts_init_sys_common_misc(); erts_init_process(ncpu); - erts_init_scheduling(use_multi_run_queue, - no_schedulers, + erts_init_scheduling(no_schedulers, no_schedulers_online); erts_init_cpu_topology(); /* Must be after init_scheduling */ + erts_alloc_late_init(); + H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0); @@ -281,8 +256,8 @@ erl_init(int ncpu) erts_init_node_tables(); init_dist(); erl_drv_thr_init(); + erts_init_async(); init_io(); - init_copy(); init_load(); erts_init_bif(); erts_init_bif_chksum(); @@ -303,45 +278,6 @@ erl_init(int ncpu) } static void -init_shared_memory(int argc, char **argv) -{ -#ifdef HYBRID - int arg_size = 0; - - global_heap_sz = erts_next_heap_size(global_heap_sz,0); - - /* Make sure arguments will fit on the heap, no one else will check! */ - while (argc--) - arg_size += 2 + strlen(argv[argc]); - if (global_heap_sz < arg_size) - global_heap_sz = erts_next_heap_size(arg_size,1); - -#ifndef INCREMENTAL - global_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, - sizeof(Eterm) * global_heap_sz); - global_hend = global_heap + global_heap_sz; - global_htop = global_heap; - global_high_water = global_heap; - global_old_hend = global_old_htop = global_old_heap = NULL; -#endif - - global_gen_gcs = 0; - global_max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); - global_gc_flags = erts_default_process_flags; - - erts_global_offheap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - erts_global_offheap.funs = NULL; -#endif - erts_global_offheap.overhead = 0; -#endif - -#ifdef INCREMENTAL - erts_init_incgc(); -#endif -} - -static void erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv) { int i; @@ -435,7 +371,7 @@ static void load_preloaded(void) { int i; - int res; + Eterm res; Preload* preload_p; Eterm module_name; byte* code; @@ -454,8 +390,9 @@ load_preloaded(void) name); res = erts_load_module(NULL, 0, NIL, &module_name, code, length); sys_preload_end(&preload_p[i]); - if (res < 0) - erl_exit(1,"Failed loading preloaded module %s\n", name); + if (res != NIL) + erl_exit(1,"Failed loading preloaded module %s (%T)\n", + name, res); i++; } } @@ -497,8 +434,6 @@ void erts_usage(void) erts_fprintf(stderr, "-K boolean enable or disable kernel poll\n"); - erts_fprintf(stderr, "-l turn on auto load tracing\n"); - erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); erts_fprintf(stderr, " see the erts_alloc(3) documentation for more info.\n"); @@ -513,8 +448,14 @@ void erts_usage(void) erts_fprintf(stderr, "-rg amount set reader groups limit\n"); erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); + erts_fprintf(stderr, "-sbwt val set scheduler busy wait threshold, valid values are:\n"); + erts_fprintf(stderr, " none|very_short|short|medium|long|very_long.\n"); + erts_fprintf(stderr, "-scl bool enable/disable compaction of scheduler load,\n"); + erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); erts_fprintf(stderr, "-sct cput set cpu topology,\n"); erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); + erts_fprintf(stderr, "-sws val set scheduler wakeup strategy, valid values are:\n"); + erts_fprintf(stderr, " default|legacy|proposal.\n"); erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n"); erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); @@ -607,12 +548,13 @@ early_init(int *argc, char **argv) /* int max_main_threads; int max_reader_groups; int reader_groups; + char envbuf[21]; /* enough for any 64-bit integer */ + size_t envbufsz; - use_multi_run_queue = 1; + erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; display_items = 200; - display_loads = 0; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; erts_async_max_threads = 0; erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; @@ -644,6 +586,10 @@ early_init(int *argc, char **argv) /* erts_use_r9_pids_ports = 0; erts_sys_pre_init(); + erts_atomic_init_nob(&exiting, 0); +#ifdef ERTS_SMP + erts_thr_progress_pre_init(); +#endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_init(); @@ -675,6 +621,16 @@ early_init(int *argc, char **argv) /* schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; + envbufsz = sizeof(envbuf); + + /* erts_sys_getenv(_raw)() not initialized yet; need erts_sys_getenv__() */ + if (erts_sys_getenv__("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) + erts_async_max_threads = atoi(envbuf); + else + erts_async_max_threads = 0; + if (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS) + erts_async_max_threads = ERTS_MAX_NO_OF_ASYNC_THREADS; + if (argc && argv) { int i = 1; while (i < *argc) { @@ -702,6 +658,20 @@ early_init(int *argc, char **argv) /* } break; } + case 'A': { + /* set number of threads in thread pool */ + char *arg = get_arg(argv[i]+2, argv[i+1], &i); + if (((erts_async_max_threads = atoi(arg)) < 0) || + (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) { + erts_fprintf(stderr, + "bad number of async threads %s\n", + arg); + erts_usage(); + VERBOSE(DEBUG_SYSTEM, ("using %d async-threads\n", + erts_async_max_threads)); + } + break; + } case 'S' : { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -761,17 +731,39 @@ early_init(int *argc, char **argv) /* } } +#ifndef USE_THREADS + erts_async_max_threads = 0; +#endif + #ifdef ERTS_SMP no_schedulers = schdlrs; no_schedulers_online = schdlrs_onln; erts_no_schedulers = (Uint) no_schedulers; #endif + erts_early_init_scheduling(no_schedulers); + alloc_opts.ncpu = ncpu; erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes) -M flags. */ /* Require allocators */ - erts_early_init_scheduling(); +#ifdef ERTS_SMP + /* + * Thread progress management: + * + * * Managed threads: + * ** Scheduler threads (see erl_process.c) + * ** Aux thread (see erl_process.c) + * ** Sys message dispatcher thread (see erl_trace.c) + * + * * Unmanaged threads that need to register: + * ** Async threads (see erl_async.c) + */ + erts_thr_progress_init(no_schedulers, + no_schedulers+2, + erts_async_max_threads); +#endif + erts_thr_q_init(); erts_init_utils(); erts_early_init_cpu_topology(no_schedulers, &max_main_threads, @@ -851,27 +843,21 @@ erl_start(int argc, char **argv) int have_break_handler = 1; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; - int async_max_threads = erts_async_max_threads; int ncpu = early_init(&argc, argv); envbufsz = sizeof(envbuf); - if (erts_sys_getenv(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) + if (erts_sys_getenv_raw(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) user_requested_db_max_tabs = atoi(envbuf); else user_requested_db_max_tabs = 0; envbufsz = sizeof(envbuf); - if (erts_sys_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) { + if (erts_sys_getenv_raw("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) { Uint16 max_gen_gcs = atoi(envbuf); erts_smp_atomic32_set_nob(&erts_max_gen_gcs, (erts_aint32_t) max_gen_gcs); } - envbufsz = sizeof(envbuf); - if (erts_sys_getenv("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) { - async_max_threads = atoi(envbuf); - } - #if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) /* * The default stack size on MacOS X is too small for pcre. @@ -937,9 +923,6 @@ erl_start(int argc, char **argv) erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]); erts_usage(); } - case 'l': - display_loads++; - break; case 'L': erts_no_line_info = 1; break; @@ -953,7 +936,6 @@ erl_start(int argc, char **argv) switch (*ch) { case 's': verbose |= DEBUG_SYSTEM; break; case 'g': verbose |= DEBUG_PRIVATE_GC; break; - case 'h': verbose |= DEBUG_HYBRID_GC; break; case 'M': verbose |= DEBUG_MEMORY; break; case 'a': verbose |= DEBUG_ALLOCATION; break; case 't': verbose |= DEBUG_THREADS; break; @@ -966,7 +948,6 @@ erl_start(int argc, char **argv) erts_printf("Verbose level: "); if (verbose & DEBUG_SYSTEM) erts_printf("SYSTEM "); if (verbose & DEBUG_PRIVATE_GC) erts_printf("PRIVATE_GC "); - if (verbose & DEBUG_HYBRID_GC) erts_printf("HYBRID_GC "); if (verbose & DEBUG_MEMORY) erts_printf("PARANOID_MEMORY "); if (verbose & DEBUG_ALLOCATION) erts_printf("ALLOCATION "); if (verbose & DEBUG_THREADS) erts_printf("THREADS "); @@ -994,12 +975,6 @@ erl_start(int argc, char **argv) #ifdef HIPE strcat(tmp, ",HIPE"); #endif -#ifdef INCREMENTAL - strcat(tmp, ",INCREMENTAL_GC"); -#endif -#ifdef HYBRID - strcat(tmp, ",HYBRID"); -#endif erts_fprintf(stderr, "Erlang "); if (tmp[1]) { erts_fprintf(stderr, "(%s) ", tmp+1); @@ -1160,6 +1135,29 @@ erl_start(int argc, char **argv) erts_usage(); } } + else if (has_prefix("bwt", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if (erts_sched_set_busy_wait_threshold(arg) != 0) { + erts_fprintf(stderr, "bad scheduler busy wait threshold: %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("scheduler wakup threshold: %s\n", arg)); + } + else if (has_prefix("cl", sub_param)) { + arg = get_arg(sub_param+2, argv[i+1], &i); + if (sys_strcmp("true", arg) == 0) + erts_sched_compact_load = 1; + else if (sys_strcmp("false", arg) == 0) + erts_sched_compact_load = 0; + else { + erts_fprintf(stderr, + "bad scheduler compact load value '%s'\n", + arg); + erts_usage(); + } + } else if (has_prefix("ct", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); res = erts_init_cpu_topology_string(arg); @@ -1203,21 +1201,27 @@ erl_start(int argc, char **argv) erts_usage(); } } - else if (sys_strcmp("mrq", sub_param) == 0) - use_multi_run_queue = 1; else if (sys_strcmp("nsp", sub_param) == 0) erts_use_sender_punish = 0; - else if (sys_strcmp("srq", sub_param) == 0) - use_multi_run_queue = 0; else if (sys_strcmp("wt", sub_param) == 0) { arg = get_arg(sub_param+2, argv[i+1], &i); - if (erts_sched_set_wakeup_limit(arg) != 0) { + if (erts_sched_set_wakeup_other_thresold(arg) != 0) { erts_fprintf(stderr, "scheduler wakeup threshold: %s\n", arg); erts_usage(); } VERBOSE(DEBUG_SYSTEM, - ("scheduler wakup threshold: %s\n", arg)); + ("scheduler wakeup threshold: %s\n", arg)); + } + else if (sys_strcmp("ws", sub_param) == 0) { + arg = get_arg(sub_param+2, argv[i+1], &i); + if (erts_sched_set_wakeup_other_type(arg) != 0) { + erts_fprintf(stderr, "scheduler wakeup strategy: %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("scheduler wakeup threshold: %s\n", arg)); } else if (has_prefix("ss", sub_param)) { /* suggested stack size (Kilo Words) for scheduler threads */ @@ -1301,17 +1305,8 @@ erl_start(int argc, char **argv) break; } - case 'A': - /* set number of threads in thread pool */ - arg = get_arg(argv[i]+2, argv[i+1], &i); - if (((async_max_threads = atoi(arg)) < 0) || - (async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) { - erts_fprintf(stderr, "bad number of async threads %s\n", arg); - erts_usage(); - } - - VERBOSE(DEBUG_SYSTEM, ("using %d async-threads\n", - async_max_threads)); + case 'A': /* Was handled in early init just read past it */ + (void) get_arg(argv[i]+2, argv[i+1], &i); break; case 'a': @@ -1400,10 +1395,6 @@ erl_start(int argc, char **argv) i++; } -#ifdef USE_THREADS - erts_async_max_threads = async_max_threads; -#endif - /* Delayed check of +P flag */ if (erts_max_processes < ERTS_MIN_PROCESSES || erts_max_processes > ERTS_MAX_PROCESSES @@ -1435,7 +1426,6 @@ erl_start(int argc, char **argv) erl_init(ncpu); - init_shared_memory(boot_argc, boot_argv); load_preloaded(); erts_initialized = 1; @@ -1449,6 +1439,10 @@ erl_start(int argc, char **argv) erts_sys_main_thread(); /* May or may not return! */ #else erts_thr_set_main_status(1, 1); +#if ERTS_USE_ASYNC_READY_Q + erts_get_scheduler_data()->aux_work_data.async_ready.queue + = erts_get_async_ready_queue(1); +#endif set_main_stack_size(); process_main(); #endif @@ -1472,8 +1466,31 @@ __decl_noreturn void erts_thr_fatal_error(int err, char *what) #endif static void -system_cleanup(int exit_code) +system_cleanup(int flush_async) { + /* + * Make sure only one thread exits the runtime system. + */ + if (erts_atomic_inc_read_nob(&exiting) != 1) { + /* + * Another thread is currently exiting the system; + * wait for it to do its job. + */ +#ifdef ERTS_SMP + if (erts_thr_progress_is_managed_thread()) { + /* + * The exiting thread might be waiting for + * us to block; need to update status... + */ + erts_thr_progress_active(NULL, 0); + erts_thr_progress_prepare_wait(NULL); + } +#endif + /* Wait forever... */ + while (1) + erts_milli_sleep(10000000); + } + /* No cleanup wanted if ... * 1. we are about to do an abnormal exit * 2. we haven't finished initializing, or @@ -1481,7 +1498,7 @@ system_cleanup(int exit_code) * (in threaded non smp case). */ - if (exit_code != 0 + if (!flush_async || !erts_initialized #if defined(USE_THREADS) && !defined(ERTS_SMP) || !erts_equal_tids(main_thread, erts_thr_self()) @@ -1493,132 +1510,62 @@ system_cleanup(int exit_code) #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); #endif - erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); /* We never release it... */ -#endif - -#ifdef HYBRID - if (ma_src_stack) erts_free(ERTS_ALC_T_OBJECT_STACK, - (void *)ma_src_stack); - if (ma_dst_stack) erts_free(ERTS_ALC_T_OBJECT_STACK, - (void *)ma_dst_stack); - if (ma_offset_stack) erts_free(ERTS_ALC_T_OBJECT_STACK, - (void *)ma_offset_stack); - ma_src_stack = NULL; - ma_dst_stack = NULL; - ma_offset_stack = NULL; - erts_cleanup_offheap(&erts_global_offheap); -#endif - -#if defined(HYBRID) && !defined(INCREMENTAL) - if (global_heap) { - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (void*) global_heap, - sizeof(Eterm) * global_heap_sz); - } - global_heap = NULL; #endif -#ifdef INCREMENTAL - erts_cleanup_incgc(); -#endif - -#if defined(USE_THREADS) - exit_async(); -#endif -#if HAVE_ERTS_MSEG - erts_mseg_exit(); -#endif - - /* - * A lot more cleaning could/should have been done... - */ - + erts_exit_flush_async(); } -/* - * Common exit function, all exits from the system go through here. - * n <= 0 -> normal exit with status n; - * n = 127 -> Erlang crash dump produced, exit with status 1; - * other positive n -> Erlang crash dump and core dump produced. - */ - -__decl_noreturn void erl_exit0(char *file, int line, int n, char *fmt,...) +static __decl_noreturn void __noreturn +erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) { unsigned int an; - va_list args; - va_start(args, fmt); + system_cleanup(flush_async); save_statistics(); - system_cleanup(n); - an = abs(n); if (erts_mtrace_enabled) erts_mtrace_exit((Uint32) an); /* Produce an Erlang core dump if error */ - if (n > 0 && erts_initialized && - (erts_no_crash_dump == 0 || n == ERTS_DUMP_EXIT)) { - erl_crash_dump_v(file, line, fmt, args); + if (((n > 0 && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) + && erts_initialized) { + erl_crash_dump_v((char*) NULL, 0, fmt, args1); } - /* need to reinitialize va_args thing */ - va_end(args); - va_start(args, fmt); - if (fmt != NULL && *fmt != '\0') - erl_error(fmt, args); /* Print error message. */ - va_end(args); + erl_error(fmt, args2); /* Print error message. */ sys_tty_reset(n); if (n == ERTS_INTR_EXIT) exit(0); - else if (n == 127) + else if (n == ERTS_DUMP_EXIT) ERTS_EXIT_AFTER_DUMP(1); else if (n > 0 || n == ERTS_ABORT_EXIT) abort(); exit(an); } -__decl_noreturn void erl_exit(int n, char *fmt,...) +/* Exit without flushing async threads */ +__decl_noreturn void __noreturn erl_exit(int n, char *fmt, ...) { - unsigned int an; - va_list args; - - va_start(args, fmt); - - save_statistics(); - - system_cleanup(n); - - an = abs(n); - - if (erts_mtrace_enabled) - erts_mtrace_exit((Uint32) an); - - /* Produce an Erlang core dump if error */ - if (n > 0 && erts_initialized && - (erts_no_crash_dump == 0 || n == ERTS_DUMP_EXIT)) { - erl_crash_dump_v((char*) NULL, 0, fmt, args); - } - - /* need to reinitialize va_args thing */ - va_end(args); - va_start(args, fmt); - - if (fmt != NULL && *fmt != '\0') - erl_error(fmt, args); /* Print error message. */ - va_end(args); - sys_tty_reset(n); - - if (n == ERTS_INTR_EXIT) - exit(0); - else if (n == ERTS_DUMP_EXIT) - ERTS_EXIT_AFTER_DUMP(1); - else if (n > 0 || n == ERTS_ABORT_EXIT) - abort(); - exit(an); + va_list args1, args2; + va_start(args1, fmt); + va_start(args2, fmt); + erl_exit_vv(n, 0, fmt, args1, args2); + va_end(args2); + va_end(args1); } +/* Exit after flushing async threads */ +__decl_noreturn void __noreturn erl_exit_flush_async(int n, char *fmt, ...) +{ + va_list args1, args2; + va_start(args1, fmt); + va_start(args2, fmt); + erl_exit_vv(n, 1, fmt, args1, args2); + va_end(args2); + va_end(args1); +} |