erts: Improve crash dumps

This commit improves crash dumps in several ways:
  *  Suspends schedulers to get a current snapshot
  *  Dumps information about scheduler
  *  Dumps stack trace of current running process
     (including Garbing processes)

1 files changed, 98 insertions, 30 deletions

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 5aee85174f..41765dad12 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -210,25 +210,12 @@ print_process_info(int to, void *to_arg, Process *p)
     erts_print(to, to_arg, "State: ");
 
     state = erts_smp_atomic32_read_acqb(&p->state);
-    if (state & ERTS_PSFLG_FREE)
-	erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */
-    else if (state & ERTS_PSFLG_EXITING)
-	erts_print(to, to_arg, "Exiting\n");
-    else if (state & ERTS_PSFLG_GC) {
-	garbing = 1;
-	running = 1;
-	erts_print(to, to_arg, "Garbing\n");
-    }
-    else if (state & ERTS_PSFLG_SUSPENDED)
-	erts_print(to, to_arg, "Suspended\n");
-    else if (state & ERTS_PSFLG_RUNNING) {
-	running = 1;
-	erts_print(to, to_arg, "Running\n");
-    }
-    else if (state & ERTS_PSFLG_ACTIVE)
-	erts_print(to, to_arg, "Scheduled\n");
-    else
-	erts_print(to, to_arg, "Waiting\n");
+    erts_dump_process_state(to, to_arg, state);
+    if (state & ERTS_PSFLG_GC) {
+        garbing = 1;
+        running = 1;
+    } else if (state & ERTS_PSFLG_RUNNING)
+        running = 1;
 
     /*
      * If the process is registered as a global process, display the
@@ -352,6 +339,10 @@ print_process_info(int to, void *to_arg, Process *p)
 #endif
 	    erts_stack_dump(to, to_arg, p);
     }
+
+    /* Display all states */
+    erts_print(to, to_arg, "Internal State: ");
+    erts_dump_extended_process_state(to, to_arg, state);
 }
 
 static void
@@ -671,6 +662,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 {
 #ifdef ERTS_SMP
     ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */
+    int bc;
 #endif
     int fd;
     size_t envsz;
@@ -681,27 +673,39 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     char* dumpname;
     int secs;
     int env_erl_crash_dump_seconds_set = 1;
+    int i;
 
     if (ERTS_SOMEONE_IS_CRASH_DUMPING)
 	return;
 
 #ifdef ERTS_SMP
+    /* Order all managed threads to block, this has to be done
+       first to guarantee that this is the only thread to generate
+       crash dump. */
+    bc = erts_thr_progress_fatal_error_block(&tpd_buf);
+
+#ifdef ERTS_THR_HAVE_SIG_FUNCS
     /*
-     * Wait for all managed threads to block. If all threads haven't blocked
-     * after a minute, we go anyway and hope for the best...
-     *
-     * We do not release system again. We expect an exit() or abort() after
-     * dump has been written.
+     * We suspend all scheduler threads so that we can dump some
+     * data about the currently running processes and scheduler data.
+     * We have to be very very careful when doing this as the schedulers
+     * could be anywhere.
      */
-    erts_thr_progress_fatal_error_block(60000, &tpd_buf);
-    /* Either worked or not... */
+    for (i = 0; i < erts_no_schedulers; i++) {
+        erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid;
+        if (!erts_equal_tids(tid,erts_thr_self()))
+            sys_thr_suspend(tid);
+    }
+
+#endif
 
     /* Allow us to pass certain places without locking... */
     erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1);
     erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1);
-#else
+
+#else /* !ERTS_SMP */
     erts_writing_erl_crash_dump = 1;
-#endif
+#endif /* ERTS_SMP */
 
     envsz = sizeof(env);
     /* ERL_CRASH_DUMP_SECONDS not set
@@ -758,9 +762,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     erts_fprintf(stderr,"\nCrash dump is being written to: %s...", dumpname);
 
     fd = open(dumpname,O_WRONLY | O_CREAT | O_TRUNC,0640);
-    if (fd < 0) 
+    if (fd < 0)
 	return; /* Can't create the crash dump, skip it */
-    
     time(&now);
     erts_fdprintf(fd, "=erl_crash_dump:0.3\n%s", ctime(&now));
 
@@ -774,9 +777,74 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     erts_fdprintf(fd, "System version: ");
     erts_print_system_version(fd, NULL, NULL);
     erts_fdprintf(fd, "%s\n", "Compiled: " ERLANG_COMPILE_DATE);
+
     erts_fdprintf(fd, "Taints: ");
     erts_print_nif_taints(fd, NULL);
     erts_fdprintf(fd, "Atoms: %d\n", atom_table_size());
+
+#ifdef USE_THREADS
+    /* We want to note which thread it was that called erl_exit */
+    if (erts_get_scheduler_data()) {
+        erts_fdprintf(fd, "Calling Thread: scheduler:%d\n",
+                      erts_get_scheduler_data()->no);
+    } else {
+        if (!erts_thr_getname(erts_thr_self(), dumpnamebuf, MAXPATHLEN))
+            erts_fdprintf(fd, "Calling Thread: %s\n", dumpnamebuf);
+        else
+            erts_fdprintf(fd, "Calling Thread: %p\n", erts_thr_self());
+    }
+#else
+    erts_fdprintf(fd, "Calling Thread: scheduler:1\n");
+#endif
+
+#if defined(ERTS_HAVE_TRY_CATCH)
+
+    /*
+     * erts_print_scheduler_info is not guaranteed to be safe to call
+     * here for all schedulers as we may have suspended a scheduler
+     * in the middle of updating the STACK_TOP and STACK_START
+     * variables and thus when scanning the stack we could get
+     * segmentation faults. We protect against this very unlikely
+     * scenario by using the ERTS_SYS_TRY_CATCH.
+     */
+    for (i = 0; i < erts_no_schedulers; i++) {
+        ERTS_SYS_TRY_CATCH(
+            erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)),
+            erts_fdprintf(fd, "** crashed **\n"));
+    }
+#endif
+
+#ifdef ERTS_SMP
+
+#if defined(ERTS_THR_HAVE_SIG_FUNCS)
+
+    /* We resume all schedulers so that we are in a known safe state
+       when we write the rest of the crash dump */
+    for (i = 0; i < erts_no_schedulers; i++) {
+        erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid;
+        if (!erts_equal_tids(tid,erts_thr_self()))
+	    sys_thr_resume(tid);
+    }
+#endif
+
+    /*
+     * Wait for all managed threads to block. If all threads haven't blocked
+     * after a minute, we go anyway and hope for the best...
+     *
+     * We do not release system again. We expect an exit() or abort() after
+     * dump has been written.
+     */
+    erts_thr_progress_fatal_error_wait(60000);
+    /* Either worked or not... */
+#endif
+
+#ifndef ERTS_HAVE_TRY_CATCH
+    /* This is safe to call here, as all schedulers are blocked */
+    for (i = 0; i < erts_no_schedulers; i++) {
+        erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i));
+    }
+#endif
+    
     info(fd, NULL); /* General system info */
     if (erts_ptab_initialized(&erts_proc))
 	process_info(fd, NULL); /* Info about each process and port */