erts_sys_hrtime() for lcnt

7 files changed, 131 insertions, 29 deletions

diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index ddeb56a6be..c6d8f4df95 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -104,16 +104,13 @@ static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) {
 }
 
 static void lcnt_time(erts_lcnt_time_t *time) {
-#if 0 || defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT)
-    ErtsMonotonicTime mtime = ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time());
+    /*
+     * erts_sys_hrtime() is the highest resolution
+     * we could find, it may or may not be monotonic...
+     */
+    ErtsMonotonicTime mtime = erts_sys_hrtime();
     time->s  = (unsigned long) (mtime / 1000000000LL);
     time->ns = (unsigned long) (mtime - 1000000000LL*time->s);
-#else
-  SysTimeval tv;
-  sys_gettimeofday(&tv);
-  time->s  = tv.tv_sec;
-  time->ns = tv.tv_usec*1000LL;
-#endif
 }
 
 static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) {
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index d83bdf8d31..c51b665db9 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -708,7 +708,6 @@ extern int erts_init_time_sup(int, ErtsTimeWarpMode);
 extern void erts_sys_init_float(void);
 extern void erts_thread_init_float(void);
 extern void erts_thread_disable_fpe(void);
-
 ERTS_GLB_INLINE int erts_block_fpe(void);
 ERTS_GLB_INLINE void erts_unblock_fpe(int);
 
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 303ebeee7c..a14a4b3eae 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -161,8 +161,10 @@ typedef struct tms SysTimes;
 
 #if SIZEOF_LONG == 8
 typedef long ErtsMonotonicTime;
+typedef long ErtsSysHrTime;
 #elif SIZEOF_LONG_LONG == 8
 typedef long long ErtsMonotonicTime;
+typedef long long ErtsSysHrTime;
 #else
 #error No signed 64-bit type found...
 #endif
@@ -201,6 +203,7 @@ ErtsMonotonicTime erts_os_monotonic_time(void);
 #elif defined(OS_MONOTONIC_TIME_USING_GETHRTIME)
 
 #define erts_os_monotonic() ((ErtsMonotonicTime) gethrtime())
+#define erts_sys_hrtime() ((ErtsSysHrTime) gethrtime())
 
 #elif defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME) \
     || defined(OS_MONOTONIC_TIME_USING_TIMES)
@@ -221,6 +224,15 @@ ErtsMonotonicTime erts_os_monotonic_time(void);
 
 #endif
 
+/*
+ * erts_sys_hrtime() is the highest resolution
+ * time function found. Time unit is nano-seconds.
+ * It may or may not be monotonic.
+ */
+#ifndef erts_sys_hrtime
+extern ErtsSysHrTime erts_sys_hrtime(void);
+#endif
+
 struct erts_sys_time_read_only_data__ {
 #ifdef ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__
     ErtsMonotonicTime (*os_monotonic_time)(void);
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 58b01c094e..3647ea6a88 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -528,18 +528,18 @@ erts_sys_pre_init(void)
 
     erts_thr_init(&eid);
 
-#endif /* USE_THREADS */
-
-    erts_init_sys_time_sup();
-
-#ifdef USE_THREADS
-
     report_exit_list = NULL;
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init();
 #endif
 
+#endif /* USE_THREADS */
+
+    erts_init_sys_time_sup();
+
+#ifdef USE_THREADS
+
 #if CHLDWTHR || defined(ERTS_SMP)
     erts_mtx_init(&chld_stat_mtx, "child_status");
 #endif
diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c
index 9db727b111..134e3a67b0 100644
--- a/erts/emulator/sys/unix/sys_time.c
+++ b/erts/emulator/sys/unix/sys_time.c
@@ -85,8 +85,8 @@ ErtsSysTimeData__ erts_sys_time_data__ erts_align_attribute(ERTS_CACHE_LINE_SIZE
 
 #define ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
 
-ErtsMonotonicTime clock_gettime_monotonic_raw(void);
-ErtsMonotonicTime clock_gettime_monotonic_verified(void);
+static ErtsMonotonicTime clock_gettime_monotonic_raw(void);
+static ErtsMonotonicTime clock_gettime_monotonic_verified(void);
 
 #endif /* defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME) */
 
@@ -306,7 +306,7 @@ clock_gettime_monotonic(void)
 
 #if defined(__linux__)
 
-ErtsMonotonicTime clock_gettime_monotonic_verified(void)
+static ErtsMonotonicTime clock_gettime_monotonic_verified(void)
 {
     ErtsMonotonicTime mtime;
 
@@ -322,7 +322,7 @@ ErtsMonotonicTime clock_gettime_monotonic_verified(void)
     return mtime;
 }
 
-ErtsMonotonicTime clock_gettime_monotonic_raw(void)
+static ErtsMonotonicTime clock_gettime_monotonic_raw(void)
 {
     return clock_gettime_monotonic();
 }
@@ -336,6 +336,12 @@ ErtsMonotonicTime erts_os_monotonic_time(void)
 
 #endif /* !defined(__linux__) */
 
+ErtsSysHrTime
+erts_sys_hrtime(void)
+{
+    return (ErtsSysHrTime) clock_gettime_monotonic();
+}
+
 #elif defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME)
 
 #include <mach/clock.h>
@@ -369,6 +375,12 @@ ErtsMonotonicTime erts_os_monotonic_time(void)
     return mtime;
 }
 
+ErtsSysHrTime
+erts_sys_hrtime(void)
+{
+    return (ErtsSysHrTime) erts_os_monotonic_time();
+}
+
 #elif defined(OS_MONOTONIC_TIME_USING_TIMES)
 
 ErtsMonotonicTime
@@ -381,7 +393,29 @@ erts_os_monotonic_time(void)
 					 ticks) << internal_state.r.o.times_shift;
 }
 
-#endif  /* !defined(OS_MONOTONIC_TIME_USING_TIMES) */
+#  define ERTS_NEED_ERTS_SYS_HRTIME_FALLBACK
+
+#else  /* !defined(OS_MONOTONIC_TIME_USING_TIMES) */
+/* No os-monotonic-time */
+#  define ERTS_NEED_ERTS_SYS_HRTIME_FALLBACK
+#endif
+
+#ifdef ERTS_NEED_ERTS_SYS_HRTIME_FALLBACK
+
+ErtsSysHrTime
+erts_sys_hrtime(void)
+{
+    ErtsSysHrTime time;
+    struct timeval tv;
+    gettimeofday(&tv);
+    time = (ErtsSysHrTime) tv.tv_sec;
+    time *= (ErtsSysHrTime) 1000*1000*1000;
+    time += ((ErtsSysHrTime) tv.tv_usec)*1000;
+    return time;
+}
+
+#endif
+
 
 #ifdef HAVE_GETHRVTIME_PROCFS_IOCTL
 
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index f04bb6a0e5..febf3b5cca 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -172,11 +172,13 @@ typedef long long          Sint64;
 #  endif
 
 typedef long long ErtsMonotonicTime;
+typedef long long ErtsSysHrTime;
 #else
 typedef ULONGLONG Uint64;
 typedef LONGLONG  Sint64;
 
 typedef LONGLONG ErtsMonotonicTime;
+typedef LONGLONG ErtsSysHrTime;
 #endif
 
 #define ERTS_MONOTONIC_TIME_MIN (((ErtsMonotonicTime) 1) << 63)
@@ -187,6 +189,7 @@ typedef LONGLONG ErtsMonotonicTime;
 
 struct erts_sys_time_read_only_data__ {
     ErtsMonotonicTime (*os_monotonic_time)(void);
+    ErtsSysHrTime (*sys_hrtime)(void);
 };
 
 typedef struct {
@@ -201,6 +204,7 @@ typedef struct {
 extern ErtsSysTimeData__ erts_sys_time_data__;
 
 ERTS_GLB_INLINE ErtsMonotonicTime erts_os_monotonic_time(void);
+ERTS_GLB_INLINE ErtsSysHrTime erts_sys_hrtime(void);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -210,6 +214,12 @@ erts_os_monotonic_time(void)
     return (*erts_sys_time_data__.r.o.os_monotonic_time)();
 }
 
+ERTS_GLB_INLINE ErtsSysHrTime
+erts_sys_hrtime(void)
+{
+    return (*erts_sys_time_data__.r.o.sys_hrtime)();
+}
+
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 extern void sys_gettimeofday(SysTimeval *tv);
diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index 5fa575fa3b..ea180f067f 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -26,6 +26,7 @@
 #include "sys.h"
 #include "assert.h"
 #include "erl_os_monotonic_time_extender.h"
+#include "erl_time.h"
 
 #define LL_LITERAL(X) ERTS_I64_LITERAL(X)
 
@@ -79,6 +80,7 @@ static int days_in_month[2][13] = {
 struct sys_time_internal_state_read_only__ {
     ULONGLONG (WINAPI *pGetTickCount64)(void);
     BOOL (WINAPI *pQueryPerformanceCounter)(LARGE_INTEGER *);
+    Sint32 pcf;
 };
 
 struct sys_time_internal_state_read_mostly__ {
@@ -148,6 +150,42 @@ os_monotonic_time_gtc64(void)
     return (ErtsMonotonicTime) ticks << 10;
 }
 
+static ErtsSysHrTime
+sys_hrtime_qpc(void)
+{
+    LARGE_INTEGER pc;
+
+    if (!(*internal_state.r.o.pQueryPerformanceCounter)(&pc))
+	erl_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n");
+
+    ASSERT(pc.QuadPart > 0);
+
+    return (ErtsSysHrTime) erts_time_unit_conversion((Uint64) pc.QuadPart,
+						     internal_state.r.o.pcf,
+						     (Uint32) 1000*1000*1000);
+}
+
+static ErtsSysHrTime
+sys_hrtime_gtc32(void)
+{
+    ErtsSysHrTime time;
+    Uint32 ticks = (Uint32) GetTickCount();
+    ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
+				      tick_count);
+    time = (ErtsSysHrTime) ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
+							 ticks);
+    time *= (ErtsSysHrTime) (1000 * 1000);
+    return time;
+}
+
+static ErtsSysHrTime
+sys_hrtime_gtc64(void)
+{
+    ErtsSysHrTime time = (*internal_state.r.o.pGetTickCount64)();
+    time *= (ErtsSysHrTime) (1000*1000);
+    return time;
+}
+
 /*
  * Init
  */
@@ -156,6 +194,7 @@ void
 sys_init_time(ErtsSysInitTimeResult *init_resp)
 {
     ErtsMonotonicTime (*os_mtime_func)(void);
+    ErtsSysHrTime (*sys_hrtime_func)(void) = NULL;
     ErtsMonotonicTime time_unit;
     char kernel_dll_name[] = "kernel32";
     HMODULE module;
@@ -180,6 +219,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
 	erts_init_os_monotonic_time_extender(&internal_state.wr.m.os_mtime_xtnd,
 					     get_tick_count,
 					     60*60*24*7); /* Check once a week */
+	if (!sys_hrtime_func)
+	    sys_hrtime_func = sys_hrtime_gtc32;
     }
     else {
 	int major, minor, build;
@@ -202,6 +243,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
 	    init_resp->os_monotonic_info.resolution = 100; /* 10 ms */
 	    time_unit = (ErtsMonotonicTime) (1000 << 10);
 	    os_mtime_func = os_monotonic_time_gtc64;
+	    if (!sys_hrtime_func)
+		sys_hrtime_func = sys_hrtime_gtc64;
 	}
 	else { /* Vista or newer... */
 
@@ -214,21 +257,28 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
 		goto get_tick_count64;
 	    if (!(*QPF)(&pf))
 		goto get_tick_count64;
-	    /*
-	     * We only use QueryPerformanceCounter() if
-	     * its frequency is equal to, or larger than
-	     * GHz in order to ensure that the user wont
-	     * be able to observe faulty order between
-	     * values retrieved on different threads.
-	     */
-	    if (pf.QuadPart < (LONGLONG) 1000*1000*1000)
-		goto get_tick_count64;
+
 	    internal_state.r.o.pQueryPerformanceCounter
 		= ((BOOL (WINAPI *)(LARGE_INTEGER *))
 		   GetProcAddress(module, "QueryPerformanceCounter"));
 	    if (!internal_state.r.o.pQueryPerformanceCounter)
 		goto get_tick_count64;
 
+	    if (pf.QuadPart < (((LONGLONG) 1) << 32)) {
+		internal_state.r.o.pcf = (Uint32) pf.QuadPart;
+		sys_hrtime_func = sys_hrtime_qpc;
+	    }
+	    
+	    /*
+	     * We only use QueryPerformanceCounter() for
+	     * os-monotonic-time if its frequency is equal
+	     * to, or larger than GHz in order to ensure
+	     * that the user wont be able to observe faulty
+	     * order between values retrieved on different threads.
+	     */
+	    if (pf.QuadPart < (LONGLONG) 1000*1000*1000)
+		goto get_tick_count64;
+
 	    init_resp->os_monotonic_info.func = "QueryPerformanceCounter";
 	    init_resp->os_monotonic_info.locked_use = 0;
 	    time_unit = (ErtsMonotonicTime) pf.QuadPart;