erts: Refactor perf counter internal interface

perf counter is now part of the function pointer interface
and also the function returns the value instead of writing
to a memory buffer.

3 files changed, 142 insertions, 21 deletions

diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 3a03d6be49..8b1822ca9f 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -161,6 +161,7 @@ typedef long long ErtsSysHrTime;
 #endif
 
 typedef ErtsMonotonicTime ErtsSystemTime;
+typedef ErtsSysHrTime ErtsSysPerfCounter;
 
 #define ERTS_MONOTONIC_TIME_MIN (((ErtsMonotonicTime) 1) << 63)
 #define ERTS_MONOTONIC_TIME_MAX (~ERTS_MONOTONIC_TIME_MIN)
@@ -209,6 +210,7 @@ ErtsSystemTime erts_os_system_time(void);
  * It may or may not be monotonic.
  */
 ErtsSysHrTime erts_sys_hrtime(void);
+#define ERTS_HRTIME_UNIT (1000*1000*1000)
 
 struct erts_sys_time_read_only_data__ {
 #ifdef ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__
@@ -217,6 +219,8 @@ struct erts_sys_time_read_only_data__ {
 #ifdef ERTS_OS_TIMES_INLINE_FUNC_PTR_CALL__
     void (*os_times)(ErtsMonotonicTime *, ErtsSystemTime *);
 #endif
+    ErtsSysPerfCounter (*perf_counter)(void);
+    ErtsSysPerfCounter perf_counter_unit;
     int ticks_per_sec;
 };
 
@@ -273,25 +277,18 @@ erts_os_times(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep)
  * Functions for getting the performance counter
  */
 
-#if defined(__x86_64__)
- /*  available on all x86_64. Best if used when we have constant_tsc and
-     nonstop_tsc cpu features. It may have been a good idea to put the
-     cpuid instruction before the rdtsc, but I decided against it
-     because it is not really needed for msacc, and it slows it down by
-     quite a bit. As a result though, this timestamp becomes much less
-     accurate as it might be re-ordered to be executed way before this
-     function is called.
- */
-#define sys_perf_counter(ts) do {                                       \
-        __asm__ __volatile__ ("rdtsc\n\t"                               \
-                              "shl $32, %%rdx\n\t"                      \
-                              "or %%rdx, %0" : "=a" (*ts) : : "rdx");   \
-    } while(0)
-#define SYS_PERF_COUNTER_UNIT erts_perf_counter_unit()
-#else
-#define sys_perf_counter(ts) *(ts) = erts_sys_hrtime()
-#define SYS_PERF_COUNTER_UNIT 1000000000LL
-#endif
+ERTS_GLB_INLINE ErtsSysPerfCounter erts_sys_perf_counter(void);
+#define erts_sys_perf_counter_unit() erts_sys_time_data__.r.o.perf_counter_unit
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE ErtsSysPerfCounter
+erts_sys_perf_counter()
+{
+    return (*erts_sys_time_data__.r.o.perf_counter)();
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 /*
  * Functions for measuring CPU time
diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c
index 6fc4fc7dc4..9738a8c352 100644
--- a/erts/emulator/sys/unix/sys_time.c
+++ b/erts/emulator/sys/unix/sys_time.c
@@ -65,6 +65,8 @@
 #  include <fcntl.h>
 #endif
 
+static void init_perf_counter(void);
+
 /******************* Routines for time measurement *********************/
 
 #undef ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
@@ -404,6 +406,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
 #  error Missing erts_os_system_time() implementation
 #endif
 
+    init_perf_counter();
+
 }
 
 void
@@ -908,6 +912,114 @@ erts_os_times(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep)
 
 #endif
 
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Performance counter functions                                             *
+\*                                                                           */
+
+
+/* What resolution to spin to in micro seconds */
+#define RESOLUTION 100
+/* How many iterations to spin */
+#define ITERATIONS 1
+/* How many significant figures to round to */
+#define SIGFIGS 3
+
+static ErtsSysPerfCounter calculate_perf_counter_unit(void) {
+    int i;
+    ErtsSysPerfCounter pre, post;
+    double value = 0;
+    double round_factor;
+#if defined(HAVE_GETHRTIME) && defined(GETHRTIME_WITH_CLOCK_GETTIME)
+    struct timespec basetime,comparetime;
+#define __GETTIME(arg) clock_gettime(CLOCK_MONOTONIC,arg)
+#define __GETUSEC(arg) (arg.tv_nsec / 1000)
+#else
+    SysTimeval basetime,comparetime;
+#define __GETTIME(arg) sys_gettimeofday(arg)
+#define __GETUSEC(arg) arg.tv_usec
+#endif
+
+    for (i = 0; i < ITERATIONS; i++) {
+        /* Make sure usec just flipped over at current resolution */
+        __GETTIME(&basetime);
+        do {
+            __GETTIME(&comparetime);
+        } while ((__GETUSEC(basetime) / RESOLUTION) == (__GETUSEC(comparetime) / RESOLUTION));
+
+        pre = erts_sys_perf_counter();
+
+        __GETTIME(&basetime);
+        do {
+            __GETTIME(&comparetime);
+        } while ((__GETUSEC(basetime) / RESOLUTION) == (__GETUSEC(comparetime) / RESOLUTION));
+
+        post = erts_sys_perf_counter();
+
+        value += post - pre;
+    }
+    /* After this value is ticks per us */
+    value /= (RESOLUTION*ITERATIONS);
+
+    /* We round to 3 significant figures */
+    round_factor = pow(10.0, SIGFIGS - ceil(log10(value)));
+    value = ((ErtsSysPerfCounter)(value * round_factor + 0.5)) / round_factor;
+
+    /* convert to ticks per second */
+    return 1000000 * value;
+}
+
+static int have_rdtscp(void)
+{
+#if defined(ETHR_X86_RUNTIME_CONF__)
+    /* On early x86 cpu's the tsc varies with
+       the current speed of the cpu, which means that the time per
+       tick vary depending on the current load of the cpu. We do not
+       want this as it would give very scewed numbers when the cpu is
+       mostly idle.
+       If we have the rdtscp feature it is a sign that the cpu is
+       relatively modern, and thus the tsc quite stable so we use it then.
+
+       If this test is not good enough, I don't know what we'll do.
+       Maybe fallback on erts_sys_hrtime always, but that would be a shame as
+       rdtsc is about 3 times faster than hrtime... */
+    return ETHR_X86_RUNTIME_CONF_HAVE_RDTSCP__;
+#else
+    return 0;
+#endif
+}
+
+static ErtsSysPerfCounter rdtsc(void)
+{
+ /*  It may have been a good idea to put the cpuid instruction before
+     the rdtsc, but I decided against it because it is not really
+     needed for msacc, and it slows it down by quite a bit (5-7 times slower).
+     As a result though, this timestamp becomes much less
+     accurate as it might be re-ordered to be executed way before or after this
+     function is called.
+ */
+    ErtsSysPerfCounter ts;
+#if defined(__x86_64__)
+    __asm__ __volatile__ ("rdtsc\n\t"
+                          "shl $32, %%rdx\n\t"
+                          "or %%rdx, %0" : "=a" (ts) : : "rdx");
+#elif defined(__i386__)
+    __asm__ __volatile__ ("rdtsc\n\t"
+                           : "=A" (ts) );
+#endif
+    return ts;
+}
+
+static void init_perf_counter(void)
+{
+    if (have_rdtscp()) {
+        erts_sys_time_data__.r.o.perf_counter = rdtsc;
+        erts_sys_time_data__.r.o.perf_counter_unit = calculate_perf_counter_unit();
+    } else {
+        erts_sys_time_data__.r.o.perf_counter = erts_sys_hrtime;
+        erts_sys_time_data__.r.o.perf_counter_unit = ERTS_HRTIME_UNIT;
+    }
+}
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #ifdef HAVE_GETHRVTIME_PROCFS_IOCTL
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index 1ab9eadefd..99c1066ab3 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -183,6 +183,7 @@ typedef LONGLONG ErtsSysHrTime;
 #endif
 
 typedef ErtsMonotonicTime ErtsSystemTime;
+typedef ErtsMonotonicTime ErtsSysPerfCounter;
 
 ErtsSystemTime erts_os_system_time(void);
 
@@ -213,6 +214,7 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_os_monotonic_time(void);
 ERTS_GLB_INLINE void erts_os_times(ErtsMonotonicTime *,
 				   ErtsSystemTime *);
 ERTS_GLB_INLINE ErtsSysHrTime erts_sys_hrtime(void);
+ERTS_GLB_INLINE ErtsSysPerfCounter erts_sys_perf_counter(void);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -234,12 +236,22 @@ erts_sys_hrtime(void)
     return (*erts_sys_time_data__.r.o.sys_hrtime)();
 }
 
+ERTS_GLB_INLINE ErtsSysPerfCounter
+erts_sys_perf_counter(void)
+{
+    return (*erts_sys_time_data__.r.o.sys_hrtime)();
+}
+
+ERTS_GLB_INLINE ErtsSysPerfCounter
+erts_sys_perf_counter_unit(void)
+{
+    return 1000 * 1000 * 1000;
+}
+
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 extern void sys_gettimeofday(SysTimeval *tv);
 extern clock_t sys_times(SysTimes *buffer);
-#define sys_perf_counter(ts) *(ts) = erts_sys_hrtime()
-#define SYS_PERF_COUNTER_UNIT ERTS_I64_LITERAL(1000000000)
 
 extern char *win_build_environment(char *);