aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/sys
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/sys')
-rw-r--r--erts/emulator/sys/common/erl_check_io.c23
-rw-r--r--erts/emulator/sys/common/erl_check_io.h6
-rw-r--r--erts/emulator/sys/common/erl_poll.c221
-rw-r--r--erts/emulator/sys/common/erl_poll.h6
-rw-r--r--erts/emulator/sys/ose/erl_poll.c69
-rw-r--r--erts/emulator/sys/ose/sys.c4
-rw-r--r--erts/emulator/sys/unix/erl_unix_sys.h120
-rw-r--r--erts/emulator/sys/unix/sys.c25
-rw-r--r--erts/emulator/sys/unix/sys_time.c398
-rw-r--r--erts/emulator/sys/win32/erl_poll.c60
-rw-r--r--erts/emulator/sys/win32/erl_win_sys.h53
-rw-r--r--erts/emulator/sys/win32/sys.c4
-rw-r--r--erts/emulator/sys/win32/sys_time.c199
13 files changed, 969 insertions, 219 deletions
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 0051b45b31..7be17d20bb 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1590,9 +1590,9 @@ ERTS_CIO_EXPORT(erts_check_io_interrupt)(int set)
void
ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set,
- erts_short_time_t msec)
+ ErtsMonotonicTime timeout_time)
{
- ERTS_CIO_POLL_INTR_TMD(pollset.ps, set, msec);
+ ERTS_CIO_POLL_INTR_TMD(pollset.ps, set, timeout_time);
}
void
@@ -1600,9 +1600,12 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
{
ErtsPollResFd *pollres;
int pollres_len;
- SysTimeval wait_time;
+ ErtsMonotonicTime timeout_time;
int poll_ret, i;
erts_aint_t current_cio_time;
+ ErtsSchedulerData *esdp = erts_get_scheduler_data();
+
+ ASSERT(esdp);
restart:
@@ -1612,12 +1615,10 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
#endif
/* Figure out timeout value */
- if (do_wait) {
- erts_time_remaining(&wait_time);
- } else { /* poll only */
- wait_time.tv_sec = 0;
- wait_time.tv_usec = 0;
- }
+ timeout_time = (do_wait
+ ? erts_check_next_timeout_time(esdp->timer_wheel,
+ ERTS_SEC_TO_MONOTONIC(10*60))
+ : ERTS_POLL_NO_TIMEOUT /* poll only */);
/*
* No need for an atomic inc op when incrementing
@@ -1640,14 +1641,12 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
erts_smp_atomic_set_nob(&pollset.in_poll_wait, 1);
- poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, &wait_time);
+ poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, timeout_time);
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_check_exact(NULL, 0); /* No locks should be locked */
#endif
- erts_deliver_time(); /* sync the machine's idea of time */
-
#ifdef ERTS_BREAK_REQUESTED
if (ERTS_BREAK_REQUESTED)
erts_do_break_handling();
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index d01297d55c..71355965aa 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -47,8 +47,8 @@ void erts_check_io_async_sig_interrupt_nkp(void);
#endif
void erts_check_io_interrupt_kp(int);
void erts_check_io_interrupt_nkp(int);
-void erts_check_io_interrupt_timed_kp(int, erts_short_time_t);
-void erts_check_io_interrupt_timed_nkp(int, erts_short_time_t);
+void erts_check_io_interrupt_timed_kp(int, ErtsMonotonicTime);
+void erts_check_io_interrupt_timed_nkp(int, ErtsMonotonicTime);
void erts_check_io_kp(int);
void erts_check_io_nkp(int);
void erts_init_check_io_kp(void);
@@ -65,7 +65,7 @@ int erts_check_io_max_files(void);
void erts_check_io_async_sig_interrupt(void);
#endif
void erts_check_io_interrupt(int);
-void erts_check_io_interrupt_timed(int, erts_short_time_t);
+void erts_check_io_interrupt_timed(int, ErtsMonotonicTime);
void erts_check_io(int);
void erts_init_check_io(void);
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index aa412a20c8..f4d4a85ca4 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -320,7 +320,7 @@ struct ErtsPollSet_ {
#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
erts_atomic32_t wakeup_state;
#endif
- erts_smp_atomic32_t timeout;
+ erts_atomic64_t timeout_time;
#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
erts_smp_atomic_t no_avoided_wakeups;
erts_smp_atomic_t no_avoided_interrupts;
@@ -384,6 +384,26 @@ static void check_poll_status(ErtsPollSet ps);
static void print_misc_debug_info(void);
#endif
+static ERTS_INLINE void
+init_timeout_time(ErtsPollSet ps)
+{
+ erts_atomic64_init_nob(&ps->timeout_time,
+ (erts_aint64_t) ERTS_MONOTONIC_TIME_MAX);
+}
+
+static ERTS_INLINE void
+set_timeout_time(ErtsPollSet ps, ErtsMonotonicTime time)
+{
+ erts_atomic64_set_relb(&ps->timeout_time,
+ (erts_aint64_t) time);
+}
+
+static ERTS_INLINE ErtsMonotonicTime
+get_timeout_time(ErtsPollSet ps)
+{
+ return (ErtsMonotonicTime) erts_atomic64_read_acqb(&ps->timeout_time);
+}
+
#define ERTS_POLL_NOT_WOKEN 0
#define ERTS_POLL_WOKEN -1
#define ERTS_POLL_WOKEN_INTR 1
@@ -1993,44 +2013,153 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
}
}
+static ERTS_INLINE ErtsMonotonicTime
+get_timeout(ErtsPollSet ps,
+ int resolution,
+ ErtsMonotonicTime timeout_time)
+{
+ ErtsMonotonicTime timeout, save_timeout_time;
+
+ if (timeout_time == ERTS_POLL_NO_TIMEOUT) {
+ save_timeout_time = ERTS_MONOTONIC_TIME_MIN;
+ timeout = 0;
+ }
+ else {
+ ErtsMonotonicTime diff_time, current_time;
+ current_time = erts_get_monotonic_time();
+ diff_time = timeout_time - current_time;
+ if (diff_time <= 0) {
+ save_timeout_time = ERTS_MONOTONIC_TIME_MIN;
+ timeout = 0;
+ }
+ else {
+ save_timeout_time = current_time;
+ switch (resolution) {
+ case 1000:
+ /* Round up to nearest even milli second */
+ timeout = ERTS_MONOTONIC_TO_MSEC(diff_time - 1) + 1;
+ if (timeout > (ErtsMonotonicTime) INT_MAX)
+ timeout = (ErtsMonotonicTime) INT_MAX;
+ save_timeout_time += ERTS_MSEC_TO_MONOTONIC(timeout);
+ break;
+ case 1000000:
+ /* Round up to nearest even micro second */
+ timeout = ERTS_MONOTONIC_TO_USEC(diff_time - 1) + 1;
+ save_timeout_time += ERTS_USEC_TO_MONOTONIC(timeout);
+ break;
+ case 1000000000:
+ /* Round up to nearest even nano second */
+ timeout = ERTS_MONOTONIC_TO_NSEC(diff_time - 1) + 1;
+ save_timeout_time += ERTS_NSEC_TO_MONOTONIC(timeout);
+ break;
+ default:
+ ERTS_INTERNAL_ERROR("Invalid resolution");
+ timeout = 0;
+ save_timeout_time = 0;
+ break;
+ }
+ }
+ }
+ set_timeout_time(ps, save_timeout_time);
+ return timeout;
+}
+
+#if ERTS_POLL_USE_SELECT
+
static ERTS_INLINE int
-check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res)
+get_timeout_timeval(ErtsPollSet ps,
+ SysTimeval *tvp,
+ ErtsMonotonicTime timeout_time)
+{
+ ErtsMonotonicTime timeout = get_timeout(ps,
+ 1000*1000,
+ timeout_time);
+
+ if (!timeout) {
+ tvp->tv_sec = 0;
+ tvp->tv_usec = 0;
+
+ return 0;
+ }
+ else {
+ ErtsMonotonicTime sec = timeout/(1000*1000);
+ tvp->tv_sec = sec;
+ tvp->tv_usec = timeout - sec*(1000*1000);
+
+ ASSERT(tvp->tv_sec >= 0);
+ ASSERT(tvp->tv_usec >= 0);
+ ASSERT(tvp->tv_usec < 1000*1000);
+
+ return !0;
+ }
+
+}
+
+#endif
+
+#if ERTS_POLL_USE_KQUEUE
+
+static ERTS_INLINE int
+get_timeout_timespec(ErtsPollSet ps,
+ struct timespec *tsp,
+ ErtsMonotonicTime timeout_time)
+{
+ ErtsMonotonicTime timeout = get_timeout(ps,
+ 1000*1000*1000,
+ timeout_time);
+
+ if (!timeout) {
+ tsp->tv_sec = 0;
+ tsp->tv_nsec = 0;
+ return 0;
+ }
+ else {
+ ErtsMonotonicTime sec = timeout/(1000*1000*1000);
+ tsp->tv_sec = sec;
+ tsp->tv_nsec = timeout - sec*(1000*1000*1000);
+
+ ASSERT(tsp->tv_sec >= 0);
+ ASSERT(tsp->tv_nsec >= 0);
+ ASSERT(tsp->tv_nsec < 1000*1000);
+
+ return !0;
+ }
+}
+
+#endif
+
+static ERTS_INLINE int
+check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
{
int res;
if (erts_smp_atomic_read_nob(&ps->no_of_user_fds) == 0
- && tv->tv_usec == 0 && tv->tv_sec == 0) {
+ && timeout_time == ERTS_POLL_NO_TIMEOUT) {
/* Nothing to poll and zero timeout; done... */
return 0;
}
else {
- long timeout = tv->tv_sec*1000 + tv->tv_usec/1000;
- if (timeout > ERTS_AINT32_T_MAX)
- timeout = ERTS_AINT32_T_MAX;
- ASSERT(timeout >= 0);
- erts_smp_atomic32_set_relb(&ps->timeout, (erts_aint32_t) timeout);
+ int timeout;
#if ERTS_POLL_USE_FALLBACK
if (!(ps->fallback_used = ERTS_POLL_NEED_FALLBACK(ps))) {
#if ERTS_POLL_USE_EPOLL /* --- epoll ------------------------------- */
- if (timeout > INT_MAX)
- timeout = INT_MAX;
if (max_res > ps->res_events_len)
grow_res_events(ps, max_res);
+ timeout = (int) get_timeout(ps, 1000, timeout_time);
#ifdef ERTS_SMP
if (timeout)
erts_thr_progress_prepare_wait(NULL);
#endif
- res = epoll_wait(ps->kp_fd, ps->res_events, max_res, (int)timeout);
+ res = epoll_wait(ps->kp_fd, ps->res_events, max_res, timeout);
#elif ERTS_POLL_USE_KQUEUE /* --- kqueue ------------------------------ */
struct timespec ts;
if (max_res > ps->res_events_len)
grow_res_events(ps, max_res);
+ timeout = get_timeout_timespec(ps, &ts, timeout_time);
#ifdef ERTS_SMP
if (timeout)
erts_thr_progress_prepare_wait(NULL);
#endif
- ts.tv_sec = tv->tv_sec;
- ts.tv_nsec = tv->tv_usec*1000;
res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts);
#endif /* ----------------------------------------- */
}
@@ -2049,8 +2178,7 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res)
#if ERTS_POLL_USE_WAKEUP_PIPE
nfds++; /* Wakeup pipe */
#endif
- if (timeout > INT_MAX)
- timeout = INT_MAX;
+ timeout = (int) get_timeout(ps, 1000, timeout_time);
poll_res.dp_nfds = nfds < max_res ? nfds : max_res;
if (poll_res.dp_nfds > ps->res_events_len)
grow_res_events(ps, poll_res.dp_nfds);
@@ -2059,33 +2187,33 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res)
if (timeout)
erts_thr_progress_prepare_wait(NULL);
#endif
- poll_res.dp_timeout = (int) timeout;
+ poll_res.dp_timeout = timeout;
res = ioctl(ps->kp_fd, DP_POLL, &poll_res);
#elif ERTS_POLL_USE_POLL /* --- poll -------------------------------- */
- if (timeout > INT_MAX)
- timeout = INT_MAX;
+ timeout = (int) get_timeout(ps, 1000, timeout_time);
#ifdef ERTS_SMP
if (timeout)
erts_thr_progress_prepare_wait(NULL);
#endif
- res = poll(ps->poll_fds, ps->no_poll_fds, (int) timeout);
+ res = poll(ps->poll_fds, ps->no_poll_fds, timeout);
#elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */
- SysTimeval to = *tv;
+ SysTimeval to;
+ timeout = get_timeout_timeval(ps, &to, timeout_time);
ERTS_FD_COPY(&ps->input_fds, &ps->res_input_fds);
ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds);
#ifdef ERTS_SMP
- if (to.tv_sec || to.tv_usec)
+ if (timeout)
erts_thr_progress_prepare_wait(NULL);
#endif
res = ERTS_SELECT(ps->max_fd + 1,
- &ps->res_input_fds,
- &ps->res_output_fds,
- NULL,
- &to);
+ &ps->res_input_fds,
+ &ps->res_output_fds,
+ NULL,
+ &to);
#ifdef ERTS_SMP
- if (to.tv_sec || to.tv_usec)
+ if (timeout)
erts_thr_progress_finalize_wait(NULL);
if (res < 0
&& errno == EBADF
@@ -2108,10 +2236,10 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res)
handle_update_requests(ps);
ERTS_POLLSET_UNLOCK(ps);
res = ERTS_SELECT(ps->max_fd + 1,
- &ps->res_input_fds,
- &ps->res_output_fds,
- NULL,
- &to);
+ &ps->res_input_fds,
+ &ps->res_output_fds,
+ NULL,
+ &to);
if (res == 0) {
errno = EAGAIN;
res = -1;
@@ -2133,15 +2261,14 @@ int
ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
ErtsPollResFd pr[],
int *len,
- SysTimeval *utvp)
+ ErtsMonotonicTime timeout_time)
{
+ ErtsMonotonicTime to;
int res, no_fds;
int ebadf = 0;
#ifdef ERTS_SMP
int ps_locked = 0;
#endif
- SysTimeval *tvp;
- SysTimeval itv;
no_fds = *len;
#ifdef ERTS_POLL_MAX_RES
@@ -2151,13 +2278,9 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
*len = 0;
- ASSERT(utvp);
-
- tvp = utvp;
-
#ifdef ERTS_POLL_DEBUG_PRINT
- erts_printf("Entering erts_poll_wait(), timeout=%d\n",
- (int) tvp->tv_sec*1000 + tvp->tv_usec/1000);
+ erts_printf("Entering erts_poll_wait(), timeout_time=%bps\n",
+ timeout_time);
#endif
if (ERTS_POLLSET_SET_POLLED_CHK(ps)) {
@@ -2166,12 +2289,9 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
goto done;
}
- if (is_woken(ps)) {
- /* Use zero timeout */
- itv.tv_sec = 0;
- itv.tv_usec = 0;
- tvp = &itv;
- }
+ to = (is_woken(ps)
+ ? ERTS_POLL_NO_TIMEOUT /* Use zero timeout */
+ : timeout_time);
#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
if (ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) {
@@ -2181,7 +2301,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
}
#endif
- res = check_fd_events(ps, tvp, no_fds);
+ res = check_fd_events(ps, to, no_fds);
woke_up(ps);
@@ -2224,7 +2344,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
#endif
done:
- erts_smp_atomic32_set_relb(&ps->timeout, ERTS_AINT32_T_MAX);
+ set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX);
#ifdef ERTS_POLL_DEBUG_PRINT
erts_printf("Leaving %s = erts_poll_wait()\n",
res == 0 ? "0" : erl_errno_id(res));
@@ -2268,13 +2388,14 @@ ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet ps)
void
ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps,
int set,
- erts_short_time_t msec)
+ ErtsMonotonicTime timeout_time)
{
#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP)
if (!set)
reset_wakeup_state(ps);
else {
- if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec)
+ ErtsMonotonicTime max_wait_time = get_timeout_time(ps);
+ if (max_wait_time > timeout_time)
wake_poller(ps, 1, 0);
#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
else {
@@ -2431,7 +2552,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
ps->internal_fd_limit = kp_fd + 1;
ps->kp_fd = kp_fd;
#endif
- erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX);
+ init_timeout_time(ps);
#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
erts_smp_atomic_init_nob(&ps->no_avoided_wakeups, 0);
erts_smp_atomic_init_nob(&ps->no_avoided_interrupts, 0);
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index 2f1c05f401..d02ed2396b 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -29,6 +29,8 @@
#include "sys.h"
+#define ERTS_POLL_NO_TIMEOUT ERTS_MONOTONIC_TIME_MIN
+
#if 0
#define ERTS_POLL_COUNT_AVOIDED_WAKEUPS
#endif
@@ -241,7 +243,7 @@ void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet,
int);
void ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet,
int,
- erts_short_time_t);
+ ErtsMonotonicTime);
ErtsPollEvents ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet,
ErtsSysFdType,
ErtsPollEvents,
@@ -254,7 +256,7 @@ void ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet,
int ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet,
ErtsPollResFd [],
int *,
- SysTimeval *);
+ ErtsMonotonicTime);
int ERTS_POLL_EXPORT(erts_poll_max_fds)(void);
void ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet,
ErtsPollInfo *);
diff --git a/erts/emulator/sys/ose/erl_poll.c b/erts/emulator/sys/ose/erl_poll.c
index 7d2a3d1e0b..36ee2557e8 100644
--- a/erts/emulator/sys/ose/erl_poll.c
+++ b/erts/emulator/sys/ose/erl_poll.c
@@ -114,7 +114,7 @@ struct ErtsPollSet_ {
Uint item_count;
PROCESS interrupt;
erts_atomic32_t wakeup_state;
- erts_smp_atomic32_t timeout;
+ erts_atomic64_t timeout_time;
#ifdef ERTS_SMP
erts_smp_mtx_t mtx;
#endif
@@ -122,6 +122,26 @@ struct ErtsPollSet_ {
static int max_fds = -1;
+static ERTS_INLINE void
+init_timeout_time(ErtsPollSet ps)
+{
+ erts_atomic64_init_nob(&ps->timeout_time,
+ (erts_aint64_t) ERTS_MONOTONIC_TIME_MAX);
+}
+
+static ERTS_INLINE void
+set_timeout_time(ErtsPollSet ps, ErtsMonotonicTime time)
+{
+ erts_atomic64_set_relb(&ps->timeout_time,
+ (erts_aint64_t) time);
+}
+
+static ERTS_INLINE ErtsMonotonicTime
+get_timeout_time(ErtsPollSet ps)
+{
+ return (ErtsMonotonicTime) erts_atomic64_read_acqb(&ps->timeout_time);
+}
+
#define ERTS_POLL_NOT_WOKEN ((erts_aint32_t) (1 << 0))
#define ERTS_POLL_WOKEN_INTR ((erts_aint32_t) (1 << 1))
#define ERTS_POLL_WOKEN_TIMEDOUT ((erts_aint32_t) (1 << 2))
@@ -386,12 +406,14 @@ void erts_poll_interrupt(ErtsPollSet ps,int set) {
}
-void erts_poll_interrupt_timed(ErtsPollSet ps,int set,erts_short_time_t msec) {
+void erts_poll_interrupt_timed(ErtsPollSet ps,
+ int set,
+ ErtsTimeoutTime timeout_time) {
HARDTRACEF("erts_poll_interrupt_timed called!\n");
if (!set)
reset_interrupt(ps);
- else if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec)
+ else if (get_timeout_time(ps) > timeout_time)
set_interrupt(ps);
}
@@ -453,12 +475,14 @@ done:
}
int erts_poll_wait(ErtsPollSet ps,
- ErtsPollResFd pr[],
- int *len,
- SysTimeval *utvp) {
+ ErtsPollResFd pr[],
+ int *len,
+ ErtsMonotonicTime timeout_time)
+{
int res = ETIMEDOUT, no_fds, currid = 0;
OSTIME timeout;
union SIGNAL *sig;
+ ErtsMonotonicTime current_time, diff_time, timeout;
// HARDTRACEF("%ux: In erts_poll_wait",ps);
if (ps->interrupt == (PROCESS)0)
ps->interrupt = current_process();
@@ -472,16 +496,29 @@ int erts_poll_wait(ErtsPollSet ps,
*len = 0;
- ASSERT(utvp);
+ /* erts_printf("Entering erts_poll_wait(), timeout_time=%bps\n",
+ timeout_time); */
- /* erts_printf("Entering erts_poll_wait(), timeout=%d\n",
- (int) utvp->tv_sec*1000 + utvp->tv_usec/1000); */
-
- timeout = utvp->tv_sec*1000 + utvp->tv_usec/1000;
+ if (timeout_time == ERTS_POLL_NO_TIMEOUT) {
+ no_timeout:
+ timeout = (OSTIME) 0;
+ save_timeout_time = ERTS_MONOTONIC_TIME_MIN;
+ }
+ else {
+ ErtsMonotonicTime current_time, diff_time;
+ current_time = erts_get_monotonic_time();
+ diff_time = timeout_time - current_time;
+ if (diff_time <= 0)
+ goto no_timeout;
+ diff_time = (ERTS_MONOTONIC_TO_MSEC(diff_time - 1) + 1);
+ if (diff_time > INT_MAX)
+ diff_time = INT_MAX;
+ timeout = (OSTIME) diff_time;
+ save_timeout_time = current_time;
+ save_timeout_time += ERTS_MSEC_TO_MONOTONIC(diff_time);
+ }
- if (timeout > ((time_t) ERTS_AINT32_T_MAX))
- timeout = ERTS_AINT32_T_MAX;
- erts_smp_atomic32_set_relb(&ps->timeout, (erts_aint32_t) timeout);
+ set_timeout_time(ps, save_timeout_time);
while (currid < no_fds) {
if (timeout > 0) {
@@ -627,7 +664,7 @@ int erts_poll_wait(ErtsPollSet ps,
}
erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
- erts_smp_atomic32_set_nob(&ps->timeout, ERTS_AINT32_T_MAX);
+ set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX);
*len = currid;
@@ -690,7 +727,7 @@ ErtsPollSet erts_poll_create_pollset(void)
ps->info = NULL;
ps->interrupt = (PROCESS)0;
erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
- erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX);
+ init_timeout_time(ps);
#ifdef ERTS_SMP
erts_smp_mtx_init(&ps->mtx, "pollset");
#endif
diff --git a/erts/emulator/sys/ose/sys.c b/erts/emulator/sys/ose/sys.c
index 5b950a7dae..13a5b71496 100644
--- a/erts/emulator/sys/ose/sys.c
+++ b/erts/emulator/sys/ose/sys.c
@@ -298,9 +298,9 @@ erts_sys_schedule_interrupt(int set)
#ifdef ERTS_SMP
void
-erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec)
+erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time)
{
- ERTS_CHK_IO_INTR_TMD(set, msec);
+ ERTS_CHK_IO_INTR_TMD(set, timeout_time);
}
#endif
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index f0050db114..46d0b8dc9b 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -114,11 +114,6 @@
/*
* Make sure that MAXPATHLEN is defined.
*/
-#ifdef GETHRTIME_WITH_CLOCK_GETTIME
-#undef HAVE_GETHRTIME
-#define HAVE_GETHRTIME 1
-#endif
-
#ifndef MAXPATHLEN
# ifdef PATH_MAX
# define MAXPATHLEN PATH_MAX
@@ -160,33 +155,112 @@ typedef struct timeval SysTimeval;
typedef struct tms SysTimes;
-extern int erts_ticks_per_sec;
-
-#define SYS_CLK_TCK (erts_ticks_per_sec)
+#define SYS_CLK_TCK (erts_sys_time_data__.r.o.ticks_per_sec)
#define sys_times(Arg) times(Arg)
-#define ERTS_WRAP_SYS_TIMES 1
-extern int erts_ticks_per_sec_wrap;
-#define SYS_CLK_TCK_WRAP (erts_ticks_per_sec_wrap)
-extern clock_t sys_times_wrap(void);
+#if SIZEOF_LONG == 8
+typedef long ErtsMonotonicTime;
+#elif SIZEOF_LONG_LONG == 8
+typedef long long ErtsMonotonicTime;
+#else
+#error No signed 64-bit type found...
+#endif
+
+#define ERTS_MONOTONIC_TIME_MIN (((ErtsMonotonicTime) 1) << 63)
+#define ERTS_MONOTONIC_TIME_MAX (~ERTS_MONOTONIC_TIME_MIN)
+
+/*
+ * OS monotonic time
+ */
+
+/*
+ * Most common with os monotonic time using nano second
+ * time unit. These defines are modified below if this
+ * isn't the case...
+ */
+#define ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT 1
+#define ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT (1000*1000*1000)
+
+#undef ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__
+#undef ERTS_HAVE_CORRECTED_OS_MONOTONIC
+
+#if defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
+
+#if defined(__linux__)
-#ifdef HAVE_GETHRTIME
-#ifdef GETHRTIME_WITH_CLOCK_GETTIME
-typedef long long SysHrTime;
+#define ERTS_HAVE_CORRECTED_OS_MONOTONIC 1
+#define ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__ 1
-extern SysHrTime sys_gethrtime(void);
-#define sys_init_hrtime() /* Nothing */
+#else /* !defined(__linux__) */
-#else /* Real gethrtime (Solaris) */
+ErtsMonotonicTime erts_os_monotonic_time(void);
-typedef hrtime_t SysHrTime;
+#endif /* !defined(__linux__) */
+
+#elif defined(OS_MONOTONIC_TIME_USING_GETHRTIME)
+
+#define erts_os_monotonic() ((ErtsMonotonicTime) gethrtime())
+
+#elif defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME) \
+ || defined(OS_MONOTONIC_TIME_USING_TIMES)
+
+#if defined(OS_MONOTONIC_TIME_USING_TIMES)
+# undef ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
+# define ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT (1000*1000)
+# define ERTS_HAVE_ERTS_OS_TIME_OFFSET_FINALIZE 1
+void erts_os_time_offset_finalize(void);
+# define ERTS_HAVE_ERTS_OS_MONOTONIC_TIME_INIT
+void erts_os_monotonic_time_init(void);
+#endif
-#define sys_gethrtime() gethrtime()
-#define sys_init_hrtime() /* Nothing */
+ErtsMonotonicTime erts_os_monotonic_time(void);
-#endif /* GETHRTIME_WITH_CLOCK_GETTIME */
-#endif /* HAVE_GETHRTIME */
+#else /* No OS monotonic available... */
+
+#undef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+#undef ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
+#define ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT (1000*1000)
+
+#endif
+
+struct erts_sys_time_read_only_data__ {
+#ifdef ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__
+ ErtsMonotonicTime (*os_monotonic_time)(void);
+#endif
+ int ticks_per_sec;
+};
+
+typedef struct {
+ union {
+ struct erts_sys_time_read_only_data__ o;
+ char align__[(((sizeof(struct erts_sys_time_read_only_data__) - 1)
+ / ASSUMED_CACHE_LINE_SIZE) + 1)
+ * ASSUMED_CACHE_LINE_SIZE];
+ } r;
+} ErtsSysTimeData__;
+
+extern ErtsSysTimeData__ erts_sys_time_data__;
+
+#ifdef ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__
+
+ERTS_GLB_INLINE ErtsMonotonicTime erts_os_monotonic_time(void);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE ErtsMonotonicTime
+erts_os_monotonic_time(void)
+{
+ return (*erts_sys_time_data__.r.o.os_monotonic_time)();
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__ */
+
+/*
+ *
+ */
#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME))
typedef long long SysCpuTime;
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 0d9c743c0c..2eb0ffa905 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -308,7 +308,7 @@ struct {
int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
void (*check_io_as_interrupt)(void);
void (*check_io_interrupt)(int);
- void (*check_io_interrupt_tmd)(int, erts_short_time_t);
+ void (*check_io_interrupt_tmd)(int, ErtsMonotonicTime);
void (*check_io)(int);
Uint (*size)(void);
Eterm (*info)(void *);
@@ -414,9 +414,9 @@ erts_sys_schedule_interrupt(int set)
#ifdef ERTS_SMP
void
-erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec)
+erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time)
{
- ERTS_CHK_IO_INTR_TMD(set, msec);
+ ERTS_CHK_IO_INTR_TMD(set, timeout_time);
}
#endif
@@ -1003,25 +1003,6 @@ static void unblock_signals(void)
}
-/************************** Time stuff **************************/
-#ifdef HAVE_GETHRTIME
-#ifdef GETHRTIME_WITH_CLOCK_GETTIME
-
-SysHrTime sys_gethrtime(void)
-{
- struct timespec ts;
- long long result;
- if (clock_gettime(CLOCK_MONOTONIC,&ts) != 0) {
- erl_exit(1,"Fatal, could not get clock_monotonic value!, "
- "errno = %d\n", errno);
- }
- result = ((long long) ts.tv_sec) * 1000000000LL +
- ((long long) ts.tv_nsec);
- return (SysHrTime) result;
-}
-#endif
-#endif
-
/************************** OS info *******************************/
/* Used by erlang:info/1. */
diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c
index fcce54a2c4..9fdb1930b7 100644
--- a/erts/emulator/sys/unix/sys_time.c
+++ b/erts/emulator/sys/unix/sys_time.c
@@ -53,9 +53,40 @@
/******************* Routines for time measurement *********************/
-int erts_ticks_per_sec = 0; /* Will be SYS_CLK_TCK in erl_unix_sys.h */
-int erts_ticks_per_sec_wrap = 0; /* Will be SYS_CLK_TCK_WRAP */
-static int ticks_bsr = 0; /* Shift wrapped tick value this much to the right */
+#undef ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
+#undef ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__
+
+#if defined(OS_MONOTONIC_TIME_USING_TIMES)
+
+#define ERTS_WRAP_SYS_TIMES 1
+#define ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
+#define ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__
+
+/*
+ * Not sure there is a need to use times() anymore, perhaps drop
+ * support for this soon...
+ *
+ * sys_times() might need to be wrapped and the values shifted (right)
+ * a bit to cope with faster ticks, this has to be taken care
+ * of dynamically to start with, a special version that uses
+ * the times() return value as a high resolution timer can be made
+ * to fully utilize the faster ticks, like on windows, but for now, we'll
+ * settle with this silly workaround
+ */
+#ifdef ERTS_WRAP_SYS_TIMES
+static clock_t sys_times_wrap(void);
+#define KERNEL_TICKS() (sys_times_wrap() & \
+ ((1UL << ((sizeof(clock_t) * 8) - 1)) - 1))
+#define ERTS_KERNEL_TICK_TO_USEC(TCKS) (((TCKS)*(1000*1000)) \
+ / internal_state.r.o.ticks_per_sec_wrap)
+#else
+
+#define KERNEL_TICKS() (sys_times(&internal_state.w.f.dummy_tms) & \
+ ((1UL << ((sizeof(clock_t) * 8) - 1)) - 1))
+#define ERTS_KERNEL_TICK_TO_USEC(TCKS) (((TCKS)*(1000*1000))/SYS_CLK_TCK)
+#endif
+
+#endif
/*
* init timers, chose a tick length, and return it.
@@ -63,37 +94,374 @@ static int ticks_bsr = 0; /* Shift wrapped tick value this much to the right */
* does almost everything. Other platforms have to
* emulate Unix in this sense.
*/
-int sys_init_time(void)
+
+ErtsSysTimeData__ erts_sys_time_data__ erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+#if defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
+
+#define ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
+
+ErtsMonotonicTime clock_gettime_monotonic_raw(void);
+ErtsMonotonicTime clock_gettime_monotonic_verified(void);
+
+#endif /* defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME) */
+
+#ifdef ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__
+struct sys_time_internal_state_read_only__ {
+#if defined(OS_MONOTONIC_TIME_USING_TIMES)
+ int ticks_bsr;
+ int ticks_per_sec_wrap;
+#endif
+};
+#endif
+
+#ifdef ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
+struct sys_time_internal_state_write_freq__ {
+ erts_smp_mtx_t mtx;
+#if defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
+ ErtsMonotonicTime last_delivered;
+#endif
+#if defined(OS_MONOTONIC_TIME_USING_TIMES)
+ ErtsMonotonicTime last_tick_count;
+ ErtsMonotonicTime last_tick_wrap_count;
+ ErtsMonotonicTime last_tick_monotonic_time;
+ ErtsMonotonicTime last_timeofday_usec;
+#ifndef ERTS_WRAP_SYS_TIMES
+ SysTimes dummy_tms;
+#endif
+#endif
+};
+#endif
+
+#if defined(ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__) \
+ || defined(ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__)
+static struct {
+#ifdef ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__
+ union {
+ struct sys_time_internal_state_read_only__ o;
+ char align__[(((sizeof(struct sys_time_internal_state_read_only__) - 1)
+ / ASSUMED_CACHE_LINE_SIZE) + 1)
+ * ASSUMED_CACHE_LINE_SIZE];
+ } r;
+#endif
+#ifdef ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
+ union {
+ struct sys_time_internal_state_write_freq__ f;
+ char align__[(((sizeof(struct sys_time_internal_state_write_freq__) - 1)
+ / ASSUMED_CACHE_LINE_SIZE) + 1)
+ * ASSUMED_CACHE_LINE_SIZE];
+ } w;
+#endif
+} internal_state erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+#endif
+
+void
+sys_init_time(ErtsSysInitTimeResult *init_resp)
{
+#if !defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT)
+
+ init_resp->have_os_monotonic = 0;
+
+#else /* defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT) */
+
+ int major, minor, build, vsn;
+
+ init_resp->os_monotonic_info.resolution = (Uint64) 1000*1000*1000;
+#if defined(HAVE_CLOCK_GETRES) && defined(MONOTONIC_CLOCK_ID)
+ {
+ struct timespec ts;
+ if (clock_getres(MONOTONIC_CLOCK_ID, &ts) == 0
+ && ts.tv_sec == 0 && ts.tv_nsec != 0) {
+ init_resp->os_monotonic_info.resolution /= ts.tv_nsec;
+ }
+ }
+#endif
+
+#ifdef MONOTONIC_CLOCK_ID_STR
+ init_resp->os_monotonic_info.clock_id = MONOTONIC_CLOCK_ID_STR;
+#else
+ init_resp->os_monotonic_info.clock_id = NULL;
+#endif
+
+ init_resp->os_monotonic_info.locked_use = 0;
+
+#if defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
+ init_resp->os_monotonic_info.func = "clock_gettime";
+#elif defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME)
+ init_resp->os_monotonic_info.func = "clock_get_time";
+#elif defined(OS_MONOTONIC_TIME_USING_GETHRTIME)
+ init_resp->os_monotonic_info.func = "gethrtime";
+#elif defined(OS_MONOTONIC_TIME_USING_TIMES)
+ init_resp->os_monotonic_info.func = "times";
+ init_resp->os_monotonic_info.locked_use = 1;
+ init_resp->os_monotonic_info.resolution = TICKS_PER_SEC();
+#else
+# error Unknown erts_os_monotonic_time() implementation
+#endif
+
+ init_resp->have_os_monotonic = 1;
+
+ os_version(&major, &minor, &build);
+
+ vsn = ERTS_MK_VSN_INT(major, minor, build);
+
+
+#if defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
+ if (vsn >= ERTS_MK_VSN_INT(2, 6, 33))
+ erts_sys_time_data__.r.o.os_monotonic_time =
+ clock_gettime_monotonic_raw;
+ else {
+ /*
+ * Linux versions prior to 2.6.33 have a
+ * known bug that sometimes cause monotonic
+ * time to take small steps backwards.
+ */
+ erts_sys_time_data__.r.o.os_monotonic_time =
+ clock_gettime_monotonic_verified;
+ erts_smp_mtx_init(&internal_state.w.f.mtx,
+ "os_monotonic_time");
+ internal_state.w.f.last_delivered
+ = clock_gettime_monotonic_raw();
+ init_resp->os_monotonic_info.locked_use = 1;
+ }
+#else /* !(defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)) */
+ {
+ char flavor[1024];
+
+ os_flavor(flavor, sizeof(flavor));
+
+ if (sys_strcmp(flavor, "sunos") == 0) {
+ /*
+ * Don't trust hrtime on multi processors
+ * on SunOS prior to SunOS 5.8
+ */
+ if (vsn < ERTS_MK_VSN_INT(5, 8, 0)) {
+#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_CONF)
+ if (sysconf(_SC_NPROCESSORS_CONF) > 1)
+#endif
+ init_resp->have_os_monotonic = 0;
+ }
+ }
+ }
+#endif /* !(defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)) */
+
+#endif /* defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT) */
+
+ init_resp->os_monotonic_time_unit = ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT;
+ init_resp->sys_clock_resolution = SYS_CLOCK_RESOLUTION;
+
/*
- * This (erts_ticks_per_sec) is only for times() (CLK_TCK),
- * the resolution is always one millisecond..
+ * This (erts_sys_time_data__.r.o.ticks_per_sec) is only for
+ * times() (CLK_TCK), the resolution is always one millisecond..
*/
- if ((erts_ticks_per_sec = TICKS_PER_SEC()) < 0)
- erl_exit(1, "Can't get clock ticks/sec\n");
- if (erts_ticks_per_sec >= 1000) {
+ if ((erts_sys_time_data__.r.o.ticks_per_sec = TICKS_PER_SEC()) < 0)
+ erl_exit(ERTS_ABORT_EXIT, "Can't get clock ticks/sec\n");
+
+#if defined(OS_MONOTONIC_TIME_USING_TIMES)
+
+ if (erts_sys_time_data__.r.o.ticks_per_sec >= 1000) {
/* Workaround for beta linux kernels, need to be done in runtime
to make erlang run on both 2.4 and 2.5 kernels. In the future,
the kernel ticks might as
well be used as a high res timer instead, but that's for when the
majority uses kernels with HZ == 1024 */
- ticks_bsr = 3;
+ internal_state.r.o.ticks_bsr = 3;
} else {
- ticks_bsr = 0;
+ internal_state.r.o.ticks_bsr = 0;
}
- erts_ticks_per_sec_wrap = (erts_ticks_per_sec >> ticks_bsr);
- return SYS_CLOCK_RESOLUTION;
+
+ internal_state.r.o.ticks_per_sec_wrap
+ = (erts_sys_time_data__.r.o.ticks_per_sec
+ >> internal_state.r.o.ticks_bsr);
+
+ erts_smp_mtx_init(&internal_state.w.f.mtx, "os_monotonic_time");
+ internal_state.w.f.last_tick_count = KERNEL_TICKS();
+ internal_state.w.f.last_tick_wrap_count = 0;
+ internal_state.w.f.last_tick_monotonic_time
+ = ERTS_KERNEL_TICK_TO_USEC(internal_state.w.f.last_tick_count);
+ {
+ SysTimeval tv;
+ sys_gettimeofday(&tv);
+ internal_state.w.f.last_timeofday_usec = tv.tv_sec*(1000*1000);
+ internal_state.w.f.last_timeofday_usec += tv.tv_usec;
+ }
+
+#endif /* defined(OS_MONOTONIC_TIME_USING_TIMES) */
+
+}
+
+#if defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
+
+static ERTS_INLINE ErtsMonotonicTime
+clock_gettime_monotonic(void)
+{
+ ErtsMonotonicTime mtime;
+ struct timespec ts;
+
+ if (clock_gettime(MONOTONIC_CLOCK_ID,&ts) != 0) {
+ int err = errno;
+ char *errstr = err ? strerror(err) : "unknown";
+ erl_exit(ERTS_ABORT_EXIT,
+ "clock_gettime(%s, _) failed: %s (%d)\n",
+ MONOTONIC_CLOCK_ID_STR, errstr, err);
+
+ }
+ mtime = (ErtsMonotonicTime) ts.tv_sec;
+ mtime *= (ErtsMonotonicTime) 1000*1000*1000;
+ mtime += (ErtsMonotonicTime) ts.tv_nsec;
+ return mtime;
+}
+
+#if defined(__linux__)
+
+ErtsMonotonicTime clock_gettime_monotonic_verified(void)
+{
+ ErtsMonotonicTime mtime;
+
+ mtime = clock_gettime_monotonic();
+
+ erts_smp_mtx_lock(&internal_state.w.f.mtx);
+ if (mtime < internal_state.w.f.last_delivered)
+ mtime = internal_state.w.f.last_delivered;
+ else
+ internal_state.w.f.last_delivered = mtime;
+ erts_smp_mtx_unlock(&internal_state.w.f.mtx);
+
+ return mtime;
+}
+
+ErtsMonotonicTime clock_gettime_monotonic_raw(void)
+{
+ return clock_gettime_monotonic();
}
-clock_t sys_times_wrap(void)
+#else /* !defined(__linux__) */
+
+ErtsMonotonicTime erts_os_monotonic_time(void)
+{
+ return clock_gettime_monotonic();
+}
+
+#endif /* !defined(__linux__) */
+
+#elif defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME)
+
+#include <mach/clock.h>
+#include <mach/mach.h>
+
+ErtsMonotonicTime erts_os_monotonic_time(void)
+{
+ ErtsMonotonicTime mtime;
+ kern_return_t res;
+ clock_serv_t clk_srv;
+ mach_timespec_t time_spec;
+ int err;
+
+ host_get_clock_service(mach_host_self(),
+ MONOTONIC_CLOCK_ID,
+ &clk_srv);
+ errno = 0;
+ res = clock_get_time(clk_srv, &time_spec);
+ err = errno;
+ mach_port_deallocate(mach_task_self(), clk_srv);
+ if (res != KERN_SUCCESS) {
+ char *errstr = err ? strerror(err) : "unknown";
+ erl_exit(ERTS_ABORT_EXIT,
+ "clock_get_time(%s, _) failed: %s (%d)\n",
+ MONOTONIC_CLOCK_ID_STR, errstr, err);
+ }
+
+ mtime = (ErtsMonotonicTime) time_spec.tv_sec;
+ mtime *= (ErtsMonotonicTime) 1000*1000*1000;
+ mtime += (ErtsMonotonicTime) time_spec.tv_nsec;
+ return mtime;
+}
+
+#elif defined(OS_MONOTONIC_TIME_USING_TIMES)
+
+static clock_t sys_times_wrap(void)
{
SysTimes dummy;
- clock_t result = (sys_times(&dummy) >> ticks_bsr);
+ clock_t result = (sys_times(&dummy) >> internal_state.r.o.ticks_bsr);
return result;
}
+void
+erts_os_time_offset_finalize(void)
+{
+ erts_smp_mtx_lock(&internal_state.w.f.mtx);
+ internal_state.w.f.last_tick_wrap_count = 0;
+ erts_smp_mtx_unlock(&internal_state.w.f.mtx);
+}
+
+#define ERTS_TIME_EXCEED_TICK_LIMIT(SYS_TIME, TCK_TIME) \
+ (((Uint64) (SYS_TIME)) - (((Uint64) (TCK_TIME)) \
+ - ERTS_KERNEL_TICK_TO_USEC(1)) \
+ > ERTS_KERNEL_TICK_TO_USEC(2))
+
+/* Returns monotonic time in micro seconds */
+ErtsMonotonicTime
+erts_os_monotonic_time(void)
+{
+ SysTimeval tv;
+ ErtsMonotonicTime res;
+ ErtsMonotonicTime tick_count;
+ ErtsMonotonicTime tick_count_usec;
+ ErtsMonotonicTime tick_monotonic_time;
+ ErtsMonotonicTime timeofday_usec;
+ ErtsMonotonicTime timeofday_diff_usec;
+
+ erts_smp_mtx_lock(&internal_state.w.f.mtx);
+ tick_count = (ErtsMonotonicTime) KERNEL_TICKS();
+ sys_gettimeofday(&tv);
+
+ if (internal_state.w.f.last_tick_count > tick_count) {
+ internal_state.w.f.last_tick_wrap_count
+ += (((ErtsMonotonicTime) 1) << ((sizeof(clock_t) * 8) - 1));
+ }
+ internal_state.w.f.last_tick_count = tick_count;
+ tick_count += internal_state.w.f.last_tick_wrap_count;
+
+ tick_count_usec = ERTS_KERNEL_TICK_TO_USEC(tick_count);
+
+ timeofday_usec = (ErtsMonotonicTime) tv.tv_sec*(1000*1000);
+ timeofday_usec += (ErtsMonotonicTime) tv.tv_usec;
+ timeofday_diff_usec = timeofday_usec;
+ timeofday_diff_usec -= internal_state.w.f.last_timeofday_usec;
+ internal_state.w.f.last_timeofday_usec = timeofday_usec;
+
+ if (timeofday_diff_usec < 0) {
+ /* timeofday jumped backwards use tick count only... */
+ tick_monotonic_time = tick_count_usec;
+ }
+ else {
+ /* Use time diff from of timeofday if not off by too much... */
+ tick_monotonic_time = internal_state.w.f.last_tick_monotonic_time;
+ tick_monotonic_time += timeofday_diff_usec;
+
+ if (ERTS_TIME_EXCEED_TICK_LIMIT(tick_monotonic_time, tick_count_usec)) {
+ /*
+ * Value off by more than one tick from tick_count, i.e.
+ * timofday leaped one way or the other. We use
+ * tick_count_usec as is instead and unfortunately
+ * get lousy precision.
+ */
+ tick_monotonic_time = tick_count_usec;
+ }
+ }
+
+ if (internal_state.w.f.last_tick_monotonic_time < tick_monotonic_time)
+ internal_state.w.f.last_tick_monotonic_time = tick_monotonic_time;
+
+ res = internal_state.w.f.last_tick_monotonic_time;
+
+ erts_smp_mtx_unlock(&internal_state.w.f.mtx);
+
+ return res;
+}
+#endif /* !defined(OS_MONOTONIC_TIME_USING_TIMES) */
#ifdef HAVE_GETHRVTIME_PROCFS_IOCTL
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 972170d465..5a62b00a68 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -285,7 +285,7 @@ struct ErtsPollSet_ {
#ifdef ERTS_SMP
erts_smp_mtx_t mtx;
#endif
- erts_smp_atomic32_t timeout;
+ erts_atomic64_t timeout_time;
};
#ifdef ERTS_SMP
@@ -363,6 +363,26 @@ do { \
wait_standby(PS); \
} while(0)
+static ERTS_INLINE void
+init_timeout_time(ErtsPollSet ps)
+{
+ erts_atomic64_init_nob(&ps->timeout_time,
+ (erts_aint64_t) ERTS_MONOTONIC_TIME_MAX);
+}
+
+static ERTS_INLINE void
+set_timeout_time(ErtsPollSet ps, ErtsMonotonicTime time)
+{
+ erts_atomic64_set_relb(&ps->timeout_time,
+ (erts_aint64_t) time);
+}
+
+static ERTS_INLINE ErtsMonotonicTime
+get_timeout_time(ErtsPollSet ps)
+{
+ return (ErtsMonotonicTime) erts_atomic64_read_acqb(&ps->timeout_time);
+}
+
#define ERTS_POLL_NOT_WOKEN ((erts_aint32_t) 0)
#define ERTS_POLL_WOKEN_IO_READY ((erts_aint32_t) 1)
#define ERTS_POLL_WOKEN_INTR ((erts_aint32_t) 2)
@@ -422,15 +442,29 @@ wakeup_cause(ErtsPollSet ps)
}
static ERTS_INLINE DWORD
-poll_wait_timeout(ErtsPollSet ps, SysTimeval *tvp)
+poll_wait_timeout(ErtsPollSet ps, ErtsMonotonicTime timeout_time)
{
- time_t timeout = tvp->tv_sec * 1000 + tvp->tv_usec / 1000;
+ ErtsMonotonicTime current_time, diff_time, timeout;
- if (timeout <= 0) {
+ if (timeout_time == ERTS_POLL_NO_TIMEOUT) {
+ no_timeout:
+ set_timeout_time(ps, ERTS_MONOTONIC_TIME_MIN);
woke_up(ps);
return (DWORD) 0;
}
+ current_time = erts_get_monotonic_time();
+ diff_time = timeout_time - current_time;
+ if (diff_time <= 0)
+ goto no_timeout;
+
+ /* Round up to nearest milli second */
+ timeout = (ERTS_MONOTONIC_TO_MSEC(diff_time - 1) + 1);
+ if (timeout > INT_MAX)
+ timeout = INT_MAX; /* Also prevents DWORD overflow */
+
+ set_timeout_time(ps, current_time + ERTS_MSEC_TO_MONOTONIC(timeout));
+
ResetEvent(ps->event_io_ready);
/*
* Since we don't know the internals of ResetEvent() we issue
@@ -442,10 +476,6 @@ poll_wait_timeout(ErtsPollSet ps, SysTimeval *tvp)
if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN)
return (DWORD) 0;
- if (timeout > ((time_t) ERTS_AINT32_T_MAX))
- timeout = ERTS_AINT32_T_MAX; /* Also prevents DWORD overflow */
-
- erts_smp_atomic32_set_relb(&ps->timeout, (erts_aint32_t) timeout);
return (DWORD) timeout;
}
@@ -1012,12 +1042,12 @@ void erts_poll_interrupt(ErtsPollSet ps, int set /* bool */)
void erts_poll_interrupt_timed(ErtsPollSet ps,
int set /* bool */,
- erts_short_time_t msec)
+ ErtsMonotonicTime timeout_time)
{
- HARDTRACEF(("In erts_poll_interrupt_timed(%d,%ld)",set,msec));
+ HARDTRACEF(("In erts_poll_interrupt_timed(%d,%ld)",set,timeout_time));
if (!set)
reset_interrupt(ps);
- else if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec)
+ else if (get_timeout_time(ps) > timeout_time)
set_interrupt(ps);
HARDTRACEF(("Out erts_poll_interrupt_timed"));
}
@@ -1092,7 +1122,7 @@ void erts_poll_controlv(ErtsPollSet ps,
int erts_poll_wait(ErtsPollSet ps,
ErtsPollResFd pr[],
int *len,
- SysTimeval *tvp)
+ ErtsMonotonicTime timeout_time)
{
int no_fds;
DWORD timeout;
@@ -1149,7 +1179,7 @@ int erts_poll_wait(ErtsPollSet ps,
no_fds = ERTS_POLL_MAX_RES;
#endif
- timeout = poll_wait_timeout(ps, tvp);
+ timeout = poll_wait_timeout(ps, timeout_time);
/*HARDDEBUGF(("timeout = %ld",(long) timeout));*/
@@ -1242,7 +1272,7 @@ int erts_poll_wait(ErtsPollSet ps,
erts_mtx_unlock(&w->mtx);
}
done:
- erts_smp_atomic32_set_nob(&ps->timeout, ERTS_AINT32_T_MAX);
+ set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX);
*len = num;
ERTS_POLLSET_UNLOCK(ps);
HARDTRACEF(("Out erts_poll_wait"));
@@ -1326,7 +1356,7 @@ ErtsPollSet erts_poll_create_pollset(void)
#ifdef ERTS_SMP
erts_smp_mtx_init(&ps->mtx, "pollset");
#endif
- erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX);
+ init_timeout_time(ps);
HARDTRACEF(("Out erts_poll_create_pollset"));
return ps;
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index fde32c8684..33aa88ab5f 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -120,9 +120,6 @@
/*
* For erl_time_sup
*/
-#define HAVE_GETHRTIME
-
-#define sys_init_hrtime() /* Nothing */
#define SYS_CLK_TCK 1000
#define SYS_CLOCK_RESOLUTION 1
@@ -164,18 +161,58 @@ typedef struct {
#if defined (__GNUC__)
typedef unsigned long long Uint64;
typedef long long Sint64;
-
-typedef long long SysHrTime;
+# ifdef ULLONG_MAX
+# define ERTS_UINT64_MAX ULLONG_MAX
+# endif
+# ifdef LLONG_MAX
+# define ERTS_SINT64_MAX LLONG_MAX
+# endif
+# ifdef LLONG_MIN
+# define ERTS_SINT64_MIN LLONG_MIN
+# endif
+
+typedef long long ErtsMonotonicTime;
#else
typedef ULONGLONG Uint64;
typedef LONGLONG Sint64;
-typedef LONGLONG SysHrTime;
+typedef LONGLONG ErtsMonotonicTime;
#endif
-extern int sys_init_time(void);
+#define ERTS_MONOTONIC_TIME_MIN (((ErtsMonotonicTime) 1) << 63)
+#define ERTS_MONOTONIC_TIME_MAX (~ERTS_MONOTONIC_TIME_MIN)
+
+#define ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT 1
+#define ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT 0
+
+struct erts_sys_time_read_only_data__ {
+ ErtsMonotonicTime (*os_monotonic_time)(void);
+};
+
+typedef struct {
+ union {
+ struct erts_sys_time_read_only_data__ o;
+ char align__[(((sizeof(struct erts_sys_time_read_only_data__) - 1)
+ / ASSUMED_CACHE_LINE_SIZE) + 1)
+ * ASSUMED_CACHE_LINE_SIZE];
+ } r;
+} ErtsSysTimeData__;
+
+extern ErtsSysTimeData__ erts_sys_time_data__;
+
+ERTS_GLB_INLINE ErtsMonotonicTime erts_os_monotonic_time(void);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE ErtsMonotonicTime
+erts_os_monotonic_time(void)
+{
+ return (*erts_sys_time_data__.r.o.os_monotonic_time)();
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
extern void sys_gettimeofday(SysTimeval *tv);
-extern SysHrTime sys_gethrtime(void);
extern clock_t sys_times(SysTimes *buffer);
extern char *win_build_environment(char *);
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 5d51659b4e..1ba87b8b0b 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -3273,9 +3273,9 @@ erts_sys_schedule_interrupt(int set)
#ifdef ERTS_SMP
void
-erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec)
+erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time)
{
- erts_check_io_interrupt_timed(set, msec);
+ erts_check_io_interrupt_timed(set, timeout_time);
}
#endif
diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index b84c8f85ce..3a10125c81 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -61,11 +61,6 @@
(epoch) = ((ull.QuadPart / TICKS_PER_SECOND) - EPOCH_JULIAN_DIFF); \
} while(0)
-static SysHrTime wrap = 0;
-static DWORD last_tick_count = 0;
-static erts_smp_mtx_t wrap_lock;
-static ULONGLONG (WINAPI *pGetTickCount64)(void) = NULL;
-
/* Getting timezone information is a heavy operation, so we want to do this
only once */
@@ -76,17 +71,161 @@ static int days_in_month[2][13] = {
{0,31,28,31,30,31,30,31,31,30,31,30,31},
{0,31,29,31,30,31,30,31,31,30,31,30,31}};
-int
-sys_init_time(void)
+/*
+ * erts_os_monotonic_time()
+ */
+
+struct sys_time_internal_state_read_only__ {
+ ULONGLONG (WINAPI *pGetTickCount64)(void);
+ BOOL (WINAPI *pQueryPerformanceCounter)(LARGE_INTEGER *);
+};
+
+struct sys_time_internal_state_write_freq__ {
+ erts_smp_mtx_t mtime_mtx;
+ ULONGLONG wrap;
+ ULONGLONG last_tick_count;
+};
+
+__declspec(align(ASSUMED_CACHE_LINE_SIZE)) struct {
+ union {
+ struct sys_time_internal_state_read_only__ o;
+ char align__[(((sizeof(struct sys_time_internal_state_read_only__) - 1)
+ / ASSUMED_CACHE_LINE_SIZE) + 1)
+ * ASSUMED_CACHE_LINE_SIZE];
+ } r;
+ union {
+ struct sys_time_internal_state_write_freq__ f;
+ char align__[(((sizeof(struct sys_time_internal_state_write_freq__) - 1)
+ / ASSUMED_CACHE_LINE_SIZE) + 1)
+ * ASSUMED_CACHE_LINE_SIZE];
+ } w;
+} internal_state;
+
+__declspec(align(ASSUMED_CACHE_LINE_SIZE)) ErtsSysTimeData__ erts_sys_time_data__;
+
+static ErtsMonotonicTime
+os_monotonic_time_qpc(void)
{
+ LARGE_INTEGER pc;
+
+ if (!(*internal_state.r.o.pQueryPerformanceCounter)(&pc))
+ erl_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n");
+
+ return (ErtsMonotonicTime) pc.QuadPart;
+}
+
+static ErtsMonotonicTime
+os_monotonic_time_gtc32(void)
+{
+ ULONGLONG res, ticks;
+
+ erts_smp_mtx_lock(&internal_state.w.f.mtime_mtx);
+
+ ticks = (ULONGLONG) (GetTickCount() & 0x7FFFFFFF);
+ if (ticks < internal_state.w.f.last_tick_count)
+ internal_state.w.f.wrap += (ULONGLONG) LL_LITERAL(1) << 31;
+ internal_state.w.f.last_tick_count = ticks;
+ res = ticks + internal_state.w.f.wrap;
+
+ erts_smp_mtx_unlock(&internal_state.w.f.mtime_mtx);
+
+ return (ErtsMonotonicTime) res*1000;
+}
+
+static ErtsMonotonicTime
+os_monotonic_time_gtc64(void)
+{
+ ULONGLONG ticks = (*internal_state.r.o.pGetTickCount64)();
+ return (ErtsMonotonicTime) ticks*1000;
+}
+
+/*
+ * Init
+ */
+
+void
+sys_init_time(ErtsSysInitTimeResult *init_resp)
+{
+ ErtsMonotonicTime (*os_mtime_func)(void);
+ ErtsMonotonicTime time_unit;
char kernel_dll_name[] = "kernel32";
HMODULE module;
+ init_resp->os_monotonic_info.clock_id = NULL;
+
module = GetModuleHandle(kernel_dll_name);
- pGetTickCount64 = (module != NULL) ?
- (ULONGLONG (WINAPI *)(void))
- GetProcAddress(module,"GetTickCount64") :
- NULL;
+ if (!module) {
+ get_tick_count:
+ erts_smp_mtx_init(&internal_state.w.f.mtime_mtx,
+ "os_monotonic_time");
+ internal_state.w.f.wrap = 0;
+ internal_state.w.f.last_tick_count = 0;
+
+ init_resp->os_monotonic_info.func = "GetTickCount";
+ init_resp->os_monotonic_info.locked_use = 1;
+ init_resp->os_monotonic_info.resolution = 1000;
+ time_unit = (ErtsMonotonicTime) 1000*1000;
+ os_mtime_func = os_monotonic_time_gtc32;
+ }
+ else {
+ int major, minor, build;
+
+ os_version(&major, &minor, &build);
+
+ if (major < 6) {
+
+ get_tick_count64:
+
+ internal_state.r.o.pGetTickCount64
+ = ((ULONGLONG (WINAPI *)(void))
+ GetProcAddress(module, "GetTickCount64"));
+ if (!internal_state.r.o.pGetTickCount64)
+ goto get_tick_count;
+
+ init_resp->os_monotonic_info.func = "GetTickCount64";
+ init_resp->os_monotonic_info.locked_use = 0;
+ init_resp->os_monotonic_info.resolution = 1000;
+ time_unit = (ErtsMonotonicTime) 1000*1000;
+ os_mtime_func = os_monotonic_time_gtc64;
+ }
+ else { /* Vista or newer... */
+
+ LARGE_INTEGER pf;
+ BOOL (WINAPI *QPF)(LARGE_INTEGER *);
+
+ QPF = ((BOOL (WINAPI *)(LARGE_INTEGER *))
+ GetProcAddress(module, "QueryPerformanceFrequency"));
+ if (!QPF)
+ goto get_tick_count64;
+ if (!(*QPF)(&pf))
+ goto get_tick_count64;
+ /*
+ * We only use QueryPerformanceCounter() if
+ * its frequency is equal to, or larger than
+ * GHz in order to ensure that the user wont
+ * be able to observe faulty order between
+ * values retrieved on different threads.
+ */
+ if (pf.QuadPart < (LONGLONG) 1000*1000*1000)
+ goto get_tick_count64;
+ internal_state.r.o.pQueryPerformanceCounter
+ = ((BOOL (WINAPI *)(LARGE_INTEGER *))
+ GetProcAddress(module, "QueryPerformanceCounter"));
+ if (!internal_state.r.o.pQueryPerformanceCounter)
+ goto get_tick_count64;
+
+ init_resp->os_monotonic_info.func = "QueryPerformanceCounter";
+ init_resp->os_monotonic_info.locked_use = 0;
+ time_unit = (ErtsMonotonicTime) pf.QuadPart;
+ init_resp->os_monotonic_info.resolution = time_unit;
+ os_mtime_func = os_monotonic_time_qpc;
+ }
+ }
+
+ erts_sys_time_data__.r.o.os_monotonic_time = os_mtime_func;
+ init_resp->os_monotonic_time_unit = time_unit;
+ init_resp->have_os_monotonic = 1;
+ init_resp->sys_clock_resolution = 1;
if(GetTimeZoneInformation(&static_tzi) &&
static_tzi.StandardDate.wMonth != 0 &&
@@ -94,9 +233,6 @@ sys_init_time(void)
have_static_tzi = 1;
}
- erts_smp_mtx_init(&wrap_lock, "sys_gethrtime");
-
- return 1;
}
/* Returns a switchtimes for DST as UTC filetimes given data from a
@@ -377,41 +513,6 @@ sys_gettimeofday(SysTimeval *tv)
EPOCH_JULIAN_DIFF);
}
-extern int erts_initialized;
-SysHrTime
-sys_gethrtime(void)
-{
- if (pGetTickCount64 != NULL) {
- return ((SysHrTime) pGetTickCount64()) * LL_LITERAL(1000000);
- } else {
- DWORD ticks;
- SysHrTime res;
- erts_smp_mtx_lock(&wrap_lock);
- ticks = (SysHrTime) (GetTickCount() & 0x7FFFFFFF);
- if (ticks < (SysHrTime) last_tick_count) {
- /* Detect a race that should no longer be here... */
- if ((((SysHrTime) last_tick_count) - ((SysHrTime) ticks)) > 1000) {
- wrap += LL_LITERAL(1) << 31;
- } else {
- /*
- * XXX Debug: Violates locking order, remove all this,
- * after testing!
- */
- erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
- erts_dsprintf(dsbufp, "Did not wrap when last_tick %d "
- "and tick %d",
- last_tick_count, ticks);
- erts_send_error_to_logger_nogl(dsbufp);
- ticks = last_tick_count;
- }
- }
- last_tick_count = ticks;
- res = ((((LONGLONG) ticks) + wrap) * LL_LITERAL(1000000));
- erts_smp_mtx_unlock(&wrap_lock);
- return res;
- }
-}
-
clock_t
sys_times(SysTimes *buffer) {
clock_t kernel_ticks = (GetTickCount() /
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 22:48:30 +0000