Use critical sections as mutex implementation on Windows

Windows native critical sections are now used internally in the
runtime system as mutex implementation. This since they perform
better under extreme contention than our own implementation.

2 files changed, 111 insertions, 5 deletions

diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h
index a0685ea3c0..86a1e9fbdf 100644
--- a/erts/include/internal/ethr_mutex.h
+++ b/erts/include/internal/ethr_mutex.h
@@ -22,6 +22,23 @@
  * Author: Rickard Green
  */
 
+/*
+ * IMPORTANT note about ethr_cond_signal() and ethr_cond_broadcast()
+ *
+ * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
+ * even though the associated mutex/mutexes isn't/aren't locked by the
+ * caller. We do not allow that by default in order to avoid a performance
+ * penalty on some platforms.
+ *
+ * Mutexes and condition variables can, however, be initialized as POSIX
+ * compliant. When initialized as such ethr_cond_signal(), and
+ * ethr_cond_broadcast() are allowed to be called even though the associated
+ * mutexes aren't locked. This will, however, incur a performance penalty on
+ * some platforms.
+ *
+ * POSIX compliant mutexes and condition variables *need* to be used together.
+ */
+
 #ifndef ETHR_MUTEX_H__
 #define ETHR_MUTEX_H__
 
@@ -40,6 +57,14 @@
 #endif
 #endif
 
+/* #define ETHR_DBG_WIN_MTX_WITH_PTHREADS */
+#ifdef ETHR_DBG_WIN_MTX_WITH_PTHREADS
+typedef pthread_mutex_t CRITICAL_SECTION;
+int TryEnterCriticalSection(CRITICAL_SECTION *);
+void EnterCriticalSection(CRITICAL_SECTION *);
+void LeaveCriticalSection(CRITICAL_SECTION *);
+#endif
+
 #ifdef ETHR_MTX_HARD_DEBUG
 #  ifdef __GNUC__
 #    warning ETHR_MTX_HARD_DEBUG
@@ -140,13 +165,19 @@ struct ethr_mutex_base_ {
 typedef struct {
     int main_spincount;
     int aux_spincount;
+    int posix_compliant;
 } ethr_mutex_opt;
 
+#define ETHR_MUTEX_OPT_DEFAULT_INITER {-1, -1, 0}
+
 typedef struct {
     int main_spincount;
     int aux_spincount;
+    int posix_compliant;
 } ethr_cond_opt;
 
+#define ETHR_COND_OPT_DEFAULT_INITER {-1, -1, 0}
+
 #ifdef ETHR_USE_OWN_MTX_IMPL__
 
 typedef struct ethr_mutex_ ethr_mutex;
@@ -179,7 +210,7 @@ struct ethr_cond_ {
 #endif
 };
 
-#else /* pthread */
+#elif defined(ETHR_PTHREADS) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
 
 typedef struct ethr_mutex_ ethr_mutex;
 struct ethr_mutex_ {
@@ -197,7 +228,36 @@ struct ethr_cond_ {
 #endif
 };
 
-#endif /* pthread */
+#elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
+#  define ETHR_WIN_MUTEX__
+
+typedef struct ethr_mutex_ ethr_mutex;
+struct ethr_mutex_ {
+    int posix_compliant;
+    CRITICAL_SECTION cs;
+    ethr_ts_event *wakeups;
+    ethr_atomic32_t have_wakeups; /* only when posix compliant */
+    ethr_atomic32_t locked;       /* only when posix compliant */
+    ethr_spinlock_t lock;         /* only when posix compliant */
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+typedef struct ethr_cond_ ethr_cond;
+struct ethr_cond_ {
+    int posix_compliant;
+    CRITICAL_SECTION cs;
+    ethr_ts_event *waiters;
+    int spincount;
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+#else
+#  error "no mutex implementation"
+#endif
 
 int ethr_mutex_init_opt(ethr_mutex *, ethr_mutex_opt *);
 int ethr_mutex_init(ethr_mutex *);
@@ -573,7 +633,7 @@ ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
 
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
-#else /* pthread_mutex */
+#elif defined(ETHR_PTHREADS) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
 
 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
 
@@ -605,7 +665,54 @@ ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
 
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
-#endif /* pthread_mutex */
+#elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
+
+static ETHR_INLINE int
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
+{
+    if (!TryEnterCriticalSection(&mtx->cs))
+	return EBUSY;
+    if (mtx->posix_compliant)
+	ethr_atomic32_set(&mtx->locked, 1);
+    return 0;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
+{
+    EnterCriticalSection(&mtx->cs);
+    if (mtx->posix_compliant)
+	ethr_atomic32_set(&mtx->locked, 1);
+}
+
+void ethr_mutex_cond_wakeup__(ethr_mutex *mtx);
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
+{
+    if (mtx->posix_compliant) {
+	ethr_atomic32_set_mb(&mtx->locked, 0);
+	if (ethr_atomic32_read_acqb(&mtx->have_wakeups))
+	    goto cond_wakeup;
+	else
+	    goto leave_cs;
+    }
+
+    if (mtx->wakeups) {
+    cond_wakeup:
+	ethr_mutex_cond_wakeup__(mtx);
+    }
+    else {
+    leave_cs:
+	LeaveCriticalSection(&mtx->cs);
+    }
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif
 
 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
 
diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h
index 8ad0ded144..142c26c0ca 100644
--- a/erts/include/internal/ethread.h
+++ b/erts/include/internal/ethread.h
@@ -191,7 +191,6 @@ typedef DWORD ethr_tsd_key;
 #undef ETHR_HAVE_ETHR_SIG_FUNCS
 
 #define ETHR_USE_OWN_RWMTX_IMPL__
-#define ETHR_USE_OWN_MTX_IMPL__
 
 #define ETHR_YIELD() (Sleep(0), 0)