aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/sys
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/sys')
-rw-r--r--erts/emulator/sys/common/erl_mseg.c315
-rw-r--r--erts/emulator/sys/common/erl_mseg.h9
-rw-r--r--erts/emulator/sys/common/erl_sys_common_misc.c204
-rw-r--r--erts/emulator/sys/common/erl_util_queue.h77
-rwxr-xr-xerts/emulator/sys/win32/sys.c82
5 files changed, 438 insertions, 249 deletions
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 538eea88d1..bd8ba82a5f 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -39,6 +39,7 @@
#include "erl_alloc.h"
#include "big.h"
#include "erl_thr_progress.h"
+#include "erl_util_queue.h"
#if HAVE_ERTS_MSEG
@@ -175,6 +176,7 @@ struct cache_t_ {
Uint size;
void *seg;
cache_t *next;
+ cache_t *prev;
};
@@ -183,9 +185,9 @@ typedef struct ErtsMsegAllctr_t_ ErtsMsegAllctr_t;
struct mem_kind_t {
cache_t cache[MAX_CACHE_SIZE];
- cache_t *cache_unpowered;
- cache_t *cache_area[CACHE_AREAS];
- cache_t *cache_free;
+ cache_t cache_unpowered_node;
+ cache_t cache_powered_node[CACHE_AREAS];
+ cache_t cache_free;
Sint cache_size;
Uint cache_hits;
@@ -516,67 +518,94 @@ do { \
#define ERTS_DBG_MK_CHK_THR_ACCESS(MK)
#endif
-/* NEW CACHE interface */
+/* Cache interface */
-static ERTS_INLINE cache_t *mseg_cache_alloc_descriptor(MemKind *mk) {
- cache_t *c = mk->cache_free;
- ERTS_DBG_MK_CHK_THR_ACCESS(mk);
- if (c)
- mk->cache_free = c->next;
-
- return c;
-}
-
-static ERTS_INLINE void mseg_cache_free_descriptor(MemKind *mk, cache_t *c) {
- ERTS_DBG_MK_CHK_THR_ACCESS(mk);
- ASSERT(c);
-
- c->seg = NULL;
+static ERTS_INLINE void mseg_cache_clear_node(cache_t *c) {
+ c->seg = NULL;
c->size = 0;
- c->next = mk->cache_free;
- mk->cache_free = c;
+ c->next = c;
+ c->prev = c;
}
-static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size) {
+static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Uint flags) {
cache_t *c;
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
- if (mk->cache_free && MAP_IS_ALIGNED(seg)) {
- if (IS_2POW(size)) {
+ ASSERT(!MSEG_FLG_IS_2POW(flags) || (MSEG_FLG_IS_2POW(flags) && MAP_IS_ALIGNED(seg) && IS_2POW(size)));
+
+ /* The idea is that sbc caching is prefered over mbc caching.
+ * Blocks are normally allocated in mb carriers and thus cached there.
+ * Large blocks has no such cache and it is up to mseg to cache them to speed things up.
+ */
+
+ if (!erts_circleq_is_empty(&(mk->cache_free))) {
+
+ /* We have free slots, use one to cache the segment */
+
+ c = erts_circleq_head(&(mk->cache_free));
+ erts_circleq_remove(c);
+
+ c->seg = seg;
+ c->size = size;
+
+ if (MSEG_FLG_IS_2POW(flags)) {
int ix = SIZE_TO_CACHE_AREA_IDX(size);
ASSERT(ix < CACHE_AREAS);
ASSERT((1 << (ix + MSEG_ALIGN_BITS)) == size);
- /* unlink from free cache list */
- c = mseg_cache_alloc_descriptor(mk);
+ erts_circleq_push_head(&(mk->cache_powered_node[ix]), c);
- /* link to cache area */
- c->seg = seg;
- c->size = size;
- c->next = mk->cache_area[ix];
+ } else
+ erts_circleq_push_head(&(mk->cache_unpowered_node), c);
- mk->cache_area[ix] = c;
- mk->cache_size++;
+ mk->cache_size++;
+ ASSERT(mk->cache_size <= mk->ma->max_cache_size);
- ASSERT(mk->cache_size <= mk->ma->max_cache_size);
+ return 1;
+ } else if (!MSEG_FLG_IS_2POW(flags) && !erts_circleq_is_empty(&(mk->cache_unpowered_node))) {
- return 1;
- } else {
- /* unlink from free cache list */
- c = mseg_cache_alloc_descriptor(mk);
+ /* No free slots.
+ * Evict oldest slot from unpowered cache so we can cache an unpowered (sbc) segment */
+
+ c = erts_circleq_tail(&(mk->cache_unpowered_node));
+ erts_circleq_remove(c);
+
+ mseg_destroy(mk->ma, mk, c->seg, c->size);
+ mseg_cache_clear_node(c);
+
+ c->seg = seg;
+ c->size = size;
+
+ erts_circleq_push_head(&(mk->cache_unpowered_node), c);
+
+ return 1;
+ } else if (!MSEG_FLG_IS_2POW(flags)) {
+
+ /* No free slots and no unpowered (sbc) slots.
+ * Evict smallest slot from powered cache so we can cache an unpowered (sbc) segment.
+ * Note: Though this is the wanted policy I don't think it is used significantly.
+ * This branch could probably be removed without serious impact.
+ */
+
+ int i;
+
+ for( i = 0; i < CACHE_AREAS; i++) {
+ if (erts_circleq_is_empty(&(mk->cache_powered_node[i])))
+ continue;
+
+ c = erts_circleq_tail(&(mk->cache_powered_node[i]));
+ erts_circleq_remove(c);
+
+ mseg_destroy(mk->ma, mk, c->seg, c->size);
+ mseg_cache_clear_node(c);
- /* link to cache area */
c->seg = seg;
c->size = size;
- c->next = mk->cache_unpowered;
-
- mk->cache_unpowered = c;
- mk->cache_size++;
- ASSERT(mk->cache_size <= mk->ma->max_cache_size);
+ erts_circleq_push_head(&(mk->cache_unpowered_node), c);
return 1;
}
@@ -585,90 +614,110 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size) {
return 0;
}
-static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p) {
+static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p, Uint flags) {
Uint size = *size_p;
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
- if (IS_2POW(size)) {
+
+ if (MSEG_FLG_IS_2POW(flags)) {
int i, ix = SIZE_TO_CACHE_AREA_IDX(size);
void *seg;
cache_t *c;
Uint csize;
+ ASSERT(IS_2POW(size));
+
for( i = ix; i < CACHE_AREAS; i++) {
- if ((c = mk->cache_area[i]) == NULL)
+ if (erts_circleq_is_empty(&(mk->cache_powered_node[i])))
continue;
+ c = erts_circleq_head(&(mk->cache_powered_node[i]));
+ erts_circleq_remove(c);
+
ASSERT(IS_2POW(c->size));
+ ASSERT(MAP_IS_ALIGNED(c->seg));
+
+ csize = c->size;
+ seg = c->seg;
- /* unlink from cache area */
- csize = c->size;
- seg = c->seg;
- mk->cache_area[i] = c->next;
- c->next = NULL;
mk->cache_size--;
mk->cache_hits++;
/* link to free cache list */
- mseg_cache_free_descriptor(mk, c);
+ mseg_cache_clear_node(c);
+ erts_circleq_push_head(&(mk->cache_free), c);
ASSERT(!(mk->cache_size < 0));
- /* divvy up the cache - if needed */
- while( i > ix) {
- csize = csize >> 1;
- /* try to cache half of it */
- if (!cache_bless_segment(mk, (char *)seg + csize, csize)) {
- /* wouldn't cache .. destroy it instead */
- mseg_destroy(mk->ma, mk, (char *)seg + csize, csize);
- }
- i--;
+ if (csize != size) {
+ mseg_destroy(mk->ma, mk, (char *)seg + size, csize - size);
}
- ASSERT(csize == size);
+
return seg;
}
}
- else if (mk->cache_unpowered) {
+ else if (!erts_circleq_is_empty(&(mk->cache_unpowered_node))) {
void *seg;
- cache_t *c, *pc;
+ cache_t *c;
+ cache_t *best = NULL;
+ Uint bdiff = 0;
Uint csize;
Uint bad_max_abs = mk->ma->abs_max_cache_bad_fit;
Uint bad_max_rel = mk->ma->rel_max_cache_bad_fit;
- c = mk->cache_unpowered;
- pc = c;
-
- while (c) {
+ erts_circleq_foreach(c, &(mk->cache_unpowered_node)) {
csize = c->size;
- if (csize >= size &&
- ((csize - size)*100 < bad_max_rel*size) &&
- (csize - size) < bad_max_abs ) {
+ if (csize >= size) {
+ if (((csize - size)*100 < bad_max_rel*size) && (csize - size) < bad_max_abs ) {
- /* unlink from cache area */
- seg = c->seg;
+ seg = c->seg;
- if (pc == c) {
- mk->cache_unpowered = c->next;
- } else {
- pc->next = c->next;
- }
+ erts_circleq_remove(c);
+
+ mk->cache_size--;
+ mk->cache_hits++;
- c->next = NULL;
- mk->cache_size--;
- mk->cache_hits++;
+ mseg_cache_clear_node(c);
+ erts_circleq_push_head(&(mk->cache_free), c);
- /* link to free cache list */
- mseg_cache_free_descriptor(mk, c);
- *size_p = csize;
+ *size_p = csize;
- return seg;
+ return seg;
+
+ } else if (!best || (csize - size) < bdiff) {
+ best = c;
+ bdiff = csize - size;
+ }
}
+ }
+
+ /* No cached segment met our criteria, use the best one found and trim it */
+
+ if (best) {
+
+ seg = best->seg;
+ csize = best->size;
+
+ ASSERT(best->seg);
+ ASSERT(best->size > 0);
+
+ mk->cache_hits++;
+
+ /* Use current cache placement for remaining segment space */
+
+ best->seg = seg + size;
+ best->size = csize - size;
+
+ ASSERT((size % GET_PAGE_SIZE) == 0);
+ ASSERT((best->size % GET_PAGE_SIZE) == 0);
+
+ *size_p = size;
+
+ return seg;
- pc = c;
- c = c->next;
}
}
return NULL;
@@ -679,20 +728,18 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p) {
* using callbacks from aux-work in the scheduler.
*/
-static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, cache_t **head) {
+static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, cache_t *head) {
cache_t *c = NULL;
- c = *head;
-
- ASSERT( c != NULL );
-
- *head = c->next;
+ c = erts_circleq_tail(head);
+ erts_circleq_remove(c);
if (erts_mtrace_enabled)
erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg);
mseg_destroy(mk->ma, mk, c->seg, c->size);
- mseg_cache_free_descriptor(mk, c);
+ mseg_cache_clear_node(c);
+ erts_circleq_push_head(&(mk->cache_free), c);
mk->segments.current.watermark--;
mk->cache_size--;
@@ -702,30 +749,27 @@ static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, cache_t **
return mk->cache_size;
}
-static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, cache_t **head) {
- cache_t *c = NULL, *next = NULL;
-
- c = *head;
- ASSERT( c != NULL );
+static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, cache_t *head) {
+ cache_t *c = NULL;
- while (c) {
+ while (!erts_circleq_is_empty(head)) {
- next = c->next;
+ c = erts_circleq_tail(head);
+ erts_circleq_remove(c);
if (erts_mtrace_enabled)
erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg);
mseg_destroy(mk->ma, mk, c->seg, c->size);
- mseg_cache_free_descriptor(mk, c);
+
+ mseg_cache_clear_node(c);
+ erts_circleq_push_head(&(mk->cache_free), c);
mk->segments.current.watermark--;
mk->cache_size--;
- c = next;
}
- *head = NULL;
-
ASSERT( mk->cache_size >= 0 );
return mk->cache_size;
@@ -743,12 +787,12 @@ static Uint mseg_check_memkind_cache(MemKind *mk) {
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
for (i = 0; i < CACHE_AREAS; i++) {
- if (mk->cache_area[i] != NULL)
- return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_area[i]));
+ if (!erts_circleq_is_empty(&(mk->cache_powered_node[i])))
+ return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_powered_node[i]));
}
- if (mk->cache_unpowered)
- return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_unpowered));
+ if (!erts_circleq_is_empty(&(mk->cache_unpowered_node)))
+ return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_unpowered_node));
return 0;
}
@@ -804,17 +848,17 @@ static void mseg_clear_memkind_cache(MemKind *mk) {
/* drop pow2 caches */
for (i = 0; i < CACHE_AREAS; i++) {
- if (mk->cache_area[i] == NULL)
+ if (erts_circleq_is_empty(&(mk->cache_powered_node[i])))
continue;
- mseg_drop_memkind_cache_size(mk, &(mk->cache_area[i]));
- ASSERT(mk->cache_area[i] == NULL);
+ mseg_drop_memkind_cache_size(mk, &(mk->cache_powered_node[i]));
+ ASSERT(erts_circleq_is_empty(&(mk->cache_powered_node[i])));
}
/* drop varied caches */
- if(mk->cache_unpowered)
- mseg_drop_memkind_cache_size(mk, &(mk->cache_unpowered));
+ if (!erts_circleq_is_empty(&(mk->cache_unpowered_node)))
+ mseg_drop_memkind_cache_size(mk, &(mk->cache_unpowered_node));
- ASSERT(mk->cache_unpowered == NULL);
+ ASSERT(erts_circleq_is_empty(&(mk->cache_unpowered_node)));
ASSERT(mk->cache_size == 0);
}
@@ -873,7 +917,7 @@ mseg_alloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, Uint *size_p,
ma->min_seg_size = size;
#endif
- if (opt->cache && mk->cache_size > 0 && (seg = cache_get_segment(mk, &size)) != NULL)
+ if (opt->cache && mk->cache_size > 0 && (seg = cache_get_segment(mk, &size, flags)) != NULL)
goto done;
if ((seg = mseg_create(ma, mk, size)) == NULL)
@@ -894,14 +938,13 @@ done:
static void
mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, Uint size,
- const ErtsMsegOpt_t *opt)
+ Uint flags, const ErtsMsegOpt_t *opt)
{
MemKind* mk = memkind(ma, opt);
-
ERTS_MSEG_DEALLOC_STAT(mk,size);
- if (opt->cache && cache_bless_segment(mk, seg, size)) {
+ if (opt->cache && cache_bless_segment(mk, seg, size, flags)) {
schedule_cache_check(ma);
goto done;
}
@@ -934,7 +977,7 @@ mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg,
/* Dealloc old segment if new segment is of size 0 */
if (!(*new_size_p)) {
- mseg_dealloc(ma, atype, seg, old_size, opt);
+ mseg_dealloc(ma, atype, seg, old_size, flags, opt);
DEC_CC(ma, dealloc);
return NULL;
}
@@ -993,7 +1036,7 @@ mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg,
else {
if (!opt->preserv) {
- mseg_dealloc(ma, atype, seg, old_size, opt);
+ mseg_dealloc(ma, atype, seg, old_size, flags, opt);
new_seg = mseg_alloc(ma, atype, &new_size, flags, opt);
ASSERT(MAP_IS_ALIGNED(new_seg) || !new_seg);
}
@@ -1020,7 +1063,7 @@ mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg,
new_size = old_size;
else {
sys_memcpy(((char *) new_seg), ((char *) seg), MIN(new_size, old_size));
- mseg_dealloc(ma, atype, seg, old_size, opt);
+ mseg_dealloc(ma, atype, seg, old_size, flags, opt);
}
#endif
}
@@ -1497,19 +1540,19 @@ erts_mseg_alloc(ErtsAlcType_t atype, Uint *size_p, Uint flags)
void
erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg,
- Uint size, const ErtsMsegOpt_t *opt)
+ Uint size, Uint flags, const ErtsMsegOpt_t *opt)
{
ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt);
ERTS_MSEG_LOCK(ma);
ERTS_DBG_MA_CHK_THR_ACCESS(ma);
- mseg_dealloc(ma, atype, seg, size, opt);
+ mseg_dealloc(ma, atype, seg, size, flags, opt);
ERTS_MSEG_UNLOCK(ma);
}
void
-erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size)
+erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size, Uint flags)
{
- erts_mseg_dealloc_opt(atype, seg, size, &erts_mseg_default_opt);
+ erts_mseg_dealloc_opt(atype, seg, size, flags, &erts_mseg_default_opt);
}
void *
@@ -1556,27 +1599,28 @@ erts_mseg_unit_size(void)
return MSEG_ALIGNED_SIZE;
}
+
static void mem_kind_init(ErtsMsegAllctr_t *ma, MemKind* mk, const char* name)
{
int i;
+ /* Clear all cache headers */
+ mseg_cache_clear_node(&(mk->cache_free));
+ mseg_cache_clear_node(&(mk->cache_unpowered_node));
+
for (i = 0; i < CACHE_AREAS; i++) {
- mk->cache_area[i] = NULL;
+ mseg_cache_clear_node(&(mk->cache_powered_node[i]));
}
- mk->cache_free = NULL;
+ /* Populate cache free list */
ASSERT(ma->max_cache_size <= MAX_CACHE_SIZE);
for (i = 0; i < ma->max_cache_size; i++) {
- mk->cache[i].seg = NULL;
- mk->cache[i].size = 0;
- mk->cache[i].next = mk->cache_free;
- mk->cache_free = &(mk->cache[i]);
+ mseg_cache_clear_node(&(mk->cache[i]));
+ erts_circleq_push_head(&(mk->cache_free), &(mk->cache[i]));
}
- mk->cache_unpowered = NULL;
-
mk->cache_size = 0;
mk->cache_hits = 0;
@@ -1594,9 +1638,6 @@ static void mem_kind_init(ErtsMsegAllctr_t *ma, MemKind* mk, const char* name)
ma->mk_list = mk;
}
-
-
-
void
erts_mseg_init(ErtsMsegInit_t *init)
{
@@ -1721,7 +1762,7 @@ erts_mseg_test(unsigned long op,
case 0x401:
return (unsigned long) erts_mseg_alloc(ERTS_ALC_A_INVALID, (Uint *) a1, (Uint) 0);
case 0x402:
- erts_mseg_dealloc(ERTS_ALC_A_INVALID, (void *) a1, (Uint) a2);
+ erts_mseg_dealloc(ERTS_ALC_A_INVALID, (void *) a1, (Uint) a2, (Uint) 0);
return (unsigned long) 0;
case 0x403:
return (unsigned long) erts_mseg_realloc(ERTS_ALC_A_INVALID,
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index 3d0b0f0355..3cab9e18da 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -74,7 +74,7 @@ typedef struct {
{ \
4*1024*1024, /* amcbf: Absolute max cache bad fit */ \
20, /* rmcbf: Relative max cache bad fit */ \
- 5, /* mcs: Max cache size */ \
+ 10, /* mcs: Max cache size */ \
1000 /* cci: Cache check interval */ \
}
@@ -93,11 +93,10 @@ extern const ErtsMsegOpt_t erts_mseg_default_opt;
void *erts_mseg_alloc(ErtsAlcType_t, Uint *, Uint);
void *erts_mseg_alloc_opt(ErtsAlcType_t, Uint *, Uint, const ErtsMsegOpt_t *);
-void erts_mseg_dealloc(ErtsAlcType_t, void *, Uint);
-void erts_mseg_dealloc_opt(ErtsAlcType_t, void *, Uint, const ErtsMsegOpt_t *);
+void erts_mseg_dealloc(ErtsAlcType_t, void *, Uint, Uint);
+void erts_mseg_dealloc_opt(ErtsAlcType_t, void *, Uint, Uint, const ErtsMsegOpt_t *);
void *erts_mseg_realloc(ErtsAlcType_t, void *, Uint, Uint *, Uint);
-void *erts_mseg_realloc_opt(ErtsAlcType_t, void *, Uint, Uint *,
- Uint, const ErtsMsegOpt_t *);
+void *erts_mseg_realloc_opt(ErtsAlcType_t, void *, Uint, Uint *, Uint, const ErtsMsegOpt_t *);
void erts_mseg_clear_cache(void);
void erts_mseg_cache_check(void);
Uint erts_mseg_no( const ErtsMsegOpt_t *);
diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 1bf5fa89f4..0b31c125e5 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -47,14 +47,16 @@
/* Written once and only once */
static int filename_encoding = ERL_FILENAME_UNKNOWN;
+static int filename_warning = ERL_FILENAME_WARNING_WARNING;
#if defined(__WIN32__) || defined(__DARWIN__)
static int user_filename_encoding = ERL_FILENAME_UTF8; /* Default unicode on windows */
#else
static int user_filename_encoding = ERL_FILENAME_LATIN1;
#endif
-void erts_set_user_requested_filename_encoding(int encoding)
+void erts_set_user_requested_filename_encoding(int encoding, int warning)
{
user_filename_encoding = encoding;
+ filename_warning = warning;
}
int erts_get_user_requested_filename_encoding(void)
@@ -62,6 +64,11 @@ int erts_get_user_requested_filename_encoding(void)
return user_filename_encoding;
}
+int erts_get_filename_warning_type(void)
+{
+ return filename_warning;
+}
+
void erts_init_sys_common_misc(void)
{
#if defined(__WIN32__)
@@ -107,9 +114,9 @@ int erts_get_native_filename_encoding(void)
}
/* For internal use by sys_double_to_chars_fast() */
-static char* float_first_trailing_zero(char* p)
+static char* find_first_trailing_zero(char* p)
{
- for (--p; *p == '0' && *(p-1) == '0'; --p);
+ for (; *(p-1) == '0'; --p);
if (*(p-1) == '.') ++p;
return p;
}
@@ -120,34 +127,83 @@ sys_double_to_chars(double fp, char *buffer, size_t buffer_size)
return sys_double_to_chars_ext(fp, buffer, buffer_size, SYS_DEFAULT_FLOAT_DECIMALS);
}
+/* Convert float to string using fixed point notation.
+ * decimals must be >= 0
+ * if compact != 0, the trailing 0's will be truncated
+ */
int
-sys_double_to_chars_fast(double f, char *outbuf, int maxlen, int decimals, int compact)
+sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
+ int compact)
{
- enum {
- FRAC_SIZE = 52
- , EXP_SIZE = 11
- , EXP_MASK = (1ll << EXP_SIZE) - 1
- , FRAC_MASK = (1ll << FRAC_SIZE) - 1
- , FRAC_MASK2 = (1ll << (FRAC_SIZE + 1)) - 1
- , MAX_FLOAT = 1ll << (FRAC_SIZE+1)
+ /* Note that some C compilers don't support "static const" propagation
+ * so we use a defines */
+ #define SYS_DOUBLE_RND_CONST 0.55555555555555555
+ #define FRAC_SIZE 52
+ #define EXP_SIZE 11
+ #define EXP_MASK ((1ll << EXP_SIZE) - 1)
+ #define MAX_DECIMALS (sizeof(cs_sys_double_pow10) \
+ / sizeof(cs_sys_double_pow10[0]))
+ #define FRAC_MASK ((1ll << FRAC_SIZE) - 1)
+ #define FRAC_MASK2 ((1ll << (FRAC_SIZE + 1)) - 1)
+ #define MAX_FLOAT (1ll << (FRAC_SIZE+1))
+
+ static const double cs_sys_double_pow10[] = {
+ SYS_DOUBLE_RND_CONST / 1ll,
+ SYS_DOUBLE_RND_CONST / 10ll,
+ SYS_DOUBLE_RND_CONST / 100ll,
+ SYS_DOUBLE_RND_CONST / 1000ll,
+ SYS_DOUBLE_RND_CONST / 10000ll,
+ SYS_DOUBLE_RND_CONST / 100000ll,
+ SYS_DOUBLE_RND_CONST / 1000000ll,
+ SYS_DOUBLE_RND_CONST / 10000000ll,
+ SYS_DOUBLE_RND_CONST / 100000000ll,
+ SYS_DOUBLE_RND_CONST / 1000000000ll,
+ SYS_DOUBLE_RND_CONST / 10000000000ll,
+ SYS_DOUBLE_RND_CONST / 100000000000ll,
+ SYS_DOUBLE_RND_CONST / 1000000000000ll,
+ SYS_DOUBLE_RND_CONST / 10000000000000ll,
+ SYS_DOUBLE_RND_CONST / 100000000000000ll,
+ SYS_DOUBLE_RND_CONST / 1000000000000000ll,
+ SYS_DOUBLE_RND_CONST / 10000000000000000ll,
+ SYS_DOUBLE_RND_CONST / 100000000000000000ll,
+ SYS_DOUBLE_RND_CONST / 1000000000000000000ll
};
- long long mantissa, int_part, int_part2, frac_part;
+ long long mantissa, int_part = 0, frac_part = 0;
short exp;
- int sign, i, n, m, max;
- double absf;
+ int max;
+ int neg;
+ double fr;
union { long long L; double F; } x;
- char c, *p = outbuf;
- int digit, roundup;
+ char *p = buffer;
+
+ if (decimals < 0)
+ return -1;
- x.F = f;
+ /* Round the number to given decimal places. The number of 5's in the
+ * SYS_DOUBLE_RND_CONST constant is chosen such that adding any more 5's doesn't
+ * change the double precision of the number, i.e.:
+ * 1> term_to_binary(0.55555555555555555, [{minor_version, 1}]).
+ * <<131,70,63,225,199,28,113,199,28,114>>
+ * 2> term_to_binary(0.5555555555555555555, [{minor_version, 1}]).
+ * <<131,70,63,225,199,28,113,199,28,114>>
+ */
+ if (f >= 0) {
+ neg = 0;
+ fr = decimals < MAX_DECIMALS ? (f + cs_sys_double_pow10[decimals]) : f;
+ x.F = fr;
+ } else {
+ neg = 1;
+ fr = decimals < MAX_DECIMALS ? (f - cs_sys_double_pow10[decimals]) : f;
+ x.F = -fr;
+ }
exp = (x.L >> FRAC_SIZE) & EXP_MASK;
mantissa = x.L & FRAC_MASK;
- sign = x.L >= 0 ? 1 : -1;
+
if (exp == EXP_MASK) {
if (mantissa == 0) {
- if (sign == -1)
+ if (neg)
*p++ = '-';
*p++ = 'i';
*p++ = 'n';
@@ -158,101 +214,79 @@ sys_double_to_chars_fast(double f, char *outbuf, int maxlen, int decimals, int c
*p++ = 'n';
}
*p = '\0';
- return p - outbuf;
+ return p - buffer;
}
exp -= EXP_MASK >> 1;
mantissa |= (1ll << FRAC_SIZE);
- frac_part = 0;
- int_part = 0;
- absf = f * sign;
-
- /* Don't bother with optimizing too large numbers and decimals */
- if (absf > MAX_FLOAT || decimals > maxlen-17) {
- int len = erts_snprintf(outbuf, maxlen, "%.*f", decimals, f);
- if (len >= maxlen)
+
+ /* Don't bother with optimizing too large numbers or too large precision */
+ if (x.F > MAX_FLOAT || decimals >= MAX_DECIMALS) {
+ int len = erts_snprintf(buffer, buffer_size, "%.*f", decimals, f);
+ char* p = buffer + len;
+ if (len >= buffer_size)
return -1;
- p = outbuf + len;
/* Delete trailing zeroes */
if (compact)
- p = float_first_trailing_zero(outbuf + len);
+ p = find_first_trailing_zero(p);
*p = '\0';
- return p - outbuf;
- }
-
- if (exp >= FRAC_SIZE)
+ return p - buffer;
+ } else if (exp >= FRAC_SIZE) {
int_part = mantissa << (exp - FRAC_SIZE);
- else if (exp >= 0) {
+ } else if (exp >= 0) {
int_part = mantissa >> (FRAC_SIZE - exp);
frac_part = (mantissa << (exp + 1)) & FRAC_MASK2;
- }
- else /* if (exp < 0) */
+ } else /* if (exp < 0) */ {
frac_part = (mantissa & FRAC_MASK2) >> -(exp + 1);
+ }
- if (int_part == 0) {
- if (sign == -1)
+ if (!int_part) {
+ if (neg)
*p++ = '-';
*p++ = '0';
} else {
- int ret;
+ int ret, i, n;
while (int_part != 0) {
- int_part2 = int_part / 10;
- *p++ = (char)(int_part - ((int_part2 << 3) + (int_part2 << 1)) + '0');
- int_part = int_part2;
+ long long j = int_part / 10;
+ *p++ = (char)(int_part - ((j << 3) + (j << 1)) + '0');
+ int_part = j;
}
- if (sign == -1)
+ if (neg)
*p++ = '-';
/* Reverse string */
- ret = p - outbuf;
+ ret = p - buffer;
for (i = 0, n = ret/2; i < n; i++) {
- int j = ret - i - 1;
- c = outbuf[i];
- outbuf[i] = outbuf[j];
- outbuf[j] = c;
+ int j = ret - i - 1;
+ char c = buffer[i];
+ buffer[i] = buffer[j];
+ buffer[j] = c;
}
}
- if (decimals != 0)
+
+ if (decimals > 0) {
+ int i;
*p++ = '.';
- max = maxlen - (p - outbuf) - 1 /* leave room for trailing '\0' */;
- if (max > decimals)
+ max = buffer_size - (p - buffer) - 1 /* leave room for trailing '\0' */;
+
+ if (decimals > max)
+ return -1; /* the number is not large enough to fit in the buffer */
+
max = decimals;
- for (m = 0; m < max; m++) {
- /* frac_part *= 10; */
- frac_part = (frac_part << 3) + (frac_part << 1);
- *p++ = (char)((frac_part >> (FRAC_SIZE + 1)) + '0');
- frac_part &= FRAC_MASK2;
- }
+ for (i = 0; i < max; i++) {
+ /* frac_part *= 10; */
+ frac_part = (frac_part << 3) + (frac_part << 1);
- roundup = 0;
- /* Rounding - look at the next digit */
- frac_part = (frac_part << 3) + (frac_part << 1);
- digit = (frac_part >> (FRAC_SIZE + 1));
- if (digit > 5)
- roundup = 1;
- else if (digit == 5) {
- frac_part &= FRAC_MASK2;
- if (frac_part != 0) roundup = 1;
- }
- if (roundup) {
- char d;
- int pos = p - outbuf - 1;
- do {
- d = outbuf[pos];
- if (d == '-') break;
- if (d == '.') continue;
- if (++d != ':') {
- outbuf[pos] = d;
- break;
- }
- outbuf[pos] = '0';
- } while (--pos);
+ *p++ = (char)((frac_part >> (FRAC_SIZE + 1)) + '0');
+ frac_part &= FRAC_MASK2;
+ }
+
+ /* Delete trailing zeroes */
+ if (compact)
+ p = find_first_trailing_zero(p);
}
- /* Delete trailing zeroes */
- if (compact && *(p - 1) == '0')
- p = float_first_trailing_zero(--p);
*p = '\0';
- return p - outbuf;
+ return p - buffer;
}
diff --git a/erts/emulator/sys/common/erl_util_queue.h b/erts/emulator/sys/common/erl_util_queue.h
new file mode 100644
index 0000000000..47925e2264
--- /dev/null
+++ b/erts/emulator/sys/common/erl_util_queue.h
@@ -0,0 +1,77 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2013. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifndef ERL_UTIL_QUEUE_H_
+#define ERL_UTIL_QUEUE_H_
+
+#define erts_circleq_head(Q) ((Q)->next)
+#define erts_circleq_tail(Q) ((Q)->prev)
+#define erts_circleq_next(Q) ((Q)->next)
+#define erts_circleq_prev(Q) ((Q)->prev)
+#define erts_circleq_is_empty(Q) ((Q)->next == (void *)(Q))
+
+#define erts_circleq_remove(N) \
+ do { \
+ (N)->next->prev = (N)->prev; \
+ (N)->prev->next = (N)->next; \
+ (N)->next = (N); \
+ (N)->prev = (N); \
+ } while(0)
+
+#define erts_circleq_pop_head(Q, N) \
+ do { \
+ (N) = (Q)->next; \
+ (N)->next->prev = (N)->prev; \
+ (N)->prev->next = (N)->next; \
+ (N)->next = (N); \
+ (N)->prev = (N); \
+ } while(0)
+
+#define erts_circleq_pop_tail(Q, N) \
+ do { \
+ (N) = (Q)->prev; \
+ (N)->next->prev = (N)->prev; \
+ (N)->prev->next = (N)->next; \
+ (N)->next = (N); \
+ (N)->prev = (N); \
+ } while(0)
+
+#define erts_circleq_push_head(Q, N) \
+ do { \
+ (N)->next = (Q)->next; \
+ (N)->prev = (void *)(Q); \
+ (Q)->next->prev = (N); \
+ (Q)->next = (N); \
+ } while(0)
+
+#define erts_circleq_push_tail(Q, N) \
+ do { \
+ (N)->prev = (Q)->prev; \
+ (N)->next = (void *)(Q); \
+ (Q)->prev->next = (N); \
+ (Q)->prev = (N); \
+ } while(0)
+
+#define erts_circleq_foreach(V, Q) \
+ for ((V) = (Q)->next; (V) != (const void *)(Q); (V) = (V)->next)
+
+#define erts_circleq_foreach_reverse(V, Q) \
+ for ((V) = (Q)->prev; (V) != (const void *)(Q); (V) = (V)->prev)
+
+#endif
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index f7756f99bc..19dffd0ea4 100755
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -57,11 +57,13 @@ extern void _dosmaperr(DWORD);
#define __argv e_argv
#endif
+typedef struct driver_data DriverData;
+
static void init_console();
static int get_and_remove_option(int* argc, char** argv, const char* option);
static char *get_and_remove_option2(int *argc, char **argv,
const char *option);
-static int init_async_io(struct async_io* aio, int use_threads);
+static int init_async_io(DriverData *dp, struct async_io* aio, int use_threads);
static void release_async_io(struct async_io* aio, ErlDrvPort);
static void async_read_file(struct async_io* aio, LPVOID buf, DWORD numToRead);
static int async_write_file(struct async_io* aio, LPVOID buf, DWORD numToWrite);
@@ -96,7 +98,7 @@ static erts_smp_atomic_t pipe_creation_counter;
static int driver_write(long, HANDLE, byte*, int);
static int create_file_thread(struct async_io* aio, int mode);
#ifdef ERTS_SMP
-static void close_active_handle(ErlDrvPort, HANDLE handle);
+static void close_active_handle(DriverData *, HANDLE handle);
static DWORD WINAPI threaded_handle_closer(LPVOID param);
#endif
static DWORD WINAPI threaded_reader(LPVOID param);
@@ -440,6 +442,8 @@ typedef struct async_io {
DWORD bytesTransferred; /* Bytes read or write in the last operation.
* Valid only when DF_OVR_READY is set.
*/
+ DriverData *dp; /* Pointer to driver data struct which
+ this struct is part of */
} AsyncIo;
@@ -458,7 +462,7 @@ static BOOL (WINAPI *fpSetHandleInformation)(HANDLE,DWORD,DWORD);
* none of the file handles.
*/
-typedef struct driver_data {
+struct driver_data {
int totalNeeded; /* Total number of bytes needed to fill
* up the packet header or packet. */
int bytesInBuffer; /* Number of bytes read so far in
@@ -476,7 +480,8 @@ typedef struct driver_data {
AsyncIo in; /* Control block for overlapped reading. */
AsyncIo out; /* Control block for overlapped writing. */
int report_exit; /* Do report exit status for the port */
-} DriverData;
+ erts_atomic32_t refc; /* References to this struct */
+};
/* Driver interfaces */
static ErlDrvData spawn_start(ErlDrvPort, char*, SysDriverOpts*);
@@ -581,6 +586,26 @@ struct erl_drv_entry vanilla_driver_entry = {
stop_select
};
+static ERTS_INLINE void
+refer_driver_data(DriverData *dp)
+{
+#ifdef DEBUG
+ erts_aint32_t refc = erts_atomic32_inc_read_nob(&dp->refc);
+ ASSERT(refc > 1);
+#else
+ erts_atomic32_inc_nob(&dp->refc);
+#endif
+}
+
+static ERTS_INLINE void
+unrefer_driver_data(DriverData *dp)
+{
+ erts_aint32_t refc = erts_atomic32_dec_read_mb(&dp->refc);
+ ASSERT(refc >= 0);
+ if (refc == 0)
+ driver_free(dp);
+}
+
/*
* Initialises a DriverData structure.
*
@@ -604,6 +629,7 @@ new_driver_data(ErlDrvPort port_num, int packet_bytes, int wait_objs_required, i
* any more, since driver_select() can't fail.
*/
+ erts_atomic32_init_nob(&dp->refc, 1);
dp->bytesInBuffer = 0;
dp->totalNeeded = packet_bytes;
dp->inBufSize = PORT_BUFSIZ;
@@ -616,9 +642,9 @@ new_driver_data(ErlDrvPort port_num, int packet_bytes, int wait_objs_required, i
dp->port_num = port_num;
dp->packet_bytes = packet_bytes;
dp->port_pid = INVALID_HANDLE_VALUE;
- if (init_async_io(&dp->in, use_threads) == -1)
+ if (init_async_io(dp, &dp->in, use_threads) == -1)
goto async_io_error1;
- if (init_async_io(&dp->out, use_threads) == -1)
+ if (init_async_io(dp, &dp->out, use_threads) == -1)
goto async_io_error2;
return dp;
@@ -662,7 +688,7 @@ release_driver_data(DriverData* dp)
dp->in.fd = INVALID_HANDLE_VALUE;
DEBUGF(("Waiting for the in event thingie"));
if (WaitForSingleObject(dp->in.ov.hEvent,timeout) == WAIT_TIMEOUT) {
- close_active_handle(dp->port_num, dp->in.ov.hEvent);
+ close_active_handle(dp, dp->in.ov.hEvent);
dp->in.ov.hEvent = NULL;
timeout = 0;
}
@@ -673,7 +699,7 @@ release_driver_data(DriverData* dp)
dp->out.fd = INVALID_HANDLE_VALUE;
DEBUGF(("Waiting for the out event thingie"));
if (WaitForSingleObject(dp->out.ov.hEvent,timeout) == WAIT_TIMEOUT) {
- close_active_handle(dp->port_num, dp->out.ov.hEvent);
+ close_active_handle(dp, dp->out.ov.hEvent);
dp->out.ov.hEvent = NULL;
}
DEBUGF(("...done\n"));
@@ -719,7 +745,7 @@ release_driver_data(DriverData* dp)
* the exit thread.
*/
- driver_free(dp);
+ unrefer_driver_data(dp);
}
#ifdef ERTS_SMP
@@ -727,11 +753,12 @@ release_driver_data(DriverData* dp)
struct handles_to_be_closed {
HANDLE handles[MAXIMUM_WAIT_OBJECTS];
unsigned cnt;
+ DriverData *dp;
};
static struct handles_to_be_closed* htbc_curr = NULL;
CRITICAL_SECTION htbc_lock;
-static void close_active_handle(ErlDrvPort port_num, HANDLE handle)
+static void close_active_handle(DriverData *dp, HANDLE handle)
{
struct handles_to_be_closed* htbc;
int i;
@@ -745,11 +772,14 @@ static void close_active_handle(ErlDrvPort port_num, HANDLE handle)
sizeof(*htbc));
htbc->handles[0] = CreateAutoEvent(FALSE);
htbc->cnt = 1;
+ htbc->dp = dp;
+ refer_driver_data(dp); /* Need to keep driver data until we have
+ closed the event; outstanding operation
+ might write into it.. */
thread = (HANDLE *) _beginthreadex(NULL, 0, threaded_handle_closer, htbc, 0, &tid);
CloseHandle(thread);
}
htbc->handles[htbc->cnt++] = handle;
- driver_select(port_num, (ErlDrvEvent)handle, ERL_DRV_USE_NO_CALLBACK, 0);
SetEvent(htbc->handles[0]);
htbc_curr = htbc;
LeaveCriticalSection(&htbc_lock);
@@ -798,6 +828,7 @@ threaded_handle_closer(LPVOID param)
}
LeaveCriticalSection(&htbc_lock);
CloseHandle(htbc->handles[0]);
+ unrefer_driver_data(htbc->dp);
erts_free(ERTS_ALC_T_DRV_TAB, htbc);
DEBUGF(("threaded_handle_closer %p terminating\r\n", htbc));
return 0;
@@ -864,8 +895,9 @@ reuse_driver_data(DriverData *dp, HANDLE ifd, HANDLE ofd, int read_write, ErlDrv
*/
static int
-init_async_io(AsyncIo* aio, int use_threads)
+init_async_io(DriverData *dp, AsyncIo* aio, int use_threads)
{
+ aio->dp = dp;
aio->flags = 0;
aio->thread = (HANDLE) -1;
aio->fd = INVALID_HANDLE_VALUE;
@@ -884,6 +916,8 @@ init_async_io(AsyncIo* aio, int use_threads)
if (aio->ov.hEvent == NULL)
return -1;
if (use_threads) {
+ OV_BUFFER_PTR(aio) = NULL;
+ OV_NUM_TO_READ(aio) = 0;
aio->ioAllowed = CreateAutoEvent(FALSE);
if (aio->ioAllowed == NULL)
return -1;
@@ -914,12 +948,8 @@ release_async_io(AsyncIo* aio, ErlDrvPort port_num)
CloseHandle(aio->fd);
aio->fd = INVALID_HANDLE_VALUE;
- if (aio->ov.hEvent != NULL) {
- (void) driver_select(port_num,
- (ErlDrvEvent)aio->ov.hEvent,
- ERL_DRV_USE, 0);
- /* was CloseHandle(aio->ov.hEvent); */
- }
+ if (aio->ov.hEvent != NULL)
+ CloseHandle(aio->ov.hEvent);
aio->ov.hEvent = NULL;
@@ -1287,12 +1317,15 @@ create_file_thread(AsyncIo* aio, int mode)
{
DWORD tid; /* Id for thread. */
+ refer_driver_data(aio->dp);
aio->thread = (HANDLE)
_beginthreadex(NULL, 0,
(mode & DO_WRITE) ? threaded_writer : threaded_reader,
aio, 0, &tid);
-
- return aio->thread != (HANDLE) -1;
+ if (aio->thread != (HANDLE) -1)
+ return 1;
+ unrefer_driver_data(aio->dp);
+ return 0;
}
/*
@@ -2078,6 +2111,7 @@ threaded_reader(LPVOID param)
if (aio->flags & DF_EXIT_THREAD)
break;
}
+ unrefer_driver_data(aio->dp);
return 0;
}
@@ -2157,6 +2191,7 @@ threaded_writer(LPVOID param)
}
CloseHandle(aio->fd);
aio->fd = INVALID_HANDLE_VALUE;
+ unrefer_driver_data(aio->dp);
return 0;
}
@@ -2297,6 +2332,7 @@ static void fd_stop(ErlDrvData data)
(void) driver_select(dp->port_num,
(ErlDrvEvent)dp->out.ov.hEvent,
ERL_DRV_WRITE, 0);
+ ASSERT(dp->out.flushEvent);
SetEvent(dp->out.flushEvent);
WaitForSingleObject(dp->out.flushReplyEvent, INFINITE);
}
@@ -2349,12 +2385,12 @@ stop(ErlDrvData data)
if (dp->in.ov.hEvent != NULL) {
(void) driver_select(dp->port_num,
(ErlDrvEvent)dp->in.ov.hEvent,
- ERL_DRV_READ, 0);
+ ERL_DRV_READ|ERL_DRV_USE_NO_CALLBACK, 0);
}
if (dp->out.ov.hEvent != NULL) {
(void) driver_select(dp->port_num,
(ErlDrvEvent)dp->out.ov.hEvent,
- ERL_DRV_WRITE, 0);
+ ERL_DRV_WRITE|ERL_DRV_USE_NO_CALLBACK, 0);
}
if (dp->out.thread == (HANDLE) -1 && dp->in.thread == (HANDLE) -1) {
@@ -2366,6 +2402,8 @@ stop(ErlDrvData data)
*/
HANDLE thread;
DWORD tid;
+
+ /* threaded_exiter implicitly takes over refc from us... */
thread = (HANDLE *) _beginthreadex(NULL, 0, threaded_exiter, dp, 0, &tid);
CloseHandle(thread);
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-03 23:43:46 +0000