path: root/erts/emulator/sys/common/erl_check_io.c

/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2006-2018. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

/*
 * Description:	Check I/O
 *
 * Author: 	Rickard Green
 */

#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

#define ERL_CHECK_IO_C__
#ifndef WANT_NONBLOCKING
#  define WANT_NONBLOCKING
#endif
#include "sys.h"
#include "global.h"
#include "erl_port.h"
#include "erl_check_io.h"
#include "erl_thr_progress.h"
#include "erl_bif_unique.h"
#include "dtrace-wrapper.h"
#include "lttng-wrapper.h"
#define ERTS_WANT_TIMER_WHEEL_API
#include "erl_time.h"

#if 0
#define DEBUG_PRINT(FMT, ...) erts_printf(FMT "\r\n", ##__VA_ARGS__)
#define DEBUG_PRINT_FD(FMT, STATE, ...)                                 \
    DEBUG_PRINT("%d: " FMT " (ev=%s, ac=%s, flg=%s)",                   \
                (STATE) ? (STATE)->fd : (ErtsSysFdType)-1, ##__VA_ARGS__, \
                ev2str((STATE) ? (STATE)->events : ERTS_POLL_EV_NONE),  \
                ev2str((STATE) ? (STATE)->active_events : ERTS_POLL_EV_NONE), \
                (STATE) ? flag2str((STATE)->flags) : ERTS_EV_FLAG_CLEAR)
#define DEBUG_PRINT_MODE
#else
#define DEBUG_PRINT(...)
#endif

#ifndef DEBUG_PRINT_FD
#define DEBUG_PRINT_FD(...)
#endif

#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
#  include "safe_hash.h"
#  define DRV_EV_STATE_HTAB_SIZE 1024
#endif

typedef enum {
    ERTS_EV_TYPE_NONE     = 0,
    ERTS_EV_TYPE_DRV_SEL  = 1, /* driver_select */
    ERTS_EV_TYPE_STOP_USE = 2, /* pending stop_select */
    ERTS_EV_TYPE_NIF      = 3, /* enif_select */
    ERTS_EV_TYPE_STOP_NIF = 4  /* pending nif stop */
} EventStateType;

typedef enum {
    ERTS_EV_FLAG_CLEAR         = 0,
    ERTS_EV_FLAG_USED          = 1,   /* ERL_DRV_USE has been turned on */
#if ERTS_POLL_USE_SCHEDULER_POLLING
    ERTS_EV_FLAG_SCHEDULER     = 2,   /* Set when the fd has been migrated
                                         to scheduler pollset */
    ERTS_EV_FLAG_IN_SCHEDULER  = 4,   /* Set when the fd is currently in
                                         scheduler pollset */
#else
    ERTS_EV_FLAG_SCHEDULER     = ERTS_EV_FLAG_CLEAR,
    ERTS_EV_FLAG_IN_SCHEDULER  = ERTS_EV_FLAG_CLEAR,
#endif
#ifdef ERTS_POLL_USE_FALLBACK
    ERTS_EV_FLAG_FALLBACK      = 8,   /* Set when kernel poll rejected fd
                                         and it was put in the nkp version */
#else
    ERTS_EV_FLAG_FALLBACK      = ERTS_EV_FLAG_CLEAR,
#endif

    /* Combinations */
    ERTS_EV_FLAG_USED_FALLBACK = ERTS_EV_FLAG_USED | ERTS_EV_FLAG_FALLBACK,
    ERTS_EV_FLAG_USED_SCHEDULER = ERTS_EV_FLAG_USED | ERTS_EV_FLAG_SCHEDULER,
    ERTS_EV_FLAG_USED_IN_SCHEDULER = ERTS_EV_FLAG_USED | ERTS_EV_FLAG_SCHEDULER | ERTS_EV_FLAG_IN_SCHEDULER,
    ERTS_EV_FLAG_UNUSED_SCHEDULER = ERTS_EV_FLAG_SCHEDULER,
    ERTS_EV_FLAG_UNUSED_IN_SCHEDULER = ERTS_EV_FLAG_SCHEDULER | ERTS_EV_FLAG_IN_SCHEDULER
} EventStateFlags;

#define flag2str(flags)                                                 \
    ((flags) == ERTS_EV_FLAG_CLEAR ? "CLEAR" :                          \
     ((flags) == ERTS_EV_FLAG_USED ? "USED" :                           \
      ((flags) == ERTS_EV_FLAG_FALLBACK ? "FLBK" :                      \
       ((flags) == ERTS_EV_FLAG_USED_FALLBACK ? "USED|FLBK" :           \
        ((flags) == ERTS_EV_FLAG_USED_SCHEDULER ? "USED|SCHD" :         \
         ((flags) == ERTS_EV_FLAG_UNUSED_SCHEDULER ? "SCHD" :           \
          ((flags) == ERTS_EV_FLAG_USED_IN_SCHEDULER ? "USED|IN_SCHD" : \
           ((flags) == ERTS_EV_FLAG_UNUSED_IN_SCHEDULER ? "IN_SCHD" :   \
            "ERROR"))))))))

/* How many events that can be handled at once by one erts_poll_wait call */
#define ERTS_CHECK_IO_POLL_RES_LEN 512

/* Each I/O Poll Thread has one ErtsPollThread each. The ps field
   can point to either a private ErtsPollSet or a shared one.
   At the moment only kqueue and epoll pollsets can be
   shared across threads.
*/
typedef struct erts_poll_thread
{
    ErtsPollSet *ps;
    ErtsPollResFd *pollres;
    ErtsThrPrgrData *tpd;
    int pollres_len;
} ErtsPollThread;

/* pollsetv contains pointers to the ErtsPollSets that are in use.
 * Which pollset to use is determined by hashing the fd.
 */
static ErtsPollSet **pollsetv;
static ErtsPollThread *psiv;
#if ERTS_POLL_USE_FALLBACK
static ErtsPollSet *flbk_pollset;
#endif
#if ERTS_POLL_USE_SCHEDULER_POLLING
static ErtsPollSet *sched_pollset;
#endif

typedef struct {
#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
    SafeHashBucket hb;
#endif
    ErtsSysFdType fd;
    struct {
	ErtsDrvSelectDataState *select;   /* ERTS_EV_TYPE_DRV_SEL */
        ErtsNifSelectDataState *nif;      /* ERTS_EV_TYPE_NIF */
        union {
            erts_driver_t*  drv_ptr;    /* ERTS_EV_TYPE_STOP_USE */
            ErtsResource* resource;   /* ERTS_EV_TYPE_STOP_NIF */
        } stop;
    } driver;
    ErtsPollEvents events;        /* The events that have been selected upon */
    ErtsPollEvents active_events; /* The events currently active in the pollset */
    EventStateType type;
    EventStateFlags flags;
    int count;                    /* Number of times this fd has triggered
                                     without being deselected. */
} ErtsDrvEventState;

struct drv_ev_state_shared {

    union {
        erts_mtx_t lck;
        byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))];
    } locks[ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT];

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    int max_fds;
    erts_atomic_t len;
    ErtsDrvEventState *v;
    erts_mtx_t grow_lock; /* prevent lock-hogging of racing growers */
#else
    SafeHash tab;
    int num_prealloc;
    ErtsDrvEventState *prealloc_first;
    erts_spinlock_t prealloc_lock;
#endif
};

int ERTS_WRITE_UNLIKELY(erts_no_pollsets) = 1;
int ERTS_WRITE_UNLIKELY(erts_no_poll_threads) = 1;
struct drv_ev_state_shared drv_ev_state;

static ERTS_INLINE int fd_hash(ErtsSysFdType fd) {
    int hash = (int)fd;
# ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
    hash ^= (hash >> 9);
# endif
    return hash;
}

static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd)
{
    return &drv_ev_state.locks[fd_hash(fd) % ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT].lck;
}

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS

static ERTS_INLINE ErtsDrvEventState *get_drv_ev_state(ErtsSysFdType fd)
{
    return &drv_ev_state.v[(int) fd];
}

#define new_drv_ev_state(State, fd) (State)
#define erase_drv_ev_state(State)

static ERTS_INLINE int grow_drv_ev_state(ErtsSysFdType fd) {
    int i;
    int old_len;
    int new_len;

    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state.len)) {

	if (fd < 0 || fd >= drv_ev_state.max_fds)
            return 0;

        erts_mtx_lock(&drv_ev_state.grow_lock);
        old_len = erts_atomic_read_nob(&drv_ev_state.len);
        if (fd >= old_len) {
            new_len = erts_poll_new_table_len(old_len, fd + 1);
            if (new_len > drv_ev_state.max_fds)
                new_len = drv_ev_state.max_fds;

            for (i=0; i<ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */
                erts_mtx_lock(&drv_ev_state.locks[i].lck);
            }
            drv_ev_state.v = (drv_ev_state.v
                              ? erts_realloc(ERTS_ALC_T_DRV_EV_STATE,
                                             drv_ev_state.v,
                                             sizeof(ErtsDrvEventState)*new_len)
                              : erts_alloc(ERTS_ALC_T_DRV_EV_STATE,
                                           sizeof(ErtsDrvEventState)*new_len));
            ERTS_CT_ASSERT(ERTS_EV_TYPE_NONE == 0);
            sys_memzero(drv_ev_state.v+old_len,
                        sizeof(ErtsDrvEventState) * (new_len - old_len));
            for (i = old_len; i < new_len; i++) {
                drv_ev_state.v[i].fd = (ErtsSysFdType) i;
            }
            erts_atomic_set_nob(&drv_ev_state.len, new_len);
            for (i=0; i<ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) {
                erts_mtx_unlock(&drv_ev_state.locks[i].lck);
            }
        }
        /*else already grown by racing thread */

        erts_mtx_unlock(&drv_ev_state.grow_lock);
    }
    return 1;
}

static int drv_ev_state_len(void)
{
    return erts_atomic_read_nob(&drv_ev_state.len);
}

#else /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */

static ERTS_INLINE ErtsDrvEventState *get_drv_ev_state(ErtsSysFdType fd)
{
    ErtsDrvEventState tmpl;
    tmpl.fd = fd;
    return  (ErtsDrvEventState *) safe_hash_get(&drv_ev_state.tab, (void *) &tmpl);
}

static ERTS_INLINE ErtsDrvEventState* new_drv_ev_state(ErtsDrvEventState *state,
                                                       ErtsSysFdType fd)
{
    ErtsDrvEventState tmpl;

    if (state)
        return state;

    tmpl.fd = fd;
    tmpl.driver.select = NULL;
    tmpl.driver.nif = NULL;
    tmpl.driver.stop.drv_ptr = NULL;
    tmpl.events = 0;
    tmpl.active_events = 0;
    tmpl.type = ERTS_EV_TYPE_NONE;
    tmpl.flags = 0;

    return  (ErtsDrvEventState *) safe_hash_put(&drv_ev_state.tab, (void *) &tmpl);
}

static ERTS_INLINE void erase_drv_ev_state(ErtsDrvEventState *state)
{
    safe_hash_erase(&drv_ev_state.tab, (void *) state);
}

static int drv_ev_state_len(void)
{
    return erts_atomic_read_nob(&drv_ev_state.tab.nitems);
}

#endif /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */

static void stale_drv_select(Eterm id, ErtsDrvEventState *state, int mode);
static void drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state,
                             int mode, int on);
static void nif_select_steal(ErtsDrvEventState *state, int mode,
                             ErtsResource* resource, Eterm ref);

static void print_drv_select_op(erts_dsprintf_buf_t *dsbufp,
                                ErlDrvPort ix, ErtsSysFdType fd, int mode, int on);
static void print_nif_select_op(erts_dsprintf_buf_t*, ErtsSysFdType,
                                int mode, ErtsResource*, Eterm ref);

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
static void drv_select_large_fd_error(ErlDrvPort, ErtsSysFdType, int, int);
static void nif_select_large_fd_error(ErtsSysFdType, int, ErtsResource*,Eterm ref);
#endif
static void
steal_pending_stop_use(erts_dsprintf_buf_t*, ErlDrvPort, ErtsDrvEventState*,
                       int mode, int on);
static void
steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource*,
                       ErtsDrvEventState *state, int mode, int on);
static ERTS_INLINE void
check_fd_cleanup(ErtsDrvEventState *state,
		 ErtsDrvSelectDataState **free_select,
                 ErtsNifSelectDataState **free_nif);
static ERTS_INLINE void iready(Eterm id, ErtsDrvEventState *state);
static ERTS_INLINE void oready(Eterm id, ErtsDrvEventState *state);
#ifdef DEBUG_PRINT_MODE
static char *drvmode2str(int mode);
static char *nifmode2str(enum ErlNifSelectFlags mode);
#endif

static ERTS_INLINE void
init_iotask(ErtsIoTask *io_task, ErtsSysFdType fd)
{
    erts_port_task_handle_init(&io_task->task);
    io_task->fd = fd;
}

static ERTS_INLINE int
is_iotask_active(ErtsIoTask *io_task)
{
    if (erts_port_task_is_scheduled(&io_task->task))
	return 1;
    return 0;
}

static ERTS_INLINE ErtsDrvSelectDataState *
alloc_drv_select_data(ErtsSysFdType fd)
{
    ErtsDrvSelectDataState *dsp = erts_alloc(ERTS_ALC_T_DRV_SEL_D_STATE,
					     sizeof(ErtsDrvSelectDataState));
    dsp->inport = NIL;
    dsp->outport = NIL;
    init_iotask(&dsp->iniotask, fd);
    init_iotask(&dsp->outiotask, fd);
    return dsp;
}

static ERTS_INLINE ErtsNifSelectDataState *
alloc_nif_select_data(void)
{
    ErtsNifSelectDataState *dsp = erts_alloc(ERTS_ALC_T_NIF_SEL_D_STATE,
					     sizeof(ErtsNifSelectDataState));
    dsp->in.pid = NIL;
    dsp->out.pid = NIL;
    return dsp;
}

static ERTS_INLINE void
free_drv_select_data(ErtsDrvSelectDataState *dsp)
{
    ASSERT(!erts_port_task_is_scheduled(&dsp->iniotask.task));
    ASSERT(!erts_port_task_is_scheduled(&dsp->outiotask.task));
    erts_free(ERTS_ALC_T_DRV_SEL_D_STATE, dsp);   
}

static ERTS_INLINE void
free_nif_select_data(ErtsNifSelectDataState *dsp)
{
    erts_free(ERTS_ALC_T_NIF_SEL_D_STATE, dsp);
}

static ERTS_INLINE int
get_pollset_id(ErtsSysFdType fd)
{
    return fd_hash(fd) % erts_no_pollsets;
}

static ERTS_INLINE ErtsPollSet *
get_pollset(ErtsSysFdType fd)
{
    return pollsetv[get_pollset_id(fd)];
}

#if ERTS_POLL_USE_FALLBACK
static ERTS_INLINE ErtsPollSet *
get_fallback_pollset(void)
{
    return flbk_pollset;
}
#endif

static ERTS_INLINE ErtsPollSet *
get_scheduler_pollset(ErtsSysFdType fd)
{
#if ERTS_POLL_USE_SCHEDULER_POLLING
    return sched_pollset;
#else
    return get_pollset(fd);
#endif
}

/*
 * Place a fd within a pollset. This will automatically use
 * the fallback ps if needed.
 */
static ERTS_INLINE ErtsPollEvents
erts_io_control_wakeup(ErtsDrvEventState *state, ErtsPollOp op,
                       ErtsPollEvents pe, int *wake_poller)
{
    ErtsSysFdType fd = state->fd;
    ErtsPollEvents res = 0;
    EventStateFlags flags = state->flags;

    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));

    if (!(flags & ERTS_EV_FLAG_FALLBACK)) {

        if (op == ERTS_POLL_OP_DEL && (flags & ERTS_EV_FLAG_SCHEDULER)) {
            erts_poll_control(get_scheduler_pollset(fd), fd, op, pe, wake_poller);
            flags &= ~ERTS_EV_FLAG_IN_SCHEDULER;
        }
        if (!(flags & ERTS_EV_FLAG_IN_SCHEDULER) || (pe & ERTS_POLL_EV_OUT)) {
            res = erts_poll_control(get_pollset(fd), fd, op, pe, wake_poller);
        } else {
            res = erts_poll_control(get_scheduler_pollset(fd), fd, op, pe, wake_poller);
        }

#if ERTS_POLL_USE_FALLBACK
        if (op == ERTS_POLL_OP_ADD && res == ERTS_POLL_EV_NVAL) {
            /* When an add fails with NVAL, the poll/kevent operation could not
               put that fd in the pollset, so we instead put it into a fallback pollset */
            state->flags |= ERTS_EV_FLAG_FALLBACK;
            res = erts_poll_control_flbk(get_fallback_pollset(), fd, op, pe, wake_poller);
        }
    } else {
        ASSERT(op != ERTS_POLL_OP_ADD);
        res = erts_poll_control_flbk(get_fallback_pollset(), fd, op, pe, wake_poller);
#endif
    }

    return res;
}

static ERTS_INLINE ErtsPollEvents
erts_io_control(ErtsDrvEventState *state, ErtsPollOp op, ErtsPollEvents pe)
{
    int wake_poller = 0;
    return erts_io_control_wakeup(state, op, pe, &wake_poller);
}

/* ToDo: Was inline in erl_check_io.h but now need struct erts_poll_thread */
void
erts_io_notify_port_task_executed(ErtsPortTaskType type,
                                  ErtsPortTaskHandle *pthp,
                                  void (*reset_handle)(ErtsPortTaskHandle *))
{
    ErtsIoTask *itp = ErtsContainerStruct(pthp, ErtsIoTask, task);
    ErtsSysFdType fd = itp->fd;
    erts_mtx_t *mtx = fd_mtx(fd);
    ErtsPollOp op = ERTS_POLL_OP_MOD;
    int active_events, new_events = 0;
    ErtsDrvEventState *state;
    ErtsDrvSelectDataState *free_select = NULL;
    ErtsNifSelectDataState *free_nif = NULL;

    ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_CHECK_IO);

    erts_mtx_lock(mtx);
    state = get_drv_ev_state(fd);

    reset_handle(pthp);

    active_events = state->active_events;

    if (!(state->flags & ERTS_EV_FLAG_IN_SCHEDULER) || type == ERTS_PORT_TASK_OUTPUT) {
        switch (type) {
        case ERTS_PORT_TASK_INPUT:

            DEBUG_PRINT_FD("executed ready_input", state);

            ASSERT(!(state->active_events & ERTS_POLL_EV_IN));
            if (state->events & ERTS_POLL_EV_IN) {
                active_events |= ERTS_POLL_EV_IN;
                if (state->count > 10 && ERTS_POLL_USE_SCHEDULER_POLLING) {
                    if (!(state->flags & ERTS_EV_FLAG_SCHEDULER))
                        op = ERTS_POLL_OP_ADD;
                    state->flags |= ERTS_EV_FLAG_IN_SCHEDULER|ERTS_EV_FLAG_SCHEDULER;
                    new_events = ERTS_POLL_EV_IN;
                    DEBUG_PRINT_FD("moving to scheduler ps", state);
                } else
                    new_events = active_events;
                if (!(state->flags & ERTS_EV_FLAG_FALLBACK) && ERTS_POLL_USE_SCHEDULER_POLLING)
                    state->count++;
            }
            break;
        case ERTS_PORT_TASK_OUTPUT:

            DEBUG_PRINT_FD("executed ready_output", state);

            ASSERT(!(state->active_events & ERTS_POLL_EV_OUT));
            if (state->events & ERTS_POLL_EV_OUT) {
                active_events |= ERTS_POLL_EV_OUT;
                if (state->flags & ERTS_EV_FLAG_IN_SCHEDULER && active_events & ERTS_POLL_EV_IN)
                    new_events = ERTS_POLL_EV_OUT;
                else
                    new_events = active_events;
            }
            break;
        default:
            erts_exit(ERTS_ABORT_EXIT, "Invalid IO port task type");
            break;
        }

        if (state->active_events != active_events && new_events) {
            state->active_events = active_events;
            new_events = erts_io_control(state, op, new_events);
        }

        /* We were unable to re-insert the fd into the pollset, signal the callback. */
        if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
            if (state->active_events & ERTS_POLL_EV_IN)
                iready(state->driver.select->inport, state);
            if (state->active_events & ERTS_POLL_EV_OUT)
                oready(state->driver.select->outport, state);
            state->active_events = 0;
        }
    }

    if (!active_events)
        check_fd_cleanup(state, &free_select, &free_nif);

    erts_mtx_unlock(mtx);

    if (free_select)
        free_drv_select_data(free_select);
    if (free_nif)
        free_nif_select_data(free_nif);

    ERTS_MSACC_POP_STATE_M_X();
}

static ERTS_INLINE void
abort_task(Eterm id, ErtsPortTaskHandle *pthp, EventStateType type)
{
    if (is_not_nil(id) && erts_port_task_is_scheduled(pthp)) {
	erts_port_task_abort(pthp);
	ASSERT(erts_is_port_alive(id));
    }
}

static ERTS_INLINE void
abort_tasks(ErtsDrvEventState *state, int mode)
{
    switch (mode) {
    case 0: check_type:
	switch (state->type) {
        case ERTS_EV_TYPE_NIF:
	case ERTS_EV_TYPE_NONE:
	    return;
	default:
	    ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL);
	    /* Fall through */
	}
    case ERL_DRV_READ|ERL_DRV_WRITE:
    case ERL_DRV_WRITE:
	ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL);
	abort_task(state->driver.select->outport,
		   &state->driver.select->outiotask.task,
		   state->type);
	if (mode == ERL_DRV_WRITE)
	    break;
    case ERL_DRV_READ:
	ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL);
	abort_task(state->driver.select->inport,
		   &state->driver.select->iniotask.task,
		   state->type);
	break;
    default:
	goto check_type;
    }
}

static void
deselect(ErtsDrvEventState *state, int mode)
{
    ErtsPollEvents rm_events;
    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));

    abort_tasks(state, mode);

    if (!mode) {
	rm_events = state->events;
    } else {
	rm_events = 0;
	ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL);
	if (mode & ERL_DRV_READ) {
	    state->driver.select->inport = NIL;
	    rm_events |= ERTS_POLL_EV_IN;
	}
	if (mode & ERL_DRV_WRITE) {
	    state->driver.select->outport = NIL;
	    rm_events |= ERTS_POLL_EV_OUT;
	}
    }

    state->events &= ~rm_events;
    state->active_events &= ~rm_events;

    if (!(state->events)) {
        erts_io_control(state, ERTS_POLL_OP_DEL, 0);
	switch (state->type) {
        case ERTS_EV_TYPE_NIF:
            state->driver.nif->in.pid = NIL;
            state->driver.nif->out.pid = NIL;
            enif_release_resource(state->driver.stop.resource->data);
            state->driver.stop.resource = NULL;
            break;
	case ERTS_EV_TYPE_DRV_SEL:
	    state->driver.select->inport = NIL;
	    state->driver.select->outport = NIL;
	    break;
	case ERTS_EV_TYPE_NONE:
	    break;
	default:
	    ASSERT(0);
	    break;
	}
	state->type = ERTS_EV_TYPE_NONE;
	state->flags = 0;
    } else {
        ErtsPollEvents new_events =
            erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events);

        /* We were unable to re-insert the fd into the pollset, signal the callback. */
        if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
            if (state->active_events & ERTS_POLL_EV_IN)
                iready(state->driver.select->inport, state);
            if (state->active_events & ERTS_POLL_EV_OUT)
                oready(state->driver.select->outport, state);
            state->active_events = 0;
        }
    }
}

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
#  define IS_FD_UNKNOWN(state) ((state)->type == ERTS_EV_TYPE_NONE)
#else
#  define IS_FD_UNKNOWN(state) ((state) == NULL)
#endif

static ERTS_INLINE void
check_fd_cleanup(ErtsDrvEventState *state,
		 ErtsDrvSelectDataState **free_select,
                 ErtsNifSelectDataState **free_nif)
{
    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
    *free_select = NULL;
    if (state->driver.select
	&& (state->type != ERTS_EV_TYPE_DRV_SEL)
	&& !is_iotask_active(&state->driver.select->iniotask)
	&& !is_iotask_active(&state->driver.select->outiotask)) {

	*free_select = state->driver.select;
	state->driver.select = NULL;
    }

    *free_nif = NULL;
    if (state->driver.nif && (state->type != ERTS_EV_TYPE_NIF)) {
        *free_nif = state->driver.nif;
        state->driver.nif = NULL;
    }

    if (((state->type != ERTS_EV_TYPE_NONE)
         | (state->driver.nif != NULL)
	 | (state->driver.select != NULL)) == 0) {

	erase_drv_ev_state(state);
    }
}

#ifdef __WIN32__
# define MUST_DEFER(MAY_SLEEP) 1
#else
# define MUST_DEFER(MAY_SLEEP) (MAY_SLEEP)
#endif

int
driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on)
{
    void (*stop_select_fn)(ErlDrvEvent, void*) = NULL;
    Port *prt = erts_drvport2port(ix);
    Eterm id = erts_drvport2id(ix);
    ErtsSysFdType fd = (ErtsSysFdType) e;
    ErtsPollEvents ctl_events = (ErtsPollEvents) 0;
    ErtsPollEvents old_events;
    ErtsPollEvents new_events;
    ErtsPollOp ctl_op = ERTS_POLL_OP_MOD;
    ErtsDrvEventState *state;
    int wake_poller = 0;
    int ret;
    ErtsDrvSelectDataState *free_select = NULL;
    ErtsNifSelectDataState *free_nif = NULL;
#ifdef USE_VM_PROBES
    DTRACE_CHARBUF(name, 64);
#endif
    ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_CHECK_IO);

    if (prt == ERTS_INVALID_ERL_DRV_PORT) {
        ERTS_MSACC_POP_STATE();
	return -1;
    }

    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    if (!grow_drv_ev_state(fd)) {
        if (fd > 0) drv_select_large_fd_error(ix, fd, mode, on);
        ERTS_MSACC_POP_STATE();
        return -1;
    }
#endif

    erts_mtx_lock(fd_mtx(fd));

    state = get_drv_ev_state(fd); /* may be NULL! */

    DEBUG_PRINT_FD("driver_select(%T, %p, %s, %d)",
                   state, id, fd, drvmode2str(mode), on);

    if (!on) {
        if (IS_FD_UNKNOWN(state)) {
            if ((mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
                /* fast track to stop_select callback */
                stop_select_fn = prt->drv_ptr->stop_select;
        #ifdef USE_VM_PROBES
                strncpy(name, prt->drv_ptr->name,
                        sizeof(DTRACE_CHARBUF_NAME(name))-1);
                name[sizeof(name)-1] = '\0';
        #endif
            }
            ret = 0;
            goto done_unknown;
        }
        else if ((mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
            mode |= (ERL_DRV_READ | ERL_DRV_WRITE);
        }
    }

    state = new_drv_ev_state(state, fd);

    switch (state->type) {
    case ERTS_EV_TYPE_NIF:
        drv_select_steal(ix, state, mode, on);
        break;
    case ERTS_EV_TYPE_STOP_USE: {
        erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
        print_drv_select_op(dsbufp, ix, state->fd, mode, on);
        steal_pending_stop_use(dsbufp, ix, state, mode, on);
        if (state->type == ERTS_EV_TYPE_STOP_USE) {
            ret = 0;
            goto done; /* stop_select still pending */
        }
        ASSERT(state->type == ERTS_EV_TYPE_NONE);
        break;
    }
    case ERTS_EV_TYPE_STOP_NIF: {
        erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
        print_drv_select_op(dsbufp, ix, state->fd, mode, on);
        steal_pending_stop_nif(dsbufp, NULL, state, mode, on);
        ASSERT(state->type == ERTS_EV_TYPE_NONE);
        break;

    }
    default: break;
    }

    if (mode & ERL_DRV_READ) {
	if (state->type == ERTS_EV_TYPE_DRV_SEL) {
	    Eterm owner = state->driver.select->inport;
	    if (owner != id && is_not_nil(owner))
		drv_select_steal(ix, state, mode, on);
	}
        ctl_events = ERTS_POLL_EV_IN;
    }
    if (mode & ERL_DRV_WRITE) {
	if (state->type == ERTS_EV_TYPE_DRV_SEL) {
	    Eterm owner = state->driver.select->outport;
	    if (owner != id && is_not_nil(owner))
		drv_select_steal(ix, state, mode, on);
	}
	ctl_events |= ERTS_POLL_EV_OUT;
    }


    ASSERT((state->type == ERTS_EV_TYPE_DRV_SEL) ||
	   (state->type == ERTS_EV_TYPE_NONE && !state->events));

    old_events = state->events;

    if (on) {
        ctl_events &= ~old_events;
        state->events |= ctl_events;
        if (ctl_events & ERTS_POLL_EV_IN && (!state->driver.select || !is_iotask_active(&state->driver.select->iniotask)))
            state->active_events |= ERTS_POLL_EV_IN;
        if (ctl_events & ERTS_POLL_EV_OUT && (!state->driver.select || !is_iotask_active(&state->driver.select->outiotask)))
            state->active_events |= ERTS_POLL_EV_OUT;
        if (old_events == 0 && !(state->flags & ERTS_EV_FLAG_USED)) {
            ctl_op = ERTS_POLL_OP_ADD;
        }
        new_events = state->active_events;
        if (state->flags & ERTS_EV_FLAG_IN_SCHEDULER)
            new_events &= ~ERTS_POLL_EV_IN;
    }
    else {
        ctl_events &= old_events;
        state->events &= ~ctl_events;
        state->active_events &= ~ctl_events;
        new_events = state->active_events;

        if (ctl_events & ERTS_POLL_EV_IN) {
            state->count = 0;
            if (state->flags & ERTS_EV_FLAG_IN_SCHEDULER) {
                new_events = 0;
            }
        }

        if (!state->events) {
            if (!(state->flags & ERTS_EV_FLAG_USED) || mode & ERL_DRV_USE)
                ctl_op = ERTS_POLL_OP_DEL;
        }
    }

    if (ctl_events || ctl_op == ERTS_POLL_OP_DEL) {

        new_events = erts_io_control_wakeup(state, ctl_op,
                                            new_events,
                                            &wake_poller);

        ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL || state->type == ERTS_EV_TYPE_NONE);
    }

    if (on) {
        if (ctl_events) {
	    if (!state->driver.select)
		state->driver.select = alloc_drv_select_data(state->fd);
	    if (state->type == ERTS_EV_TYPE_NONE)
		state->type = ERTS_EV_TYPE_DRV_SEL;
	    ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL);
	    if (ctl_events & ERTS_POLL_EV_IN) {
		state->driver.select->inport = id;
                if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL))
                    iready(id, state);
            }
	    if (ctl_events & ERTS_POLL_EV_OUT) {
		state->driver.select->outport = id;
                if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL))
                    oready(id, state);
            }
	    if (mode & ERL_DRV_USE)
		state->flags |= ERTS_EV_FLAG_USED;
        }
    }
    else { /* off */
        if (state->type == ERTS_EV_TYPE_DRV_SEL) {
            if (ctl_events & ERTS_POLL_EV_IN) {
                abort_tasks(state, ERL_DRV_READ);
                state->driver.select->inport = NIL;
                state->flags &= ~ERTS_EV_FLAG_IN_SCHEDULER;
            }
            if (ctl_events & ERTS_POLL_EV_OUT) {
                abort_tasks(state, ERL_DRV_WRITE);
                state->driver.select->outport = NIL;
            }
            if (state->events == 0) {
                if ((mode & ERL_DRV_USE) || !(state->flags & ERTS_EV_FLAG_USED)) {
                    state->type = ERTS_EV_TYPE_NONE;
                    if (state->flags & ERTS_EV_FLAG_SCHEDULER)
                        erts_atomic32_read_bor_nob(&prt->state, ERTS_PORT_SFLG_CHECK_FD_CLEANUP);
                    state->flags = 0;
                }
                /*else keep it, as fd will probably be selected upon again */
            }
        }
        if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
            erts_driver_t* drv_ptr = prt->drv_ptr;
            ASSERT(state->events==0);
            if (!wake_poller) {
                /* Safe to close fd now as it is not in pollset
                   or there was no need to eject fd (kernel poll) */
                stop_select_fn = drv_ptr->stop_select;
#ifdef USE_VM_PROBES
                strncpy(name, prt->drv_ptr->name, sizeof(name)-1);
                name[sizeof(name)-1] = '\0';
#endif
            }
            else {
                /* Not safe to close fd, postpone stop_select callback. */
                state->type = ERTS_EV_TYPE_STOP_USE;
                state->driver.stop.drv_ptr = drv_ptr;
                if (drv_ptr->handle) {
                    erts_ddll_reference_referenced_driver(drv_ptr->handle);
                }
            }
        }
    }
   
    ret = 0;

done:

    check_fd_cleanup(state,
		     &free_select,
                     &free_nif);

done_unknown:
    erts_mtx_unlock(fd_mtx(fd));
    if (stop_select_fn) {
	int was_unmasked = erts_block_fpe();
	DTRACE1(driver_stop_select, name);
	LTTNG1(driver_stop_select, "unknown");
	(*stop_select_fn)(e, NULL);
	erts_unblock_fpe(was_unmasked);
    }
    if (free_select)
	free_drv_select_data(free_select);
    if (free_nif)
        free_nif_select_data(free_nif);

    ERTS_MSACC_POP_STATE();

    return ret;
}

int
enif_select(ErlNifEnv* env,
            ErlNifEvent e,
            enum ErlNifSelectFlags mode,
            void* obj,
            const ErlNifPid* pid,
            Eterm ref)
{
    int on;
    ErtsResource* resource = DATA_TO_RESOURCE(obj);
    ErtsSysFdType fd = (ErtsSysFdType) e;
    ErtsPollEvents ctl_events = (ErtsPollEvents) 0;
    ErtsPollEvents old_events;
    ErtsPollOp ctl_op = ERTS_POLL_OP_MOD;
    ErtsDrvEventState *state;
    int ret, wake_poller = 0;
    enum { NO_STOP=0, CALL_STOP, CALL_STOP_AND_RELEASE } call_stop = NO_STOP;
    ErtsDrvSelectDataState *free_select = NULL;
    ErtsNifSelectDataState *free_nif = NULL;

    ASSERT(!resource->monitors);

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    if (!grow_drv_ev_state(fd)) {
        if (fd > 0) nif_select_large_fd_error(fd, mode, resource, ref);
        return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT;
    }
#endif

    erts_mtx_lock(fd_mtx(fd));

    state = get_drv_ev_state(fd); /* may be NULL! */

    DEBUG_PRINT_FD("enif_select(%T, %d, %s, %p, %T, %T)",
                   state, env->proc->common.id, fd, nifmode2str(mode), resource,
                   pid ? pid->pid : THE_NON_VALUE, ref);

    if (mode & ERL_NIF_SELECT_STOP) {
        ASSERT(resource->type->stop);
        if (IS_FD_UNKNOWN(state)) {
            /* fast track to stop callback */
            call_stop = CALL_STOP;
            ret = ERL_NIF_SELECT_STOP_CALLED;
            goto done_unknown;
        }
        on = 0;
        mode = ERL_DRV_READ | ERL_DRV_WRITE | ERL_DRV_USE;
        ctl_events = ERTS_POLL_EV_IN | ERTS_POLL_EV_OUT;
        ctl_op = ERTS_POLL_OP_DEL;
    }
    else {
        on = !(mode & ERL_NIF_SELECT_CANCEL);
        ASSERT(mode);
        if (mode & ERL_DRV_READ) {
            ctl_events |= ERTS_POLL_EV_IN;
        }
        if (mode & ERL_DRV_WRITE) {
            ctl_events |= ERTS_POLL_EV_OUT;
        }
    }

    state = new_drv_ev_state(state,fd);

    switch (state->type) {
    case ERTS_EV_TYPE_NIF:
        /*
         * Changing resource is considered stealing.
         * Changing process and/or ref is ok (I think?).
         */
        if (state->driver.stop.resource != resource)
            nif_select_steal(state, ERL_DRV_READ | ERL_DRV_WRITE, resource, ref);
        break;
    case ERTS_EV_TYPE_DRV_SEL:
        nif_select_steal(state, mode, resource, ref);
        break;
    case ERTS_EV_TYPE_STOP_USE: {
        erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
        print_nif_select_op(dsbufp, fd, mode, resource, ref);
        steal_pending_stop_use(dsbufp, ERTS_INVALID_ERL_DRV_PORT, state, mode, on);
        ASSERT(state->type == ERTS_EV_TYPE_NONE);
        break;
    }
    case ERTS_EV_TYPE_STOP_NIF: {
        erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
        print_nif_select_op(dsbufp, fd, mode, resource, ref);
        steal_pending_stop_nif(dsbufp, resource, state, mode, on);
        if (state->type == ERTS_EV_TYPE_STOP_NIF) {
            ret = ERL_NIF_SELECT_STOP_SCHEDULED;  /* ?? */
            goto done;
        }
        ASSERT(state->type == ERTS_EV_TYPE_NONE);
        break;
    }
    default: break;
    }

    ASSERT((state->type == ERTS_EV_TYPE_NIF) ||
	   (state->type == ERTS_EV_TYPE_NONE && !state->events));

    old_events = state->events;

    if (on) {
        ctl_events &= ~old_events;
        state->events |= ctl_events;
        state->active_events |= ctl_events;
        if (state->type == ERTS_EV_TYPE_NONE)
            ctl_op = ERTS_POLL_OP_ADD;
    }
    else {
        ctl_events &= old_events;
        state->events &= ~ctl_events;
        state->active_events &= ~ctl_events;
    }

    if (ctl_events || ctl_op == ERTS_POLL_OP_DEL) {
        ErtsPollEvents new_events;

        new_events = erts_io_control_wakeup(state, ctl_op,
                                            state->active_events,
                                            &wake_poller);

        if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
            if (state->type == ERTS_EV_TYPE_NIF && !old_events) {
                state->type = ERTS_EV_TYPE_NONE;
                state->flags = 0;
                state->driver.nif->in.pid = NIL;
                state->driver.nif->out.pid = NIL;
                state->driver.stop.resource = NULL;
            }
            ret = INT_MIN | ERL_NIF_SELECT_FAILED;
            goto done;
        }
        ASSERT(new_events == state->events);
    }

    ASSERT(state->type == ERTS_EV_TYPE_NIF
	   || state->type == ERTS_EV_TYPE_NONE);

    if (on) {
        const Eterm recipient = pid ? pid->pid : env->proc->common.id;
        Uint32* refn;
        if (!state->driver.nif)
            state->driver.nif = alloc_nif_select_data();
        if (state->type == ERTS_EV_TYPE_NONE) {
            state->type = ERTS_EV_TYPE_NIF;
            state->driver.stop.resource = resource;
            enif_keep_resource(resource->data);
        }
        ASSERT(state->type == ERTS_EV_TYPE_NIF);
        ASSERT(state->driver.stop.resource == resource);
        if (mode & ERL_DRV_READ) {
            state->driver.nif->in.pid = recipient;
            if (is_immed(ref)) {
                state->driver.nif->in.immed = ref;
            } else {
                ASSERT(is_internal_ref(ref));
                refn = internal_ref_numbers(ref);
                state->driver.nif->in.immed = THE_NON_VALUE;
                sys_memcpy(state->driver.nif->in.refn, refn,
                           sizeof(state->driver.nif->in.refn));
            }
        }
        if (mode & ERL_DRV_WRITE) {
            state->driver.nif->out.pid = recipient;
            if (is_immed(ref)) {
                state->driver.nif->out.immed = ref;
            } else {
                ASSERT(is_internal_ref(ref));
                refn = internal_ref_numbers(ref);
                state->driver.nif->out.immed = THE_NON_VALUE;
                sys_memcpy(state->driver.nif->out.refn, refn,
                           sizeof(state->driver.nif->out.refn));
            }
        }
        ret = 0;
    }
    else { /* off */
        ret = 0;
        if (state->type == ERTS_EV_TYPE_NIF) {
            if (mode & ERL_NIF_SELECT_READ
                && is_not_nil(state->driver.nif->in.pid)) {
                state->driver.nif->in.pid = NIL;
                ret |= ERL_NIF_SELECT_READ_CANCELLED;
            }
            if (mode & ERL_NIF_SELECT_WRITE
                && is_not_nil(state->driver.nif->out.pid)) {
                state->driver.nif->out.pid = NIL;
                ret |= ERL_NIF_SELECT_WRITE_CANCELLED;
            }
        }
        if (mode & ERL_NIF_SELECT_STOP) {
            ASSERT(state->events==0);
            if (!wake_poller) {
                /*
                 * Safe to close fd now as it is not in pollset
                 * or there was no need to eject fd (kernel poll)
                 */
                if (state->type == ERTS_EV_TYPE_NIF) {
                    ASSERT(state->driver.stop.resource == resource);
                    call_stop = CALL_STOP_AND_RELEASE;
                    state->driver.stop.resource = NULL;
                }
                else {
                    ASSERT(!state->driver.stop.resource);
                    call_stop = CALL_STOP;
                }
                state->type = ERTS_EV_TYPE_NONE;
                ret |= ERL_NIF_SELECT_STOP_CALLED;
            }
            else {
                /* Not safe to close fd, postpone stop_select callback. */
                if (state->type == ERTS_EV_TYPE_NONE) {
                    ASSERT(!state->driver.stop.resource);
                    state->driver.stop.resource = resource;
                    enif_keep_resource(resource);
                }
                state->type = ERTS_EV_TYPE_STOP_NIF;
                ret |= ERL_NIF_SELECT_STOP_SCHEDULED;
            }
        }
        else
            ASSERT(mode & ERL_NIF_SELECT_CANCEL);
    }

done:

    check_fd_cleanup(state,
		     &free_select,
                     &free_nif);

done_unknown:
    erts_mtx_unlock(fd_mtx(fd));
    if (call_stop) {
        erts_resource_stop(resource, (ErlNifEvent)fd, 1);
        if (call_stop == CALL_STOP_AND_RELEASE) {
            enif_release_resource(resource->data);
        }
    }
    if (free_select)
	free_drv_select_data(free_select);
    if (free_nif)
        free_nif_select_data(free_nif);

    return ret;
}

static ERTS_INLINE int
chk_stale(Eterm id, ErtsDrvEventState *state, int mode)
{
    if (is_nil(id))
	return 0;
    if (erts_is_port_alive(id))
	return 1; /* Steal */
    stale_drv_select(id, state, mode);
    return 0;
}

static int
need2steal(ErtsDrvEventState *state, int mode)
{
    int do_steal = 0;
    switch (state->type) {
    case ERTS_EV_TYPE_DRV_SEL:
	if (mode & ERL_DRV_READ)
	    do_steal |= chk_stale(state->driver.select->inport,
				  state,
				  ERL_DRV_READ);
	if (mode & ERL_DRV_WRITE)
	    do_steal |= chk_stale(state->driver.select->outport,
				  state,
				  ERL_DRV_WRITE);
	break;
    case ERTS_EV_TYPE_NIF:
        ASSERT(state->driver.stop.resource);
        do_steal = 1;
        break;

    case ERTS_EV_TYPE_STOP_USE:
    case ERTS_EV_TYPE_STOP_NIF:
        ASSERT(0);
	break;
    default:
	break;
    }
    return do_steal;
}

static void
print_driver_name(erts_dsprintf_buf_t *dsbufp, Eterm id)
{
    ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT);
    if (!pnp->name && !pnp->driver_name)
	erts_dsprintf(dsbufp, "%s ", "<unknown>");
    else {
	if (pnp->name) {
	    if (!pnp->driver_name || strcmp(pnp->driver_name, pnp->name) == 0)
		erts_dsprintf(dsbufp, "%s ", pnp->name);
	    else
		erts_dsprintf(dsbufp, "%s (%s) ", pnp->driver_name, pnp->name);
	}
	else if (pnp->driver_name) {
	    erts_dsprintf(dsbufp, "%s ", pnp->driver_name);
	}
    }
    erts_free_port_names(pnp);
}

static void
steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode)
{
    erts_dsprintf(dsbufp, "stealing control of fd=%d from ", (int) state->fd);
    switch (state->type) {
    case ERTS_EV_TYPE_DRV_SEL: {
	int deselect_mode = 0;
	Eterm iid = state->driver.select->inport;
	Eterm oid = state->driver.select->outport;
	if ((mode & ERL_DRV_READ) && (is_not_nil(iid))) {
	    erts_dsprintf(dsbufp, "input driver ");
	    print_driver_name(dsbufp, iid);
	    erts_dsprintf(dsbufp, "%T ", iid);
	    deselect_mode |= ERL_DRV_READ;
	}
	if ((mode & ERL_DRV_WRITE) && is_not_nil(oid)) {
	    if (deselect_mode) {
	    erts_dsprintf(dsbufp, "and ");
	    }
	    erts_dsprintf(dsbufp, "output driver ");
	    print_driver_name(dsbufp, oid);
	    erts_dsprintf(dsbufp, "%T ", oid);
	    deselect_mode |= ERL_DRV_WRITE;
	}
	if (deselect_mode)
	    deselect(state, deselect_mode);
	else {
	    erts_dsprintf(dsbufp, "no one", (int) state->fd);
	    ASSERT(0);
	}
	erts_dsprintf(dsbufp, "\n");
	break;
    }
    case ERTS_EV_TYPE_NIF: {
        Eterm iid = state->driver.nif->in.pid;
        Eterm oid = state->driver.nif->out.pid;
        const char* with = "with";
        ErlNifResourceType* rt = state->driver.stop.resource->type;

        erts_dsprintf(dsbufp, "resource %T:%T", rt->module, rt->name);

        if (is_not_nil(iid)) {
            erts_dsprintf(dsbufp, " %s in-pid %T", with, iid);
            with = "and";
        }
        if (is_not_nil(oid)) {
            erts_dsprintf(dsbufp, " %s out-pid %T", with, oid);
        }
        deselect(state, 0);
        erts_dsprintf(dsbufp, "\n");
        break;
    }
    case ERTS_EV_TYPE_STOP_USE:
    case ERTS_EV_TYPE_STOP_NIF: {
	ASSERT(0);
	break;
    }
    default:
	erts_dsprintf(dsbufp, "no one\n", (int) state->fd);
	ASSERT(0);
    }
}

static void
print_drv_select_op(erts_dsprintf_buf_t *dsbufp,
                    ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
{
    Port *pp = erts_drvport2port(ix);
    erts_dsprintf(dsbufp,
		  "driver_select(%p, %d,%s%s%s%s, %d) "
		  "by ",
		  ix,
		  (int) fd,
		  mode & ERL_DRV_READ ? " ERL_DRV_READ" : "",
		  mode & ERL_DRV_WRITE ? " ERL_DRV_WRITE" : "",
		  mode & ERL_DRV_USE ? " ERL_DRV_USE" : "",
		  mode & (ERL_DRV_USE_NO_CALLBACK & ~ERL_DRV_USE) ? "_NO_CALLBACK" : "",
		  on);
    print_driver_name(dsbufp, pp != ERTS_INVALID_ERL_DRV_PORT ? pp->common.id : NIL);
    erts_dsprintf(dsbufp, "driver %T ", pp != ERTS_INVALID_ERL_DRV_PORT ? pp->common.id : NIL);
}

static void
print_nif_select_op(erts_dsprintf_buf_t *dsbufp,
                    ErtsSysFdType fd, int mode,
                    ErtsResource* resource, Eterm ref)
{
    erts_dsprintf(dsbufp,
		  "enif_select(_, %d,%s%s%s, %T:%T, %T) ",
		  (int) fd,
		  mode & ERL_NIF_SELECT_READ ? " READ" : "",
		  mode & ERL_NIF_SELECT_WRITE ? " WRITE" : "",
		  (mode & ERL_NIF_SELECT_STOP ? " STOP"
                   : (mode & ERL_NIF_SELECT_CANCEL ? " CANCEL" : "")),
		  resource->type->module,
                  resource->type->name,
                  ref);
}


static void
drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state, int mode, int on)
{
    if (need2steal(state, mode)) {
	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
	print_drv_select_op(dsbufp, ix, state->fd, mode, on);
	steal(dsbufp, state, mode);
	erts_send_error_to_logger_nogl(dsbufp);
    }
}

static void
nif_select_steal(ErtsDrvEventState *state, int mode,
                 ErtsResource* resource, Eterm ref)
{
    if (need2steal(state, mode)) {
	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
	print_nif_select_op(dsbufp, state->fd, mode, resource, ref);
	steal(dsbufp, state, mode);
	erts_send_error_to_logger_nogl(dsbufp);
    }
}

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
static void
large_fd_error_common(erts_dsprintf_buf_t *dsbufp, ErtsSysFdType fd)
{
    erts_dsprintf(dsbufp,
		  "fd=%d is larger than the largest allowed fd=%d\n",
		  (int) fd, drv_ev_state.max_fds - 1);
}

static void
drv_select_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
{
    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
    print_drv_select_op(dsbufp, ix, fd, mode, on);
    erts_dsprintf(dsbufp, "failed: ");
    large_fd_error_common(dsbufp, fd);
    erts_send_error_to_logger_nogl(dsbufp);
}
static void
nif_select_large_fd_error(ErtsSysFdType fd, int mode,
                          ErtsResource* resource, Eterm ref)
{
    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
    print_nif_select_op(dsbufp, fd, mode, resource, ref);
    erts_dsprintf(dsbufp, "failed: ");
    large_fd_error_common(dsbufp, fd);
    erts_send_error_to_logger_nogl(dsbufp);
}
#endif /* ERTS_SYS_CONTINOUS_FD_NUMBERS */



static void
steal_pending_stop_use(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix,
                       ErtsDrvEventState *state, int mode, int on)
{
    int cancel = 0;
    ASSERT(state->type == ERTS_EV_TYPE_STOP_USE);

    if (on) {
	/* Either fd-owner changed its mind about closing
	 * or closed fd before stop_select callback and fd is now reused.
	 * In either case stop_select should not be called.
	 */
        cancel = 1;
    }
    else if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
	Port *prt = erts_drvport2port(ix);
	if (prt == ERTS_INVALID_ERL_DRV_PORT
            || prt->drv_ptr != state->driver.stop.drv_ptr) {
	    /* Some other driver or nif wants the stop_select callback */
            cancel = 1;
        }
    }

    if (cancel) {
        erts_dsprintf(dsbufp, "called before stop_select was called for driver '%s'\n",
                      state->driver.stop.drv_ptr->name);
        if (state->driver.stop.drv_ptr->handle) {
            erts_ddll_dereference_driver(state->driver.stop.drv_ptr->handle);
        }
        state->type = ERTS_EV_TYPE_NONE;
        state->flags = 0;
        state->driver.stop.drv_ptr = NULL;
    }
    else {
        erts_dsprintf(dsbufp, "ignored repeated call\n");
    }
    erts_send_error_to_logger_nogl(dsbufp);
}

static void
steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource,
                       ErtsDrvEventState *state, int mode, int on)
{
    int cancel = 0;

    ASSERT(state->type == ERTS_EV_TYPE_STOP_NIF);
    ASSERT(state->driver.stop.resource);

    if (on) {
        ASSERT(mode & (ERL_NIF_SELECT_READ | ERL_NIF_SELECT_WRITE));
        /* Either fd-owner changed its mind about closing
         * or closed fd before stop callback and fd is now reused.
         * In either case, stop should not be called.
         */
        cancel = 1;
    }
    else if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE
             && resource != state->driver.stop.resource) {
        /* Some driver or other resource wants the stop callback */
        cancel = 1;
    }

    if (cancel) {
        ErlNifResourceType* rt = state->driver.stop.resource->type;
        erts_dsprintf(dsbufp, "called before stop was called for NIF resource %T:%T\n",
                      rt->module, rt->name);

        enif_release_resource(state->driver.stop.resource->data);
        state->type = ERTS_EV_TYPE_NONE;
        state->flags = 0;
        state->driver.stop.resource = NULL;
    }
    else {
        erts_dsprintf(dsbufp, "ignored repeated call\n");
    }
    erts_send_error_to_logger_nogl(dsbufp);

}

static ERTS_INLINE int
io_task_schedule_allowed(ErtsDrvEventState *state,
			 ErtsPortTaskType type)
{
    ErtsIoTask *io_task;

    switch (type) {
    case ERTS_PORT_TASK_INPUT:
	if (!state->driver.select)
	    return 0;
	io_task = &state->driver.select->iniotask;
	break;
    case ERTS_PORT_TASK_OUTPUT:
	if (!state->driver.select)
	    return 0;
	io_task = &state->driver.select->outiotask;
	break;
    default:
	ERTS_INTERNAL_ERROR("Invalid I/O-task type");
	return 0;
    }

    return !is_iotask_active(io_task);
}

static ERTS_INLINE void
iready(Eterm id, ErtsDrvEventState *state)
{
    if (io_task_schedule_allowed(state,
				 ERTS_PORT_TASK_INPUT)) {
	ErtsIoTask *iotask = &state->driver.select->iniotask;
	if (erts_port_task_schedule(id,
				    &iotask->task,
				    ERTS_PORT_TASK_INPUT,
				    (ErlDrvEvent) state->fd,
                                    state->flags & ERTS_EV_FLAG_IN_SCHEDULER) != 0) {
	    stale_drv_select(id, state, ERL_DRV_READ);
	} else {
            DEBUG_PRINT_FD("schedule ready_input(%T, %d)",
                           state, id, state->fd);
        }
    }
}

static ERTS_INLINE void
oready(Eterm id, ErtsDrvEventState *state)
{
    if (io_task_schedule_allowed(state,
				 ERTS_PORT_TASK_OUTPUT)) {
	ErtsIoTask *iotask = &state->driver.select->outiotask;
	if (erts_port_task_schedule(id,
				    &iotask->task,
				    ERTS_PORT_TASK_OUTPUT,
				    (ErlDrvEvent) state->fd,
                                    0) != 0) {
	    stale_drv_select(id, state, ERL_DRV_WRITE);
	} else {
            DEBUG_PRINT_FD("schedule ready_output(%T, %d)", state, id, state->fd);
        }
    }
}

static ERTS_INLINE void
send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource,
                 Eterm event_atom)
{
    Process* rp = erts_proc_lookup(e->pid);
    ErtsProcLocks rp_locks = 0;
    ErtsMessage* mp;
    ErlOffHeap* ohp;
    ErtsBinary* bin;
    Eterm* hp;
    Uint hsz;
    Eterm resource_term, ref_term, tuple;

    if (!rp) {
        return;
    }

    bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);

     /* {select, Resource, Ref, EventAtom} */
    if (is_value(e->immed)) {
        hsz = 5 + ERTS_MAGIC_REF_THING_SIZE;
    }
    else {
        hsz = 5 + ERTS_MAGIC_REF_THING_SIZE + ERTS_REF_THING_SIZE;
    }

    mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp);

    resource_term = erts_mk_magic_ref(&hp, ohp, &bin->binary);
    if (is_value(e->immed)) {
        ASSERT(is_immed(e->immed));
        ref_term = e->immed;
    }
    else {
        write_ref_thing(hp, e->refn[0], e->refn[1], e->refn[2]);
        ref_term = make_internal_ref(hp);
        hp += ERTS_REF_THING_SIZE;
    }
    tuple = TUPLE4(hp, am_select, resource_term, ref_term, event_atom);

    erts_queue_message(rp, rp_locks, mp, tuple, am_system);

    if (rp_locks)
        erts_proc_unlock(rp, rp_locks);
}

static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport);

void
erts_check_io_interrupt(ErtsPollThread *psi, int set)
{
    if (psi) {
#if ERTS_POLL_USE_FALLBACK
        if (psi->ps == get_fallback_pollset()) {
            erts_poll_interrupt_flbk(psi->ps, set);
            return;
        }
#endif
        erts_poll_interrupt(psi->ps, set);
    }
}

ErtsPollThread *
erts_create_pollset_thread(int id, ErtsThrPrgrData *tpd) {
    psiv[id].tpd = tpd;
    return psiv+id;
}

void
erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time)
{
    int pollres_len;
    int poll_ret, i;
    ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_CHECK_IO);

 restart:

#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_check_exact(NULL, 0); /* No locks should be locked */
#endif

    pollres_len = psi->pollres_len;

#if ERTS_POLL_USE_FALLBACK
    if (psi->ps == get_fallback_pollset()) {

        poll_ret = erts_poll_wait_flbk(psi->ps, psi->pollres, &pollres_len, psi->tpd, timeout_time);

    } else
#endif
    {
        poll_ret = erts_poll_wait(psi->ps, psi->pollres, &pollres_len, psi->tpd, timeout_time);
    }

#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_check_exact(NULL, 0); /* No locks should be locked */
#endif

    if (poll_ret != 0) {

	if (poll_ret == EAGAIN) {
	    goto restart;
	}

	if (poll_ret != ETIMEDOUT
	    && poll_ret != EINTR
#ifdef ERRNO_BLOCK
	    && poll_ret != ERRNO_BLOCK
#endif
	    ) {
	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
	    erts_dsprintf(dsbufp, "erts_poll_wait() failed: %s (%d)\n",
			  erl_errno_id(poll_ret), poll_ret);
	    erts_send_error_to_logger_nogl(dsbufp);
	}
        ERTS_MSACC_POP_STATE();
	return;
    }

    for (i = 0; i < pollres_len; i++) {

        erts_driver_t* drv_ptr = NULL;
        ErtsResource* resource = NULL;
        ErtsDrvSelectDataState *free_select = NULL;
        ErtsNifSelectDataState *free_nif = NULL;
	ErtsSysFdType fd = (ErtsSysFdType) ERTS_POLL_RES_GET_FD(&psi->pollres[i]);
	ErtsDrvEventState *state;
        ErtsPollEvents revents = ERTS_POLL_RES_GET_EVTS(&psi->pollres[i]);

        /* The fd will be set to -1 if a pollset internal fd was triggered
           that was determined to be too expensive to remove from the result.
        */
        if (fd == -1) continue;

	erts_mtx_lock(fd_mtx(fd));

	state = get_drv_ev_state(fd);

	if (!state) {
            erts_mtx_unlock(fd_mtx(fd));
            continue;
	}

        DEBUG_PRINT_FD("triggered %s", state, ev2str(revents));

        if (revents & ERTS_POLL_EV_ERR) {
            /*
             * Handle error events by triggering all in/out events
             * that has been selected on.
             * We *do not* want to call a callback that corresponds
             * to an event not selected.
             */
            revents = state->active_events;
            state->active_events = 0;

            if (state->flags & ERTS_EV_FLAG_IN_SCHEDULER) {
                erts_io_control(state, ERTS_POLL_OP_MOD, 0);
                state->flags &= ~ERTS_EV_FLAG_IN_SCHEDULER;
            }
        } else {

            /* Disregard any events that are not active at the moment,
               for instance this could happen if the driver/nif does
               select/deselect in rapid succession. */
            revents &= state->active_events | ERTS_POLL_EV_NVAL;

            if (psi->ps != get_scheduler_pollset(fd) || !ERTS_POLL_USE_SCHEDULER_POLLING) {
                ErtsPollEvents reactive_events;
                state->active_events &= ~revents;

                reactive_events = state->active_events;

                if (state->flags & ERTS_EV_FLAG_IN_SCHEDULER)
                    reactive_events &= ~ERTS_POLL_EV_IN;

                /* Reactivate the poll op if there are still active events */
                if (reactive_events) {
                    ErtsPollEvents new_events;
                    DEBUG_PRINT_FD("re-enable %s", state, ev2str(reactive_events));

                    new_events = erts_io_control(state, ERTS_POLL_OP_MOD, reactive_events);

                    /* Unable to re-enable the fd, signal all callbacks */
                    if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
                        revents |= reactive_events;
                        state->active_events &= ~reactive_events;
                    }
                }
            }
        }

	switch (state->type) {
	case ERTS_EV_TYPE_DRV_SEL: { /* Requested via driver_select()... */

            if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) {
		if (revents & ERTS_POLL_EV_OUT) {
		    oready(state->driver.select->outport, state);
		}
		/* Someone might have deselected input since revents
		   was read (true also on the non-smp emulator since
		   oready() may have been called); therefore, update
		   revents... */
                revents &= state->events;
		if (revents & ERTS_POLL_EV_IN) {
		    iready(state->driver.select->inport, state);
		}
	    }
	    else if (revents & ERTS_POLL_EV_NVAL) {
		bad_fd_in_pollset(state,
                                  state->driver.select->inport,
                                  state->driver.select->outport);
                check_fd_cleanup(state, &free_select, &free_nif);
	    }
	    break;
	}

        case ERTS_EV_TYPE_NIF: { /* Requested via enif_select()... */
            struct erts_nif_select_event in = {NIL};
            struct erts_nif_select_event out = {NIL};
            ErtsResource* resource = NULL;

            if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) {
                if (revents & ERTS_POLL_EV_OUT) {
                    if (is_not_nil(state->driver.nif->out.pid)) {
                        out = state->driver.nif->out;
                        resource = state->driver.stop.resource;
                        state->driver.nif->out.pid = NIL;
                    }
                }
                if (revents & ERTS_POLL_EV_IN) {
                    if (is_not_nil(state->driver.nif->in.pid)) {
                        in = state->driver.nif->in;
                        resource = state->driver.stop.resource;
                        state->driver.nif->in.pid = NIL;
                    }
                }
                state->events &= ~revents;
            }
            else if (revents & ERTS_POLL_EV_NVAL) {
                bad_fd_in_pollset(state, NIL, NIL);
                check_fd_cleanup(state, &free_select, &free_nif);
            }

            erts_mtx_unlock(fd_mtx(fd));

            if (is_not_nil(in.pid)) {
                send_event_tuple(&in, resource, am_ready_input);
            }
            if (is_not_nil(out.pid)) {
                send_event_tuple(&out, resource, am_ready_output);
            }
            continue;
        }

        case ERTS_EV_TYPE_STOP_NIF: {
            resource = state->driver.stop.resource;
            state->type = ERTS_EV_TYPE_NONE;
            goto case_ERTS_EV_TYPE_NONE;
        }

        case ERTS_EV_TYPE_STOP_USE: {
#if ERTS_POLL_USE_FALLBACK
            ASSERT(psi->ps == get_fallback_pollset());
#endif
            drv_ptr = state->driver.stop.drv_ptr;
            state->type = ERTS_EV_TYPE_NONE;
            /* fallthrough */
	case ERTS_EV_TYPE_NONE: /* Deselected ... */
        case_ERTS_EV_TYPE_NONE:
            ASSERT(!state->events && !state->active_events && !state->flags);
            check_fd_cleanup(state, &free_select, &free_nif);
	    break;
        }

	default: { /* Error */
	    erts_dsprintf_buf_t *dsbufp;
	    dsbufp = erts_create_logger_dsbuf();
	    erts_dsprintf(dsbufp,
			  "Invalid event request type for fd in erts_poll()! "
			  "fd=%d, event request type=%d\n", (int) state->fd,
			  (int) state->type);
	    ASSERT(0);
	    deselect(state, 0);
	    break;
	}
	}

	erts_mtx_unlock(fd_mtx(fd));

        if (drv_ptr) {
            int was_unmasked = erts_block_fpe();
            DTRACE1(driver_stop_select, drv_ptr->name);
            LTTNG1(driver_stop_select, drv_ptr->name);
            (*drv_ptr->stop_select)((ErlDrvEvent) fd, NULL);
            erts_unblock_fpe(was_unmasked);
            if (drv_ptr->handle) {
		erts_ddll_dereference_driver(drv_ptr->handle);
	    }
        }
        if (resource) {
            erts_resource_stop(resource, (ErlNifEvent)fd, 0);
            enif_release_resource(resource->data);
        }
        if (free_select)
            free_drv_select_data(free_select);
        if (free_nif)
            free_nif_select_data(free_nif);
    }

    /* The entire pollres array was filled with events,
     * grow it for the next call. We do this for two reasons:
     * 1. Pulling out more events in on go will increase throughput
     * 2. If the polling implementation is not fair, this will make
     *    sure that we get all fds that we can. i.e. if 12 fds are
     *    constantly active, but we only have a pollres_len of 10,
     *    two of the fds may never be triggered depending on what the
     *    kernel decides to do.
     **/
    if (pollres_len == psi->pollres_len) {
        int ev_state_len = drv_ev_state_len();
        erts_free(ERTS_ALC_T_POLLSET, psi->pollres);
        psi->pollres_len *= 2;
        /* Never grow it larger than the current drv_ev_state.len size */
        if (psi->pollres_len > ev_state_len)
            psi->pollres_len = ev_state_len;
        psi->pollres = erts_alloc(ERTS_ALC_T_POLLSET,
                                  sizeof(ErtsPollResFd) * psi->pollres_len);
    }

    ERTS_MSACC_POP_STATE();
}

static void
bad_fd_in_pollset(ErtsDrvEventState *state, Eterm inport, Eterm outport)
{
    ErtsPollEvents events = state->events;
    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();

    if (events & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) {
	char *io_str;
	Eterm port = NIL;
	if ((events & ERTS_POLL_EV_IN) && (events & ERTS_POLL_EV_OUT)) {
	    io_str = "input/output";
	    if (inport == outport)
		port = inport;
	}
	else {
	    if (events & ERTS_POLL_EV_IN) {
		io_str = "input";
		port = inport;
	    }
	    else {
		io_str = "output";
		port = outport;
	    }
	}
	erts_dsprintf(dsbufp,
		      "Bad %s fd in erts_poll()! fd=%d, ",
		      io_str, (int) state->fd);
        if (state->type == ERTS_EV_TYPE_DRV_SEL) {
            if (is_nil(port)) {
                ErtsPortNames *ipnp = erts_get_port_names(inport, ERTS_INVALID_ERL_DRV_PORT);
                ErtsPortNames *opnp = erts_get_port_names(outport, ERTS_INVALID_ERL_DRV_PORT);
                erts_dsprintf(dsbufp, "ports=%T/%T, drivers=%s/%s, names=%s/%s\n",
                              is_nil(inport) ? am_undefined : inport,
                              is_nil(outport) ? am_undefined : outport,
                              ipnp->driver_name ? ipnp->driver_name : "<unknown>",
                              opnp->driver_name ? opnp->driver_name : "<unknown>",
                              ipnp->name ? ipnp->name : "<unknown>",
                              opnp->name ? opnp->name : "<unknown>");
                erts_free_port_names(ipnp);
                erts_free_port_names(opnp);
            }
            else {
                ErtsPortNames *pnp = erts_get_port_names(port, ERTS_INVALID_ERL_DRV_PORT);
                erts_dsprintf(dsbufp, "port=%T, driver=%s, name=%s\n",
                              is_nil(port) ? am_undefined : port,
                              pnp->driver_name ? pnp->driver_name : "<unknown>",
                              pnp->name ? pnp->name : "<unknown>");
                erts_free_port_names(pnp);
            }
        }
        else {
            ErlNifResourceType* rt;
            ASSERT(state->type == ERTS_EV_TYPE_NIF);
            ASSERT(state->driver.stop.resource);
            rt = state->driver.stop.resource->type;
            erts_dsprintf(dsbufp, "resource={%T,%T}\n", rt->module, rt->name);
        }
    }
    else {
	erts_dsprintf(dsbufp, "Bad fd in erts_poll()! fd=%d\n", (int) state->fd);
    }
    erts_send_error_to_logger_nogl(dsbufp);

    /* unmap entry */
    deselect(state, 0);
}

static void
stale_drv_select(Eterm id, ErtsDrvEventState *state, int mode)
{
    erts_stale_drv_select(id, ERTS_INVALID_ERL_DRV_PORT, (ErlDrvEvent) state->fd, mode, 0);
    deselect(state, mode);
}

#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS

static SafeHashValue drv_ev_state_hash(void *des)
{
    SafeHashValue val = (SafeHashValue) ((ErtsDrvEventState *) des)->fd;
    return val ^ (val >> 8);  /* Good enough for aligned pointer values? */
}

static int drv_ev_state_cmp(void *des1, void *des2)
{
    return ( ((ErtsDrvEventState *) des1)->fd == ((ErtsDrvEventState *) des2)->fd
	    ? 0 : 1);
}

static void *drv_ev_state_alloc(void *des_tmpl)
{
    ErtsDrvEventState *evstate;
    erts_spin_lock(&drv_ev_state.prealloc_lock);
    if (drv_ev_state.prealloc_first == NULL) {
	erts_spin_unlock(&drv_ev_state.prealloc_lock);
	evstate = (ErtsDrvEventState *) 
	    erts_alloc(ERTS_ALC_T_DRV_EV_STATE, sizeof(ErtsDrvEventState));
    } else {
	evstate = drv_ev_state.prealloc_first;
	drv_ev_state.prealloc_first = (ErtsDrvEventState *) evstate->hb.next;
	--drv_ev_state.num_prealloc;
	erts_spin_unlock(&drv_ev_state.prealloc_lock);
    }
    /* XXX: Already valid data if prealloced, could ignore template! */
    *evstate = *((ErtsDrvEventState *) des_tmpl);

    return (void *) evstate;
}

static void drv_ev_state_free(void *des)
{
    erts_spin_lock(&drv_ev_state.prealloc_lock);
    ((ErtsDrvEventState *) des)->hb.next = &drv_ev_state.prealloc_first->hb;
    drv_ev_state.prealloc_first = (ErtsDrvEventState *) des;
    ++drv_ev_state.num_prealloc;
    erts_spin_unlock(&drv_ev_state.prealloc_lock);
}
#endif

#define ERTS_MAX_NO_OF_POLL_THREADS ERTS_MAX_NO_OF_SCHEDULERS

static char *
get_arg(char* rest, char** argv, int* ip)
{
    int i = *ip;
    if (*rest == '\0') {
	if (argv[i+1] == NULL) {
	    erts_fprintf(stderr, "too few arguments\n");
	    erts_usage();
	}
        argv[i++] = NULL;
        rest = argv[i];
    }
    argv[i] = NULL;
    *ip = i;
    return rest;
}

static void
parse_args(int *argc, char **argv, int concurrent_waiters)
{
    int i = 0, j;
    int no_pollsets = 0, no_poll_threads = 0,
        no_pollsets_percentage = 0,
        no_poll_threads_percentage = 0;
    ASSERT(argc && argv);
    while (i < *argc) {
	if(argv[i][0] == '-') {
	    switch (argv[i][1]) {
            case 'I': {
                if (strncmp(argv[i]+2, "Ot", 2) == 0) {
                    char *arg = get_arg(argv[i]+4, argv, &i);
                    if (sscanf(arg, "%d", &no_poll_threads) != 1 ||
                        no_poll_threads < 1 ||
                        ERTS_MAX_NO_OF_POLL_THREADS < no_poll_threads) {
                        erts_fprintf(stderr,"bad I/O poll threads number: %s\n", arg);
                        erts_usage();
                    }
                } else if (strncmp(argv[i]+2, "Op", 3) == 0) {
                    char *arg = get_arg(argv[i]+4, argv, &i);
                    if (sscanf(arg, "%d", &no_pollsets) != 1 ||
                        no_pollsets < 1) {
                        erts_fprintf(stderr,"bad I/O pollset number: %s\n", arg);
                        erts_usage();
                    }
                } else if (strncmp(argv[i]+2, "OPt", 4) == 0) {
                    char *arg = get_arg(argv[i]+5, argv, &i);
                    if (sscanf(arg, "%d", &no_poll_threads_percentage) != 1 ||
                        no_poll_threads_percentage < 0 ||
                        no_poll_threads_percentage > 100) {
                        erts_fprintf(stderr,"bad I/O poll thread percentage number: %s\n", arg);
                        erts_usage();
                    }
                } else if (strncmp(argv[i]+2, "OPp", 4) == 0) {
                    char *arg = get_arg(argv[i]+5, argv, &i);
                    if (sscanf(arg, "%d", &no_pollsets_percentage) != 1 ||
                        no_pollsets_percentage < 0 ||
                        no_pollsets_percentage > 100) {
                        erts_fprintf(stderr,"bad I/O pollset percentage number: %s\n", arg);
                        erts_usage();
                    }
                } else {
                    break;
                }
                break;
            }
            case 'K':
                (void)get_arg(argv[i]+2, argv, &i);
                break;
            case '-':
                goto args_parsed;
            default:
                break;
            }
        }
        i++;
    }

args_parsed:

    if (!concurrent_waiters) {
        no_pollsets = no_poll_threads;
        no_pollsets_percentage = 100;
    }

    if (no_poll_threads == 0) {
        if (no_poll_threads_percentage == 0)
            no_poll_threads = 1; /* This is the default */
        else {
            no_poll_threads = erts_no_schedulers * no_poll_threads_percentage / 100;
            if (no_poll_threads < 1)
                no_poll_threads = 1;
        }
    }

    if (no_pollsets == 0) {
        if (no_pollsets_percentage == 0)
            no_pollsets = 1; /* This is the default */
        else {
            no_pollsets = no_poll_threads * no_pollsets_percentage / 100;
            if (no_pollsets < 1)
                no_pollsets = 1;
        }
    }

    if (no_poll_threads < no_pollsets) {
        erts_fprintf(stderr,
                     "number of IO poll threads has to be greater or equal to "
                     "the number of \nIO pollsets. Current values are set to: \n"
                     "  -IOt %d -IOp %d\n",
                     no_poll_threads, no_pollsets);
        erts_usage();
    }

    /* Handled arguments have been marked with NULL. Slide arguments
       not handled towards the beginning of argv. */
    for (i = 0, j = 0; i < *argc; i++) {
	if (argv[i])
	    argv[j++] = argv[i];
    }
    *argc = j;

    erts_no_pollsets = no_pollsets;
    erts_no_poll_threads = no_poll_threads;
}

void
erts_init_check_io(int *argc, char **argv)
{
    int j, concurrent_waiters, no_poll_threads;
    ERTS_CT_ASSERT((INT_MIN & (ERL_NIF_SELECT_STOP_CALLED |
                               ERL_NIF_SELECT_STOP_SCHEDULED |
                               ERL_NIF_SELECT_INVALID_EVENT |
                               ERL_NIF_SELECT_FAILED)) == 0);


    erts_poll_init(&concurrent_waiters);
#if ERTS_POLL_USE_FALLBACK
    erts_poll_init_flbk(NULL);
#endif

    parse_args(argc, argv, concurrent_waiters);

    /* Create the actual pollsets */
    pollsetv = erts_alloc(ERTS_ALC_T_POLLSET,sizeof(ErtsPollSet *) * erts_no_pollsets);

    for (j=0; j < erts_no_pollsets; j++)
        pollsetv[j] = erts_poll_create_pollset(j);

    no_poll_threads = erts_no_poll_threads;

    j = -1;

#if ERTS_POLL_USE_SCHEDULER_POLLING
    sched_pollset = erts_poll_create_pollset(j--);
    no_poll_threads++;
#endif

#if ERTS_POLL_USE_FALLBACK
    flbk_pollset = erts_poll_create_pollset_flbk(j--);
    no_poll_threads++;
#endif

    psiv = erts_alloc(ERTS_ALC_T_POLLSET, sizeof(ErtsPollThread) * no_poll_threads);

#if ERTS_POLL_USE_FALLBACK
    psiv[0].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN;
    psiv[0].pollres = erts_alloc(ERTS_ALC_T_POLLSET,
        sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN);
    psiv[0].ps = get_fallback_pollset();
    psiv++;
#endif

#if ERTS_POLL_USE_SCHEDULER_POLLING
    psiv[0].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN;
    psiv[0].pollres = erts_alloc(ERTS_ALC_T_POLLSET,
        sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN);
    psiv[0].ps = get_scheduler_pollset(0);
    psiv++;
#endif

    for (j = 0; j < erts_no_poll_threads; j++) {
        psiv[j].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN;
        psiv[j].pollres = erts_alloc(ERTS_ALC_T_POLLSET,
                                      sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN);
        psiv[j].ps = pollsetv[j % erts_no_pollsets];
    }

    for (j=0; j < ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; j++) {
        erts_mtx_init(&drv_ev_state.locks[j].lck, "drv_ev_state", make_small(j),
                          ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
    }

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    drv_ev_state.max_fds = erts_poll_max_fds();
    erts_atomic_init_nob(&drv_ev_state.len, 0);
    drv_ev_state.v = NULL;
    erts_mtx_init(&drv_ev_state.grow_lock, "drv_ev_state_grow", NIL,
        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
#else
    {
	SafeHashFunctions hf;
	hf.hash = &drv_ev_state_hash;
	hf.cmp = &drv_ev_state_cmp;
	hf.alloc = &drv_ev_state_alloc;
	hf.free = &drv_ev_state_free;
	drv_ev_state.num_prealloc = 0;
	drv_ev_state.prealloc_first = NULL;
	erts_spinlock_init(&drv_ev_state.prealloc_lock, "state_prealloc", NIL,
                               ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
	safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state.tab, "drv_ev_state_tab",
            ERTS_LOCK_FLAGS_CATEGORY_IO, DRV_EV_STATE_HTAB_SIZE, hf);
    }
#endif
}

int
erts_check_io_max_files(void)
{
#ifdef  ERTS_SYS_CONTINOUS_FD_NUMBERS
    return drv_ev_state.max_fds;
#else
    return erts_poll_max_fds();
#endif
}

Uint
erts_check_io_size(void)
{
    Uint res = 0;
    ErtsPollInfo pi;
    int i;

#if ERTS_POLL_USE_FALLBACK
    erts_poll_info(get_fallback_pollset(), &pi);
    res += pi.memory_size;
#endif

#if ERTS_POLL_USE_SCHEDULER_POLLING
    erts_poll_info(get_scheduler_pollset(0), &pi);
    res += pi.memory_size;
#endif

    for (i = 0; i < erts_no_pollsets; i++) {
        erts_poll_info(pollsetv[i], &pi);
        res += pi.memory_size;
    }
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len);
#else
    res += safe_hash_table_sz(&drv_ev_state.tab);
    {
	SafeHashInfo hi;
	safe_hash_get_info(&hi, &drv_ev_state.tab);
	res += hi.objs * sizeof(ErtsDrvEventState);
    }
    erts_spin_lock(&drv_ev_state.prealloc_lock);
    res += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState);
    erts_spin_unlock(&drv_ev_state.prealloc_lock);
#endif
    return res;
}

Eterm
erts_check_io_info(void *proc)
{
    Process *p = (Process *) proc;
    Eterm tags[16], values[16], res, list = NIL;
    Uint sz, *szp, *hp, **hpp;
    ErtsPollInfo *piv;
    Sint i, j = 0, len;
    int no_pollsets = erts_no_pollsets + ERTS_POLL_USE_FALLBACK + ERTS_POLL_USE_SCHEDULER_POLLING;
    ERTS_CT_ASSERT(ERTS_POLL_USE_FALLBACK == 0 || ERTS_POLL_USE_FALLBACK == 1);
    ERTS_CT_ASSERT(ERTS_POLL_USE_SCHEDULER_POLLING == 0 || ERTS_POLL_USE_SCHEDULER_POLLING == 1);

    piv = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollInfo) * no_pollsets);

#if ERTS_POLL_USE_FALLBACK
    erts_poll_info_flbk(get_fallback_pollset(), &piv[0]);
    piv[0].poll_threads = 1;
    piv[0].active_fds = 0;
    piv++;
#endif

#if ERTS_POLL_USE_SCHEDULER_POLLING
    erts_poll_info(get_scheduler_pollset(0), &piv[0]);
    piv[0].poll_threads = 1;
    piv[0].active_fds = 0;
    piv++;
#endif

    for (j = 0; j < erts_no_pollsets; j++) {
        erts_poll_info(pollsetv[j], &piv[j]);
        piv[j].active_fds = 0;
        piv[j].poll_threads = erts_no_poll_threads / erts_no_pollsets;
        if (erts_no_poll_threads % erts_no_pollsets > j)
            piv[j].poll_threads++;
    }

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    i = 0;
    erts_mtx_lock(&drv_ev_state.grow_lock);
    len = erts_atomic_read_nob(&drv_ev_state.len);
    for (i = 0; i < ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) {
        erts_mtx_lock(&drv_ev_state.locks[i].lck);
        for (j = i; j < len; j+=ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT) {
            ErtsDrvEventState *state = get_drv_ev_state(j);
            int pollsetid = get_pollset_id(j);
            ASSERT(fd_mtx(j) == &drv_ev_state.locks[i].lck);
            if (state->flags & ERTS_EV_FLAG_FALLBACK)
                pollsetid = -1;
            if (state->driver.select
                && (state->type == ERTS_EV_TYPE_DRV_SEL)
                && (is_iotask_active(&state->driver.select->iniotask)
                    || is_iotask_active(&state->driver.select->outiotask)))
                piv[pollsetid].active_fds++;
        }
        erts_mtx_unlock(&drv_ev_state.locks[i].lck);
    }
    erts_mtx_unlock(&drv_ev_state.grow_lock);

    piv[0].memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len);
#else
    piv[0].memory_size += safe_hash_table_sz(&drv_ev_state.tab);
    {
        SafeHashInfo hi;
        safe_hash_get_info(&hi, &drv_ev_state.tab);
        piv[0].memory_size += hi.objs * sizeof(ErtsDrvEventState);
    }
    erts_spin_lock(&drv_ev_state.prealloc_lock);
    piv[0].memory_size += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState);
    erts_spin_unlock(&drv_ev_state.prealloc_lock);
#endif

    hpp = NULL;
    szp = &sz;
    sz = 0;

    piv -= ERTS_POLL_USE_FALLBACK;
    piv -= ERTS_POLL_USE_SCHEDULER_POLLING;

 bld_it:

    for (j = no_pollsets-1; j >= 0; j--) {
        i = 0;

        tags[i] = erts_bld_atom(hpp, szp, "name");
        values[i++] = erts_bld_atom(hpp, szp, "erts_poll");

        tags[i] = erts_bld_atom(hpp, szp, "primary");
        values[i++] = erts_bld_atom(hpp, szp, piv[j].primary);

        tags[i] = erts_bld_atom(hpp, szp, "kernel_poll");
        values[i++] = erts_bld_atom(hpp, szp,
                                    piv[j].kernel_poll ? piv[j].kernel_poll : "false");

        tags[i] = erts_bld_atom(hpp, szp, "memory_size");
        values[i++] = erts_bld_uint(hpp, szp, piv[j].memory_size);

        tags[i] = erts_bld_atom(hpp, szp, "total_poll_set_size");
        values[i++] = erts_bld_uint(hpp, szp, piv[j].poll_set_size);

        tags[i] = erts_bld_atom(hpp, szp, "lazy_updates");
        values[i++] = piv[j].lazy_updates ? am_true : am_false;

        tags[i] = erts_bld_atom(hpp, szp, "pending_updates");
        values[i++] = erts_bld_uint(hpp, szp, piv[j].pending_updates);

        tags[i] = erts_bld_atom(hpp, szp, "batch_updates");
        values[i++] = piv[j].batch_updates ? am_true : am_false;

        tags[i] = erts_bld_atom(hpp, szp, "concurrent_updates");
        values[i++] = piv[j].concurrent_updates ? am_true : am_false;

        tags[i] = erts_bld_atom(hpp, szp, "fallback");
        values[i++] = piv[j].is_fallback ? am_true : am_false;

        tags[i] = erts_bld_atom(hpp, szp, "max_fds");
        values[i++] = erts_bld_uint(hpp, szp, piv[j].max_fds);

        tags[i] = erts_bld_atom(hpp, szp, "active_fds");
        values[i++] = erts_bld_uint(hpp, szp, piv[j].active_fds);

        tags[i] = erts_bld_atom(hpp, szp, "poll_threads");
        values[i++] = erts_bld_uint(hpp, szp, piv[j].poll_threads);

        res = erts_bld_2tup_list(hpp, szp, i, tags, values);

        if (!hpp) {
            *szp += 2;
        }
        else {
            list = CONS(*hpp, res, list);
            *hpp += 2;
        }
    }

    if (!hpp) {
	hp = HAlloc(p, sz);
	hpp = &hp;
	szp = NULL;
	goto bld_it;
    }

    erts_free(ERTS_ALC_T_TMP, piv);

    return list;
}

static ERTS_INLINE ErtsPollEvents
print_events(erts_dsprintf_buf_t *dsbufp, ErtsPollEvents ev)
{
    int first = 1;
    if(ev == ERTS_POLL_EV_NONE) {
        erts_dsprintf(dsbufp, "N/A");
        return 0;
    }
    if(ev & ERTS_POLL_EV_IN) {
	ev &= ~ERTS_POLL_EV_IN;
	erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "IN");
	first = 0;
    }
    if(ev & ERTS_POLL_EV_OUT) {
	ev &= ~ERTS_POLL_EV_OUT;
	erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "OUT");
	first = 0;
    }
    /* The following should not appear... */
    if(ev & ERTS_POLL_EV_NVAL) {
	erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "NVAL");
	first = 0;
    }
    if(ev & ERTS_POLL_EV_ERR) {
	erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "ERR");
	first = 0;
    }
    if (ev)
	erts_dsprintf(dsbufp, "%s0x%b32x", first ? "" : "|", (Uint32) ev);
    return ev;
}

static ERTS_INLINE void
print_flags(erts_dsprintf_buf_t *dsbufp, EventStateFlags f)
{
    erts_dsprintf(dsbufp, "%s", flag2str(f));
}

#ifdef DEBUG_PRINT_MODE

static ERTS_INLINE char *
drvmode2str(int mode) {
    switch (mode) {
    case ERL_DRV_READ|ERL_DRV_USE: return "READ|USE";
    case ERL_DRV_WRITE|ERL_DRV_USE: return "WRITE|USE";
    case ERL_DRV_READ|ERL_DRV_WRITE|ERL_DRV_USE: return "READ|WRITE|USE";
    case ERL_DRV_USE: return "USE";
    case ERL_DRV_READ: return "READ";
    case ERL_DRV_WRITE: return "WRITE";
    case ERL_DRV_READ|ERL_DRV_WRITE: return "READ|WRITE";
    default: return "UNKNOWN";
    }
}

static ERTS_INLINE char *
nifmode2str(enum ErlNifSelectFlags mode) {
    if (mode & ERL_NIF_SELECT_STOP)
        return "STOP";
    switch (mode) {
    case ERL_NIF_SELECT_READ: return "READ";
    case ERL_NIF_SELECT_WRITE: return "WRITE";
    case ERL_NIF_SELECT_READ|ERL_NIF_SELECT_WRITE: return "READ|WRITE";
    case ERL_NIF_SELECT_CANCEL|ERL_NIF_SELECT_READ: return "CANCEL|READ";
    case ERL_NIF_SELECT_CANCEL|ERL_NIF_SELECT_WRITE: return "CANCEL|WRITE";
    case ERL_NIF_SELECT_CANCEL|ERL_NIF_SELECT_READ|ERL_NIF_SELECT_WRITE:
        return "CANCEL|READ|WRITE";
    default: return "UNKNOWN";
    }
}

#endif

typedef struct {
    int used_fds;
    int num_errors;
    int no_driver_select_structs;
    int no_enif_select_structs;
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    int internal_fds;
    ErtsPollEvents *epep;
#endif
} IterDebugCounters;

static int erts_debug_print_checkio_state(erts_dsprintf_buf_t *dsbufp,
                                          ErtsDrvEventState *state,
                                          ErtsPollEvents ep_events,
                                          int internal)
{
#if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE)
    struct stat stat_buf;
#endif
    ErtsSysFdType fd = state->fd;
    ErtsPollEvents cio_events = state->events;
    int err = 0;
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    ErtsPollEvents aio_events = state->active_events;
#endif
    erts_dsprintf(dsbufp, "pollset=%d fd=%d ",
                state->flags & ERTS_EV_FLAG_FALLBACK ? -1 : get_pollset_id(fd), (int) fd);

#if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE)
    if (fstat((int) fd, &stat_buf) < 0)
        erts_dsprintf(dsbufp, "type=unknown ");
    else {
        erts_dsprintf(dsbufp, "type=");
#ifdef S_ISSOCK
        if (S_ISSOCK(stat_buf.st_mode))
            erts_dsprintf(dsbufp, "sock ");
        else
#endif
#ifdef S_ISFIFO
	    if (S_ISFIFO(stat_buf.st_mode))
		erts_dsprintf(dsbufp, "fifo ");
	    else
#endif
#ifdef S_ISCHR
                if (S_ISCHR(stat_buf.st_mode))
                    erts_dsprintf(dsbufp, "chr ");
                else
#endif
#ifdef S_ISDIR
                    if (S_ISDIR(stat_buf.st_mode))
                        erts_dsprintf(dsbufp, "dir ");
                    else
#endif
#ifdef S_ISBLK
                        if (S_ISBLK(stat_buf.st_mode))
                            erts_dsprintf(dsbufp, "blk ");
                        else
#endif
#ifdef S_ISREG
                            if (S_ISREG(stat_buf.st_mode))
                                erts_dsprintf(dsbufp, "reg ");
                            else
#endif
#ifdef S_ISLNK
                                if (S_ISLNK(stat_buf.st_mode))
                                    erts_dsprintf(dsbufp, "lnk ");
                                else
#endif
#ifdef S_ISDOOR
                                    if (S_ISDOOR(stat_buf.st_mode))
                                        erts_dsprintf(dsbufp, "door ");
                                    else
#endif
#ifdef S_ISWHT
                                        if (S_ISWHT(stat_buf.st_mode))
                                            erts_dsprintf(dsbufp, "wht ");
                                        else
#endif
#ifdef S_ISXATTR
                                            if (S_ISXATTR(stat_buf.st_mode))
                                                erts_dsprintf(dsbufp, "xattr ");
                                            else
#endif
                                                erts_dsprintf(dsbufp, "unknown ");
    }
#else
    erts_dsprintf(dsbufp, "type=unknown ");
#endif

    if (state->type == ERTS_EV_TYPE_DRV_SEL) {
        erts_dsprintf(dsbufp, "driver_select ");

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
        if (internal) {
            erts_dsprintf(dsbufp, "internal ");
            err = 1;
        }
        if (aio_events == cio_events) {
            if (cio_events == ep_events) {
                erts_dsprintf(dsbufp, "ev=");
                if (print_events(dsbufp, cio_events) != 0)
                    err = 1;
            }
            else {
                ErtsPollEvents ev = cio_events;
                if (ev != ep_events && ep_events != ERTS_POLL_EV_NONE)
                    err = 1;
                erts_dsprintf(dsbufp, "cio_ev=");
                print_events(dsbufp, cio_events);
                erts_dsprintf(dsbufp, " ep_ev=");
                print_events(dsbufp, ep_events);
            }
        } else {
            erts_dsprintf(dsbufp, "cio_ev=");
            print_events(dsbufp, cio_events);
            erts_dsprintf(dsbufp, " aio_ev=");
            print_events(dsbufp, aio_events);
            if ((aio_events != ep_events && ep_events != ERTS_POLL_EV_NONE) ||
                (aio_events != 0 && ep_events == ERTS_POLL_EV_NONE)) {
                erts_dsprintf(dsbufp, " ep_ev=");
                print_events(dsbufp, ep_events);
                err = 1;
            }
        }
#else
        if (print_events(dsbufp, cio_events) != 0)
            err = 1;
#endif
        erts_dsprintf(dsbufp, " ");
        if (cio_events & ERTS_POLL_EV_IN) {
            Eterm id = state->driver.select->inport;
            if (is_nil(id)) {
                erts_dsprintf(dsbufp, "inport=none inname=none indrv=none ");
                err = 1;
            }
            else {
                ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT);
                erts_dsprintf(dsbufp, " inport=%T inname=%s indrv=%s ",
                            id,
                            pnp->name ? pnp->name : "unknown",
                            (pnp->driver_name
                             ? pnp->driver_name
                             : "unknown"));
                erts_free_port_names(pnp);
            }
        }
        if (cio_events & ERTS_POLL_EV_OUT) {
            Eterm id = state->driver.select->outport;
            if (is_nil(id)) {
                erts_dsprintf(dsbufp, "outport=none outname=none outdrv=none ");
                err = 1;
            }
            else {
                ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT);
                erts_dsprintf(dsbufp, " outport=%T outname=%s outdrv=%s ",
                            id,
                            pnp->name ? pnp->name : "unknown",
                            (pnp->driver_name
                             ? pnp->driver_name
                             : "unknown"));
                erts_free_port_names(pnp);
            }
        }
    }
    else if (state->type == ERTS_EV_TYPE_NIF) {
        ErtsResource* r;
        erts_dsprintf(dsbufp, "enif_select ");

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
        if (internal) {
            erts_dsprintf(dsbufp, "internal ");
            err = 1;
        }

        if (cio_events == ep_events) {
            erts_dsprintf(dsbufp, "ev=");
            if (print_events(dsbufp, cio_events) != 0)
                err = 1;
        }
        else {
            err = 1;
            erts_dsprintf(dsbufp, "cio_ev=");
            print_events(dsbufp, cio_events);
            erts_dsprintf(dsbufp, " ep_ev=");
            print_events(dsbufp, ep_events);
        }
#else
        if (print_events(dsbufp, cio_events) != 0)
            err = 1;
#endif
        erts_dsprintf(dsbufp, " inpid=%T", state->driver.nif->in.pid);
        erts_dsprintf(dsbufp, " outpid=%T", state->driver.nif->out.pid);
        r = state->driver.stop.resource;
        erts_dsprintf(dsbufp, " resource=%p(%T:%T)", r, r->type->module, r->type->name);
    }
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    else if (internal) {
        erts_dsprintf(dsbufp, "internal ");
        if (cio_events) {
            err = 1;
            erts_dsprintf(dsbufp, "cio_ev=");
            print_events(dsbufp, cio_events);
        }
        if (ep_events) {
            erts_dsprintf(dsbufp, "ep_ev=");
            print_events(dsbufp, ep_events);
        }
    }
#endif
    else {
        err = 1;
        erts_dsprintf(dsbufp, "control_type=%d ", (int)state->type);
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
        if (cio_events == ep_events) {
            erts_dsprintf(dsbufp, "ev=");
            print_events(dsbufp, cio_events);
        }
        else {
            erts_dsprintf(dsbufp, "cio_ev="); print_events(dsbufp, cio_events);
            erts_dsprintf(dsbufp, " ep_ev="); print_events(dsbufp, ep_events);
        }
#else
        erts_dsprintf(dsbufp, "ev=0x%b32x", (Uint32) cio_events);
#endif
    }

    erts_dsprintf(dsbufp, " flags="); print_flags(dsbufp, state->flags);
    if (err) {
        erts_dsprintf(dsbufp, " ERROR");
    }
    erts_dsprintf(dsbufp, "\r\n");
    return err;
}

static void doit_erts_check_io_debug(void *vstate, void *vcounters,
                                     erts_dsprintf_buf_t *dsbufp)
{
    ErtsDrvEventState *state = (ErtsDrvEventState *) vstate;
    IterDebugCounters *counters = (IterDebugCounters *) vcounters;
    int internal = 0;
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    ErtsSysFdType fd = state->fd;
    ErtsPollEvents ep_events = counters->epep[(int) fd];
#else
    ErtsPollEvents ep_events = ERTS_POLL_EV_NONE;
#endif

    if (state->driver.select) {
	counters->no_driver_select_structs++;
        ASSERT(state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE));
    }
    if (state->driver.nif) {
        counters->no_enif_select_structs++;
        ASSERT(state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE));
    }

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    if (state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE)) {
	if (ep_events & ERTS_POLL_EV_NVAL) {
	    ep_events &= ~ERTS_POLL_EV_NVAL;
	    internal = 1;
	    counters->internal_fds++;
	}
	else
	    counters->used_fds++;
#else
    if (state->events) {
	counters->used_fds++;
#endif
	if (erts_debug_print_checkio_state(dsbufp, state, ep_events, internal)) {
	    counters->num_errors++;
	}
    }
}

/* ciodpi can be NULL when called from etp-commands */
int
erts_check_io_debug(ErtsCheckIoDebugInfo *ciodip)
{
    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    int fd, len, i;
#endif
    IterDebugCounters counters = {0};
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    ErtsDrvEventState null_des;

    null_des.driver.select = NULL;
    null_des.driver.nif = NULL;
    null_des.driver.stop.drv_ptr = NULL;
    null_des.events = 0;
    null_des.type = ERTS_EV_TYPE_NONE;
    null_des.flags = 0;

    counters.epep = erts_alloc(ERTS_ALC_T_TMP,
                               sizeof(ErtsPollEvents)*drv_ev_state.max_fds);
#endif


#if defined(ERTS_ENABLE_LOCK_CHECK)
    erts_lc_check_exact(NULL, 0); /* No locks should be locked */
#endif

    if (ciodip)
        erts_thr_progress_block(); /* stop the world to avoid messy locking */

#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    len = erts_atomic_read_nob(&drv_ev_state.len);

#if ERTS_POLL_USE_FALLBACK
    erts_dsprintf(dsbufp, "--- fds in flbk pollset ---------------------------------\n");
    erts_poll_get_selected_events_flbk(get_fallback_pollset(), counters.epep,
                                       drv_ev_state.max_fds);
    for (fd = 0; fd < len; fd++) {
        if (drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK)
            doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp);
    }
#endif
#if ERTS_POLL_USE_SCHEDULER_POLLING
    erts_dsprintf(dsbufp, "--- fds in scheduler pollset ----------------------------\n");
    erts_poll_get_selected_events(get_scheduler_pollset(0), counters.epep,
                                  drv_ev_state.max_fds);
    for (fd = 0; fd < len; fd++) {
        if (drv_ev_state.v[fd].flags & ERTS_EV_FLAG_SCHEDULER) {
            if (drv_ev_state.v[fd].events && drv_ev_state.v[fd].events != ERTS_POLL_EV_NONE)
                counters.epep[fd] &= ~ERTS_POLL_EV_OUT;
            doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp);
        }
    }
#endif

    erts_dsprintf(dsbufp, "--- fds in pollset --------------------------------------\n");

    for (i = 0; i < erts_no_pollsets; i++) {
        erts_poll_get_selected_events(pollsetv[i],
                                      counters.epep,
                                      drv_ev_state.max_fds);
        for (fd = 0; fd < len; fd++) {
            if (!(drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK)
                && get_pollset_id(fd) == i) {
                if (counters.epep[fd] != ERTS_POLL_EV_NONE &&
                    drv_ev_state.v[fd].flags & ERTS_EV_FLAG_IN_SCHEDULER) {
                    /* We add the in flag if it is enabled in the scheduler pollset
                       and get_selected_events works on the platform */
                    counters.epep[fd] |= ERTS_POLL_EV_IN;
                }
                doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp);
            }
        }
    }
    for (fd = len ; fd < drv_ev_state.max_fds; fd++) {
        null_des.fd = fd;
        doit_erts_check_io_debug(&null_des, &counters, dsbufp);
    }
#else
    safe_hash_for_each(&drv_ev_state.tab, &doit_erts_check_io_debug,
                       &counters, dsbufp);
#endif

    if (ciodip)
        erts_thr_progress_unblock();

    if (ciodip) {
        ciodip->no_used_fds = counters.used_fds;
        ciodip->no_driver_select_structs = counters.no_driver_select_structs;
        ciodip->no_enif_select_structs = counters.no_enif_select_structs;
    }

    erts_dsprintf(dsbufp, "\n");
    erts_dsprintf(dsbufp, "used fds=%d\n", counters.used_fds);
    erts_dsprintf(dsbufp, "Number of driver_select() structures=%d\n", counters.no_driver_select_structs);
    erts_dsprintf(dsbufp, "Number of enif_select() structures=%d\n", counters.no_enif_select_structs);
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    erts_dsprintf(dsbufp, "internal fds=%d\n", counters.internal_fds);
#endif
    erts_dsprintf(dsbufp, "---------------------------------------------------------\n");
    erts_send_error_to_logger_nogl(dsbufp);
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
    erts_free(ERTS_ALC_T_TMP, (void *) counters.epep);
#endif

    return counters.num_errors;
}

#ifdef ERTS_ENABLE_LOCK_COUNT
void erts_lcnt_update_cio_locks(int enable) {
    int i;
#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
    erts_lcnt_enable_hash_lock_count(&drv_ev_state.tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable);
#else
    (void)enable;
#endif

#if ERTS_POLL_USE_FALLBACK
    erts_lcnt_enable_pollset_lock_count_flbk(get_fallback_pollset(), enable);
#endif

    for (i = 0; i < erts_no_pollsets; i++)
        erts_lcnt_enable_pollset_lock_count(pollsetv[i], enable);
}
#endif /* ERTS_ENABLE_LOCK_COUNT */