aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/binary.c5
-rw-r--r--erts/emulator/beam/erl_io_queue.c36
-rw-r--r--erts/emulator/beam/erl_nif.c124
-rw-r--r--erts/emulator/beam/erl_trace.c30
4 files changed, 119 insertions, 76 deletions
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index d4db1a4188..95d324d2c1 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -953,7 +953,10 @@ HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1)
BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1)
{
if (is_binary(BIF_ARG_1)) {
- BIF_RET(BIF_ARG_1);
+ if (binary_bitsize(BIF_ARG_1) == 0) {
+ BIF_RET(BIF_ARG_1);
+ }
+ BIF_ERROR(BIF_P, BADARG);
}
return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]);
}
diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c
index 190ba6bbb9..eb156cc578 100644
--- a/erts/emulator/beam/erl_io_queue.c
+++ b/erts/emulator/beam/erl_io_queue.c
@@ -801,12 +801,11 @@ static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term,
ERTS_GET_REAL_BIN(bin_term, orig_pb_term,
byte_offset, bit_offset, bit_size);
- (void)bit_offset;
- (void)bit_size;
+ ASSERT(bit_size == 0);
sb->thing_word = HEADER_SUB_BIN;
+ sb->bitoffs = bit_offset;
sb->bitsize = 0;
- sb->bitoffs = 0;
sb->orig = orig_pb_term;
sb->is_writable = 0;
@@ -984,7 +983,7 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) {
parent_header = binary_val(parent_binary);
binary_size = binary_size(bin_term);
- if (bit_offset != 0 || bit_size != 0) {
+ if (bit_size != 0) {
return 0;
} else if (binary_size == 0) {
state->bytereds_spent += 1;
@@ -1026,8 +1025,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) {
* then just copy it into the accumulator. */
iol2v_expand_acc(state, binary_size);
- sys_memcpy(&(state->acc)->orig_bytes[state->acc_size],
- binary_data, binary_size);
+ if (ERTS_LIKELY(bit_offset == 0)) {
+ sys_memcpy(&(state->acc)->orig_bytes[state->acc_size],
+ binary_data, binary_size);
+ } else {
+ ASSERT(binary_size <= ERTS_UWORD_MAX / 8);
+
+ erts_copy_bits(binary_data, bit_offset, 1,
+ (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1,
+ binary_size * 8);
+ }
state->acc_size += binary_size;
} else {
@@ -1038,8 +1045,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) {
iol2v_expand_acc(state, spill);
- sys_memcpy(&(state->acc)->orig_bytes[state->acc_size],
- binary_data, spill);
+ if (ERTS_LIKELY(bit_offset == 0)) {
+ sys_memcpy(&(state->acc)->orig_bytes[state->acc_size],
+ binary_data, spill);
+ } else {
+ ASSERT(binary_size <= ERTS_UWORD_MAX / 8);
+
+ erts_copy_bits(binary_data, bit_offset, 1,
+ (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1,
+ spill * 8);
+ }
state->acc_size += spill;
@@ -1188,7 +1203,10 @@ BIF_RETTYPE iolist_to_iovec_1(BIF_ALIST_1) {
if (is_nil(BIF_ARG_1)) {
BIF_RET(NIL);
} else if (is_binary(BIF_ARG_1)) {
- if (binary_size(BIF_ARG_1) != 0) {
+ if (binary_bitsize(BIF_ARG_1) != 0) {
+ ASSERT(!(BIF_P->flags & F_DISABLE_GC));
+ BIF_ERROR(BIF_P, BADARG);
+ } else if (binary_size(BIF_ARG_1) != 0) {
Eterm *hp = HAlloc(BIF_P, 2);
BIF_RET(CONS(hp, BIF_ARG_1, NIL));
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index f67b67325d..b96db5d054 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3396,8 +3396,8 @@ typedef struct {
Eterm sublist_start;
Eterm sublist_end;
- UWord offheap_size;
- UWord onheap_size;
+ UWord referenced_size;
+ UWord copied_size;
UWord iovec_len;
} iovec_slice_t;
@@ -3407,16 +3407,16 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul
result->sublist_start = list;
result->sublist_length = 0;
- result->offheap_size = 0;
- result->onheap_size = 0;
+ result->referenced_size = 0;
+ result->copied_size = 0;
result->iovec_len = 0;
lookahead = result->sublist_start;
while (is_list(lookahead)) {
- Eterm *binary_header, binary;
+ UWord byte_size;
+ Eterm binary;
Eterm *cell;
- UWord size;
cell = list_val(lookahead);
binary = CAR(cell);
@@ -3425,35 +3425,36 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul
return 0;
}
- size = binary_size(binary);
- binary_header = binary_val(binary);
+ byte_size = binary_size(binary);
- if (size > 0) {
- /* If we're a sub-binary we'll need to check our underlying binary
- * to determine whether we're on-heap or not. */
- if (thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) {
- ErlSubBin *sb = (ErlSubBin*)binary_header;
+ if (byte_size > 0) {
+ int bit_offset, bit_size;
+ Eterm parent_binary;
+ UWord byte_offset;
- /* Reject bitstrings */
- if((sb->bitoffs + sb->bitsize) > 0) {
- return 0;
- }
+ int requires_copying;
- ASSERT(size <= binary_size(sb->orig));
- binary_header = binary_val(sb->orig);
+ ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset,
+ bit_offset, bit_size);
+
+ (void)byte_offset;
+
+ if (bit_size != 0) {
+ return 0;
}
- if (thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) {
- ASSERT(size <= ERL_ONHEAP_BIN_LIMIT);
+ /* If we're unaligned or an on-heap binary we'll need to copy
+ * ourselves over to a temporary buffer. */
+ requires_copying = (bit_offset != 0) ||
+ thing_subtag(*binary_val(parent_binary)) == HEAP_BINARY_SUBTAG;
- result->iovec_len += 1;
- result->onheap_size += size;
+ if (requires_copying) {
+ result->copied_size += byte_size;
} else {
- ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG);
-
- result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN;
- result->offheap_size += size;
+ result->referenced_size += byte_size;
}
+
+ result->iovec_len += 1 + byte_size / MAX_SYSIOVEC_IOVLEN;
}
result->sublist_length += 1;
@@ -3473,7 +3474,9 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul
return 1;
}
-static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) {
+static void marshal_iovec_binary(Eterm binary, ErlNifBinary *copy_buffer,
+ UWord *copy_offset, ErlNifBinary *result) {
+
Eterm *parent_header;
Eterm parent_binary;
@@ -3484,6 +3487,8 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) {
ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, bit_offset, bit_size);
+ ASSERT(bit_size == 0);
+
parent_header = binary_val(parent_binary);
result->size = binary_size(binary);
@@ -3510,24 +3515,50 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) {
result->data = &((unsigned char*)&hb->data)[byte_offset];
result->ref_bin = NULL;
}
+
+ /* If this isn't an *aligned* refc binary, copy its contents to the buffer
+ * and reference that instead. */
+
+ if (result->ref_bin == NULL || bit_offset != 0) {
+ ASSERT(result->size <= (copy_buffer->size - *copy_offset));
+
+ if (bit_offset == 0) {
+ sys_memcpy(&copy_buffer->data[*copy_offset],
+ result->data, result->size);
+ } else {
+ erts_copy_bits(result->data, bit_offset, 1,
+ (byte*)&copy_buffer->data[*copy_offset], 0, 1,
+ result->size * 8);
+ }
+
+ result->data = &copy_buffer->data[*copy_offset];
+ result->ref_bin = copy_buffer->ref_bin;
+
+ *copy_offset += result->size;
+ }
}
static int fill_iovec_with_slice(ErlNifEnv *env,
iovec_slice_t *slice,
ErlNifIOVec *iovec) {
- UWord onheap_offset, iovec_idx;
- ErlNifBinary onheap_data;
+ UWord copy_offset, iovec_idx;
+ ErlNifBinary copy_buffer;
Eterm sublist_iterator;
- /* Set up a common refc binary for all on-heap binaries. */
- if (slice->onheap_size > 0) {
- if (!enif_alloc_binary(slice->onheap_size, &onheap_data)) {
+ /* Set up a common refc binary for all on-heap and unaligned binaries. */
+ if (slice->copied_size > 0) {
+ if (!enif_alloc_binary(slice->copied_size, &copy_buffer)) {
return 0;
}
+ } else {
+#ifdef DEBUG
+ copy_buffer.data = NULL;
+ copy_buffer.size = 0;
+#endif
}
sublist_iterator = slice->sublist_start;
- onheap_offset = 0;
+ copy_offset = 0;
iovec_idx = 0;
while (sublist_iterator != slice->sublist_end) {
@@ -3535,20 +3566,7 @@ static int fill_iovec_with_slice(ErlNifEnv *env,
Eterm *cell;
cell = list_val(sublist_iterator);
- inspect_raw_binary_data(CAR(cell), &raw_data);
-
- /* If this isn't a refc binary, copy its contents to the onheap buffer
- * and reference that instead. */
- if (raw_data.size > 0 && raw_data.ref_bin == NULL) {
- ASSERT(onheap_offset < onheap_data.size);
- ASSERT(slice->onheap_size > 0);
-
- sys_memcpy(&onheap_data.data[onheap_offset],
- raw_data.data, raw_data.size);
-
- raw_data.data = &onheap_data.data[onheap_offset];
- raw_data.ref_bin = onheap_data.ref_bin;
- }
+ marshal_iovec_binary(CAR(cell), &copy_buffer, &copy_offset, &raw_data);
while (raw_data.size > 0) {
UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN);
@@ -3579,16 +3597,16 @@ static int fill_iovec_with_slice(ErlNifEnv *env,
erts_refc_inc(&refc_binary->intern.refc, 1);
}
- if (slice->onheap_size > 0) {
+ if (slice->copied_size > 0) {
/* Transfer ownership to the iovec; we've taken references to it in
* the above loop. */
- enif_release_binary(&onheap_data);
+ enif_release_binary(&copy_buffer);
}
} else {
- if (slice->onheap_size > 0) {
+ if (slice->copied_size > 0) {
/* Attach the binary to our environment and let the GC take care of
* it after returning. */
- enif_make_binary(env, &onheap_data);
+ enif_make_binary(env, &copy_buffer);
}
}
@@ -3635,7 +3653,7 @@ static int create_iovec_from_slice(ErlNifEnv *env,
iovec->flags = 0;
}
- iovec->size = slice->offheap_size + slice->onheap_size;
+ iovec->size = slice->referenced_size + slice->copied_size;
iovec->iovcnt = slice->iovec_len;
if(!fill_iovec_with_slice(env, slice, iovec)) {
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index db7d0ac449..9fda4ff6ff 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3009,24 +3009,28 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
ASSERT(0);
}
- /* Only remove tracer on self() and ports */
+ /* Only remove tracer on (self() or ports) AND we are on a normal scheduler */
if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) {
+ ErtsSchedulerData *esdp = erts_get_scheduler_data();
ErtsProcLocks c_p_xlocks = 0;
- if (is_internal_pid(t_p->id)) {
- ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
- if (c_p_locks != ERTS_PROC_LOCKS_ALL) {
- c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL;
- if (erts_smp_proc_trylock(c_p, c_p_xlocks) == EBUSY) {
- erts_smp_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN);
- erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+ if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+ if (is_internal_pid(t_p->id)) {
+ ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
+ if (c_p_locks != ERTS_PROC_LOCKS_ALL) {
+ c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL;
+ if (erts_smp_proc_trylock(c_p, c_p_xlocks) == EBUSY) {
+ erts_smp_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN);
+ erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+ }
}
}
- }
- erts_tracer_replace(t_p, erts_tracer_nil);
- t_p->trace_flags &= ~TRACEE_FLAGS;
- if (c_p_xlocks)
- erts_smp_proc_unlock(c_p, c_p_xlocks);
+ erts_tracer_replace(t_p, erts_tracer_nil);
+ t_p->trace_flags &= ~TRACEE_FLAGS;
+
+ if (c_p_xlocks)
+ erts_smp_proc_unlock(c_p, c_p_xlocks);
+ }
}
return 0;
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-02 19:49:57 +0000