From ca0425c6ff85262bc15367f5fd9cbc51cde52b20 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 2 Oct 2013 10:07:27 +0200 Subject: Execution of system tasks in context of another process A process requesting a system task to be executed in the context of another process will be notified by a message when the task has executed. This message will be on the form: {RequestType, RequestId, Pid, Result}. A process requesting a system task to be executed can set priority on the system task. The requester typically set the same priority on the task as its own process priority, and by this avoiding priority inversion. A request for execution of a system task is made by calling the statically linked in NIF erts_internal:request_system_task(Pid, Prio, Request). This is an undocumented ERTS internal function that should remain so. It should *only* be called from BIF implementations. Currently defined system tasks are: * garbage_collect * check_process_code Further system tasks can and will be implemented in the future. The erlang:garbage_collect/[1,2] and erlang:check_process_code/[2,3] BIFs are now implemented using system tasks. Both the 'garbage_collect' and the 'check_process_code' operations perform or may perform garbage_collections. By doing these via the system task functionality all garbage collect operations in the system will be performed solely in the context of the process being garbage collected. This makes it possible to later implement functionality for disabling garbage collection of a process over context switches. Newly introduced BIFs: * erlang:garbage_collect/2 - The new second argument is an option list. Introduced option: * {async, RequestId} - making it possible for users to issue asynchronous garbage collect requests. * erlang:check_process_code/3 - The new third argument is an option list. Introduced options: * {async, RequestId} - making it possible for users to issue asynchronous check process code requests. * {allow_gc, boolean()} - making it possible to issue requests that aren't allowed to garbage collect (operation will abort if gc should be needed). These options have been introduced as a preparation for parallelization of check_process_code operations when the code_server is about to purge a module. --- erts/emulator/beam/erl_message.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 771eba431f..0f3bb8d281 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -46,6 +46,11 @@ typedef struct erl_off_heap { Uint64 overhead; /* Administrative overhead (used to force GC). */ } ErlOffHeap; +#define ERTS_INIT_OFF_HEAP(OHP) \ + do { \ + (OHP)->first = NULL; \ + (OHP)->overhead = 0; \ + } while (0) #include "external.h" #include "erl_process.h" -- cgit v1.2.3 From f8dbf0c0ff3bd68d720faca230356d281e4e3e42 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 3 Mar 2015 19:32:00 +0100 Subject: First stab at binary_to_term for hamt with over estimation of heap size. --- erts/emulator/beam/erl_message.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 0f3bb8d281..ece75a5ee4 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -68,6 +68,23 @@ struct erl_heap_fragment { Eterm mem[1]; /* Data */ }; +typedef struct { + Process* p; + Eterm* hp; + Eterm* hp_end; + /* more to come... */ +} ErtsHeapFactory; + +Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); +#ifdef CHECK_FOR_HOLES +# define ERTS_FACTORY_HOLE_CHECK(f) do { \ + if ((f)->p) erts_check_for_holes((f)->p); \ + } while (0) +#else +# define ERTS_FACTORY_HOLE_CHECK(p) +#endif + + typedef struct erl_mesg { struct erl_mesg* next; /* Next message */ union { -- cgit v1.2.3 From 12fc63bcaf68b4a9e89ce91e1235aafb8bcdaee5 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 13 Mar 2015 18:00:33 +0100 Subject: erts: Fix map bug in dec_term for 32-bit debug VM Adding ERTS_SWORD_MAX to a pointer does not work as a way to disable a bound check. Remove the hp_end from ErtsHeapFactory as it isn't really used anyway. --- erts/emulator/beam/erl_message.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index ece75a5ee4..6b8c3cebc7 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -71,8 +71,6 @@ struct erl_heap_fragment { typedef struct { Process* p; Eterm* hp; - Eterm* hp_end; - /* more to come... */ } ErtsHeapFactory; Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); -- cgit v1.2.3 From 6b1921d767de5cd1a980234f83b36dbfa13d9fc7 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 13 Feb 2015 00:25:57 +0100 Subject: Erlang based BIF timer implementation for scalability --- erts/emulator/beam/erl_message.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 0f3bb8d281..a2a7193ea9 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -198,15 +198,25 @@ do { \ if ((M)->data.attached) { \ Uint need__ = erts_msg_attached_data_size((M)); \ if ((ST) - (HT) >= need__) { \ - Uint *htop__ = (HT); \ + Uint *htop__; \ + move__attached__msg__data____: \ + htop__ = (HT); \ erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\ ASSERT(htop__ - (HT) <= need__); \ (HT) = htop__; \ } \ else { \ + int off_heap_msgs__ = (int) (P)->flags & F_OFF_HEAP_MSGS; \ + if (!off_heap_msgs__) \ + need__ = 0; \ { SWPO ; } \ - (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ + (FC) -= erts_garbage_collect((P), need__, NULL, 0); \ { SWPI ; } \ + if (off_heap_msgs__) { \ + ASSERT((M)->data.attached); \ + ASSERT((ST) - (HT) >= need__); \ + goto move__attached__msg__data____; \ + } \ } \ ASSERT(!(M)->data.attached); \ } \ -- cgit v1.2.3 From 0a1e2b5e6c92cbea2d2e853facb8b4d6f058d541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 14 Apr 2015 13:29:08 +0200 Subject: erts: Refactor erts_queue_message --- erts/emulator/beam/erl_message.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 8713941769..8f9ea939e8 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -258,11 +258,17 @@ ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); -void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, Eterm, Eterm #ifdef USE_VM_PROBES - , Eterm dt_utag +void erts_queue_message_probe(Process*, ErtsProcLocks*, ErlHeapFragment*, + Eterm message, Eterm seq_trace_token, Eterm dt_utag); +#define erts_queue_message(RP,RL,BP,Msg,SEQ) \ + erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL) +#else +void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, + Eterm message, Eterm seq_trace_token); +#define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \ + erts_queue_message((RP),(RL),(BP),(Msg),(SEQ)) #endif -); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); -- cgit v1.2.3 From 9c78f149517dc02457d4c59e90bc9b03d411e28c Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 5 May 2015 20:05:00 +0200 Subject: Optimized timer implementation --- erts/emulator/beam/erl_message.h | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 8713941769..6b8c3cebc7 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -213,25 +213,15 @@ do { \ if ((M)->data.attached) { \ Uint need__ = erts_msg_attached_data_size((M)); \ if ((ST) - (HT) >= need__) { \ - Uint *htop__; \ - move__attached__msg__data____: \ - htop__ = (HT); \ + Uint *htop__ = (HT); \ erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\ ASSERT(htop__ - (HT) <= need__); \ (HT) = htop__; \ } \ else { \ - int off_heap_msgs__ = (int) (P)->flags & F_OFF_HEAP_MSGS; \ - if (!off_heap_msgs__) \ - need__ = 0; \ { SWPO ; } \ - (FC) -= erts_garbage_collect((P), need__, NULL, 0); \ + (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ { SWPI ; } \ - if (off_heap_msgs__) { \ - ASSERT((M)->data.attached); \ - ASSERT((ST) - (HT) >= need__); \ - goto move__attached__msg__data____; \ - } \ } \ ASSERT(!(M)->data.attached); \ } \ -- cgit v1.2.3 From 02d0ce034598297565f2b35ecc3d1af121787f33 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 2 Jun 2015 19:23:49 +0200 Subject: erts: Remove hashmap probabilistic heap overestimation by adding a dynamic heap factory. "binary_to_term" is now a hybrid solution with both a call to decoded_size() to calculate needed heap space AND possible dynamic allocation of more heap space if needed for big maps. The heap size returned from decoded_size() is guaranteed to be sufficient for all term heap data except for hashmap nodes. All hashmap nodes are created at the end of dec_term() by invoking the heap factory interface that may allocate more heap space on process heap or in fragments. With this commit it is no longer guaranteed that a message is confined to only one heap fragment. --- erts/emulator/beam/erl_message.h | 85 ++++++++++++++++++++++++++-------------- 1 file changed, 56 insertions(+), 29 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 1e1dafee90..39946f2a14 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -51,6 +51,44 @@ typedef struct erl_off_heap { (OHP)->first = NULL; \ (OHP)->overhead = 0; \ } while (0) + +typedef struct { + enum { + FACTORY_CLOSED = 0, + FACTORY_HALLOC, + FACTORY_HEAP_FRAGS, + FACTORY_STATIC + } mode; + Process* p; + Eterm* hp_start; + Eterm* hp; + Eterm* hp_end; + struct erl_heap_fragment* heap_frags; + struct erl_heap_fragment* heap_frags_saved; + ErlOffHeap* off_heap; + ErlOffHeap off_heap_saved; + Uint32 alloc_type; +} ErtsHeapFactory; + +void erts_factory_proc_init(ErtsHeapFactory*, Process*); +void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size); +void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*); +void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*); +void erts_factory_dummy_init(ErtsHeapFactory*); + +Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); +Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need); +void erts_factory_close(ErtsHeapFactory*); +void erts_factory_undo(ErtsHeapFactory*); + +#ifdef CHECK_FOR_HOLES +# define ERTS_FACTORY_HOLE_CHECK(f) do { \ + /*if ((f)->p) erts_check_for_holes((f)->p);*/ \ + } while (0) +#else +# define ERTS_FACTORY_HOLE_CHECK(p) +#endif + #include "external.h" #include "erl_process.h" @@ -68,21 +106,6 @@ struct erl_heap_fragment { Eterm mem[1]; /* Data */ }; -typedef struct { - Process* p; - Eterm* hp; -} ErtsHeapFactory; - -Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); -#ifdef CHECK_FOR_HOLES -# define ERTS_FACTORY_HOLE_CHECK(f) do { \ - if ((f)->p) erts_check_for_holes((f)->p); \ - } while (0) -#else -# define ERTS_FACTORY_HOLE_CHECK(p) -#endif - - typedef struct erl_mesg { struct erl_mesg* next; /* Next message */ union { @@ -139,7 +162,7 @@ typedef struct { *(p)->msg.last = (mp); \ (p)->msg.last = &(mp)->next; \ (p)->msg.len++; \ -} while(0) +} while (0) #ifdef ERTS_SMP @@ -212,17 +235,23 @@ do { \ do { \ if ((M)->data.attached) { \ Uint need__ = erts_msg_attached_data_size((M)); \ + { SWPO ; } \ if ((ST) - (HT) >= need__) { \ - Uint *htop__ = (HT); \ - erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\ - ASSERT(htop__ - (HT) <= need__); \ - (HT) = htop__; \ + ErtsHeapFactory factory__; \ + erts_factory_proc_prealloc_init(&factory__, (P), need__); \ + erts_move_msg_attached_data_to_heap(&factory__, (M)); \ + erts_factory_close(&factory__); \ + if ((P)->mbuf != NULL) { \ + /* Heap was exhausted by messages. This is a rare case */ \ + /* that can currently (OTP 18) only happen if hamts are */ \ + /* far exceeding the estimated heap size. Do GC. */ \ + (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ + } \ } \ else { \ - { SWPO ; } \ (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ - { SWPI ; } \ } \ + { SWPI ; } \ ASSERT(!(M)->data.attached); \ } \ } while (0) @@ -266,23 +295,21 @@ void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *); Uint erts_msg_attached_data_size_aux(ErlMessage *msg); -void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *); - +void erts_move_msg_attached_data_to_heap(ErtsHeapFactory*, ErlMessage *); Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, Eterm *, ErtsDistExternal *); void erts_cleanup_offheap(ErlOffHeap *offheap); -ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg); +ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg) +ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp) { - const ErlHeapFragment *bp; Uint sz = 0; - for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) { + for ( ; bp!=NULL; bp=bp->next) { sz += bp->used_size; } return sz; @@ -292,7 +319,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) { ASSERT(msg->data.attached); if (is_value(ERL_MESSAGE_TERM(msg))) - return erts_msg_used_frag_sz(msg); + return erts_used_frag_sz(msg->data.heap_frag); else if (msg->data.dist_ext->heap_size < 0) return erts_msg_attached_data_size_aux(msg); else { -- cgit v1.2.3 From f5138184479bd16c0ee0a5e583479378451401c8 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 10 Jun 2015 20:59:05 +0200 Subject: erts: Add erts_factory_trim_and_close --- erts/emulator/beam/erl_message.h | 1 + 1 file changed, 1 insertion(+) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 39946f2a14..705ba5e506 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -79,6 +79,7 @@ void erts_factory_dummy_init(ErtsHeapFactory*); Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need); void erts_factory_close(ErtsHeapFactory*); +void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size); void erts_factory_undo(ErtsHeapFactory*); #ifdef CHECK_FOR_HOLES -- cgit v1.2.3 From 738c34d4bb8f1a3811acd00af8c6c12107f8315b Mon Sep 17 00:00:00 2001 From: Bruce Yinhe Date: Thu, 18 Jun 2015 11:31:02 +0200 Subject: Change license text to APLv2 --- erts/emulator/beam/erl_message.h | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 705ba5e506..fbdf3fb0e2 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ -- cgit v1.2.3 From 1c86a620d74f4f9383c4956dafd3e2486300dc0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 17 Jun 2015 17:33:29 +0200 Subject: erts: Remove HALFWORD_HEAP definition --- erts/emulator/beam/erl_message.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index fbdf3fb0e2..f37b430d27 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -32,9 +32,6 @@ struct external_thing_; struct erl_off_heap_header { Eterm thing_word; Uint size; -#if HALFWORD_HEAP - void* dummy_ptr_padding__; -#endif struct erl_off_heap_header* next; }; -- cgit v1.2.3 From 3ac08f9b668613a4292436979eacc61863c2ab94 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 16 Sep 2015 15:02:50 +0200 Subject: Fragmented young heap generation and off_heap_message_queue option * The youngest generation of the heap can now consist of multiple blocks. Heap fragments and message fragments are added to the youngest generation when needed without triggering a GC. After a GC the youngest generation is contained in one single block. * The off_heap_message_queue process flag has been added. When enabled all message data in the queue is kept off heap. When a message is selected from the queue, the message fragment (or heap fragment) containing the actual message is attached to the youngest generation. Messages stored off heap is not part of GC. --- erts/emulator/beam/erl_message.h | 277 +++++++++++++++++++++++++++------------ 1 file changed, 192 insertions(+), 85 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index f37b430d27..740ae46a0f 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -24,6 +24,8 @@ struct proc_bin; struct external_thing_; +typedef struct erl_mesg ErtsMessage; + /* * This struct represents data that must be updated by structure copy, * but is stored outside of any heap. @@ -54,6 +56,7 @@ typedef struct { enum { FACTORY_CLOSED = 0, FACTORY_HALLOC, + FACTORY_MESSAGE, FACTORY_HEAP_FRAGS, FACTORY_STATIC } mode; @@ -61,8 +64,10 @@ typedef struct { Eterm* hp_start; Eterm* hp; Eterm* hp_end; + ErtsMessage *message; struct erl_heap_fragment* heap_frags; struct erl_heap_fragment* heap_frags_saved; + Uint heap_frags_saved_used; ErlOffHeap* off_heap; ErlOffHeap off_heap_saved; Uint32 alloc_type; @@ -70,7 +75,10 @@ typedef struct { void erts_factory_proc_init(ErtsHeapFactory*, Process*); void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size); -void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*); +void erts_factory_heap_frag_init(ErtsHeapFactory*, struct erl_heap_fragment*); +ErtsMessage *erts_factory_message_create(ErtsHeapFactory *, Process *, + ErtsProcLocks *, Uint sz); +void erts_factory_selfcontained_message_init(ErtsHeapFactory*, ErtsMessage *, Eterm *); void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*); void erts_factory_dummy_init(ErtsHeapFactory*); @@ -91,6 +99,8 @@ void erts_factory_undo(ErtsHeapFactory*); #include "external.h" #include "erl_process.h" +#define ERTS_INVALID_HFRAG_PTR ((ErlHeapFragment *) ~((UWord) 7)) + /* * This struct represents a heap fragment, which is used when there * isn't sufficient room in the process heap and we can't do a GC. @@ -105,33 +115,46 @@ struct erl_heap_fragment { Eterm mem[1]; /* Data */ }; -typedef struct erl_mesg { - struct erl_mesg* next; /* Next message */ - union { - ErtsDistExternal *dist_ext; - ErlHeapFragment *heap_frag; - void *attached; - } data; -#ifdef USE_VM_PROBES - Eterm m[3]; /* m[0] = message, m[1] = seq trace token, m[3] = dynamic trace user tag */ -#else - Eterm m[2]; /* m[0] = message, m[1] = seq trace token */ -#endif -} ErlMessage; - +/* m[0] = message, m[1] = seq trace token */ +#define ERL_MESSAGE_REF_ARRAY_SZ 2 #define ERL_MESSAGE_TERM(mp) ((mp)->m[0]) #define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1]) + #ifdef USE_VM_PROBES +/* m[2] = dynamic trace user tag */ +#undef ERL_MESSAGE_REF_ARRAY_SZ +#define ERL_MESSAGE_REF_ARRAY_SZ 3 #define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2]) +#else #endif +#define ERL_MESSAGE_REF_FIELDS__ \ + ErtsMessage *next; /* Next message */ \ + union { \ + ErtsDistExternal *dist_ext; \ + ErlHeapFragment *heap_frag; \ + void *attached; \ + } data; \ + Eterm m[ERL_MESSAGE_REF_ARRAY_SZ] + + +typedef struct erl_msg_ref__ { + ERL_MESSAGE_REF_FIELDS__; +} ErtsMessageRef; + +struct erl_mesg { + ERL_MESSAGE_REF_FIELDS__; + + ErlHeapFragment hfrag; +}; + /* Size of default message buffer (erl_message.c) */ #define ERL_MESSAGE_BUF_SZ 500 typedef struct { - ErlMessage* first; - ErlMessage** last; /* point to the last next pointer */ - ErlMessage** save; + ErtsMessage* first; + ErtsMessage** last; /* point to the last next pointer */ + ErtsMessage** save; Sint len; /* queue length */ /* @@ -139,14 +162,14 @@ typedef struct { * recv_set/1 instructions. */ BeamInstr* mark; /* address to rec_loop/2 instruction */ - ErlMessage** saved_last; /* saved last pointer */ + ErtsMessage** saved_last; /* saved last pointer */ } ErlMessageQueue; #ifdef ERTS_SMP typedef struct { - ErlMessage* first; - ErlMessage** last; /* point to the last next pointer */ + ErtsMessage* first; + ErtsMessage** last; /* point to the last next pointer */ Sint len; /* queue length */ } ErlMessageInQueue; @@ -197,7 +220,7 @@ do { \ /* Unlink current message */ #define UNLINK_MESSAGE(p,msgp) do { \ - ErlMessage* __mp = (msgp)->next; \ + ErtsMessage* __mp = (msgp)->next; \ *(p)->msg.save = __mp; \ (p)->msg.len--; \ if (__mp == NULL) \ @@ -213,76 +236,33 @@ do { \ #define SAVE_MESSAGE(p) \ (p)->msg.save = &(*(p)->msg.save)->next -/* - * ErtsMoveMsgAttachmentIntoProc() moves data attached to a message - * onto the heap of a process. The attached data is the content of - * the the message either on the internal format or on the external - * format, and also possibly a seq trace token on the internal format. - * If the message content is on the external format, the decode might - * fail. If the decoding fails, ERL_MESSAGE_TERM(M) will contain - * THE_NON_VALUE. That is, ERL_MESSAGE_TERM(M) *has* to be checked - * afterwards and taken care of appropriately. - * - * ErtsMoveMsgAttachmentIntoProc() will shallow copy to heap if - * possible; otherwise, move to heap via garbage collection. - * - * ErtsMoveMsgAttachmentIntoProc() is used when receiveing messages - * in process_main() and in hipe_check_get_msg(). - */ - -#define ErtsMoveMsgAttachmentIntoProc(M, P, ST, HT, FC, SWPO, SWPI) \ -do { \ - if ((M)->data.attached) { \ - Uint need__ = erts_msg_attached_data_size((M)); \ - { SWPO ; } \ - if ((ST) - (HT) >= need__) { \ - ErtsHeapFactory factory__; \ - erts_factory_proc_prealloc_init(&factory__, (P), need__); \ - erts_move_msg_attached_data_to_heap(&factory__, (M)); \ - erts_factory_close(&factory__); \ - if ((P)->mbuf != NULL) { \ - /* Heap was exhausted by messages. This is a rare case */ \ - /* that can currently (OTP 18) only happen if hamts are */ \ - /* far exceeding the estimated heap size. Do GC. */ \ - (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ - } \ - } \ - else { \ - (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ - } \ - { SWPI ; } \ - ASSERT(!(M)->data.attached); \ - } \ -} while (0) - #define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) -#define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS) \ -do { \ - (HEAP_FRAG_P)->next = NULL; \ - (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \ - (HEAP_FRAG_P)->used_size = (DATA_WORDS); \ - (HEAP_FRAG_P)->off_heap.first = NULL; \ - (HEAP_FRAG_P)->off_heap.overhead = 0; \ -} while (0) +#define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, USED_WORDS, DATA_WORDS) \ + do { \ + (HEAP_FRAG_P)->next = NULL; \ + (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \ + (HEAP_FRAG_P)->used_size = (USED_WORDS); \ + (HEAP_FRAG_P)->off_heap.first = NULL; \ + (HEAP_FRAG_P)->off_heap.overhead = 0; \ + } while (0) void init_message(void); -void free_message(ErlMessage *); ErlHeapFragment* new_message_buffer(Uint); ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); #ifdef USE_VM_PROBES -void erts_queue_message_probe(Process*, ErtsProcLocks*, ErlHeapFragment*, +void erts_queue_message_probe(Process*, ErtsProcLocks*, ErtsMessage*, Eterm message, Eterm seq_trace_token, Eterm dt_utag); #define erts_queue_message(RP,RL,BP,Msg,SEQ) \ erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL) #else -void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, +void erts_queue_message(Process*, ErtsProcLocks*, ErtsMessage*, Eterm message, Eterm seq_trace_token); #define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \ erts_queue_message((RP),(RL),(BP),(Msg),(SEQ)) @@ -291,20 +271,141 @@ void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); -void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *); - -Uint erts_msg_attached_data_size_aux(ErlMessage *msg); -void erts_move_msg_attached_data_to_heap(ErtsHeapFactory*, ErlMessage *); -Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, - Eterm *, ErtsDistExternal *); +Uint erts_msg_attached_data_size_aux(ErtsMessage *msg); void erts_cleanup_offheap(ErlOffHeap *offheap); +void erts_save_message_in_proc(Process *p, ErtsMessage *msg); +Sint erts_move_messages_off_heap(Process *c_p); +Sint erts_complete_off_heap_message_queue_change(Process *c_p); +Eterm erts_change_off_heap_message_queue_state(Process *c_p, int enable); + +int erts_decode_dist_message(Process *, ErtsProcLocks, ErtsMessage *, int); + +void erts_cleanup_messages(ErtsMessage *mp); + +typedef struct { + Uint size; + ErtsMessage *msgp; +} ErtsMessageInfo; + +Uint erts_prep_msgq_for_inspection(Process *c_p, + Process *rp, + ErtsProcLocks rp_locks, + ErtsMessageInfo *mip); +void *erts_alloc_message_ref(void); +void erts_free_message_ref(void *); +#define ERTS_SMALL_FIX_MSG_SZ 10 +#define ERTS_MEDIUM_FIX_MSG_SZ 20 +#define ERTS_LARGE_FIX_MSG_SZ 30 + +void *erts_alloc_small_message(void); +void erts_free_small_message(void *mp); + +typedef struct { + ErtsMessage m; + Eterm data[ERTS_SMALL_FIX_MSG_SZ-1]; +} ErtsSmallFixSzMessage; + +typedef struct { + ErtsMessage m; + Eterm data[ERTS_MEDIUM_FIX_MSG_SZ-1]; +} ErtsMediumFixSzMessage; + +typedef struct { + ErtsMessage m; + Eterm data[ERTS_LARGE_FIX_MSG_SZ-1]; +} ErtsLargeFixSzMessage; + +ErtsMessage *erts_try_alloc_message_on_heap(Process *pp, + erts_aint32_t *psp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp, + int *on_heap_p); +ErtsMessage *erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, + Eterm *brefs, Uint brefs_size); + +ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp); +ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz, + Eterm *brefs, Uint brefs_size); +ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp); ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*); -ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); +ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg); + +#define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1) #if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp) +{ + ErtsMessage *mp; + + if (sz == 0) { + mp = erts_alloc_message_ref(); + mp->next = NULL; + ERL_MESSAGE_TERM(mp) = NIL; + mp->data.attached = NULL; + if (hpp) + *hpp = NULL; + return mp; + } + + mp = erts_alloc(ERTS_ALC_T_MSG, + sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm)); + + mp->next = NULL; + ERL_MESSAGE_TERM(mp) = NIL; + mp->data.attached = ERTS_MSG_COMBINED_HFRAG; + ERTS_INIT_HEAP_FRAG(&mp->hfrag, sz, sz); + + if (hpp) + *hpp = &mp->hfrag.mem[0]; + + return mp; +} + +ERTS_GLB_FORCE_INLINE ErtsMessage * +erts_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_size) +{ + if (sz == 0) { + ErtsMessage *nmp; + if (!mp->data.attached) + return mp; + ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG); + nmp = erts_alloc_message_ref(); +#ifdef DEBUG + if (brefs && brefs_size) { + int i; + for (i = 0; i < brefs_size; i++) + ASSERT(is_non_value(brefs[i]) || is_immed(brefs[i])); + } +#endif + erts_free(ERTS_ALC_T_MSG, mp); + return nmp; + } + + ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG); + ASSERT(mp->hfrag.used_size >= sz); + + if (sz >= (mp->hfrag.alloc_size - mp->hfrag.alloc_size / 16)) { + mp->hfrag.used_size = sz; + return mp; + } + + return erts_realloc_shrink_message(mp, sz, brefs, brefs_size); +} + +ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp) +{ + if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) + erts_free_message_ref(mp); + else + erts_free(ERTS_ALC_T_MSG, mp); +} + ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp) { Uint sz = 0; @@ -314,11 +415,17 @@ ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp) return sz; } -ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) +ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg) { ASSERT(msg->data.attached); - if (is_value(ERL_MESSAGE_TERM(msg))) - return erts_used_frag_sz(msg->data.heap_frag); + if (is_value(ERL_MESSAGE_TERM(msg))) { + ErlHeapFragment *bp; + if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG) + bp = &msg->hfrag; + else + bp = msg->data.heap_frag; + return erts_used_frag_sz(bp); + } else if (msg->data.dist_ext->heap_size < 0) return erts_msg_attached_data_size_aux(msg); else { -- cgit v1.2.3 From f3ae60838a5e8c83cfdd3000e25562dfcbbd438d Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Sat, 9 Jun 2012 01:12:50 +0300 Subject: Enable shcopy for sending messages --- erts/emulator/beam/erl_message.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 740ae46a0f..0de36c3199 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -238,6 +238,11 @@ do { \ #define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) +#define ERTS_SND_FLG_SHCOPY_SHIFT 1 +#define ERTS_SND_FLG_SHCOPY_MASK (ERTS_SHCOPY_FLG_MASK << ERTS_SND_FLG_SHCOPY_SHIFT) +#define ERTS_SND_FLG_SHCOPY_NONE (ERTS_SHCOPY_FLG_NONE << ERTS_SND_FLG_SHCOPY_SHIFT) +#define ERTS_SND_FLG_SHCOPY_TMP_BUF (ERTS_SHCOPY_FLG_TMP_BUF << ERTS_SND_FLG_SHCOPY_SHIFT) + #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) -- cgit v1.2.3 From 748c73f1687b2375d4c607487f40036ba990c4c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 17 Sep 2015 16:17:10 +0200 Subject: Refactor copy sharing --- erts/emulator/beam/erl_message.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 0de36c3199..52a648fe0b 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -239,9 +239,9 @@ do { \ #define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) #define ERTS_SND_FLG_SHCOPY_SHIFT 1 -#define ERTS_SND_FLG_SHCOPY_MASK (ERTS_SHCOPY_FLG_MASK << ERTS_SND_FLG_SHCOPY_SHIFT) -#define ERTS_SND_FLG_SHCOPY_NONE (ERTS_SHCOPY_FLG_NONE << ERTS_SND_FLG_SHCOPY_SHIFT) -#define ERTS_SND_FLG_SHCOPY_TMP_BUF (ERTS_SHCOPY_FLG_TMP_BUF << ERTS_SND_FLG_SHCOPY_SHIFT) +#define ERTS_SND_FLG_SHCOPY_MASK (ERTS_SHCOPY_FLG_MASK << ERTS_SND_FLG_SHCOPY_SHIFT) +#define ERTS_SND_FLG_SHCOPY_NONE (ERTS_SHCOPY_FLG_NONE << ERTS_SND_FLG_SHCOPY_SHIFT) +#define ERTS_SND_FLG_SHCOPY_TMPBUF (ERTS_SHCOPY_FLG_TMPBUF << ERTS_SND_FLG_SHCOPY_SHIFT) #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) -- cgit v1.2.3 From 5d764f988ab09326d24e39a172083b09ab364c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 18 Nov 2015 17:58:14 +0100 Subject: Refactor sharing preserved copy flags The TMPBUF option is no longer needed due to is_literal test and NONE was only used for initial debugging. So we remove the entire option. --- erts/emulator/beam/erl_message.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 52a648fe0b..740ae46a0f 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -238,11 +238,6 @@ do { \ #define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) -#define ERTS_SND_FLG_SHCOPY_SHIFT 1 -#define ERTS_SND_FLG_SHCOPY_MASK (ERTS_SHCOPY_FLG_MASK << ERTS_SND_FLG_SHCOPY_SHIFT) -#define ERTS_SND_FLG_SHCOPY_NONE (ERTS_SHCOPY_FLG_NONE << ERTS_SND_FLG_SHCOPY_SHIFT) -#define ERTS_SND_FLG_SHCOPY_TMPBUF (ERTS_SHCOPY_FLG_TMPBUF << ERTS_SND_FLG_SHCOPY_SHIFT) - #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) -- cgit v1.2.3 From 6ff15f23c68db356bbad6ab5f939c191b58d453d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 19 Nov 2015 19:36:08 +0100 Subject: Refactor have seq_trace token test --- erts/emulator/beam/erl_message.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 740ae46a0f..76387bc34c 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -128,6 +128,13 @@ struct erl_heap_fragment { #else #endif +#ifdef USE_VM_PROBES +#define have_no_seqtrace(T) ((T) == NIL || (T) == am_have_dt_utag) +#else +#define have_no_seqtrace(T) ((T) == NIL) +#endif +#define have_seqtrace(T) (!have_no_seqtrace(T)) + #define ERL_MESSAGE_REF_FIELDS__ \ ErtsMessage *next; /* Next message */ \ union { \ -- cgit v1.2.3 From ce8279d6a48d41f9de577825844f499bb3084b96 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 3 Dec 2015 15:34:14 +0100 Subject: erts: Fix bug for remote control message containing fat maps that could cause the static factory to overflow Fix: Introduce a new factory mode FACTORY_TMP --- erts/emulator/beam/erl_message.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index fbdf3fb0e2..92ba3e571c 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -58,7 +58,8 @@ typedef struct { FACTORY_CLOSED = 0, FACTORY_HALLOC, FACTORY_HEAP_FRAGS, - FACTORY_STATIC + FACTORY_STATIC, + FACTORY_TMP } mode; Process* p; Eterm* hp_start; @@ -75,6 +76,7 @@ void erts_factory_proc_init(ErtsHeapFactory*, Process*); void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size); void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*); void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*); +void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype); void erts_factory_dummy_init(ErtsHeapFactory*); Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); -- cgit v1.2.3 From 19c4689eea86f26c5af9b8f712c227ce4f62310b Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Nov 2015 15:57:55 +0100 Subject: Replace off_heap_message_queue option with message_queue_data option The message_queue_data option can have the values - off_heap - on_heap - mixed --- erts/emulator/beam/erl_message.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 740ae46a0f..e241926638 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -277,7 +277,7 @@ void erts_cleanup_offheap(ErlOffHeap *offheap); void erts_save_message_in_proc(Process *p, ErtsMessage *msg); Sint erts_move_messages_off_heap(Process *c_p); Sint erts_complete_off_heap_message_queue_change(Process *c_p); -Eterm erts_change_off_heap_message_queue_state(Process *c_p, int enable); +Eterm erts_change_message_queue_management(Process *c_p, Eterm new_state); int erts_decode_dist_message(Process *, ErtsProcLocks, ErtsMessage *, int); -- cgit v1.2.3 From 6664eed554974336909d3ffe03f20349cc4c38fd Mon Sep 17 00:00:00 2001 From: Henrik Nord Date: Tue, 15 Mar 2016 15:19:56 +0100 Subject: update copyright-year --- erts/emulator/beam/erl_message.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 60035d15ae..ad4e65274c 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2012. All Rights Reserved. + * Copyright Ericsson AB 1997-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. -- cgit v1.2.3 From 37092dab15448ef6a078800e3ff0cc41880ea765 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Thu, 10 Dec 2015 11:10:46 +0100 Subject: erts: Implement tracer modules Add the possibility to use modules as trace data receivers. The functions in the module have to be nifs as otherwise complex trace probes will be very hard to handle (complex means trace probes for ports for example). This commit changes the way that the ptab->tracer field works from always being an immediate, to now be NIL if no tracer is present or else be the tuple {TracerModule, TracerState} where TracerModule is an atom that is later used to lookup the appropriate tracer callbacks to call and TracerState is just passed to the tracer callback. The default process and port tracers have been rewritten to use the new API. This commit also changes the order which trace messages are delivered to the potential tracer process. Any enif_send done in a tracer module may be delayed indefinitely because of lock order issues. If a message is delayed any other trace message send from that process is also delayed so that order is preserved for each traced entity. This means that for some trace events (i.e. send/receive) the events may come in an unintuitive order (receive before send) to the trace receiver. Timestamps are taken when the trace message is generated so trace messages from differented processes may arrive with the timestamp out of order. Both the erlang:trace and seq_trace:set_system_tracer accept the new tracer module tracers and also the backwards compatible arguments. OTP-10267 --- erts/emulator/beam/erl_message.h | 110 +++++++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 45 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index ad4e65274c..608cf552a2 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -182,48 +182,64 @@ typedef struct { Sint len; /* queue length */ } ErlMessageInQueue; +typedef struct erl_trace_message_queue__ { + struct erl_trace_message_queue__ *next; /* point to the next receiver */ + Eterm receiver; + ErtsMessage* first; + ErtsMessage** last; /* point to the last next pointer */ + Sint len; /* queue length */ +} ErlTraceMessageQueue; + #endif /* Get "current" message */ #define PEEK_MESSAGE(p) (*(p)->msg.save) +#ifdef USE_VM_PROBES +#define LINK_MESSAGE_DTAG(mp, dt) ERL_MESSAGE_DT_UTAG(mp) = dt +#else +#define LINK_MESSAGE_DTAG(mp, dt) +#endif -/* Add message last in private message queue */ -#define LINK_MESSAGE_PRIVQ(p, mp) do { \ - *(p)->msg.last = (mp); \ - (p)->msg.last = &(mp)->next; \ - (p)->msg.len++; \ -} while (0) - +#define LINK_MESSAGE_IMPL(p, first_msg, last_msg, num_msgs, where) do { \ + *(p)->where.last = (first_msg); \ + (p)->where.last = (last_msg); \ + (p)->where.len += (num_msgs); \ + } while(0) #ifdef ERTS_SMP -/* Move in message queue to end of private message queue */ -#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(P) \ -do { \ - if ((P)->msg_inq.first) { \ - *(P)->msg.last = (P)->msg_inq.first; \ - (P)->msg.last = (P)->msg_inq.last; \ - (P)->msg.len += (P)->msg_inq.len; \ - (P)->msg_inq.first = NULL; \ - (P)->msg_inq.last = &(P)->msg_inq.first; \ - (P)->msg_inq.len = 0; \ - } \ -} while (0) - -/* Add message last in message queue */ -#define LINK_MESSAGE(p, mp) do { \ - *(p)->msg_inq.last = (mp); \ - (p)->msg_inq.last = &(mp)->next; \ - (p)->msg_inq.len++; \ -} while(0) +/* Add message last in private message queue */ +#define LINK_MESSAGE_PRIVQ(p, first_msg, last_msg, len) \ + do { \ + LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg); \ + } while (0) + +/* Add message last_msg in message queue */ +#define LINK_MESSAGE(p, first_msg, last_msg, len) \ + LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg_inq) + +#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p) \ + do { \ + if (p->msg_inq.first) { \ + *p->msg.last = p->msg_inq.first; \ + p->msg.last = p->msg_inq.last; \ + p->msg.len += p->msg_inq.len; \ + p->msg_inq.first = NULL; \ + p->msg_inq.last = &p->msg_inq.first; \ + p->msg_inq.len = 0; \ + } \ + } while (0) #else -#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(P) +#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p) -/* Add message last in message queue */ -#define LINK_MESSAGE(p, mp) LINK_MESSAGE_PRIVQ((p), (mp)) +/* Add message last_msg in message queue */ +#define LINK_MESSAGE(p, first_msg, last_msg, len) \ + do { \ + LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg); \ + } while(0) #endif @@ -259,23 +275,30 @@ do { \ (HEAP_FRAG_P)->off_heap.overhead = 0; \ } while (0) +#ifdef USE_VM_PROBES +#define ERL_MESSAGE_DT_UTAG_INIT(MP) ERL_MESSAGE_DT_UTAG(MP) = NIL +#else +#define ERL_MESSAGE_DT_UTAG_INIT(MP) do{ } while (0) +#endif + +#define ERTS_INIT_MESSAGE(MP) \ + do { \ + (MP)->next = NULL; \ + ERL_MESSAGE_TERM(MP) = THE_NON_VALUE; \ + ERL_MESSAGE_TOKEN(MP) = NIL; \ + ERL_MESSAGE_DT_UTAG_INIT(MP); \ + MP->data.attached = NULL; \ + } while (0) + void init_message(void); ErlHeapFragment* new_message_buffer(Uint); ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); -#ifdef USE_VM_PROBES -void erts_queue_message_probe(Process*, ErtsProcLocks*, ErtsMessage*, - Eterm message, Eterm seq_trace_token, Eterm dt_utag); -#define erts_queue_message(RP,RL,BP,Msg,SEQ) \ - erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL) -#else -void erts_queue_message(Process*, ErtsProcLocks*, ErtsMessage*, - Eterm message, Eterm seq_trace_token); -#define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \ - erts_queue_message((RP),(RL),(BP),(Msg),(SEQ)) -#endif +Sint erts_queue_message(Process*, ErtsProcLocks*,ErtsMessage*, Eterm); +Sint erts_queue_messages(Process*, ErtsProcLocks*, + ErtsMessage*, ErtsMessage**, Uint); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); @@ -354,9 +377,7 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp) if (sz == 0) { mp = erts_alloc_message_ref(); - mp->next = NULL; - ERL_MESSAGE_TERM(mp) = NIL; - mp->data.attached = NULL; + ERTS_INIT_MESSAGE(mp); if (hpp) *hpp = NULL; return mp; @@ -365,8 +386,7 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp) mp = erts_alloc(ERTS_ALC_T_MSG, sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm)); - mp->next = NULL; - ERL_MESSAGE_TERM(mp) = NIL; + ERTS_INIT_MESSAGE(mp); mp->data.attached = ERTS_MSG_COMBINED_HFRAG; ERTS_INIT_HEAP_FRAG(&mp->hfrag, sz, sz); -- cgit v1.2.3 From 9627711cc39fd311a573a172e6d0e8a6b55dbd17 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 1 Apr 2016 15:38:16 +0200 Subject: erts: Add Sender in 'receive' trace matchspec All 'EXIT' and monitor messages are sent from 'system' Timeouts are "sent" from 'clock_service' --- erts/emulator/beam/erl_message.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 608cf552a2..851ac37fda 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -295,10 +295,10 @@ ErlHeapFragment* new_message_buffer(Uint); ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); -void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); -Sint erts_queue_message(Process*, ErtsProcLocks*,ErtsMessage*, Eterm); -Sint erts_queue_messages(Process*, ErtsProcLocks*, - ErtsMessage*, ErtsMessage**, Uint); +void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *, Eterm, Eterm); +Sint erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm); +Sint erts_queue_messages(Process*, ErtsProcLocks, + ErtsMessage*, ErtsMessage**, Uint, Eterm); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); -- cgit v1.2.3 From d053fed5d3f546b2f1f19e16a60c6330f8526c95 Mon Sep 17 00:00:00 2001 From: Mikael Pettersson Date: Tue, 26 Apr 2016 09:13:45 +0200 Subject: do not limit heap fragments to 4 giga-words - struct erl_heap_fragment: use Uint for sizes - erl_nif.c: use size_t for sizes - erts_heap_frag_shrink(): remove now redundant cast --- erts/emulator/beam/erl_message.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 608cf552a2..60cf9a24e1 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -112,8 +112,8 @@ typedef struct erl_heap_fragment ErlHeapFragment; struct erl_heap_fragment { ErlHeapFragment* next; /* Next heap fragment */ ErlOffHeap off_heap; /* Offset heap data. */ - unsigned alloc_size; /* Size in (half)words of mem */ - unsigned used_size; /* With terms to be moved to heap by GC */ + Uint alloc_size; /* Size in (half)words of mem */ + Uint used_size; /* With terms to be moved to heap by GC */ Eterm mem[1]; /* Data */ }; -- cgit v1.2.3 From 703eea866831dea0753c5cd151d6d8d55debd470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 18 May 2016 19:45:42 +0200 Subject: erts: Refactor ERTS_MSG_COMBINED_HFRAG to heap fragment --- erts/emulator/beam/erl_message.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 851ac37fda..ecf65ca324 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -369,6 +369,10 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg); #define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1) +#define erts_message_to_heap_frag(MP) \ + (((MP)->data.attached == ERTS_MSG_COMBINED_HFRAG) ? \ + &(MP)->hfrag : (MP)->data.heap_frag) + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp) @@ -449,10 +453,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg) ASSERT(msg->data.attached); if (is_value(ERL_MESSAGE_TERM(msg))) { ErlHeapFragment *bp; - if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG) - bp = &msg->hfrag; - else - bp = msg->data.heap_frag; + bp = erts_message_to_heap_frag(msg); return erts_used_frag_sz(bp); } else if (msg->data.dist_ext->heap_size < 0) -- cgit v1.2.3 From 8241eb36a733d6fee7c4a6f12c8bdfc206889795 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 26 May 2016 16:17:45 +0200 Subject: Fix message queue update on replacement and removal of message --- erts/emulator/beam/erl_message.h | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'erts/emulator/beam/erl_message.h') diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 46063ea0c2..6df969367b 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -366,6 +366,12 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz, ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp); ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg); +ERTS_GLB_INLINE void erts_msgq_update_internal_pointers(ErlMessageQueue *msgq, + ErtsMessage **newpp, + ErtsMessage **oldpp); +ERTS_GLB_INLINE void erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, + ErtsMessage *newp, + ErtsMessage **oldpp); #define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1) @@ -468,6 +474,29 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg) return sz; } } + +ERTS_GLB_INLINE void +erts_msgq_update_internal_pointers(ErlMessageQueue *msgq, + ErtsMessage **newpp, + ErtsMessage **oldpp) +{ + if (msgq->save == oldpp) + msgq->save = newpp; + if (msgq->last == oldpp) + msgq->last = newpp; + if (msgq->saved_last == oldpp) + msgq->saved_last = newpp; +} + +ERTS_GLB_INLINE void +erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, ErtsMessage *newp, ErtsMessage **oldpp) +{ + ErtsMessage *oldp = *oldpp; + newp->next = oldp->next; + erts_msgq_update_internal_pointers(msgq, &newp->next, &oldp->next); + *oldpp = newp; +} + #endif #endif -- cgit v1.2.3