diff options
226 files changed, 12976 insertions, 8837 deletions
diff --git a/bootstrap/lib/stdlib/ebin/erl_parse.beam b/bootstrap/lib/stdlib/ebin/erl_parse.beam Binary files differindex adf1cfb43e..0522f5c05e 100644 --- a/bootstrap/lib/stdlib/ebin/erl_parse.beam +++ b/bootstrap/lib/stdlib/ebin/erl_parse.beam diff --git a/erts/doc/src/absform.xml b/erts/doc/src/absform.xml index 835a4fc692..e1a8c2e517 100644 --- a/erts/doc/src/absform.xml +++ b/erts/doc/src/absform.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2001</year><year>2013</year> + <year>2001</year><year>2015</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -80,6 +80,28 @@ <item>If F is a record declaration <c><![CDATA[-record(Name,{V_1, ..., V_k})]]></c>, then Rep(F) = <c><![CDATA[{attribute,LINE,record,{Name,[Rep(V_1), ..., Rep(V_k)]}}]]></c>. For Rep(V), see below.</item> + <item>If F is a type attribute (i.e. <c><![CDATA[opaque]]></c> or + <c><![CDATA[type]]></c>) + <c><![CDATA[-Attr Name(A_1, ..., A_k) :: T]]></c> where each + <c><![CDATA[A_i]]></c> is a variable, then Rep(F) = + <c><![CDATA[{attribute,LINE,Attr,{Name,Rep(T),[Rep(A_1), ..., Rep(A_k)]}}]]></c>. + For Rep(T), see below.</item> + <item>If F is a type spec (i.e. <c><![CDATA[callback]]></c> or + <c><![CDATA[spec]]></c>) + <c><![CDATA[-Attr F Tc_1; ...; Tc_k]]></c>, + where each <c><![CDATA[Tc_i]]></c> is a fun type clause with an + argument sequence of the same length <c><![CDATA[Arity]]></c>, then + Rep(F) = + <c><![CDATA[{Attr,LINE,{{F,Arity},[Rep(Tc_1), ..., Rep(Tc_k)]}}]]></c>. + For Rep(Tc_i), see below.</item> + <item>If F is a type spec (i.e. <c><![CDATA[callback]]></c> or + <c><![CDATA[spec]]></c>) + <c><![CDATA[-Attr Mod:F Tc_1; ...; Tc_k]]></c>, + where each <c><![CDATA[Tc_i]]></c> is a fun type clause with an + argument sequence of the same length <c><![CDATA[Arity]]></c>, then + Rep(F) = + <c><![CDATA[{Attr,LINE,{{Mod,F,Arity},[Rep(Tc_1), ..., Rep(Tc_k)]}}]]></c>. + For Rep(Tc_i), see below.</item> <item>If F is a wild attribute <c><![CDATA[-A(T)]]></c>, then Rep(F) = <c><![CDATA[{attribute,LINE,A,T}]]></c>. <br></br></item> @@ -90,6 +112,127 @@ </list> <section> + <title>Type clauses</title> + <list type="bulleted"> + <item>If T is a fun type clause + <c><![CDATA[(A_1, ..., A_n) -> Ret]]></c>, where each + <c><![CDATA[A_i]]></c> and <c><![CDATA[Ret]]></c> are types, then + Rep(T) = + <c><![CDATA[{type,LINE,'fun',[{type,LINE,product,[Rep(A_1), ..., Rep(A_n)]},Rep(Ret)]}]]></c>. + </item> + <item>If T is a bounded fun type clause <c><![CDATA[Tc when Tg]]></c>, + where <c><![CDATA[Tc]]></c> is an unbounded fun type clause and + <c><![CDATA[Tg]]></c> is a type guard sequence, then Rep(T) = + <c><![CDATA[{type,LINE,bounded_fun,[Rep(Tc),Rep(Tg)]}]]></c>.</item> + </list> + </section> + + <section> + <title>Type guards</title> + <list type="bulleted"> + <item>If G is a constraint <c><![CDATA[F(A_1, ..., A_k)]]></c>, where + <c><![CDATA[F]]></c> is an atom and each <c><![CDATA[A_i]]></c> is a + type, then Rep(G) = + <c><![CDATA[{type,LINE,constraint,[Rep(F),[Rep(A_1), ..., Rep(A_k)]]}]]></c>. + </item> + <item>If G is a type definition <c><![CDATA[Name :: Type]]></c>, + where <c><![CDATA[Name]]></c> is a variable and + <c><![CDATA[Type]]></c> is a type, then Rep(G) = + <c><![CDATA[{type,LINE,constraint,[{atom,LINE,is_subtype},[Rep(Name),Rep(Type)]]}]]></c>.</item> + </list> + </section> + + <section> + <title>Types</title> + <list type="bulleted"> + <item>If T is a type definition <c><![CDATA[Name :: Type]]></c>, + where <c><![CDATA[Name]]></c> is a variable and + <c><![CDATA[Type]]></c> is a type, then Rep(T) = + <c><![CDATA[{ann_type,LINE,[Rep(Name),Rep(Type)]}]]></c>.</item> + <item>If T is a type union <c><![CDATA[A_1 | ... | A_k]]></c>, + where each <c><![CDATA[A_i]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,union,[Rep(A_1), ..., Rep(A_k)]}]]></c>.</item> + <item>If T is a type range <c><![CDATA[L .. R]]></c>, + where <c><![CDATA[L]]></c> and <c><![CDATA[R]]></c> are types, then + Rep(T) = <c><![CDATA[{type,LINE,range,[Rep(L), Rep(R)]}]]></c>.</item> + <item>If T is a binary operation <c><![CDATA[L Op R]]></c>, + where <c><![CDATA[Op]]></c> is an arithmetic or bitwise binary operator + and <c><![CDATA[L]]></c> and <c><![CDATA[R]]></c> are types, then + Rep(T) = <c><![CDATA[{op,LINE,Op,Rep(L),Rep(R)}]]></c>.</item> + <item>If T is <c><![CDATA[Op A]]></c>, where <c><![CDATA[Op]]></c> is an + arithmetic or bitwise unary operator and <c><![CDATA[A]]></c> is a + type, then Rep(T) = <c><![CDATA[{op,LINE,Op,Rep(A)}]]></c>.</item> + <item>If T is a fun type <c><![CDATA[fun()]]></c>, then Rep(T) = + <c><![CDATA[{type,LINE,'fun',[]}]]></c>.</item> + <item>If T is a variable <c><![CDATA[V]]></c>, then Rep(T) = + <c><![CDATA[{var,LINE,A}]]></c>, where <c><![CDATA[A]]></c> is an atom + with a printname consisting of the same characters as + <c><![CDATA[V]]></c>.</item> + <item>If T is an atomic literal L and L is not a string literal, then + Rep(T) = Rep(L).</item> + <item>If T is a tuple or map type <c><![CDATA[F()]]></c> (i.e. + <c><![CDATA[tuple]]></c> or <c><![CDATA[map]]></c>), then Rep(T) = + <c><![CDATA[{type,LINE,F,any}]]></c>.</item> + <item>If T is a type <c><![CDATA[F(A_1, ..., A_k)]]></c>, where each + <c><![CDATA[A_i]]></c> is a type, then Rep(T) = + <c><![CDATA[{user_type,LINE,F,[Rep(A_1), ..., Rep(A_k)]}]]></c>.</item> + <item>If T is a remote type <c><![CDATA[M:F(A_1, ..., A_k)]]></c>, where + each <c><![CDATA[A_i]]></c> is a type and <c><![CDATA[M]]></c> and + <c><![CDATA[F]]></c>, then Rep(T) = + <c><![CDATA[{remote_type,LINE,[Rep(M),Rep(F),[Rep(A_1), ..., Rep(A_k)]]}]]></c>. + </item> + <item>If T is the nil type <c><![CDATA[[]]]></c>, then Rep(T) = + <c><![CDATA[{type,LINE,nil,[]}]]></c>.</item> + <item>If T is a list type <c><![CDATA[[A]]]></c>, where + <c><![CDATA[A]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,list,[Rep(A)]}]]></c>.</item> + <item>If T is a non-empty list type <c><![CDATA[[A, ...]]]></c>, where + <c><![CDATA[A]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,nonempty_list,[Rep(A)]}]]></c>.</item> + <item>If T is a map type <c><![CDATA[#{P_1, ..., P_k}]]></c>, where each + <c><![CDATA[P_i]]></c> is a map pair type, then Rep(T) = + <c><![CDATA[{type,LINE,map,[Rep(P_1), ..., Rep(P_k)]}]]></c>.</item> + <item>If T is a map pair type <c><![CDATA[K => V]]></c>, where + <c><![CDATA[K]]></c> and <c><![CDATA[V]]></c> are types, + then Rep(T) = + <c><![CDATA[{type,LINE,map_field_assoc,[Rep(K),Rep(V)]}]]></c>.</item> + <item>If T is a tuple type <c><![CDATA[{A_1, ..., A_k}]]></c>, where + each <c><![CDATA[A_i]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,tuple,[Rep(A_1), ..., Rep(A_k)]}]]></c>.</item> + <item>If T is a record type <c><![CDATA[#Name{}]]></c>, where + <c><![CDATA[Name]]></c> is an atom, then Rep(T) = + <c><![CDATA[{type,LINE,record,[Rep(Name)]}]]></c>.</item> + <item>If T is a record type <c><![CDATA[#Name{F_1, ..., F_k}]]></c>, + where <c><![CDATA[Name]]></c> is an atom, then Rep(T) = + <c><![CDATA[{type,LINE,record,[Rep(Name),[Rep(F_1), ..., Rep(F_k)]]}]]></c>. + </item> + <item>If T is a record field type <c><![CDATA[Name :: Type]]></c>, + where <c><![CDATA[Name]]></c> is an atom, then Rep(T) = + <c><![CDATA[{type,LINE,field_type,[Rep(Name),Rep(Type)]}]]></c>.</item> + <item>If T is a record field type <c><![CDATA[<<>>]]></c>, then Rep(T) = + <c><![CDATA[{type,LINE,binary,[{integer,LINE,0},{integer,LINE,0}]}]]></c>. + </item> + <item>If T is a binary type <c><![CDATA[<< _ : B >>]]></c>, where + <c><![CDATA[B]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,binary,[Rep(B),{integer,LINE,0}]}]]></c>.</item> + <item>If T is a binary type <c><![CDATA[<< _ : _ * U >>]]></c>, + where <c><![CDATA[U]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,binary,[{integer,LINE,0},Rep(U)]}]]></c>.</item> + <item>If T is a binary type <c><![CDATA[<< _ : B , _ : _ * U >>]]></c>, + where <c><![CDATA[B]]></c> and <c><![CDATA[U]]></c> is a type, then + Rep(T) = + <c><![CDATA[{type,LINE,binary,[Rep(B),Rep(U)]}]]></c>.</item> + + <item>If T is a fun type <c><![CDATA[fun((...) -> Ret)]]></c>, then + Rep(T) = <c><![CDATA[{type,LINE,'fun',[{type,LINE,product,[]},Rep(Ret)]}]]></c>. + </item> + <item>If T is a fun type <c><![CDATA[fun(Tc)]]></c>, where + <c><![CDATA[Tc]]></c> is an unbounded fun type clause, + then Rep(T) = <c><![CDATA[Rep(Tc)]]></c>.</item> + </list> + </section> + + <section> <title>Record fields</title> <p>Each field in a record declaration may have an optional explicit default initializer expression</p> @@ -98,6 +241,21 @@ Rep(V) = <c><![CDATA[{record_field,LINE,Rep(A)}]]></c>.</item> <item>If V is <c><![CDATA[A = E]]></c>, then Rep(V) = <c><![CDATA[{record_field,LINE,Rep(A),Rep(E)}]]></c>.</item> + <item>If V is <c><![CDATA[A :: T]]></c>, where <c><![CDATA[A]]></c> is + an atom and <c><![CDATA[T]]></c> is a type and it does not contain + <c><![CDATA[undefined]]></c> syntactically, then Rep(V) = + <c><![CDATA[{typed_record_field,{record_field,LINE,Rep(A)},Rep(undefined | T)}]]></c>. + Note that if <![CDATA[T]]> is an annotated type, it will be wrapped in + parentheses.</item> + <item>If V is <c><![CDATA[A :: T]]></c>, where <c><![CDATA[A]]></c> is + an atom and <c><![CDATA[T]]></c> is a type, then Rep(V) = + <c><![CDATA[{typed_record_field,{record_field,LINE,Rep(A)},Rep(T)}]]></c>. + </item> + <item>If V is <c><![CDATA[A = E :: T]]></c>, where <c><![CDATA[A]]></c> + is an atom, <c><![CDATA[E]]></c> is an expression and + <c><![CDATA[T]]></c> is a type, then Rep(V) = + <c><![CDATA[{typed_record_field,{record_field,LINE,Rep(A),Rep(E)},Rep(T)}]]></c>. + </item> </list> </section> diff --git a/erts/doc/src/crash_dump.xml b/erts/doc/src/crash_dump.xml index 8291bf38b7..e13d468ee6 100644 --- a/erts/doc/src/crash_dump.xml +++ b/erts/doc/src/crash_dump.xml @@ -88,22 +88,20 @@ operating system.</p> <list type="bulleted"> <item>"<em><A></em>: Cannot allocate <em><N></em> - bytes of memory (of type "<em><T></em>", thread - <em><I></em>em>)." - The system has run out of memory. <A> - is the allocator that failed to allocate memory, <N> is the - number of bytes that <A> tried to allocate, <T> is the - memory block type that the memory was needed for, and <I> is the - thread identifier. The most common case is that a process stores huge - amounts of data. In this case <T> is most often - <c><![CDATA[heap]]></c>, <c><![CDATA[old_heap]]></c>, - <c><![CDATA[heap_frag]]></c>, or <c><![CDATA[binary]]></c>. - For more information on allocators see - <seealso marker="erts_alloc">erts_alloc(3)</seealso>.</item> + bytes of memory (of type "<em><T></em>")." - The system + has run out of memory. <A> is the allocator that failed + to allocate memory, <N> is the number of bytes that + <A> tried to allocate, and <T> is the memory block + type that the memory was needed for. The most common case is + that a process stores huge amounts of data. In this case + <T> is most often <c><![CDATA[heap]]></c>, <c><![CDATA[old_heap]]></c>, + <c><![CDATA[heap_frag]]></c>, or <c><![CDATA[binary]]></c>. For more information on + allocators see + <seealso marker="erts_alloc">erts_alloc(3)</seealso>.</item> <item>"<em><A></em>: Cannot reallocate <em><N></em> - bytes of memory (of type "<em><T></em>", thread - <em><I></em>em>)." - Same as above with the exception that memory - was being reallocated instead of being allocated when the system ran - out of memory.</item> + bytes of memory (of type "<em><T></em>")." - Same as + above with the exception that memory was being reallocated + instead of being allocated when the system ran out of memory.</item> <item>"Unexpected op code <em>N</em>" - Error in compiled code, <c><![CDATA[beam]]></c> file damaged or error in the compiler.</item> <item>"Module <em>Name</em> undefined" <c><![CDATA[|]]></c> "Function @@ -306,9 +304,6 @@ <tag><em>Last scheduled in for | Current call</em></tag> <item>The current function of the process. These fields will not always exist.</item> - <tag><em>Run queue</em></tag> - <item>The identifier of the scheduler run queue in which the process is - running.</item> <tag><em>Spawned by</em></tag> <item>The parent of the process, i.e. the process which executed <c><![CDATA[spawn]]></c> or <c><![CDATA[spawn_link]]></c>.</item> diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 98d05dc7de..f41b6e6149 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -1351,6 +1351,18 @@ give lower latency and higher throughput at the expense of higher memory usage.</p> </item> + <tag><marker id="+zdntgc"><c>+zdntgc time</c></marker></tag> + <item> + <p>Set the delayed node table garbage collection time + (<seealso marker="erlang#system_info_delayed_node_table_gc">delayed_node_table_gc</seealso>) + in seconds. Valid values are either <c>infinity</c> or + an integer in the range [0-100000000]. Default is 60.</p> + <p>Node table entries that are not referred will linger + in the table for at least the amount of time that this + parameter determines. The lingering prevents repeated + deletions and insertions in the tables from occurring. + </p> + </item> </taglist> </item> </taglist> diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index f64381c99d..412c0e02ac 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -1177,7 +1177,7 @@ typedef enum { <code type="none"> ERL_NIF_TERM key, value; ErlNifMapIterator iter; -enif_map_iterator_create(env, my_map, ERL_NIF_MAP_ITERATOR_FIRST); +enif_map_iterator_create(env, my_map, &iter, ERL_NIF_MAP_ITERATOR_FIRST); while (enif_map_iterator_get_pair(env, &iter, &key, &value)) { do_something(key,value); diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 3fea64cef5..50a26781c4 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -6222,6 +6222,7 @@ ok <name name="system_info" arity="1" clause_i="64"/> <name name="system_info" arity="1" clause_i="65"/> <name name="system_info" arity="1" clause_i="66"/> + <name name="system_info" arity="1" clause_i="67"/> <fsummary>Information about the system</fsummary> <desc> <p>Returns various information about the current system @@ -6291,6 +6292,15 @@ ok compiled; otherwise, <c>false</c>. </p> </item> + <tag><marker id="system_info_delayed_node_table_gc"><c>delayed_node_table_gc</c></marker></tag> + <item> + <p>Returns the amount of time in seconds that garbage collection + of an entry in a node table will be delayed. This limit can be set + on startup by passing the + <seealso marker="erts:erl#+zdntgc">+zdntgc</seealso> command line + flag to <c>erl</c>. For more information see the documentation of the + command line flag.</p> + </item> <tag><marker id="system_info_dirty_cpu_schedulers"><c>dirty_cpu_schedulers</c></marker></tag> <item> <p>Returns the number of dirty CPU scheduler threads used by diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 5ec1409adf..0cf21b4635 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -265,6 +265,7 @@ atom get_seq_token atom get_tcw atom getenv atom gather_gc_info_result +atom gather_io_bytes atom gather_sched_wall_time_result atom getting_linked atom getting_unlinked @@ -350,6 +351,7 @@ atom message atom message_binary atom message_queue_len atom messages +atom merge_trap atom meta atom meta_match_spec atom micro_seconds diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 500a98195b..c769428266 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -33,6 +33,7 @@ #include "beam_catches.h" #include "erl_binary.h" #include "erl_nif.h" +#include "erl_bits.h" #include "erl_thr_progress.h" static void set_default_trace_pattern(Eterm module); @@ -937,7 +938,15 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) break; case TAG_PRIMARY_HEADER: if (!header_is_transparent(val)) { - Eterm* new_p = p + thing_arityval(val); + Eterm* new_p; + if (header_is_bin_matchstate(val)) { + ErlBinMatchState *ms = (ErlBinMatchState*) p; + ErlBinMatchBuffer *mb = &(ms->mb); + if (in_area(EXPAND_POINTER(mb->orig), mod_start, mod_size)) { + return 1; + } + } + new_p = p + thing_arityval(val); ASSERT(start <= new_p && new_p < end); p = new_p; } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index a21622f424..2d4bf4240c 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6653,8 +6653,9 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) p->htop = mhp; - factory.p = p; + erts_factory_proc_init(&factory, p); res = erts_hashmap_from_array(&factory, thp, n/2, 0); + erts_factory_close(&factory); if (p->mbuf) { Uint live = Arg(2); reg[live] = res; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0d40201934..006ba5d0db 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -205,10 +205,7 @@ typedef struct { typedef struct { Eterm term; /* The tagged term (in the heap). */ - Uint heap_size; /* (Exact) size on the heap. */ - SWord offset; /* Offset from temporary location to final. */ - ErlOffHeap off_heap; /* Start of linked list of ProcBins. */ - Eterm* heap; /* Heap for term. */ + ErlHeapFragment* heap_frags; } Literal; /* @@ -477,6 +474,8 @@ typedef struct LoaderState { static void free_loader_state(Binary* magic); +static ErlHeapFragment* new_literal_fragment(Uint size); +static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, @@ -525,13 +524,16 @@ static void new_literal_patch(LoaderState* stp, int pos); static void new_string_patch(LoaderState* stp, int pos); static Uint new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size); static int genopargcompare(GenOpArg* a, GenOpArg* b); -static Eterm exported_from_module(Process* p, Eterm mod); -static Eterm functions_in_module(Process* p, Eterm mod); -static Eterm attributes_for_module(Process* p, Eterm mod); -static Eterm compilation_info_for_module(Process* p, Eterm mod); -static Eterm md5_of_module(Process* p, Eterm mod); -static Eterm has_native(Process* p, Eterm mod); -static Eterm native_addresses(Process* p, Eterm mod); +static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix, + BeamInstr* code, Eterm module, Eterm what); +static Eterm exported_from_module(Process* p, ErtsCodeIndex code_ix, + Eterm mod); +static Eterm functions_in_module(Process* p, BeamInstr* code); +static Eterm attributes_for_module(Process* p, BeamInstr* code); +static Eterm compilation_info_for_module(Process* p, BeamInstr* code); +static Eterm md5_of_module(Process* p, BeamInstr* code); +static Eterm has_native(BeamInstr* code); +static Eterm native_addresses(Process* p, BeamInstr* code); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); @@ -883,6 +885,28 @@ free_loader_state(Binary* magic) } } +static ErlHeapFragment* new_literal_fragment(Uint size) +{ + ErlHeapFragment* bp; + bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_PREPARED_CODE, + ERTS_HEAP_FRAG_SIZE(size)); + ERTS_INIT_HEAP_FRAG(bp, size); + return bp; +} + +static void free_literal_fragment(ErlHeapFragment* bp) +{ + ASSERT(bp != NULL); + do { + ErlHeapFragment* next_bp = bp->next; + + erts_cleanup_offheap(&bp->off_heap); + ERTS_HEAP_FREE(ERTS_ALC_T_PREPARED_CODE, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->size)); + bp = next_bp; + }while (bp != NULL); +} + /* * This destructor function can safely be called multiple times. */ @@ -922,10 +946,9 @@ loader_state_dtor(Binary* magic) if (stp->literals != 0) { int i; for (i = 0; i < stp->num_literals; i++) { - if (stp->literals[i].heap != 0) { - erts_free(ERTS_ALC_T_PREPARED_CODE, - (void *) stp->literals[i].heap); - stp->literals[i].heap = 0; + if (stp->literals[i].heap_frags != 0) { + free_literal_fragment(stp->literals[i].heap_frags); + stp->literals[i].heap_frags = 0; } } erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals); @@ -1450,6 +1473,7 @@ read_lambda_table(LoaderState* stp) return 0; } + static int read_literal_table(LoaderState* stp) { @@ -1471,7 +1495,7 @@ read_literal_table(LoaderState* stp) stp->allocated_literals = stp->num_literals; for (i = 0; i < stp->num_literals; i++) { - stp->literals[i].heap = 0; + stp->literals[i].heap_frags = 0; } for (i = 0; i < stp->num_literals; i++) { @@ -1479,28 +1503,38 @@ read_literal_table(LoaderState* stp) Sint heap_size; byte* p; Eterm val; - Eterm* hp; + ErtsHeapFactory factory; GetInt(stp, 4, sz); /* Size of external term format. */ GetString(stp, p, sz); if ((heap_size = erts_decode_ext_size(p, sz)) < 0) { LoadError1(stp, "literal %d: bad external format", i); } - hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, - heap_size*sizeof(Eterm)); - stp->literals[i].off_heap.first = 0; - stp->literals[i].off_heap.overhead = 0; - val = erts_decode_ext(&hp, &stp->literals[i].off_heap, &p); - stp->literals[i].heap_size = hp - stp->literals[i].heap; - if (stp->literals[i].heap_size > heap_size) { - erl_exit(1, "overrun by %d word(s) for literal heap, term %d", - stp->literals[i].heap_size - heap_size, i); - } - if (is_non_value(val)) { - LoadError1(stp, "literal %d: bad external format", i); - } - stp->literals[i].term = val; - stp->total_literal_size += stp->literals[i].heap_size; + + if (heap_size > 0) { + erts_factory_message_init(&factory, NULL, NULL, + new_literal_fragment(heap_size)); + factory.alloc_type = ERTS_ALC_T_PREPARED_CODE; + val = erts_decode_ext(&factory, &p); + + if (is_non_value(val)) { + LoadError1(stp, "literal %d: bad external format", i); + } + erts_factory_close(&factory); + stp->literals[i].heap_frags = factory.heap_frags; + stp->total_literal_size += erts_used_frag_sz(factory.heap_frags); + } + else { + erts_factory_dummy_init(&factory); + val = erts_decode_ext(&factory, &p); + if (is_non_value(val)) { + LoadError1(stp, "literal %d: bad external format", i); + } + ASSERT(is_immed(val)); + stp->literals[i].heap_frags = NULL; + } + stp->literals[i].term = val; + } erts_free(ERTS_ALC_T_TMP, uncompressed); return 1; @@ -4370,8 +4404,9 @@ freeze_code(LoaderState* stp) Uint* low; Uint* high; LiteralPatch* lp; - struct erl_off_heap_header* off_heap = 0; - struct erl_off_heap_header** off_heap_last = &off_heap; + ErlOffHeap code_off_heap; + + ERTS_INIT_OFF_HEAP(&code_off_heap); low = (Uint *) (code+stp->ci); high = low + stp->total_literal_size; @@ -4379,73 +4414,21 @@ freeze_code(LoaderState* stp) code[MI_LITERALS_END] = (BeamInstr) high; ptr = low; for (i = 0; i < stp->num_literals; i++) { - SWord offset; - struct erl_off_heap_header* t_off_heap; - - sys_memcpy(ptr, stp->literals[i].heap, - stp->literals[i].heap_size*sizeof(Eterm)); - offset = ptr - stp->literals[i].heap; - stp->literals[i].offset = offset; - high = ptr + stp->literals[i].heap_size; - while (ptr < high) { - Eterm val = *ptr; - switch (primary_tag(val)) { - case TAG_PRIMARY_LIST: - case TAG_PRIMARY_BOXED: - *ptr++ = offset_ptr(val, offset); - break; - case TAG_PRIMARY_HEADER: - if (header_is_transparent(val)) { - ptr++; - } else { - if (thing_subtag(val) == REFC_BINARY_SUBTAG) { - struct erl_off_heap_header* oh; - - oh = (struct erl_off_heap_header*) ptr; - if (oh->next) { - Eterm** uptr = (Eterm **) (void *) &oh->next; - *uptr += offset; - } - } - ptr += 1 + thing_arityval(val); - } - break; - default: - ptr++; - break; - } - } - ASSERT(ptr == high); - - /* - * Re-link the off_heap list for this term onto the - * off_heap list for the entire module. - */ - t_off_heap = stp->literals[i].off_heap.first; - if (t_off_heap) { - t_off_heap = (struct erl_off_heap_header *) - offset_ptr((UWord) t_off_heap, offset); - while (t_off_heap) { - *off_heap_last = t_off_heap; - off_heap_last = &t_off_heap->next; - t_off_heap = t_off_heap->next; - } - } + if (stp->literals[i].heap_frags) { + move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags, + &stp->literals[i].term, 1); + } + else ASSERT(is_immed(stp->literals[i].term)); } - code[MI_LITERALS_OFF_HEAP] = (BeamInstr) off_heap; + code[MI_LITERALS_OFF_HEAP] = (BeamInstr) code_off_heap.first; lp = stp->literal_patches; while (lp != 0) { BeamInstr* op_ptr; - Uint literal; Literal* lit; op_ptr = code + lp->pos; lit = &stp->literals[op_ptr[0]]; - literal = lit->term; - if (is_boxed(literal) || is_list(literal)) { - literal = offset_ptr(literal, lit->offset); - } - op_ptr[0] = literal; + op_ptr[0] = lit->term; lp = lp->next; } literal_end += stp->total_literal_size; @@ -5376,19 +5359,18 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size) stp->total_literal_size += heap_size; lit = stp->literals + stp->num_literals; - lit->offset = 0; - lit->heap_size = heap_size; - lit->heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm)); - lit->term = make_boxed(lit->heap); - lit->off_heap.first = 0; - lit->off_heap.overhead = 0; - *hpp = lit->heap; + lit->heap_frags = new_literal_fragment(heap_size); + lit->term = make_boxed(lit->heap_frags->mem); + *hpp = lit->heap_frags->mem; return stp->num_literals++; } Eterm erts_module_info_0(Process* p, Eterm module) { + Module* modp; + ErtsCodeIndex code_ix = erts_active_code_ix(); + BeamInstr* code; Eterm *hp; Eterm list = NIL; Eterm tup; @@ -5397,12 +5379,18 @@ erts_module_info_0(Process* p, Eterm module) return THE_NON_VALUE; } - if (erts_get_module(module, erts_active_code_ix()) == NULL) { + modp = erts_get_module(module, code_ix); + if (modp == NULL) { return THE_NON_VALUE; } + code = modp->curr.code; + if (code == NULL) { + return THE_NON_VALUE; + } + #define BUILD_INFO(What) \ - tup = erts_module_info_1(p, module, What); \ + tup = get_module_info(p, code_ix, code, module, What); \ hp = HAlloc(p, 5); \ tup = TUPLE2(hp, What, tup); \ hp += 3; \ @@ -5423,22 +5411,47 @@ erts_module_info_0(Process* p, Eterm module) Eterm erts_module_info_1(Process* p, Eterm module, Eterm what) { + Module* modp; + ErtsCodeIndex code_ix = erts_active_code_ix(); + BeamInstr* code; + + if (is_not_atom(module)) { + return THE_NON_VALUE; + } + + modp = erts_get_module(module, code_ix); + if (modp == NULL) { + return THE_NON_VALUE; + } + + code = modp->curr.code; + if (code == NULL) { + return THE_NON_VALUE; + } + + return get_module_info(p, code_ix, code, module, what); +} + +static Eterm +get_module_info(Process* p, ErtsCodeIndex code_ix, BeamInstr* code, + Eterm module, Eterm what) +{ if (what == am_module) { return module; } else if (what == am_md5) { - return md5_of_module(p, module); + return md5_of_module(p, code); } else if (what == am_exports) { - return exported_from_module(p, module); + return exported_from_module(p, code_ix, module); } else if (what == am_functions) { - return functions_in_module(p, module); + return functions_in_module(p, code); } else if (what == am_attributes) { - return attributes_for_module(p, module); + return attributes_for_module(p, code); } else if (what == am_compile) { - return compilation_info_for_module(p, module); + return compilation_info_for_module(p, code); } else if (what == am_native_addresses) { - return native_addresses(p, module); + return native_addresses(p, code); } else if (what == am_native) { - return has_native(p, module); + return has_native(code); } return THE_NON_VALUE; } @@ -5446,16 +5459,12 @@ erts_module_info_1(Process* p, Eterm module, Eterm what) /* * Builds a list of all functions in the given module: * [{Name, Arity},...] - * - * Returns a tagged term, or 0 on error. */ Eterm functions_in_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ + BeamInstr* code) { - Module* modp; - BeamInstr* code; int i; Uint num_functions; Uint need; @@ -5463,15 +5472,6 @@ functions_in_module(Process* p, /* Process whose heap to use. */ Eterm* hp_end; Eterm result = NIL; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; num_functions = code[MI_NUM_FUNCTIONS]; need = 5*num_functions; hp = HAlloc(p, need); @@ -5503,23 +5503,12 @@ functions_in_module(Process* p, /* Process whose heap to use. */ */ static Eterm -has_native(Process* p, Eterm mod) +has_native(BeamInstr *code) { Eterm result = am_false; #ifdef HIPE - Module* modp; - - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - - if (erts_is_module_native(modp->curr.code)) { - result = am_true; + if (erts_is_module_native(code)) { + result = am_true; } #endif return result; @@ -5548,15 +5537,11 @@ erts_is_module_native(BeamInstr* code) /* * Builds a list of all functions including native addresses. * [{Name,Arity,NativeAddress},...] - * - * Returns a tagged term, or 0 on error. */ static Eterm -native_addresses(Process* p, Eterm mod) +native_addresses(Process* p, BeamInstr* code) { - Module* modp; - BeamInstr* code; int i; Eterm* hp; Uint num_functions; @@ -5564,16 +5549,6 @@ native_addresses(Process* p, Eterm mod) Eterm* hp_end; Eterm result = NIL; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - - code = modp->curr.code; num_functions = code[MI_NUM_FUNCTIONS]; need = (6+BIG_UINT_HEAP_SIZE)*num_functions; hp = HAlloc(p, need); @@ -5600,25 +5575,18 @@ native_addresses(Process* p, Eterm mod) /* * Builds a list of all exported functions in the given module: * [{Name, Arity},...] - * - * Returns a tagged term, or 0 on error. */ Eterm exported_from_module(Process* p, /* Process whose heap to use. */ + ErtsCodeIndex code_ix, Eterm mod) /* Tagged atom for module. */ { int i; Eterm* hp = NULL; Eterm* hend = NULL; Eterm result = NIL; - ErtsCodeIndex code_ix; - - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - code_ix = erts_active_code_ix(); for (i = 0; i < export_list_size(code_ix); i++) { Export* ep = export_list(i,code_ix); @@ -5648,112 +5616,59 @@ exported_from_module(Process* p, /* Process whose heap to use. */ /* * Returns a list of all attributes for the module. - * - * Returns a tagged term, or 0 on error. */ Eterm attributes_for_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ - + BeamInstr* code) { - Module* modp; - BeamInstr* code; - Eterm* hp; byte* ext; Eterm result = NIL; - Eterm* end; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; ext = (byte *) code[MI_ATTR_PTR]; if (ext != NULL) { - hp = HAlloc(p, code[MI_ATTR_SIZE_ON_HEAP]); - end = hp + code[MI_ATTR_SIZE_ON_HEAP]; - result = erts_decode_ext(&hp, &MSO(p), &ext); + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, code[MI_ATTR_SIZE_ON_HEAP]); + result = erts_decode_ext(&factory, &ext); if (is_value(result)) { - ASSERT(hp <= end); + erts_factory_close(&factory); } - HRelease(p,end,hp); } return result; } /* * Returns a list containing compilation information. - * - * Returns a tagged term, or 0 on error. */ Eterm compilation_info_for_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ + BeamInstr* code) { - Module* modp; - BeamInstr* code; - Eterm* hp; byte* ext; Eterm result = NIL; - Eterm* end; - - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; ext = (byte *) code[MI_COMPILE_PTR]; if (ext != NULL) { - hp = HAlloc(p, code[MI_COMPILE_SIZE_ON_HEAP]); - end = hp + code[MI_COMPILE_SIZE_ON_HEAP]; - result = erts_decode_ext(&hp, &MSO(p), &ext); + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, code[MI_COMPILE_SIZE_ON_HEAP]); + result = erts_decode_ext(&factory, &ext); if (is_value(result)) { - ASSERT(hp <= end); + erts_factory_close(&factory); } - HRelease(p,end,hp); } return result; } /* * Returns the MD5 checksum for a module - * - * Returns a tagged term, or 0 on error. */ Eterm md5_of_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ + BeamInstr* code) { - Module* modp; - BeamInstr* code; - Eterm res = NIL; - - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; - if (code[MI_MD5_PTR] != 0) { - res = new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE); - } else { - res = am_undefined; - } - return res; + return new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE); } /* diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 2b782f4484..bfca861830 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1914,7 +1914,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) } else if (is_external_pid(to)) { dep = external_pid_dist_entry(to); if(dep == erts_this_dist_entry) { -#if DEBUG erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Discarding message %T from %T to %T in an old " @@ -1925,7 +1924,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) external_pid_creation(to), erts_this_node->creation); erts_send_error_to_logger(p->group_leader, dsbufp); -#endif return 0; } return remote_send(p, dep, to, to, msg, ctx); @@ -1959,7 +1957,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) } else if (is_external_port(to) && (external_port_dist_entry(to) == erts_this_dist_entry)) { -#if DEBUG erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Discarding message %T from %T to %T in an old " @@ -1970,7 +1967,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) external_port_creation(to), erts_this_node->creation); erts_send_error_to_logger(p->group_leader, dsbufp); -#endif return 0; } else if (is_internal_port(to)) { int ret_val; diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 4e4611de16..5b5550da43 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -85,6 +85,7 @@ typedef Uint dsize_t; /* Vector size type */ /* The heap size needed for a bignum */ #define BIG_NEED_SIZE(x) ((x) + 1) +#define BIG_NEED_FOR_BITS(bits) BIG_NEED_SIZE(((bits)-1)/D_EXP + 1) #define BIG_UINT_HEAP_SIZE (1 + 1) /* always, since sizeof(Uint) <= sizeof(Eterm) */ diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 02e65cb9c6..3cb605834f 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -242,7 +242,6 @@ print_process_info(int to, void *to_arg, Process *p) p->current[1], p->current[2]); } - erts_print(to, to_arg, "Run queue: %d\n", erts_get_runq_proc(p)->ix); erts_print(to, to_arg, "Spawned by: %T\n", p->parent); approx_started = (time_t) p->approx_started; diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 850606dd86..cddadb346c 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -34,7 +34,7 @@ #include "erl_bits.h" #include "dtrace-wrapper.h" -static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*); +static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*); /* * Copy object "obj" to process p. @@ -661,8 +661,7 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, unsigned i; for (bp=first; bp!=NULL; bp=bp->next) { - move_one_frag(hpp, bp->mem, bp->used_size, off_heap); - OH_OVERHEAD(off_heap, bp->off_heap.overhead); + move_one_frag(hpp, bp, off_heap); } hp_end = *hpp; for (hp=hp_start; hp<hp_end; ++hp) { @@ -698,10 +697,10 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, } static void -move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) +move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap) { - Eterm* ptr = src; - Eterm* end = ptr + src_sz; + Eterm* ptr = frag->mem; + Eterm* end = ptr + frag->used_size; Eterm dummy_ref; Eterm* hp = *hpp; @@ -732,5 +731,7 @@ move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) } } *hpp = hp; + OH_OVERHEAD(off_heap, frag->off_heap.overhead); + frag->off_heap.first = NULL; } diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 142fcb3c00..1354cee267 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1149,6 +1149,7 @@ int erts_net_message(Port *prt, DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); Eterm* ctl = ctl_default; ErlOffHeap off_heap; + ErtsHeapFactory factory; Eterm* hp; Sint type; Eterm token; @@ -1225,7 +1226,8 @@ int erts_net_message(Port *prt, } hp = ctl; - arg = erts_decode_dist_ext(&hp, &off_heap, &ede); + erts_factory_static_init(&factory, ctl, ctl_len, &off_heap); + arg = erts_decode_dist_ext(&factory, &ede); if (is_non_value(arg)) { #ifdef ERTS_DIST_MSG_DBG erts_fprintf(stderr, "DIST MSG DEBUG: erts_dist_ext_size(CTL) failed:\n"); @@ -2086,6 +2088,7 @@ erts_dist_command(Port *prt, int reds_limit) DistEntry *dep = prt->dist_entry; Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); erts_aint32_t sched_flags; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -2140,12 +2143,12 @@ erts_dist_command(Port *prt, int reds_limit) ErtsDistOutputBuf *fob; size = (*send)(prt, foq.first); + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG erts_fprintf(stderr, ">> "); bw(foq.first->extp, size); #endif reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - erts_smp_atomic_add_nob(&erts_bytes_out, size); fob = foq.first; obufsize += size_obuf(fob); foq.first = foq.first->next; @@ -2225,12 +2228,12 @@ erts_dist_command(Port *prt, int reds_limit) ASSERT(&oq.first->data[0] <= oq.first->extp && oq.first->extp < oq.first->ext_endp); size = (*send)(prt, oq.first); + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG erts_fprintf(stderr, ">> "); bw(oq.first->extp, size); #endif reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - erts_smp_atomic_add_nob(&erts_bytes_out, size); fob = oq.first; obufsize += size_obuf(fob); oq.first = oq.first->next; @@ -2522,7 +2525,7 @@ info_dist_entry(int to, void *arg, DistEntry *dep, int visible, int connected) erts_print(to, arg, "Name: %T", dep->sysname); #ifdef DEBUG - erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 1)); + erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 0)); #endif erts_print(to, arg, "\n"); if (!connected && is_nil(dep->cid)) { diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index dcae5509ec..c9ac024743 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -581,8 +581,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = erts_timer_type_size(ERTS_ALC_T_HL_PTIMER); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)] = erts_timer_type_size(ERTS_ALC_T_BIF_TIMER); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)] = erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER); +#endif #ifdef HARD_DEBUG hdbg_init(); @@ -1882,8 +1884,8 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) size = va_arg(argp, Uint); va_end(argp); erl_exit(1, - "%s: Cannot %s %lu bytes of memory (of type \"%s\", thread %d).\n", - allctr_str, op, size, t_str, ERTS_ALC_GET_THR_IX()); + "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n", + allctr_str, op, size, t_str); break; } case ERTS_ALC_E_NOALLCTR: @@ -2343,10 +2345,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_BIF_TIMER); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT add_fix_values(&size.processes, &size.processes_used, fi, ERTS_ALC_T_ABIF_TIMER); +#endif } if (want.atom || want.atom_used) { diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 57c506458c..92c397ffd3 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -167,7 +167,7 @@ type TIMER_SERVICE LONG_LIVED SYSTEM timer_service type LL_PTIMER FIXED_SIZE PROCESSES ll_ptimer type HL_PTIMER FIXED_SIZE PROCESSES hl_ptimer type BIF_TIMER FIXED_SIZE PROCESSES bif_timer -type ABIF_TIMER FIXED_SIZE PROCESSES accessor_bif_timer +# type ABIF_TIMER FIXED_SIZE PROCESSES accessor_bif_timer type TIMER_REQUEST SHORT_LIVED PROCESSES timer_request type BTM_YIELD_STATE SHORT_LIVED PROCESSES btm_yield_state type REG_TABLE STANDARD SYSTEM reg_tab @@ -274,6 +274,7 @@ type BUSY_CALLER SHORT_LIVED SYSTEM busy_caller type PROC_SYS_TSK SHORT_LIVED PROCESSES proc_sys_task type PROC_SYS_TSK_QS SHORT_LIVED PROCESSES proc_sys_task_queues type NEW_TIME_OFFSET SHORT_LIVED SYSTEM new_time_offset +type IOB_REQ SHORT_LIVED SYSTEM io_bytes_request +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index f74aea80a7..7eb31fb80e 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -60,6 +60,7 @@ static Export* alloc_info_trap = NULL; static Export* alloc_sizes_trap = NULL; +static Export* gather_io_bytes_trap = NULL; static Export *gather_sched_wall_time_res_trap; static Export *gather_gc_info_res_trap; @@ -1106,19 +1107,18 @@ process_info_aux(Process *BIF_P, heap_need += mq[i].copy_struct_size; } else { - mq[i].copy_struct_size = 0; - if (mp->data.attached) - heap_need += erts_msg_attached_data_size(mp); + mq[i].copy_struct_size = mp->data.attached ? + erts_msg_attached_data_size(mp) : 0; } i++; } - hp = HAlloc(BIF_P, heap_need); - hp_end = hp + heap_need; - ASSERT(i == n); - for (i--; i >= 0; i--) { - Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); - if (rp != BIF_P) { + if (rp != BIF_P) { + hp = HAlloc(BIF_P, heap_need); + hp_end = hp + heap_need; + ASSERT(i == n); + for (i--; i >= 0; i--) { + Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); if (is_value(msg)) { if (mq[i].copy_struct_size) msg = copy_struct(msg, @@ -1152,9 +1152,9 @@ process_info_aux(Process *BIF_P, } else { /* Make our copy of the message */ - ASSERT(size_object(msg) == hfp->used_size); + ASSERT(size_object(msg) == erts_used_frag_sz(hfp)); msg = copy_struct(msg, - hfp->used_size, + erts_used_frag_sz(hfp), &hp, &MSO(BIF_P)); } @@ -1164,27 +1164,38 @@ process_info_aux(Process *BIF_P, remove_bad_messages = 1; continue; } + res = CONS(hp, msg, res); + hp += 2; } - else { + HRelease(BIF_P, hp_end, hp+3); + } + else { + for (i--; i >= 0; i--) { + ErtsHeapFactory factory; + Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); + + erts_factory_proc_prealloc_init(&factory, BIF_P, + mq[i].copy_struct_size+2); if (mq[i].msgp->data.attached) { /* Decode it on the heap */ - erts_move_msg_attached_data_to_heap(&hp, - &MSO(BIF_P), + erts_move_msg_attached_data_to_heap(&factory, mq[i].msgp); msg = ERL_MESSAGE_TERM(mq[i].msgp); ASSERT(!mq[i].msgp->data.attached); - if (is_non_value(msg)) { - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - continue; - } - } + } + if (is_value(msg)) { + hp = erts_produce_heap(&factory, 2, 0); + res = CONS(hp, msg, res); + } + else { + /* Bad distribution message; ignore */ + remove_bad_messages = 1; + continue; + } + erts_factory_close(&factory); } - - res = CONS(hp, msg, res); - hp += 2; + hp = HAlloc(BIF_P, 3); } - HRelease(BIF_P, hp_end, hp+3); erts_free(ERTS_ALC_T_TMP, mq); if (remove_bad_messages) { ErlMessage **mpp; @@ -2687,6 +2698,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = hsz ? HAlloc(BIF_P, hsz) : NULL; res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); BIF_RET(res); + } else if (ERTS_IS_ATOM_STR("delayed_node_table_gc", BIF_ARG_1)) { + Uint hsz = 0; + Uint dntgc = erts_delayed_node_table_gc(); + if (dntgc == ERTS_NODE_TAB_DELAY_GC_INFINITY) + BIF_RET(am_infinity); + (void) erts_bld_uint(NULL, &hsz, dntgc); + hp = hsz ? HAlloc(BIF_P, hsz) : NULL; + res = erts_bld_uint(&hp, NULL, dntgc); + BIF_RET(res); } else if (ERTS_IS_ATOM_STR("ethread_info", BIF_ARG_1)) { BIF_RET(erts_get_ethread_info(BIF_P)); } @@ -3201,7 +3221,6 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2) BIF_RET(am_true); } - /* this is a general call which return some possibly useful information */ BIF_RETTYPE statistics_1(BIF_ALIST_1) @@ -3274,23 +3293,8 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_io) { - Eterm r1, r2; - Eterm in, out; - Uint hsz = 9; - Uint bytes_in = (Uint) erts_smp_atomic_read_nob(&erts_bytes_in); - Uint bytes_out = (Uint) erts_smp_atomic_read_nob(&erts_bytes_out); - - (void) erts_bld_uint(NULL, &hsz, bytes_in); - (void) erts_bld_uint(NULL, &hsz, bytes_out); - hp = HAlloc(BIF_P, hsz); - in = erts_bld_uint(&hp, NULL, bytes_in); - out = erts_bld_uint(&hp, NULL, bytes_out); - - r1 = TUPLE2(hp, am_input, in); - hp += 3; - r2 = TUPLE2(hp, am_output, out); - hp += 3; - BIF_RET(TUPLE2(hp, r1, r2)); + Eterm ref = erts_request_io_bytes(BIF_P); + BIF_TRAP2(gather_io_bytes_trap, BIF_P, ref, make_small(erts_no_schedulers)); } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { Eterm res, *hp, **hpp; @@ -4050,7 +4054,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) { if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) { - if (erts_debug_wait_deallocations(BIF_P)) { + int flag = ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS; + if (erts_debug_wait_completed(BIF_P, flag)) { + ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); + } + } + if (ERTS_IS_ATOM_STR("timer_cancellations", BIF_ARG_2)) { + int flag = ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS; + if (erts_debug_wait_completed(BIF_P, flag)) { ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); } } @@ -4062,6 +4073,17 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) int res = erts_debug_set_unique_monotonic_integer_state(BIF_ARG_2); BIF_RET(res ? am_true : am_false); } + else if (ERTS_IS_ATOM_STR("node_tab_delayed_delete", BIF_ARG_1)) { + /* node_container_SUITE */ + Sint64 msecs; + if (term_to_Sint64(BIF_ARG_2, &msecs)) { + /* Negative value restore original value... */ + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_debug_test_node_tab_delayed_delete(msecs); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + BIF_RET(am_ok); + } + } } BIF_ERROR(BIF_P, BADARG); @@ -4259,59 +4281,41 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) BIF_RET(am_ok); } else if (is_tuple(BIF_ARG_1)) { - Eterm* tp = tuple_val(BIF_ARG_1); - - switch (arityval(tp[0])) { - case 2: { - int opt = 0; - int val = 0; - if (ERTS_IS_ATOM_STR("copy_save", tp[1])) { - opt = ERTS_LCNT_OPT_COPYSAVE; - } else if (ERTS_IS_ATOM_STR("process_locks", tp[1])) { - opt = ERTS_LCNT_OPT_PROCLOCK; - } else if (ERTS_IS_ATOM_STR("port_locks", tp[1])) { - opt = ERTS_LCNT_OPT_PORTLOCK; - } else if (ERTS_IS_ATOM_STR("suspend", tp[1])) { - opt = ERTS_LCNT_OPT_SUSPEND; - } else if (ERTS_IS_ATOM_STR("location", tp[1])) { - opt = ERTS_LCNT_OPT_LOCATION; - } else { - BIF_ERROR(BIF_P, BADARG); - } - if (tp[2] == am_true) { - val = 1; - } else if (tp[2] == am_false) { - val = 0; - } else { - BIF_ERROR(BIF_P, BADARG); - } - - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - - if (val) { - res = erts_lcnt_set_rt_opt(opt) ? am_true : am_false; - } else { - res = erts_lcnt_clear_rt_opt(opt) ? am_true : am_false; - } + Eterm* ptr = tuple_val(BIF_ARG_1); + + if ((arityval(ptr[0]) == 2) && (ptr[2] == am_false || ptr[2] == am_true)) { + int lock_opt = 0, enable = (ptr[2] == am_true) ? 1 : 0; + if (ERTS_IS_ATOM_STR("copy_save", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_COPYSAVE; + } else if (ERTS_IS_ATOM_STR("process_locks", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_PROCLOCK; + } else if (ERTS_IS_ATOM_STR("port_locks", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_PORTLOCK; + } else if (ERTS_IS_ATOM_STR("suspend", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_SUSPEND; + } else if (ERTS_IS_ATOM_STR("location", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_LOCATION; + } else { + BIF_ERROR(BIF_P, BADARG); + } + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + + if (enable) res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false; + else res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false; + #ifdef ERTS_SMP - if (res != tp[2]) { - if (opt == ERTS_LCNT_OPT_PORTLOCK) { - erts_lcnt_enable_io_lock_count(val); - } else if (opt == ERTS_LCNT_OPT_PROCLOCK) { - erts_lcnt_enable_proc_lock_count(val); - } - } + if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PORTLOCK) { + erts_lcnt_enable_io_lock_count(enable); + } else if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PROCLOCK) { + erts_lcnt_enable_proc_lock_count(enable); + } #endif - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - BIF_RET(res); - break; - } - - default: - break; - } + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + BIF_RET(res); + } } #endif @@ -4351,6 +4355,8 @@ erts_bif_info_init(void) = erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1); gather_gc_info_res_trap = erts_export_put(am_erlang, am_gather_gc_info_result, 1); + gather_io_bytes_trap + = erts_export_put(am_erts_internal, am_gather_io_bytes, 2); process_info_init(); os_info_init(); } diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 8d264d166e..6b96787d40 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -194,6 +194,9 @@ ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size); ERTS_GLB_INLINE void erts_bin_free(Binary *bp); +ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, + void (*destructor)(Binary *), + int unaligned); ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, void (*destructor)(Binary *)); @@ -332,21 +335,30 @@ erts_bin_free(Binary *bp) } ERTS_GLB_INLINE Binary * -erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) +erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *), + int unaligned) { - Uint bsize = ERTS_MAGIC_BIN_SIZE(size); + Uint bsize = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_SIZE(size) + : ERTS_MAGIC_BIN_SIZE(size); Binary* bptr = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize); ASSERT(bsize > size); if (!bptr) erts_alloc_n_enomem(ERTS_ALC_T2N(ERTS_ALC_T_BINARY), bsize); ERTS_CHK_BIN_ALIGNMENT(bptr); bptr->flags = BIN_FLAG_MAGIC; - bptr->orig_size = ERTS_MAGIC_BIN_ORIG_SIZE(size); + bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size) + : ERTS_MAGIC_BIN_ORIG_SIZE(size); erts_refc_init(&bptr->refc, 0); ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor; return bptr; } +ERTS_GLB_INLINE Binary * +erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) +{ + return erts_create_magic_binary_x(size, destructor, 0); +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* !__ERL_BINARY_H */ diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index b8ae93fa58..2e29bf8895 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -107,6 +107,14 @@ erts_bits_destroy_state(ERL_BITS_PROTO_0) void erts_init_bits(void) { + ERTS_CT_ASSERT(offsetof(Binary,orig_bytes) % 8 == 0); + ERTS_CT_ASSERT(offsetof(ErtsMagicBinary,u.aligned.data) % 8 == 0); + ERTS_CT_ASSERT(ERTS_MAGIC_BIN_BYTES_TO_ALIGN == + (offsetof(ErtsMagicBinary,u.aligned.data) + - offsetof(ErtsMagicBinary,u.unaligned.data))); + ERTS_CT_ASSERT(offsetof(ErtsBinary,driver.binary.orig_bytes) + == offsetof(Binary,orig_bytes)); + erts_smp_atomic_init_nob(&bits_bufs_size, 0); #if defined(ERTS_SMP) /* erl_process.c calls erts_bits_init_state() on all state instances */ diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 2e2cb98354..844272c699 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -279,6 +279,8 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; if (use_frequent_read_lock) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + if (erts_ets_rwmtx_spin_count >= 0) + rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; #endif #ifdef ERTS_SMP erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt, @@ -2856,10 +2858,11 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) ** External interface (NOT BIF's) */ +int erts_ets_rwmtx_spin_count = -1; /* Init the db */ -void init_db(void) +void init_db(ErtsDbSpinCount db_spin_count) { DbTable init_tb; int i; @@ -2868,10 +2871,48 @@ void init_db(void) size_t size; #ifdef ERTS_SMP + int max_spin_count = (1 << 15) - 1; /* internal limit */ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + switch (db_spin_count) { + case ERTS_DB_SPNCNT_NONE: + erts_ets_rwmtx_spin_count = 0; + break; + case ERTS_DB_SPNCNT_VERY_LOW: + erts_ets_rwmtx_spin_count = 100; + break; + case ERTS_DB_SPNCNT_LOW: + erts_ets_rwmtx_spin_count = 200; + erts_ets_rwmtx_spin_count += erts_no_schedulers * 50; + if (erts_ets_rwmtx_spin_count > 1000) + erts_ets_rwmtx_spin_count = 1000; + break; + case ERTS_DB_SPNCNT_HIGH: + erts_ets_rwmtx_spin_count = 2000; + erts_ets_rwmtx_spin_count += erts_no_schedulers * 100; + if (erts_ets_rwmtx_spin_count > 15000) + erts_ets_rwmtx_spin_count = 15000; + break; + case ERTS_DB_SPNCNT_VERY_HIGH: + erts_ets_rwmtx_spin_count = 15000; + erts_ets_rwmtx_spin_count += erts_no_schedulers * 500; + if (erts_ets_rwmtx_spin_count > max_spin_count) + erts_ets_rwmtx_spin_count = max_spin_count; + break; + case ERTS_DB_SPNCNT_EXTREMELY_HIGH: + erts_ets_rwmtx_spin_count = max_spin_count; + break; + case ERTS_DB_SPNCNT_NORMAL: + default: + erts_ets_rwmtx_spin_count = -1; + break; + } + + if (erts_ets_rwmtx_spin_count >= 0) + rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; + meta_main_tab_locks = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES, sizeof(erts_meta_main_tab_lock_t) diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 5b4681fc90..5039b71108 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -61,7 +61,17 @@ union db_table { "ERL_MAX_ETS_TABLES" */ #define ERL_MAX_ETS_TABLES_ENV "ERL_MAX_ETS_TABLES" -void init_db(void); +typedef enum { + ERTS_DB_SPNCNT_NONE, + ERTS_DB_SPNCNT_VERY_LOW, + ERTS_DB_SPNCNT_LOW, + ERTS_DB_SPNCNT_NORMAL, + ERTS_DB_SPNCNT_HIGH, + ERTS_DB_SPNCNT_VERY_HIGH, + ERTS_DB_SPNCNT_EXTREMELY_HIGH +} ErtsDbSpinCount; + +void init_db(ErtsDbSpinCount); int erts_db_process_exiting(Process *, ErtsProcLocks); void db_info(int, void *, int); void erts_db_foreach_table(void (*)(DbTable *, void *), void *); @@ -69,6 +79,7 @@ void erts_db_foreach_offheap(DbTable *, void (*func)(ErlOffHeap *, void *), void *); +extern int erts_ets_rwmtx_spin_count; extern int user_requested_db_max_tabs; /* set in erl_init */ extern int erts_ets_realloc_always_moves; /* set in erl_init */ extern int erts_ets_always_compress; /* set in erl_init */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 383ee7c430..dce0a3d621 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -670,6 +670,8 @@ int db_create_hash(Process *p, DbTable *tbl) int i; if (tb->common.type & DB_FREQ_READ) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + if (erts_ets_rwmtx_spin_count >= 0) + rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ (DbTable *) tb, sizeof(DbTableHashFineLocks)); @@ -1049,7 +1051,6 @@ static int db_get_element_hash(Process *p, DbTable *tbl, Eterm copy = db_copy_element_from_ets(&tb->common, p, &b->dbterm, ndex, &hp, 2); elem_list = CONS(hp, copy, elem_list); - hp += 2; } b = b->next; } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index c6c3c55a7e..d47ff03a30 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2175,11 +2175,12 @@ restart: { ErtsHeapFactory factory; Uint ix; - factory.p = build_proc; for (ix = 0; ix < 2*n; ix++){ ehp[ix] = esp[ix]; } + erts_factory_proc_init(&factory, build_proc); t = erts_hashmap_from_array(&factory, ehp, n, 0); + erts_factory_close(&factory); } *esp++ = t; break; @@ -3192,6 +3193,7 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, { Eterm* hp = *hpp; int i, arity = arityval(bp->tpl[0]); + ErtsHeapFactory factory; hp[0] = bp->tpl[0]; *hpp += arity + 1; @@ -3199,17 +3201,23 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos], size_object_rel(bp->tpl[tb->keypos], bp->tpl), hpp, off_heap, bp->tpl, NULL); + + erts_factory_static_init(&factory, *hpp, bp->size - (arity+1), off_heap); + for (i=arity; i>0; i--) { if (i != tb->keypos) { if (is_immed(bp->tpl[i])) { hp[i] = bp->tpl[i]; } else { - hp[i] = erts_decode_ext_ets(hpp, off_heap, + hp[i] = erts_decode_ext_ets(&factory, elem2ext(bp->tpl, i)); } } } + *hpp = factory.hp; + erts_factory_close(&factory); + ASSERT((*hpp - hp) <= bp->size); #ifdef DEBUG_CLONE ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone)); @@ -3228,12 +3236,13 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, if (tb->compress && pos != tb->keypos) { byte* ext = elem2ext(obj->tpl, pos); Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra; - Eterm* hp = HAlloc(p, sz); - Eterm* endp = hp + sz; - Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext); - *hpp = hp; - hp += extra; - HRelease(p, endp, hp); + Eterm copy; + ErtsHeapFactory factory; + + erts_factory_proc_prealloc_init(&factory, p, sz); + copy = erts_decode_ext_ets(&factory, ext); + *hpp = erts_produce_heap(&factory, extra, 0); + erts_factory_close(&factory); #ifdef DEBUG_CLONE ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone)); #endif diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 1785fc27be..71ca2713b2 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -108,8 +108,7 @@ static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size); static Eterm* collect_heap_frags(Process* p, Eterm* heap, Eterm* htop, Eterm* objv, int nobj); -static Uint adjust_after_fullsweep(Process *p, Uint size_before, - int need, Eterm *objv, int nobj); +static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); static void sweep_off_heap(Process *p, int fullsweep); @@ -677,7 +676,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint area_size; Eterm* old_htop; Uint n; - struct erl_off_heap_header** prev; + struct erl_off_heap_header** prev = NULL; if (p->flags & F_DISABLE_GC) return; @@ -786,10 +785,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, */ if (oh) { - prev = &MSO(p).first; - while (*prev) { - prev = &(*prev)->next; - } + prev = &MSO(p).first; + while (*prev) { + prev = &(*prev)->next; + } } /* @@ -818,6 +817,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, oh = oh->next; } + if (prev) { + *prev = NULL; + } + /* * We no longer need this temporary area. */ @@ -868,29 +871,37 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) ErlMessage *msgp; Uint size_after; Uint need_after; - Uint stack_size = STACK_SZ_ON_HEAP(p); - Uint fragments = MBUF_SIZE(p) + combined_message_size(p); - Uint size_before = fragments + (HEAP_TOP(p) - HEAP_START(p)); - Uint new_sz = next_heap_size(p, HEAP_SIZE(p) + fragments, 0); + const Uint stack_size = STACK_SZ_ON_HEAP(p); + const Uint size_before = MBUF_SIZE(p) + (HEAP_TOP(p) - HEAP_START(p)); + Uint new_sz = HEAP_SIZE(p) + MBUF_SIZE(p) + combined_message_size(p); + new_sz = next_heap_size(p, new_sz, 0); do_minor(p, new_sz, objv, nobj); - /* + size_after = HEAP_TOP(p) - HEAP_START(p); + *recl += (size_before - size_after); + + /* * Copy newly received message onto the end of the new heap. */ - ErtsGcQuickSanityCheck(p); - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); - ErtsGcQuickSanityCheck(p); - } - } + ErtsGcQuickSanityCheck(p); + for (msgp = p->msg.first; msgp; msgp = msgp->next) { + if (msgp->data.attached) { + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, + erts_msg_attached_data_size(msgp)); + erts_move_msg_attached_data_to_heap(&factory, msgp); + erts_factory_close(&factory); + ErtsGcQuickSanityCheck(p); + } + } ErtsGcQuickSanityCheck(p); GEN_GCS(p)++; - size_after = HEAP_TOP(p) - HEAP_START(p); - need_after = size_after + need + stack_size; - *recl += (size_before - size_after); + need_after = ((HEAP_TOP(p) - HEAP_START(p)) + + erts_used_frag_sz(MBUF(p)) + + need + + stack_size); /* * Excessively large heaps should be shrunk, but @@ -925,6 +936,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); + ASSERT(MBUF(p) == NULL); return 1; /* We are done. */ } @@ -933,6 +945,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * The heap size turned out to be just right. We are done. */ ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); + ASSERT(MBUF(p) == NULL); return 1; } } @@ -1212,7 +1225,9 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) { Rootset rootset; Roots* roots; - Uint size_before; + const Uint size_before = ((HEAP_TOP(p) - HEAP_START(p)) + + (OLD_HTOP(p) - OLD_HEAP(p)) + + MBUF_SIZE(p)); Eterm* n_heap; Eterm* n_htop; char* src = (char *) HEAP_START(p); @@ -1221,24 +1236,15 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) Uint oh_size = (char *) OLD_HTOP(p) - oh; Uint n; Uint new_sz; - Uint fragments = MBUF_SIZE(p) + combined_message_size(p); - - size_before = fragments + (HEAP_TOP(p) - HEAP_START(p)); /* * Do a fullsweep GC. First figure out the size of the heap * to receive all live data. */ - new_sz = HEAP_SIZE(p) + fragments + (OLD_HTOP(p) - OLD_HEAP(p)); - /* - * We used to do - * - * new_sz += STACK_SZ_ON_HEAP(p); - * - * here for no obvious reason. (The stack size is already counted once - * in HEAP_SIZE(p).) - */ + new_sz = (HEAP_SIZE(p) + MBUF_SIZE(p) + + combined_message_size(p) + + (OLD_HTOP(p) - OLD_HEAP(p))); new_sz = next_heap_size(p, new_sz, 0); /* @@ -1431,20 +1437,27 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) ErtsGcQuickSanityCheck(p); + *recl += size_before - (HEAP_TOP(p) - HEAP_START(p)); + { ErlMessage *msgp; + /* * Copy newly received message onto the end of the new heap. */ - for (msgp = p->msg.first; msgp; msgp = msgp->next) { + for (msgp = p->msg.first; msgp; msgp = msgp->next) { if (msgp->data.attached) { - erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, + erts_msg_attached_data_size(msgp)); + erts_move_msg_attached_data_to_heap(&factory, msgp); + erts_factory_close(&factory); ErtsGcQuickSanityCheck(p); } } } - *recl += adjust_after_fullsweep(p, size_before, need, objv, nobj); + adjust_after_fullsweep(p, need, objv, nobj); #ifdef HARDDEBUG disallow_heap_frag_ref_in_heap(p); @@ -1455,21 +1468,17 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) return 1; /* We are done. */ } -static Uint -adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int nobj) +static void +adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) { - Uint wanted, sz, size_after, need_after; + Uint wanted, sz, need_after; Uint stack_size = STACK_SZ_ON_HEAP(p); - Uint reclaimed_now; - - size_after = (HEAP_TOP(p) - HEAP_START(p)); - reclaimed_now = (size_before - size_after); /* * Resize the heap if needed. */ - need_after = size_after + need + stack_size; + need_after = (HEAP_TOP(p) - HEAP_START(p)) + need + stack_size; if (HEAP_SIZE(p) < need_after) { /* Too small - grow to match requested need */ sz = next_heap_size(p, need_after, 0); @@ -1492,8 +1501,6 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int shrink_new_heap(p, sz, objv, nobj); } } - - return reclaimed_now; } /* @@ -1870,6 +1877,21 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr if (!header_is_thing(gval)) { heap_ptr++; } else { + if (header_is_bin_matchstate(gval)) { + ErlBinMatchState *ms = (ErlBinMatchState*) heap_ptr; + ErlBinMatchBuffer *mb = &(ms->mb); + Eterm* origptr; + origptr = &(mb->orig); + ptr = boxed_val(*origptr); + val = *ptr; + if (IS_MOVED_BOXED(val)) { + *origptr = val; + mb->base = binary_bytes(*origptr); + } else if (in_area(ptr, src, src_size)) { + MOVE_BOXED(ptr,val,htop,origptr); + mb->base = binary_bytes(*origptr); + } + } heap_ptr += (thing_arityval(gval)+1); } break; @@ -1941,7 +1963,8 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, * until next GC. */ qb = MBUF(p); - while (qb != NULL) { + while (qb != NULL) { + ASSERT(!qb->off_heap.first); /* process fragments use the MSO(p) list */ frag_size = qb->used_size * sizeof(Eterm); if (frag_size != 0) { frag_begin = (char *) qb->mem; diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 51cd843935..8eacb921fe 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -81,6 +81,13 @@ static void hdbg_chk_srv(ErtsHLTimerService *srv); #error "ERTS_REF_NUMBERS changed. Update me..." #endif +typedef enum { + ERTS_TMR_BIF, + ERTS_TMR_PROC, + ERTS_TMR_PORT, + ERTS_TMR_CALLBACK +} ErtsTmrType; + #define ERTS_BIF_TIMER_SHORT_TIME 5000 #ifdef ERTS_SMP @@ -93,11 +100,14 @@ static void hdbg_chk_srv(ErtsHLTimerService *srv); /* Bit 0 to 9 contains scheduler id (see mask below) */ #define ERTS_TMR_ROFLG_HLT (((Uint32) 1) << 10) #define ERTS_TMR_ROFLG_BIF_TMR (((Uint32) 1) << 11) -#define ERTS_TMR_ROFLG_ABIF_TMR (((Uint32) 1) << 12) -#define ERTS_TMR_ROFLG_PRE_ALC (((Uint32) 1) << 13) -#define ERTS_TMR_ROFLG_REG_NAME (((Uint32) 1) << 14) -#define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 15) -#define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 16) +#define ERTS_TMR_ROFLG_PRE_ALC (((Uint32) 1) << 12) +#define ERTS_TMR_ROFLG_REG_NAME (((Uint32) 1) << 13) +#define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 14) +#define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 15) +#define ERTS_TMR_ROFLG_CALLBACK (((Uint32) 1) << 16) +#ifdef ERTS_BTM_ACCESSOR_SUPPORT +#define ERTS_TMR_ROFLG_ABIF_TMR (((Uint32) 1) << 17) +#endif #define ERTS_TMR_ROFLG_SID_MASK \ (ERTS_TMR_ROFLG_HLT - (Uint32) 1) @@ -141,6 +151,7 @@ typedef struct { Uint32 roflgs; erts_smp_atomic32_t refc; union { + void *arg; erts_atomic_t next; } u; } ErtsTmrHead; @@ -156,6 +167,7 @@ struct ErtsHLTimer_ { Process *proc; Port *port; Eterm name; + void (*callback)(void *); } receiver; #ifdef ERTS_HLT_HARD_DEBUG @@ -172,19 +184,28 @@ struct ErtsHLTimer_ { Eterm message; ErlHeapFragment *bp; } btm; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT struct { Eterm accessor; ErtsHLTimerTree tree; } abtm; +#endif }; #define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm) +#ifdef ERTS_BTM_ACCESSOR_SUPPORT #define ERTS_BIF_TIMER_SIZE offsetof(ErtsHLTimer, abtm) #define ERTS_ABIF_TIMER_SIZE sizeof(ErtsHLTimer) +#else +#define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer) +#endif typedef struct { ErtsTmrHead head; /* NEED to be first! */ - void *p; + union { + void *p; + void (*callback)(void *); + } u; ErtsTWheelTimer tw_tmr; } ErtsTWTimer; @@ -340,8 +361,8 @@ refn_is_lt(Uint32 *x, Uint32 *y) #define ERTS_RBT_WANT_SMALLEST #define ERTS_RBT_WANT_LOOKUP_INSERT #define ERTS_RBT_WANT_REPLACE +#define ERTS_RBT_WANT_FOREACH #ifdef ERTS_HLT_HARD_DEBUG -# define ERTS_RBT_WANT_FOREACH # define ERTS_RBT_WANT_LOOKUP #endif #define ERTS_RBT_UNDEF @@ -452,8 +473,6 @@ same_time_list_foreach_destroy_yielding(ErtsHLTimer **root, } } -#ifdef ERTS_HLT_HARD_DEBUG - static ERTS_INLINE void same_time_list_foreach(ErtsHLTimer *root, void (*op)(ErtsHLTimer *, void *), @@ -468,6 +487,8 @@ same_time_list_foreach(ErtsHLTimer *root, } } +#ifdef ERTS_HLT_HARD_DEBUG + static ERTS_INLINE ErtsHLTimer * same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) { @@ -584,6 +605,8 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #include "erl_rbtree.h" +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + #define ERTS_RBT_PREFIX abtm #define ERTS_RBT_T ErtsHLTimer #define ERTS_RBT_KEY_T Uint32 * @@ -634,6 +657,8 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #include "erl_rbtree.h" +#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ + #ifdef ERTS_SMP static void init_canceled_queue(ErtsHLTCncldTmrQ *cq); #endif @@ -673,7 +698,9 @@ erts_timer_type_size(ErtsAlcType_t type) case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer); case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE; case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT case ERTS_ALC_T_ABIF_TIMER: return ERTS_ABIF_TIMER_SIZE; +#endif default: ERTS_INTERNAL_ERROR("Unknown type"); } return 0; @@ -747,9 +774,9 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr) * dropped at once... */ if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC) - erts_proc_dec_refc((Process *) tmr->p); - else - erts_port_dec_refc((Port *) tmr->p); + erts_proc_dec_refc((Process *) tmr->u.p); + else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT) + erts_port_dec_refc((Port *) tmr->u.p); erts_schedule_thr_prgr_later_cleanup_op( scheduled_tw_timer_destroy, @@ -771,7 +798,7 @@ static void tw_proc_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Process *proc = (Process *) twtp->p; + Process *proc = (Process *) twtp->u.p; if (proc_timeout_common(proc, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -781,7 +808,7 @@ static void tw_port_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Port *port = (Port *) twtp->p; + Port *port = (Port *) twtp->u.p; if (port_timeout_common(port, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -801,13 +828,26 @@ cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr) erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->tw_tmr); } +static void +tw_callback_timeout(void *vtwtp) +{ + ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; + void (*callback)(void *) = twtp->u.callback; + void *arg = twtp->head.u.arg; + tw_timer_dec_refc(twtp); + (*callback)(arg); +} + static ErtsTWTimer * create_tw_timer(ErtsSchedulerData *esdp, - void *p, int is_proc, + ErtsTmrType type, void *p, + void (*callback)(void *), void *arg, ErtsMonotonicTime timeout_pos) { ErtsTWTimer *tmr; void (*timeout_func)(void *); + void (*cancel_func)(void *); + erts_aint32_t refc; tmr = tw_timer_alloc(); erts_twheel_init_timer(&tmr->tw_tmr); @@ -815,24 +855,48 @@ create_tw_timer(ErtsSchedulerData *esdp, tmr->head.roflgs = (Uint32) esdp->no; ERTS_HLT_ASSERT((tmr->head.roflgs & ~ERTS_TMR_ROFLG_SID_MASK) == 0); - tmr->p = p; - if (is_proc) { + + switch (type) { + + case ERTS_TMR_PROC: + tmr->u.p = p; tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; timeout_func = tw_proc_timeout; + cancel_func = tw_ptimer_cancel; erts_proc_inc_refc((Process *) p); - } - else { + refc = 2; + break; + + case ERTS_TMR_PORT: + tmr->u.p = p; tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT; timeout_func = tw_port_timeout; + cancel_func = tw_ptimer_cancel; erts_port_inc_refc((Port *) p); + refc = 2; + break; + + case ERTS_TMR_CALLBACK: + tmr->head.u.arg = arg; + tmr->u.callback = callback; + + tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK; + timeout_func = tw_callback_timeout; + cancel_func = NULL; + refc = 1; + break; + + default: + ERTS_INTERNAL_ERROR("Unsupported timer type"); + return NULL; } - erts_smp_atomic32_init_nob(&tmr->head.refc, 2); + erts_smp_atomic32_init_nob(&tmr->head.refc, refc); erts_twheel_set_timer(esdp->timer_wheel, &tmr->tw_tmr, timeout_func, - tw_ptimer_cancel, + cancel_func, tmr, timeout_pos); @@ -852,8 +916,10 @@ hl_timer_destroy(ErtsHLTimer *tmr) else { if (roflgs & ERTS_TMR_ROFLG_PRE_ALC) bif_timer_pre_free(tmr); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR) erts_free(ERTS_ALC_T_ABIF_TIMER, tmr); +#endif else erts_free(ERTS_ALC_T_BIF_TIMER, tmr); } @@ -948,6 +1014,8 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) #endif } +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + static void hlt_delete_abtm(ErtsHLTimer *tmr) { @@ -971,18 +1039,20 @@ hlt_delete_abtm(ErtsHLTimer *tmr) } } +#endif + static ErtsHLTimer * create_hl_timer(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos, - int short_time, int is_bif_tmr, + int short_time, ErtsTmrType type, void *rcvrp, Eterm rcvr, Eterm acsr, - Eterm msg, Uint32 *refn) + Eterm msg, Uint32 *refn, + void (*callback)(void *), void *arg) { ErtsHLTimerService *srv = esdp->timer_service; ErtsHLTimer *tmr, *st_tmr; erts_aint32_t refc; Uint32 roflgs; - int is_abif_tmr = is_bif_tmr && is_value(acsr) && acsr != rcvr; check_canceled_queue(esdp, srv); @@ -990,43 +1060,69 @@ create_hl_timer(ErtsSchedulerData *esdp, roflgs = ((Uint32) esdp->no) | ERTS_TMR_ROFLG_HLT; - if (!is_bif_tmr) + if (type != ERTS_TMR_BIF) { + tmr = erts_alloc(ERTS_ALC_T_HL_PTIMER, ERTS_HL_PTIMER_SIZE); - else if (short_time) { - tmr = bif_timer_pre_alloc(); - if (!tmr) - goto alloc_bif_timer; - roflgs |= ERTS_TMR_ROFLG_PRE_ALC; - } - else { - alloc_bif_timer: - if (is_abif_tmr) - tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER, - ERTS_ABIF_TIMER_SIZE); - else - tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, - ERTS_BIF_TIMER_SIZE); - } + tmr->timeout = timeout_pos; - tmr->timeout = timeout_pos; + switch (type) { + + case ERTS_TMR_PROC: + ERTS_HLT_ASSERT(is_internal_pid(rcvr)); - if (!is_bif_tmr) { - if (is_internal_pid(rcvr)) { erts_proc_inc_refc((Process *) rcvrp); tmr->receiver.proc = (Process *) rcvrp; roflgs |= ERTS_TMR_ROFLG_PROC; - } - else { - erts_port_inc_refc((Port *) rcvrp); + refc = 2; + break; + + case ERTS_TMR_PORT: ERTS_HLT_ASSERT(is_internal_port(rcvr)); + erts_port_inc_refc((Port *) rcvrp); tmr->receiver.port = (Port *) rcvrp; roflgs |= ERTS_TMR_ROFLG_PORT; + refc = 2; + break; + + case ERTS_TMR_CALLBACK: + roflgs |= ERTS_TMR_ROFLG_CALLBACK; + tmr->receiver.callback = callback; + tmr->head.u.arg = arg; + refc = 1; + break; + + default: + ERTS_INTERNAL_ERROR("Unsupported timer type"); + return NULL; } - refc = 2; + } - else { + else { /* ERTS_TMR_BIF */ Uint hsz; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + int is_abif_tmr = is_value(acsr) && acsr != rcvr; +#endif + + if (short_time) { + tmr = bif_timer_pre_alloc(); + if (!tmr) + goto alloc_bif_timer; + roflgs |= ERTS_TMR_ROFLG_PRE_ALC; + } + else { + alloc_bif_timer: +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (is_abif_tmr) + tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER, + ERTS_ABIF_TIMER_SIZE); + else +#endif + tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, + ERTS_BIF_TIMER_SIZE); + } + + tmr->timeout = timeout_pos; roflgs |= ERTS_TMR_ROFLG_BIF_TMR; if (is_internal_pid(rcvr)) { @@ -1057,6 +1153,8 @@ create_hl_timer(ErtsSchedulerData *esdp, tmr->btm.refn[2] = refn[2]; tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT if (is_abif_tmr) { Process *aproc; roflgs |= ERTS_TMR_ROFLG_ABIF_TMR; @@ -1071,6 +1169,9 @@ create_hl_timer(ErtsSchedulerData *esdp, erts_smp_proc_unlock(aproc, ERTS_PROC_LOCK_BTM); } } +#endif + + btm_rbt_insert(&srv->btm_tree, tmr); } tmr->head.roflgs = roflgs; @@ -1098,9 +1199,6 @@ create_hl_timer(ErtsSchedulerData *esdp, if (st_tmr) same_time_list_insert(&st_tmr->time.tree.same_time, tmr); - if (is_bif_tmr) - btm_rbt_insert(&srv->btm_tree, tmr); - #ifdef ERTS_HLT_HARD_DEBUG tmr->pending_timeout = 0; #endif @@ -1120,8 +1218,10 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) hlt_delete_abtm(tmr); +#endif if (is_reg_name) { Eterm pid; @@ -1203,10 +1303,13 @@ static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) hlt_bif_timer_timeout(tmr, roflgs); else if (roflgs & ERTS_TMR_ROFLG_PROC) hlt_proc_timeout(tmr); - else { - ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PORT); + else if (roflgs & ERTS_TMR_ROFLG_PORT) hlt_port_timeout(tmr); + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); + (*tmr->receiver.callback)(tmr->head.u.arg); } + } tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; @@ -1300,8 +1403,10 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; } +#ifdef ERTS_BTM_ACCESSOR_SUPPORT if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) hlt_delete_abtm(tmr); +#endif } if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) { @@ -1670,8 +1775,9 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg; - tmr = create_hl_timer(esdp, timeout_pos, short_time, 1, NULL, - rcvr, acsr, tmo_msg, internal_ref_numbers(ref)); + tmr = create_hl_timer(esdp, timeout_pos, short_time, + ERTS_TMR_BIF, NULL, rcvr, acsr, tmo_msg, + internal_ref_numbers(ref), NULL, NULL); UnUseTmpHeap(4, c_p); @@ -1943,8 +2049,10 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, */ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM); tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT if (!tmr) tmr = abtm_rbt_lookup(c_p->accessor_bif_timers, trefn); +#endif erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM); if (!tmr) return 0; @@ -2184,11 +2292,13 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info, if (!abs || !bool_arg(tp[2], abs)) return 0; break; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT case am_accessor: if (!accessor || is_not_internal_pid(tp[2])) return 0; *accessor = tp[2]; break; +#endif default: return 0; } @@ -2250,7 +2360,9 @@ typedef struct { ErtsBifTimers *bif_timers; union { proc_btm_rbt_yield_state_t proc_btm_yield_state; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT abtm_rbt_yield_state_t abtm_yield_state; +#endif } u; } ErtsBifTimerYieldState; @@ -2291,6 +2403,8 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) return res; } +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + static void detach_bif_timer(ErtsHLTimer *tmr, void *vesdp) { @@ -2335,6 +2449,8 @@ int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp return res; } +#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ + static ERTS_INLINE int parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, ErtsMonotonicTime *conv_arg, int abs, @@ -2498,6 +2614,80 @@ BIF_RETTYPE read_timer_2(BIF_ALIST_2) return ret; } +static void +start_callback_timer(ErtsSchedulerData *esdp, + int twt, + ErtsMonotonicTime timeout_pos, + void (*callback)(void *), + void *arg) + +{ + if (twt) + create_tw_timer(esdp, ERTS_TMR_CALLBACK, NULL, + callback, arg, timeout_pos); + else + create_hl_timer(esdp, timeout_pos, 0, + ERTS_TMR_CALLBACK, NULL, + NIL, THE_NON_VALUE, NIL, + NULL, callback, arg); +} + +typedef struct { + int twt; + ErtsMonotonicTime timeout_pos; + void (*callback)(void *); + void *arg; +} ErtsStartCallbackTimerRequest; + +static void +scheduled_start_callback_timer(void *vsctr) +{ + ErtsStartCallbackTimerRequest *sctr + = (ErtsStartCallbackTimerRequest *) vsctr; + + start_callback_timer(erts_get_scheduler_data(), + sctr->twt, + sctr->timeout_pos, + sctr->callback, + sctr->arg); + + erts_free(ERTS_ALC_T_TIMER_REQUEST, vsctr); +} + +void +erts_start_timer_callback(ErtsMonotonicTime tmo, + void (*callback)(void *), + void *arg) +{ + ErtsSchedulerData *esdp; + ErtsMonotonicTime timeout_pos; + int twt; + + esdp = erts_get_scheduler_data(); + timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), + tmo); + twt = tmo < ERTS_TIMER_WHEEL_MSEC; + + if (esdp) + start_callback_timer(esdp, + twt, + timeout_pos, + callback, + arg); + else { + ErtsStartCallbackTimerRequest *sctr; + sctr = erts_alloc(ERTS_ALC_T_TIMER_REQUEST, + sizeof(ErtsStartCallbackTimerRequest)); + sctr->twt = twt; + sctr->timeout_pos = timeout_pos; + sctr->callback = callback; + sctr->arg = arg; + erts_schedule_misc_aux_work(1, + scheduled_start_callback_timer, + (void *) sctr); + } +} + /* * Process and Port timer functionality. * @@ -2521,11 +2711,13 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo, c_p->flags &= ~F_TIMO; if (tmo < ERTS_TIMER_WHEEL_MSEC) - tmr = (void *) create_tw_timer(esdp, (void *) c_p, 1, timeout_pos); + tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PROC, (void *) c_p, + NULL, NULL, timeout_pos); else - tmr = (void *) create_hl_timer(esdp, timeout_pos, - short_time, 0, (void *) c_p, - c_p->common.id, NIL, NIL, NULL); + tmr = (void *) create_hl_timer(esdp, timeout_pos, short_time, + ERTS_TMR_PROC, (void *) c_p, + c_p->common.id, THE_NON_VALUE, + NIL, NULL, NULL, NULL); erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr); } } @@ -2608,13 +2800,12 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo) timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo); if (tmo < ERTS_TIMER_WHEEL_MSEC) - tmr = (void *) create_tw_timer(esdp, (void *) c_prt, 0, - timeout_pos); + tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PORT, (void *) c_prt, + NULL, NULL, timeout_pos); else - tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, 0, - (void *) c_prt, - c_prt->common.id, NIL, NIL, - NULL); + tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT, + (void *) c_prt, c_prt->common.id, + THE_NON_VALUE, NIL, NULL, NULL, NULL); erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr); } @@ -2759,6 +2950,98 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm, } } +typedef struct { + void (*tclbk)(void *); + void (*func)(void *, + ErtsMonotonicTime, + void *); + void *arg; +} ErtsDebugForeachCallbackTimer; + +static void +debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct) +{ + ErtsDebugForeachCallbackTimer *dfct + = (ErtsDebugForeachCallbackTimer *) vdfct; + + if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) + && (tmr->receiver.callback && dfct->tclbk)) + (*dfct->func)(dfct->arg, + tmr->timeout, + tmr->head.u.arg); +} + +static void +debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct) +{ + ErtsDebugForeachCallbackTimer *dfct + = (ErtsDebugForeachCallbackTimer *) vdfct; + + if (tmr->time.tree.same_time) + same_time_list_foreach(tmr->time.tree.same_time, + debug_callback_timer_foreach_list, + vdfct); + + if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) + && (tmr->receiver.callback && dfct->tclbk)) + (*dfct->func)(dfct->arg, + tmr->timeout, + tmr->head.u.arg); +} + +static void +debug_tw_callback_timer(void *vdfct, + ErtsMonotonicTime timeout_pos, + void *vtwtp) +{ + ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; + ErtsDebugForeachCallbackTimer *dfct + = (ErtsDebugForeachCallbackTimer *) vdfct; + + if (twtp->u.callback == dfct->tclbk) + (*dfct->func)(dfct->arg, + timeout_pos, + twtp->head.u.arg); +} + +void +erts_debug_callback_timer_foreach(void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg) +{ + int six; + ErtsDebugForeachCallbackTimer dfct; + + dfct.tclbk = tclbk; + dfct.func = func; + dfct.arg = arg; + + if (!erts_smp_thr_progress_is_blocking()) + ERTS_INTERNAL_ERROR("Not blocking thread progress"); + + for (six = 0; six < erts_no_schedulers; six++) { + ErtsHLTimerService *srv = + erts_aligned_scheduler_data[six].esd.timer_service; + ErtsTimerWheel *twheel = + erts_aligned_scheduler_data[six].esd.timer_wheel; + + erts_twheel_debug_foreach(twheel, + tw_callback_timeout, + debug_tw_callback_timer, + (void *) &dfct); + + if (srv->yield.root) + debug_callback_timer_foreach(srv->yield.root, + (void *) &dfct); + + time_rbt_foreach(srv->btm_tree, + debug_callback_timer_foreach, + (void *) &dfct); + } +} + #ifdef ERTS_HLT_HARD_DEBUG typedef struct { diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index 30889a71da..24c57fc873 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -59,7 +59,9 @@ int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **); int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **); ErtsHLTimerService *erts_create_timer_service(void); void erts_hl_timer_init(void); - +void erts_start_timer_callback(ErtsMonotonicTime, + void (*)(void *), + void *); #ifdef ERTS_SMP void erts_handle_canceled_timers(void *vesdp, @@ -76,5 +78,10 @@ void erts_debug_bif_timer_foreach(void (*func)(Eterm, ErlHeapFragment *, void *), void *arg); - +void +erts_debug_callback_timer_foreach(void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg); #endif /* ERL_HL_TIMER_H__ */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 33417833a9..ac1f00d2d8 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -141,7 +141,9 @@ static void erl_init(int ncpu, int port_tab_sz_ignore_files, int legacy_port_tab, int time_correction, - ErtsTimeWarpMode time_warp_mode); + ErtsTimeWarpMode time_warp_mode, + int node_tab_delete_delay, + ErtsDbSpinCount db_spin_count); static erts_atomic_t exiting; @@ -314,7 +316,9 @@ erts_short_init(void) 0, 0, time_correction, - time_warp_mode); + time_warp_mode, + ERTS_NODE_TAB_DELAY_GC_DEFAULT, + ERTS_DB_SPNCNT_NORMAL); erts_initialized = 1; } @@ -326,7 +330,9 @@ erl_init(int ncpu, int port_tab_sz_ignore_files, int legacy_port_tab, int time_correction, - ErtsTimeWarpMode time_warp_mode) + ErtsTimeWarpMode time_warp_mode, + int node_tab_delete_delay, + ErtsDbSpinCount db_spin_count) { init_benchmarking(); @@ -366,8 +372,8 @@ erl_init(int ncpu, erts_ptab_init(); /* Must be after init_emulator() */ erts_init_binary(); /* Must be after init_emulator() */ erts_bp_init(); - init_db(); /* Must be after init_emulator */ - erts_init_node_tables(); + init_db(db_spin_count); /* Must be after init_emulator */ + erts_init_node_tables(node_tab_delete_delay); init_dist(); erl_drv_thr_init(); erts_init_async(); @@ -379,6 +385,7 @@ erl_init(int ncpu, erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_init_external(); + erts_init_map(); erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG @@ -629,6 +636,13 @@ void erts_usage(void) erts_fprintf(stderr, " see error_logger documentation for details\n"); erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024); + erts_fprintf(stderr, "-zdntgc time set delayed node table gc in seconds\n"); + erts_fprintf(stderr, " valid values are infinity or intergers in the range [0-%d]\n", + ERTS_NODE_TAB_DELAY_GC_MAX); +#if 0 + erts_fprintf(stderr, "-zebwt val set ets busy wait threshold, valid values are:\n"); + erts_fprintf(stderr, " none|very_short|short|medium|long|very_long|extremely_long\n"); +#endif erts_fprintf(stderr, "\n"); erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n"); erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n"); @@ -1219,6 +1233,8 @@ erl_start(int argc, char **argv) int legacy_port_tab = 0; int time_correction; ErtsTimeWarpMode time_warp_mode; + int node_tab_delete_delay = ERTS_NODE_TAB_DELAY_GC_DEFAULT; + ErtsDbSpinCount db_spin_count = ERTS_DB_SPNCNT_NORMAL; set_default_time_adj(&time_correction, &time_warp_mode); @@ -2005,9 +2021,9 @@ erl_start(int argc, char **argv) case 'z': { char *sub_param = argv[i]+2; - int new_limit; if (has_prefix("dbbl", sub_param)) { + int new_limit; arg = get_arg(sub_param+4, argv[i+1], &i); new_limit = atoi(arg); if (new_limit < 1 || INT_MAX/1024 < new_limit) { @@ -2016,6 +2032,46 @@ erl_start(int argc, char **argv) } else { erts_dist_buf_busy_limit = new_limit*1024; } + } + else if (has_prefix("dntgc", sub_param)) { + long secs; + + arg = get_arg(sub_param+5, argv[i+1], &i); + if (sys_strcmp(arg, "infinity") == 0) + secs = ERTS_NODE_TAB_DELAY_GC_INFINITY; + else { + char *endptr; + errno = 0; + secs = strtol(arg, &endptr, 10); + if (errno != 0 || *arg == '\0' || *endptr != '\0' + || secs < 0 || ERTS_NODE_TAB_DELAY_GC_MAX < secs) { + erts_fprintf(stderr, "Invalid delayed node table gc: %s\n", arg); + erts_usage(); + } + } + node_tab_delete_delay = (int) secs; + } + else if (has_prefix("ebwt", sub_param)) { + arg = get_arg(sub_param+4, argv[i+1], &i); + if (sys_strcmp(arg, "none") == 0) + db_spin_count = ERTS_DB_SPNCNT_NONE; + else if (sys_strcmp(arg, "very_short") == 0) + db_spin_count = ERTS_DB_SPNCNT_VERY_LOW; + else if (sys_strcmp(arg, "short") == 0) + db_spin_count = ERTS_DB_SPNCNT_LOW; + else if (sys_strcmp(arg, "medium") == 0) + db_spin_count = ERTS_DB_SPNCNT_NORMAL; + else if (sys_strcmp(arg, "long") == 0) + db_spin_count = ERTS_DB_SPNCNT_HIGH; + else if (sys_strcmp(arg, "very_long") == 0) + db_spin_count = ERTS_DB_SPNCNT_VERY_HIGH; + else if (sys_strcmp(arg, "extremely_long") == 0) + db_spin_count = ERTS_DB_SPNCNT_EXTREMELY_HIGH; + else { + erts_fprintf(stderr, + "Invalid ets busy wait threshold: %s\n", arg); + erts_usage(); + } } else { erts_fprintf(stderr, "bad -z option %s\n", argv[i]); erts_usage(); @@ -2089,7 +2145,9 @@ erl_start(int argc, char **argv) port_tab_sz_ignore_files, legacy_port_tab, time_correction, - time_warp_mode); + time_warp_mode, + node_tab_delete_delay, + db_spin_count); load_preloaded(); erts_end_staging_code_ix(); diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index c6d8f4df95..c4956d8569 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -20,7 +20,7 @@ /* * Description: Statistics for locks. * - * Author: Bj�rn-Egil Dahlberg + * Author: Björn-Egil Dahlberg * Date: 2008-07-03 */ @@ -49,7 +49,7 @@ const char *str_undefined = "undefined"; static ethr_tsd_key lcnt_thr_data_key; static int lcnt_n_thr; -static erts_lcnt_thread_data_t *lcnt_thread_data[4096]; +static erts_lcnt_thread_data_t *lcnt_thread_data[2048]; /* local functions */ @@ -83,12 +83,12 @@ static ERTS_INLINE int lcnt_log2(Uint64 v) { static char* lcnt_lock_type(Uint16 flag) { switch(flag & ERTS_LCNT_LT_ALL) { - case ERTS_LCNT_LT_SPINLOCK: return "spinlock"; - case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock"; - case ERTS_LCNT_LT_MUTEX: return "mutex"; - case ERTS_LCNT_LT_RWMUTEX: return "rw_mutex"; - case ERTS_LCNT_LT_PROCLOCK: return "proclock"; - default: return ""; + case ERTS_LCNT_LT_SPINLOCK: return "spinlock"; + case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock"; + case ERTS_LCNT_LT_MUTEX: return "mutex"; + case ERTS_LCNT_LT_RWMUTEX: return "rw_mutex"; + case ERTS_LCNT_LT_PROCLOCK: return "proclock"; + default: return ""; } } @@ -116,15 +116,15 @@ static void lcnt_time(erts_lcnt_time_t *time) { static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) { long ds; long dns; - + ds = t1->s - t0->s; dns = t1->ns - t0->ns; - + /* the difference should not be able to get bigger than 1 sec in ns*/ - + if (dns < 0) { - ds -= 1; - dns += 1000000000LL; + ds -= 1; + dns += 1000000000LL; } ASSERT(ds >= 0); @@ -145,7 +145,7 @@ static void lcnt_time_add(erts_lcnt_time_t *t, erts_lcnt_time_t *d) { static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) { erts_lcnt_thread_data_t *eltd; - + eltd = (erts_lcnt_thread_data_t*)malloc(sizeof(erts_lcnt_thread_data_t)); if (!eltd) { ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); @@ -164,7 +164,6 @@ static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) { return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key); } - /* debug */ #if 0 @@ -183,15 +182,15 @@ static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action) { type = lcnt_lock_type(lock->flag); r_state = ethr_atomic_read(&lock->r_state); w_state = ethr_atomic_read(&lock->w_state); - + if (lock->flag & flag) { erts_fprintf(stderr,"%10s [%24s] [r/w state %4ld/%4ld] %2s id %T\r\n", - action, - lock->name, - r_state, - w_state, - type, - lock->id); + action, + lock->name, + r_state, + w_state, + type, + lock->id); } } #endif @@ -201,18 +200,18 @@ static erts_lcnt_lock_stats_t *lcnt_get_lock_stats(erts_lcnt_lock_t *lock, char erts_lcnt_lock_stats_t *stats = NULL; if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) { - for (i = 0; i < lock->n_stats; i++) { - if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) { - return &(lock->stats[i]); - } - } - if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) { - stats = &lock->stats[lock->n_stats]; - lock->n_stats++; - stats->file = file; - stats->line = line; - return stats; - } + for (i = 0; i < lock->n_stats; i++) { + if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) { + return &(lock->stats[i]); + } + } + if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) { + stats = &lock->stats[lock->n_stats]; + lock->n_stats++; + stats->file = file; + stats->line = line; + return stats; + } } return &lock->stats[0]; } @@ -222,53 +221,48 @@ static void lcnt_update_stats_hist(erts_lcnt_hist_t *hist, erts_lcnt_time_t *tim unsigned long r; if (time_wait->s > 0 || time_wait->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) { - idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1; + idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1; } else { - r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT; - if (r) idx = lcnt_log2(r); - else idx = 0; + r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT; + if (r) idx = lcnt_log2(r); + else idx = 0; } hist->ns[idx]++; } static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, - erts_lcnt_time_t *time_wait) { - + erts_lcnt_time_t *time_wait) { + ethr_atomic_inc(&stats->tries); if (lock_in_conflict) - ethr_atomic_inc(&stats->colls); + ethr_atomic_inc(&stats->colls); if (time_wait) { - lcnt_time_add(&(stats->timer), time_wait); - stats->timer_n++; - lcnt_update_stats_hist(&stats->hist,time_wait); + lcnt_time_add(&(stats->timer), time_wait); + stats->timer_n++; + lcnt_update_stats_hist(&stats->hist,time_wait); } } -/* - * interface - */ +/* interface */ void erts_lcnt_init() { erts_lcnt_thread_data_t *eltd = NULL; - + /* init lock */ if (ethr_mutex_init(&lcnt_data_lock) != 0) abort(); /* init tsd */ lcnt_n_thr = 0; - ethr_tsd_key_create(&lcnt_thr_data_key,"lcnt_data"); lcnt_lock(); - erts_lcnt_rt_options = ERTS_LCNT_OPT_PROCLOCK | ERTS_LCNT_OPT_LOCATION; - + erts_lcnt_rt_options = ERTS_LCNT_OPT_LOCATION | ERTS_LCNT_OPT_PROCLOCK; eltd = lcnt_thread_data_alloc(); - ethr_tsd_set(lcnt_thr_data_key, eltd); - + /* init lcnt structure */ erts_lcnt_data = (erts_lcnt_data_t*)malloc(sizeof(erts_lcnt_data_t)); if (!erts_lcnt_data) { @@ -293,7 +287,7 @@ void erts_lcnt_late_init() { erts_lcnt_lock_list_t *erts_lcnt_list_init(void) { erts_lcnt_lock_list_t *list; - + list = (erts_lcnt_lock_list_t*)malloc(sizeof(erts_lcnt_lock_list_t)); if (!list) { ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); @@ -307,14 +301,14 @@ erts_lcnt_lock_list_t *erts_lcnt_list_init(void) { /* only do this on the list with the deleted locks! */ void erts_lcnt_list_clear(erts_lcnt_lock_list_t *list) { erts_lcnt_lock_t *lock = NULL, - *next = NULL; + *next = NULL; lock = list->head; - + while(lock != NULL) { - next = lock->next; - free(lock); - lock = next; + next = lock->next; + free(lock); + lock = next; } list->head = NULL; @@ -327,26 +321,25 @@ void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) tail = list->tail; if (tail) { - tail->next = lock; - lock->prev = tail; + tail->next = lock; + lock->prev = tail; } else { - list->head = lock; - lock->prev = NULL; - ASSERT(!lock->next); + list->head = lock; + lock->prev = NULL; + ASSERT(!lock->next); } lock->next = NULL; list->tail = lock; - + list->n++; } void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) { - if (lock->next) lock->next->prev = lock->prev; if (lock->prev) lock->prev->next = lock->next; if (list->head == lock) list->head = lock->next; if (list->tail == lock) list->tail = lock->prev; - + lock->prev = NULL; lock->next = NULL; list->n--; @@ -364,12 +357,9 @@ void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { int i; - if (!name) { - lock->flag = 0; - return; - } + if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; } lcnt_lock(); - + lock->next = NULL; lock->prev = NULL; lock->flag = flag; @@ -378,46 +368,55 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter ethr_atomic_init(&lock->r_state, 0); ethr_atomic_init(&lock->w_state, 0); - #ifdef DEBUG ethr_atomic_init(&lock->flowstate, 0); #endif - + lock->n_stats = 1; for (i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { - lcnt_clear_stats(&lock->stats[i]); + lcnt_clear_stats(&lock->stats[i]); } erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock); lcnt_unlock(); } - +/* init empty, instead of zero struct */ +void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) { + lock->next = NULL; + lock->prev = NULL; + lock->flag = 0; + lock->name = NULL; + lock->id = NIL; + ethr_atomic_init(&lock->r_state, 0); + ethr_atomic_init(&lock->w_state, 0); +#ifdef DEBUG + ethr_atomic_init(&lock->flowstate, 0); +#endif + lock->n_stats = 0; + sys_memzero(lock->stats, sizeof(lock->stats)); +} +/* destroy lock */ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { - erts_lcnt_lock_t *deleted_lock; - - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; - + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; lcnt_lock(); if (erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) { - /* copy structure and insert the copy */ - - deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t)); + erts_lcnt_lock_t *deleted_lock; + /* copy structure and insert the copy */ + deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t)); if (!deleted_lock) { ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); } - memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t)); - - deleted_lock->next = NULL; - deleted_lock->prev = NULL; - - erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock); + memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t)); + deleted_lock->next = NULL; + deleted_lock->prev = NULL; + erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock); } /* delete original */ erts_lcnt_list_delete(erts_lcnt_data->current_locks, lock); - lock->flag = 0; - + ERTS_LCNT_CLEAR_FLAG(lock); + lcnt_unlock(); } @@ -426,16 +425,15 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { erts_aint_t r_state = 0, w_state = 0; erts_lcnt_thread_data_t *eltd; - + if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; eltd = lcnt_get_thread_data(); - ASSERT(eltd); - + w_state = ethr_atomic_read(&lock->w_state); - + if (option & ERTS_LCNT_LO_WRITE) { r_state = ethr_atomic_read(&lock->r_state); ethr_atomic_inc( &lock->w_state); @@ -443,48 +441,47 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { if (option & ERTS_LCNT_LO_READ) { ethr_atomic_inc( &lock->r_state); } - + /* we cannot acquire w_lock if either w or r are taken */ /* we cannot acquire r_lock if w_lock is taken */ - + if ((w_state > 0) || (r_state > 0)) { - eltd->lock_in_conflict = 1; - if (eltd->timer_set == 0) { - lcnt_time(&eltd->timer); - } - eltd->timer_set++; + eltd->lock_in_conflict = 1; + if (eltd->timer_set == 0) { + lcnt_time(&eltd->timer); + } + eltd->timer_set++; } else { - eltd->lock_in_conflict = 0; + eltd->lock_in_conflict = 0; } } void erts_lcnt_lock(erts_lcnt_lock_t *lock) { erts_aint_t w_state; erts_lcnt_thread_data_t *eltd; - + if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; w_state = ethr_atomic_read(&lock->w_state); ethr_atomic_inc(&lock->w_state); - eltd = lcnt_get_thread_data(); ASSERT(eltd); if (w_state > 0) { - eltd->lock_in_conflict = 1; - /* only set the timer if nobody else has it - * This should only happen when proc_locks aquires several locks - * 'atomicly'. All other locks will block the thread if w_state > 0 - * i.e. locked. - */ - if (eltd->timer_set == 0) { - lcnt_time(&eltd->timer); - } - eltd->timer_set++; + eltd->lock_in_conflict = 1; + /* only set the timer if nobody else has it + * This should only happen when proc_locks aquires several locks + * 'atomicly'. All other locks will block the thread if w_state > 0 + * i.e. locked. + */ + if (eltd->timer_set == 0) { + lcnt_time(&eltd->timer); + } + eltd->timer_set++; } else { - eltd->lock_in_conflict = 0; + eltd->lock_in_conflict = 0; } } @@ -493,16 +490,19 @@ void erts_lcnt_lock(erts_lcnt_lock_t *lock) { void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock) { /* should check if this thread was "waiting" */ if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; ethr_atomic_dec(&lock->w_state); } -/* erts_lcnt_lock_post - * used when we get a lock (i.e. directly after a lock operation) +/* + * erts_lcnt_lock_post + * + * Used when we get a lock (i.e. directly after a lock operation) * if the timer was set then we had to wait for the lock * lock_post will calculate the wait time. */ + void erts_lcnt_lock_post(erts_lcnt_lock_t *lock) { erts_lcnt_lock_post_x(lock, (char*)str_undefined, 0); } @@ -517,31 +517,31 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; - + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; + #ifdef DEBUG if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 0); - ethr_atomic_inc(&lock->flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); + ASSERT(flowstate == 0); + ethr_atomic_inc(&lock->flowstate); } #endif - + eltd = lcnt_get_thread_data(); - + ASSERT(eltd); /* if lock was in conflict, time it */ stats = lcnt_get_lock_stats(lock, file, line); if (eltd->timer_set) { - lcnt_time(&timer); - - lcnt_time_diff(&time_wait, &timer, &(eltd->timer)); - lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait); - eltd->timer_set--; - ASSERT(eltd->timer_set >= 0); + lcnt_time(&timer); + + lcnt_time_diff(&time_wait, &timer, &(eltd->timer)); + lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait); + eltd->timer_set--; + ASSERT(eltd->timer_set >= 0); } else { - lcnt_update_stats(stats, eltd->lock_in_conflict, NULL); + lcnt_update_stats(stats, eltd->lock_in_conflict, NULL); } } @@ -550,27 +550,28 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_dec(&lock->w_state); if (option & ERTS_LCNT_LO_READ ) ethr_atomic_dec(&lock->r_state); } void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { -#ifdef DEBUG - erts_aint_t w_state; - erts_aint_t flowstate; -#endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; #ifdef DEBUG - /* flowstate */ - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 1); - ethr_atomic_dec(&lock->flowstate); - - /* write state */ - w_state = ethr_atomic_read(&lock->w_state); - ASSERT(w_state > 0); + { + erts_aint_t w_state; + erts_aint_t flowstate; + + /* flowstate */ + flowstate = ethr_atomic_read(&lock->flowstate); + ASSERT(flowstate == 1); + ethr_atomic_dec(&lock->flowstate); + + /* write state */ + w_state = ethr_atomic_read(&lock->w_state); + ASSERT(w_state > 0); + } #endif ethr_atomic_dec(&lock->w_state); } @@ -579,35 +580,34 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; /* Determine lock_state via res instead of state */ if (res != EBUSY) { - if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state); - if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state); - lcnt_update_stats(&(lock->stats[0]), 0, NULL); + if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state); + if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state); + lcnt_update_stats(&(lock->stats[0]), 0, NULL); } else { ethr_atomic_inc(&lock->stats[0].tries); ethr_atomic_inc(&lock->stats[0].colls); } } - + void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { /* Determine lock_state via res instead of state */ -#ifdef DEBUG - erts_aint_t flowstate; -#endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; if (res != EBUSY) { - #ifdef DEBUG - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 0); - ethr_atomic_inc( &lock->flowstate); + { + erts_aint_t flowstate; + flowstate = ethr_atomic_read(&lock->flowstate); + ASSERT(flowstate == 0); + ethr_atomic_inc( &lock->flowstate); + } #endif - ethr_atomic_inc(&lock->w_state); - lcnt_update_stats(&(lock->stats[0]), 0, NULL); + ethr_atomic_inc(&lock->w_state); + lcnt_update_stats(&(lock->stats[0]), 0, NULL); } else { ethr_atomic_inc(&lock->stats[0].tries); ethr_atomic_inc(&lock->stats[0].colls); @@ -662,13 +662,13 @@ void erts_lcnt_clear_counters(void) { lcnt_lock(); list = erts_lcnt_data->current_locks; - + for (lock = list->head; lock != NULL; lock = lock->next) { - for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { - stats = &lock->stats[i]; - lcnt_clear_stats(stats); - } - lock->n_stats = 1; + for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { + stats = &lock->stats[i]; + lcnt_clear_stats(stats); + } + lock->n_stats = 1; } /* empty deleted locks in lock list */ @@ -681,14 +681,14 @@ void erts_lcnt_clear_counters(void) { erts_lcnt_data_t *erts_lcnt_get_data(void) { erts_lcnt_time_t timer_stop; - + lcnt_lock(); - + lcnt_time(&timer_stop); lcnt_time_diff(&(erts_lcnt_data->duration), &timer_stop, &timer_start); - + lcnt_unlock(); - + return erts_lcnt_data; } diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 09fadd7e9e..051f55271d 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -20,7 +20,7 @@ /* * Description: Statistics for locks. * - * Author: Bj�rn-Egil Dahlberg + * Author: Björn-Egil Dahlberg * Date: 2008-07-03 * Abstract: * Locks statistics internal representation. @@ -95,30 +95,35 @@ #define ERTS_LCNT_LO_WRITE (((Uint16) 1) << 7) #define ERTS_LCNT_LO_READ_WRITE ( ERTS_LCNT_LO_READ \ - | ERTS_LCNT_LO_WRITE ) + | ERTS_LCNT_LO_WRITE ) #define ERTS_LCNT_LT_ALL ( ERTS_LCNT_LT_SPINLOCK \ - | ERTS_LCNT_LT_RWSPINLOCK \ - | ERTS_LCNT_LT_MUTEX \ - | ERTS_LCNT_LT_RWMUTEX \ - | ERTS_LCNT_LT_PROCLOCK ) + | ERTS_LCNT_LT_RWSPINLOCK \ + | ERTS_LCNT_LT_MUTEX \ + | ERTS_LCNT_LT_RWMUTEX \ + | ERTS_LCNT_LT_PROCLOCK ) + +#define ERTS_LCNT_LOCK_TYPE(lock) ((lock)->flag & ERTS_LCNT_LT_ALL) +#define ERTS_LCNT_IS_LOCK_INVALID(lock) (!((lock)->flag & ERTS_LCNT_LT_ALL)) +#define ERTS_LCNT_CLEAR_FLAG(lock) ((lock)->flag = 0) + /* runtime options */ -#define ERTS_LCNT_OPT_SUSPEND (((Uint16) 1) << 0) -#define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1) -#define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2) -#define ERTS_LCNT_OPT_COPYSAVE (((Uint16) 1) << 3) -#define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 4) +#define ERTS_LCNT_OPT_SUSPEND (((Uint16) 1) << 0) +#define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1) +#define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2) +#define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 3) +#define ERTS_LCNT_OPT_COPYSAVE (((Uint16) 1) << 4) typedef struct { unsigned long s; unsigned long ns; } erts_lcnt_time_t; - + extern erts_lcnt_time_t timer_start; typedef struct { - Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */ + Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */ } erts_lcnt_hist_t; typedef struct erts_lcnt_lock_stats_s { @@ -129,10 +134,10 @@ typedef struct erts_lcnt_lock_stats_s { char *file; /* which file the lock was taken */ unsigned int line; /* line number in file */ - + ethr_atomic_t tries; /* n tries to get lock */ ethr_atomic_t colls; /* n collisions of tries to get lock */ - + unsigned long timer_n; /* #times waited for lock */ erts_lcnt_time_t timer; /* total wait time for lock */ erts_lcnt_hist_t hist; @@ -155,7 +160,7 @@ typedef struct erts_lcnt_lock_s { /* statistics */ unsigned int n_stats; erts_lcnt_lock_stats_t stats[ERTS_LCNT_MAX_LOCK_LOCATIONS]; /* first entry is "undefined"*/ - + /* chains for list handling */ /* data is hold by lcnt_lock */ struct erts_lcnt_lock_s *prev; @@ -167,7 +172,7 @@ typedef struct { erts_lcnt_lock_t *tail; unsigned long n; } erts_lcnt_lock_list_t; - + typedef struct { erts_lcnt_time_t duration; /* time since last clear */ erts_lcnt_lock_list_t *current_locks; @@ -205,6 +210,7 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock); /* lock operations (global) */ void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag); void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id); +void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock); void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock); void erts_lcnt_lock(erts_lcnt_lock_t *lock); @@ -226,7 +232,5 @@ void erts_lcnt_clear_counters(void); char *erts_lcnt_lock_type(Uint16 type); erts_lcnt_data_t *erts_lcnt_get_data(void); -#define ERTS_LCNT_LOCK_TYPE(lockp) ((lockp)->flag & ERTS_LCNT_LT_ALL) - #endif /* ifdef ERTS_ENABLE_LOCK_COUNT */ #endif /* ifndef ERTS_LOCK_COUNT_H__ */ diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index a1bd39dbc8..95a10daa67 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -32,6 +32,7 @@ #include "erl_process.h" #include "error.h" #include "bif.h" +#include "erl_binary.h" #include "erl_map.h" @@ -79,8 +80,13 @@ typedef struct { static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB); -static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args); -static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB); +static BIF_RETTYPE map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args); +struct HashmapMergeContext_; +static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_args, + struct HashmapMergeContext_*); +static Export hashmap_merge_trap_export; +static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1); +static Uint hashmap_subtree_size(Eterm node); static Eterm hashmap_to_list(Process *p, Eterm map); static Eterm hashmap_keys(Process *p, Eterm map); static Eterm hashmap_values(Process *p, Eterm map); @@ -95,6 +101,15 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]); static int hxnodecmp(hxnode_t* a, hxnode_t* b); static int hxnodecmpkey(hxnode_t* a, hxnode_t* b); + +void erts_init_map(void) { + erts_init_trap_export(&hashmap_merge_trap_export, + am_maps, am_merge_trap, 1, + &maps_merge_trap_1); + return; +} + + /* erlang:map_size/1 * the corresponding instruction is implemented in: * beam/erl_bif_guard.c @@ -410,8 +425,9 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) { } UnUseTmpHeap(2,p); - factory.p = p; + erts_factory_proc_init(&factory, p); res = hashmap_from_unsorted_array(&factory, hxns, size, 0); + erts_factory_close(&factory); erts_free(ERTS_ALC_T_TMP, (void *) hxns); ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); @@ -515,8 +531,9 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n hxns[i].i = i; } - factory.p = p; + erts_factory_proc_init(&factory, p); res = hashmap_from_unsorted_array(&factory, hxns, sz, 0); + erts_factory_close(&factory); erts_free(ERTS_ALC_T_TMP, (void *) hxns); ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); @@ -942,15 +959,15 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) { BIF_RET(flatmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2)); } else if (is_hashmap(BIF_ARG_2)) { /* Will always become a tree */ - BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0)); + return map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0); } BIF_P->fvalue = BIF_ARG_2; } else if (is_hashmap(BIF_ARG_1)) { if (is_hashmap(BIF_ARG_2)) { - BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2)); + return hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2, 0, NULL); } else if (is_flatmap(BIF_ARG_2)) { /* Will always become a tree */ - BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1)); + return map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1); } BIF_P->fvalue = BIF_ARG_2; } else { @@ -1063,8 +1080,9 @@ static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { hxns[i].i = i; } - factory.p = p; + erts_factory_proc_init(&factory, p); res = hashmap_from_unsorted_array(&factory, hxns, n, 0); + erts_factory_close(&factory); erts_free(ERTS_ALC_T_TMP, (void *) hxns); ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); @@ -1107,36 +1125,71 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args) hxns[i].i = i; } - factory.p = p; + erts_factory_proc_init(&factory, p); res = hashmap_from_unsorted_array(&factory, hxns, n, 0); + erts_factory_close(&factory); erts_free(ERTS_ALC_T_TMP, (void *) hxns); ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); - return swap_args ? hashmap_merge(p, tree, res) : hashmap_merge(p, res, tree); + return hashmap_merge(p, res, tree, swap_args, NULL); +} + +#define PSTACK_TYPE struct HashmapMergePStackType +struct HashmapMergePStackType { + Eterm nodeA, nodeB; + Eterm *srcA, *srcB; + Uint32 abm, bbm, rbm; /* node bitmaps */ + int mix; /* &1: there are unique A stuff in node + * &2: there are unique B stuff in node */ + int ix; + Eterm array[16]; /* temp node construction area */ +}; + +typedef struct HashmapMergeContext_ { + Uint size; /* total key-value counter */ + unsigned int lvl; + Eterm trap_bin; + ErtsPStack pstack; +#ifdef DEBUG + Eterm dbg_map_A, dbg_map_B; +#endif +} HashmapMergeContext; + +static void hashmap_merge_ctx_destructor(Binary* ctx_bin) +{ + HashmapMergeContext* ctx = (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); + + PSTACK_DESTROY_SAVED(&ctx->pstack); +} + +BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) { + Binary* ctx_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); + + return hashmap_merge(BIF_P, NIL, NIL, 0, + (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin)); } #define HALLOC_EXTRA 200 +#define MAP_MERGE_LOOP_FACTOR 8 -static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { +static BIF_RETTYPE hashmap_merge(Process *p, Eterm map_A, Eterm map_B, + int swap_args, HashmapMergeContext* ctx) { #define PSTACK_TYPE struct HashmapMergePStackType - struct HashmapMergePStackType { - Eterm *srcA, *srcB; - Uint32 abm, bbm, rbm; /* node bitmaps */ - int keepA; - int ix; - Eterm array[16]; - }; PSTACK_DECLARE(s, 4); - struct HashmapMergePStackType* sp = PSTACK_PUSH(s); - Eterm *hp, *nhp; + HashmapMergeContext local_ctx; + struct HashmapMergePStackType* sp; + Uint32 hx; + Eterm res = THE_NON_VALUE; Eterm hdrA, hdrB; - Uint32 ahx, bhx; - Uint size; /* total key-value counter */ - int keepA = 0; - unsigned int lvl = 0; + Eterm *hp, *nhp; + Eterm trap_ret; + Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * MAP_MERGE_LOOP_FACTOR); + Sint reds = initial_reds; DeclareTmpHeap(th,2,p); - Eterm res = THE_NON_VALUE; UseTmpHeap(2,p); /* @@ -1144,152 +1197,139 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { * and merge each pair of nodes. */ - { - hashmap_head_t* a = (hashmap_head_t*) hashmap_val(nodeA); - hashmap_head_t* b = (hashmap_head_t*) hashmap_val(nodeB); - size = a->size + b->size; + PSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + + if (ctx == NULL) { /* first call */ + hashmap_head_t* a = (hashmap_head_t*) hashmap_val(map_A); + hashmap_head_t* b = (hashmap_head_t*) hashmap_val(map_B); + + sp = PSTACK_PUSH(s); + sp->srcA = swap_args ? &map_B : &map_A; + sp->srcB = swap_args ? &map_A : &map_B; + sp->mix = 0; + local_ctx.size = a->size + b->size; + local_ctx.lvl = 0; + #ifdef DEBUG + local_ctx.dbg_map_A = map_A; + local_ctx.dbg_map_B = map_B; + local_ctx.trap_bin = THE_NON_VALUE; + #endif + ctx = &local_ctx; + } + else { + PSTACK_RESTORE(s, &ctx->pstack); + sp = PSTACK_TOP(s); + goto resume_from_trap; } recurse: - if (primary_tag(nodeA) == TAG_PRIMARY_BOXED && - primary_tag(nodeB) == TAG_PRIMARY_LIST) { - /* Avoid implementing this combination by switching places */ - Eterm tmp = nodeA; - nodeA = nodeB; - nodeB = tmp; - keepA = !keepA; - } - - switch (primary_tag(nodeA)) { - case TAG_PRIMARY_LIST: { - sp->srcA = list_val(nodeA); - switch (primary_tag(nodeB)) { - case TAG_PRIMARY_LIST: { /* LEAF + LEAF */ - sp->srcB = list_val(nodeB); - - if (EQ(CAR(sp->srcA), CAR(sp->srcB))) { - --size; - res = keepA ? nodeA : nodeB; - } else { - ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA)); - bhx = hashmap_restore_hash(th, lvl, CAR(sp->srcB)); - sp->abm = 1 << hashmap_index(ahx); - sp->bbm = 1 << hashmap_index(bhx); + sp->nodeA = *sp->srcA; + sp->nodeB = *sp->srcB; - sp->srcA = &nodeA; - sp->srcB = &nodeB; - } - break; - } - case TAG_PRIMARY_BOXED: { /* LEAF + NODE */ - sp->srcB = boxed_val(nodeB); - ASSERT(is_header(*sp->srcB)); - hdrB = *sp->srcB++; - - ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA)); - sp->abm = 1 << hashmap_index(ahx); - sp->srcA = &nodeA; - switch(hdrB & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: - sp->srcB++; - sp->bbm = 0xffff; - break; + if (sp->nodeA == sp->nodeB) { + res = sp->nodeA; + ctx->size -= is_list(sp->nodeB) ? 1 : hashmap_subtree_size(sp->nodeB); + } + else { + if (is_list(sp->nodeA)) { /* A is LEAF */ + Eterm keyA = CAR(list_val(sp->nodeA)); + + if (is_list(sp->nodeB)) { /* LEAF + LEAF */ + Eterm keyB = CAR(list_val(sp->nodeB)); + + if (EQ(keyA, keyB)) { + --ctx->size; + res = sp->nodeB; + sp->mix = 2; /* We assume values differ. + + Don't spend time comparing big values. + - Might waste some heap space for internal + nodes that could otherwise be reused. */ + goto merge_nodes; + } + } + hx = hashmap_restore_hash(th, ctx->lvl, keyA); + sp->abm = 1 << hashmap_index(hx); + /* keep srcA pointing at the leaf */ + } + else { /* A is NODE */ + sp->srcA = boxed_val(sp->nodeA); + hdrA = *sp->srcA++; + ASSERT(is_header(hdrA)); + switch (hdrA & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: { + sp->srcA++; + sp->abm = 0xffff; + break; + } + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++; + case HAMT_SUBTAG_NODE_BITMAP: { + sp->abm = MAP_HEADER_VAL(hdrA); + break; + } + default: + erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA); + } + } - case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; - case HAMT_SUBTAG_NODE_BITMAP: - sp->bbm = MAP_HEADER_VAL(hdrB); - break; + if (is_list(sp->nodeB)) { /* B is LEAF */ + Eterm keyB = CAR(list_val(sp->nodeB)); - default: - erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK); - break; - } - break; - } - default: - erl_exit(1, "bad primary tag %ld\r\n", nodeB); - } - break; - } - case TAG_PRIMARY_BOXED: { /* NODE + NODE */ - sp->srcA = boxed_val(nodeA); - hdrA = *sp->srcA++; - ASSERT(is_header(hdrA)); - switch (hdrA & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: { - sp->srcA++; - ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED); - sp->abm = 0xffff; - sp->srcB = boxed_val(nodeB); - hdrB = *sp->srcB++; - ASSERT(is_header(hdrB)); - switch (hdrB & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: - sp->srcB++; - sp->bbm = 0xffff; - break; - case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; - case HAMT_SUBTAG_NODE_BITMAP: - sp->bbm = MAP_HEADER_VAL(hdrB); - break; - default: - erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK); - } - break; - } - case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++; - case HAMT_SUBTAG_NODE_BITMAP: { - ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED); - sp->abm = MAP_HEADER_VAL(hdrA); - sp->srcB = boxed_val(nodeB); - hdrB = *sp->srcB++; - ASSERT(is_header(hdrB)); - switch (hdrB & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: + hx = hashmap_restore_hash(th, ctx->lvl, keyB); + sp->bbm = 1 << hashmap_index(hx); + /* keep srcB pointing at the leaf */ + } + else { /* B is NODE */ + sp->srcB = boxed_val(sp->nodeB); + hdrB = *sp->srcB++; + ASSERT(is_header(hdrB)); + switch (hdrB & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: { sp->srcB++; - sp->bbm = 0xffff; - break; - case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; - case HAMT_SUBTAG_NODE_BITMAP: - sp->bbm = MAP_HEADER_VAL(hdrB); - break; - - default: - erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK); - } - break; - } - default: - erl_exit(1, "bad primary tag %ld\r\n", nodeA); - } - break; - } - default: - erl_exit(1, "bad primary tag %ld\r\n", nodeA); + sp->bbm = 0xffff; + break; + } + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; + case HAMT_SUBTAG_NODE_BITMAP: { + sp->bbm = MAP_HEADER_VAL(hdrB); + break; + } + default: + erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB); + } + } } +merge_nodes: + for (;;) { if (is_value(res)) { /* We have a complete (sub-)tree or leaf */ - if (lvl == 0) + int child_mix; + if (ctx->lvl == 0) break; /* Pop from stack and continue build parent node */ - lvl--; + ctx->lvl--; + child_mix = sp->mix; sp = PSTACK_POP(s); sp->array[sp->ix++] = res; + sp->mix |= child_mix; res = THE_NON_VALUE; if (sp->rbm) { sp->srcA++; sp->srcB++; - keepA = sp->keepA; } } else { /* Start build a node */ sp->ix = 0; sp->rbm = sp->abm | sp->bbm; - ASSERT(!(sp->rbm == 0 && lvl > 0)); + ASSERT(!(sp->rbm == 0 && ctx->lvl > 0)); } + if (--reds <= 0) { + goto trap; + } +resume_from_trap: + while (sp->rbm) { Uint32 next = sp->rbm & (sp->rbm-1); Uint32 bit = sp->rbm ^ next; @@ -1297,43 +1337,123 @@ recurse: if (sp->abm & bit) { if (sp->bbm & bit) { /* Bit clash. Push and resolve by recursive merge */ - if (sp->rbm) { - sp->keepA = keepA; - } - nodeA = *sp->srcA; - nodeB = *sp->srcB; - lvl++; + Eterm* srcA = sp->srcA; + Eterm* srcB = sp->srcB; + ctx->lvl++; sp = PSTACK_PUSH(s); + sp->srcA = srcA; + sp->srcB = srcB; + sp->mix = 0; goto recurse; } else { sp->array[sp->ix++] = *sp->srcA++; + sp->mix |= 1; } } else { ASSERT(sp->bbm & bit); sp->array[sp->ix++] = *sp->srcB++; + sp->mix |= 2; } } - ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm)); - if (lvl == 0) { - nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA); - hp = nhp; - *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY - : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm)); - *hp++ = size; - } else { - nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA); - hp = nhp; - *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm); - } - memcpy(hp, sp->array, sp->ix * sizeof(Eterm)); - res = make_boxed(nhp); + switch (sp->mix) { + case 0: /* Nodes A and B contain the *EXACT* same sub-trees + => fall through and reuse nodeA */ + + case 1: /* Only unique A stuff => reuse nodeA */ + res = sp->nodeA; + break; + + case 2: /* Only unique B stuff => reuse nodeB */ + res = sp->nodeB; + break; + + case 3: /* We have a mix => must build new node */ + ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm)); + if (ctx->lvl == 0) { + nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA); + hp = nhp; + *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY + : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm)); + *hp++ = ctx->size; + } else { + nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA); + hp = nhp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm); + } + sys_memcpy(hp, sp->array, sp->ix * sizeof(Eterm)); + res = make_boxed(nhp); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix); + } + } + + /* Done */ + +#ifdef DEBUG + { + Eterm *head = hashmap_val(res); + Uint size = head[1]; + Uint real_size = hashmap_subtree_size(res); + ASSERT(size == real_size); + } +#endif + + if (ctx != &local_ctx) { + ASSERT(ctx->trap_bin != THE_NON_VALUE); + ASSERT(p->flags & F_DISABLE_GC); + erts_set_gc_state(p, 1); + } + else { + ASSERT(ctx->trap_bin == THE_NON_VALUE); + ASSERT(!(p->flags & F_DISABLE_GC)); } PSTACK_DESTROY(s); UnUseTmpHeap(2,p); + BUMP_REDS(p, (initial_reds - reds) / MAP_MERGE_LOOP_FACTOR); return res; + +trap: /* Yield */ + + if (ctx == &local_ctx) { + Binary* ctx_b = erts_create_magic_binary(sizeof(HashmapMergeContext), + hashmap_merge_ctx_destructor); + ctx = ERTS_MAGIC_BIN_DATA(ctx_b); + sys_memcpy(ctx, &local_ctx, sizeof(HashmapMergeContext)); + hp = HAlloc(p, PROC_BIN_SIZE); + ASSERT(ctx->trap_bin == THE_NON_VALUE); + ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), ctx_b); + + erts_set_gc_state(p, 0); + } + else { + ASSERT(ctx->trap_bin != THE_NON_VALUE); + ASSERT(p->flags & F_DISABLE_GC); + } + + PSTACK_SAVE(s, &ctx->pstack); + + BUMP_ALL_REDS(p); + ERTS_BIF_PREP_TRAP1(trap_ret, &hashmap_merge_trap_export, + p, ctx->trap_bin); + UnUseTmpHeap(2,p); + return trap_ret; } +static Uint hashmap_subtree_size(Eterm node) { + DECLARE_WSTACK(stack); + Uint size = 0; + + hashmap_iterator_init(&stack, node, 0); + while (hashmap_iterator_next(&stack)) { + size++; + } + DESTROY_WSTACK(stack); + return size; +} + + static int hash_cmp(Uint32 ha, Uint32 hb) { int i; @@ -1756,10 +1876,11 @@ void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) { sz = 16; break; case HAMT_SUBTAG_HEAD_BITMAP: - sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); break; default: - erl_exit(1, "bad header"); + erl_exit(ERTS_ABORT_EXIT, "bad header"); } WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE, /* end marker */ @@ -1796,7 +1917,7 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) { ASSERT(sz < 17); break; default: - erl_exit(1, "bad header"); + erl_exit(ERTS_ABORT_EXIT, "bad header"); } idx++; @@ -2504,6 +2625,9 @@ int erts_validate_and_sort_flatmap(flatmap_t* mp) return 1; } +#if 0 /* Can't get myself to remove this beautiful piece of code + for probabilistic overestimation of nr of nodes in a hashmap */ + /* Really rough estimate of sqrt(x) * Guaranteed not to be less than sqrt(x) */ @@ -2525,7 +2649,10 @@ static int int_sqrt_ceiling(Uint x) } } -Uint hashmap_over_estimated_heap_size(Uint k) +/* May not be enough if hashing is broken (not uniform) + * or if hell freezes over. + */ +Uint hashmap_overestimated_node_count(Uint k) { /* k is nr of key-value pairs. N(k) is expected nr of nodes in hamt. @@ -2539,12 +2666,9 @@ Uint hashmap_over_estimated_heap_size(Uint k) by 15 std.devs above the average, which gives a probability for overrun less than 1.0e-49 (same magnitude as a git SHA1 collision). */ - Uint max_nodes = 2*k/5 + (15/3)*int_sqrt_ceiling(k); - return (k*2 + /* leaf cons cells */ - k + /* leaf list terms */ - max_nodes*2); /* headers + parent boxed terms */ + return 2*k/5 + 1 + (15/3)*int_sqrt_ceiling(k); } - +#endif BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) { if (is_hashmap(BIF_ARG_1)) { diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 2cc6768bfc..b5941c5c9a 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -91,7 +91,6 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, Uint *upsz, struct ErtsEStack_ *sp); int erts_validate_and_sort_flatmap(flatmap_t* map); -Uint hashmap_over_estimated_heap_size(Uint n); void hashmap_iterator_init(struct ErtsWStack_* s, Eterm node, int reverse); Eterm* hashmap_iterator_next(struct ErtsWStack_* s); Eterm* hashmap_iterator_prev(struct ErtsWStack_* s); @@ -191,5 +190,18 @@ typedef struct hashmap_head_s { #define hashmap_index(hash) (((Uint32)hash) & 0xf) +/* hashmap heap size: + [one cons cell + one list term in parent node] per key + [one header + one boxed term in parent node] per inner node + [one header + one size word] for root node +*/ +#define HASHMAP_HEAP_SIZE(KEYS,NODES) ((KEYS)*3 + (NODES)*2) +#ifdef DEBUG +# define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (KEYS) +#else +# define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (2*(KEYS)/5) +#endif +#define HASHMAP_ESTIMATED_HEAP_SIZE(KEYS) \ + HASHMAP_HEAP_SIZE(KEYS,HASHMAP_ESTIMATED_NODE_COUNT(KEYS)) #endif diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index ccfc2e6458..f806d2c498 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -93,9 +93,6 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, #endif ErlHeapFragment* nbp; - /* ToDo: Make use of 'used_size' to avoid realloc - when shrinking just a few words */ - #ifdef DEBUG { Uint off_sz = size < bp->used_size ? size : bp->used_size; @@ -110,8 +107,10 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, } #endif - if (size == bp->used_size) + if (size >= (bp->used_size - bp->used_size / 16)) { + bp->used_size = size; return bp; + } #ifdef HARD_DEBUG dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size); @@ -237,8 +236,7 @@ erts_msg_distext2heap(Process *pp, Eterm msg; Uint tok_sz = 0; Eterm *hp = NULL; - Eterm *hp_end = NULL; - ErlOffHeap *ohp; + ErtsHeapFactory factory; Sint sz; *bpp = NULL; @@ -250,36 +248,26 @@ erts_msg_distext2heap(Process *pp, tok_sz = heap_frag->used_size; sz += tok_sz; } - if (pp) + if (pp) { + ErlOffHeap *ohp; hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp); + } else { *bpp = new_message_buffer(sz); hp = (*bpp)->mem; - ohp = &(*bpp)->off_heap; } - hp_end = hp + sz; - msg = erts_decode_dist_ext(&hp, ohp, dist_extp); + erts_factory_message_init(&factory, pp, hp, *bpp); + msg = erts_decode_dist_ext(&factory, dist_extp); if (is_non_value(msg)) goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - *tokenp = copy_struct(*tokenp, tok_sz, &hp, ohp); + hp = erts_produce_heap(&factory, tok_sz, 0); + *tokenp = copy_struct(*tokenp, tok_sz, &hp, factory.off_heap); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); - if (hp_end != hp) { - if (!(*bpp)) { - HRelease(pp, hp_end, hp); - } - else { - Uint final_size = hp - &(*bpp)->mem[0]; - Eterm brefs[2] = {msg, *tokenp}; - ASSERT(sz - (hp_end - hp) == final_size); - *bpp = erts_resize_message_buffer(*bpp, final_size, &brefs[0], 2); - msg = brefs[0]; - *tokenp = brefs[1]; - } - } + erts_factory_close(&factory); return msg; decode_error: @@ -288,13 +276,7 @@ erts_msg_distext2heap(Process *pp, erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); - if (*bpp) { - free_message_buffer(*bpp); - *bpp = NULL; - } - else if (hp) { - HRelease(pp, hp_end, hp); - } + *bpp = NULL; return THE_NON_VALUE; } @@ -851,10 +833,11 @@ erts_msg_attached_data_size_aux(ErlMessage *msg) } void -erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *msg) +erts_move_msg_attached_data_to_heap(ErtsHeapFactory* factory, + ErlMessage *msg) { if (is_value(ERL_MESSAGE_TERM(msg))) - erts_move_msg_mbuf_to_heap(hpp, ohp, msg); + erts_move_msg_mbuf_to_heap(&factory->hp, factory->off_heap, msg); else if (msg->data.dist_ext) { ASSERT(msg->data.dist_ext->heap_size >= 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { @@ -862,12 +845,11 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg), heap_frag->used_size, - hpp, - ohp); + &factory->hp, + factory->off_heap); erts_cleanup_offheap(&heap_frag->off_heap); } - ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(hpp, - ohp, + ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(factory, msg->data.dist_ext); erts_free_dist_ext_copy(msg->data.dist_ext); msg->data.dist_ext = NULL; @@ -1134,15 +1116,281 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, } } +void erts_factory_proc_init(ErtsHeapFactory* factory, + Process* p) +{ + erts_factory_proc_prealloc_init(factory, p, HEAP_LIMIT(p) - HEAP_TOP(p)); +} + +void erts_factory_proc_prealloc_init(ErtsHeapFactory* factory, + Process* p, + Sint size) +{ + factory->mode = FACTORY_HALLOC; + factory->p = p; + factory->hp_start = HAlloc(p, size); + factory->hp = factory->hp_start; + factory->hp_end = factory->hp_start + size; + factory->off_heap = &p->off_heap; + factory->off_heap_saved.first = p->off_heap.first; + factory->off_heap_saved.overhead = p->off_heap.overhead; + factory->heap_frags_saved = p->mbuf; + factory->heap_frags = NULL; /* not used */ + factory->alloc_type = 0; /* not used */ +} + +void erts_factory_message_init(ErtsHeapFactory* factory, + Process* rp, + Eterm* hp, + ErlHeapFragment* bp) +{ + if (bp) { + factory->mode = FACTORY_HEAP_FRAGS; + factory->p = NULL; + factory->hp_start = bp->mem; + factory->hp = hp ? hp : bp->mem; + factory->hp_end = bp->mem + bp->alloc_size; + factory->off_heap = &bp->off_heap; + factory->heap_frags = bp; + factory->heap_frags_saved = bp; + factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; + ASSERT(!bp->next); + } + else { + factory->mode = FACTORY_HALLOC; + factory->p = rp; + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = HEAP_TOP(rp); + factory->off_heap = &rp->off_heap; + factory->heap_frags_saved = rp->mbuf; + factory->heap_frags = NULL; /* not used */ + factory->alloc_type = 0; /* not used */ + } + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; + + ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end); +} + +void erts_factory_static_init(ErtsHeapFactory* factory, + Eterm* hp, + Uint size, + ErlOffHeap* off_heap) +{ + factory->mode = FACTORY_STATIC; + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = hp + size; + factory->off_heap = off_heap; + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; +} + +/* When we know the term is an immediate and need no heap. +*/ +void erts_factory_dummy_init(ErtsHeapFactory* factory) +{ + factory->mode = FACTORY_CLOSED; +} + +static void reserve_heap(ErtsHeapFactory*, Uint need, Uint xtra); + Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) { Eterm* res; - if (factory->p) { - res = HAllocX(factory->p, need, xtra); - } else { - res = factory->hp; - factory->hp += need; + + ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); + if (factory->hp + need > factory->hp_end) { + reserve_heap(factory, need, xtra); } + res = factory->hp; + factory->hp += need; return res; } +Eterm* erts_reserve_heap(ErtsHeapFactory* factory, Uint need) +{ + ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); + if (factory->hp + need > factory->hp_end) { + reserve_heap(factory, need, 200); + } + return factory->hp; +} + +static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) +{ + ErlHeapFragment* bp; + + switch (factory->mode) { + case FACTORY_HALLOC: + HRelease(factory->p, factory->hp_end, factory->hp); + factory->hp = HAllocX(factory->p, need, xtra); + factory->hp_end = factory->hp + need; + return; + + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + + if (bp) { + ASSERT(factory->hp > bp->mem); + ASSERT(factory->hp <= factory->hp_end); + ASSERT(factory->hp_end == bp->mem + bp->alloc_size); + + bp->used_size = factory->hp - bp->mem; + } + bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type, + ERTS_HEAP_FRAG_SIZE(need+xtra)); + bp->next = factory->heap_frags; + factory->heap_frags = bp; + bp->alloc_size = need + xtra; + bp->used_size = need; + bp->off_heap.first = NULL; + bp->off_heap.overhead = 0; + + factory->hp = bp->mem; + factory->hp_end = bp->mem + bp->alloc_size; + return; + + case FACTORY_STATIC: + case FACTORY_CLOSED: + default: + ASSERT(!"Invalid factory mode"); + } +} + +void erts_factory_close(ErtsHeapFactory* factory) +{ + ErlHeapFragment* bp; + + switch (factory->mode) { + case FACTORY_HALLOC: + HRelease(factory->p, factory->hp_end, factory->hp); + break; + + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + + if (bp) { + ASSERT(factory->hp >= bp->mem); + ASSERT(factory->hp <= factory->hp_end); + ASSERT(factory->hp_end == bp->mem + bp->alloc_size); + + bp->used_size = factory->hp - bp->mem; + } + break; + case FACTORY_STATIC: break; + case FACTORY_CLOSED: break; + default: + ASSERT(!"Invalid factory mode"); + } + factory->mode = FACTORY_CLOSED; +} + +void erts_factory_trim_and_close(ErtsHeapFactory* factory, + Eterm *brefs, Uint brefs_size) +{ + if (factory->mode == FACTORY_HEAP_FRAGS) { + ErlHeapFragment* bp = factory->heap_frags; + if (bp->next == NULL) { + Uint used_sz = factory->hp - bp->mem; + ASSERT(used_sz <= bp->alloc_size); + factory->heap_frags = erts_resize_message_buffer(bp, used_sz, + brefs, brefs_size); + factory->mode = FACTORY_CLOSED; + return; + } + /*else we don't trim multi fragmented messages for now */ + } + erts_factory_close(factory); +} + +void erts_factory_undo(ErtsHeapFactory* factory) +{ + ErlHeapFragment* bp; + struct erl_off_heap_header *hdr, **hdr_nextp; + + switch (factory->mode) { + case FACTORY_HALLOC: + case FACTORY_STATIC: + /* Cleanup off-heap + */ + hdr_nextp = NULL; + for (hdr = factory->off_heap->first; + hdr != factory->off_heap_saved.first; + hdr = hdr->next) { + + hdr_nextp = &hdr->next; + } + + if (hdr_nextp != NULL) { + *hdr_nextp = NULL; + erts_cleanup_offheap(factory->off_heap); + factory->off_heap->first = factory->off_heap_saved.first; + factory->off_heap->overhead = factory->off_heap_saved.overhead; + } + + if (factory->mode == FACTORY_HALLOC) { + /* Free heap frags + */ + bp = factory->p->mbuf; + if (bp != factory->heap_frags_saved) { + do { + ErlHeapFragment *next_bp = bp->next; + ASSERT(bp->off_heap.first == NULL); + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->alloc_size)); + bp = next_bp; + } while (bp != factory->heap_frags_saved); + + factory->p->mbuf = bp; + } + + /* Rollback heap top + */ + if (factory->heap_frags_saved == NULL) { /* No heap frags when we started */ + ASSERT(factory->hp_start >= HEAP_START(factory->p)); + ASSERT(factory->hp_start <= HEAP_LIMIT(factory->p)); + + HEAP_TOP(factory->p) = factory->hp_start; + } + else { + ASSERT(factory->heap_frags_saved == factory->p->mbuf); + if (factory->hp_start == factory->heap_frags_saved->mem) { + factory->p->mbuf = factory->p->mbuf->next; + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved, + ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size)); + } + else if (factory->hp_start != factory->hp_end) { + unsigned remains = factory->hp_start - factory->heap_frags_saved->mem; + ASSERT(remains > 0 && remains < factory->heap_frags_saved->used_size); + factory->heap_frags_saved->used_size = remains; + } + } + } + break; + + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + do { + ErlHeapFragment* next_bp = bp->next; + + erts_cleanup_offheap(&bp->off_heap); + ERTS_HEAP_FREE(factory->alloc_type, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->size)); + bp = next_bp; + }while (bp != NULL); + break; + + case FACTORY_CLOSED: break; + default: + ASSERT(!"Invalid factory mode"); + } + factory->mode = FACTORY_CLOSED; +#ifdef DEBUG + factory->p = NULL; + factory->hp = NULL; + factory->heap_frags = NULL; +#endif +} + diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 1e1dafee90..705ba5e506 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -51,6 +51,45 @@ typedef struct erl_off_heap { (OHP)->first = NULL; \ (OHP)->overhead = 0; \ } while (0) + +typedef struct { + enum { + FACTORY_CLOSED = 0, + FACTORY_HALLOC, + FACTORY_HEAP_FRAGS, + FACTORY_STATIC + } mode; + Process* p; + Eterm* hp_start; + Eterm* hp; + Eterm* hp_end; + struct erl_heap_fragment* heap_frags; + struct erl_heap_fragment* heap_frags_saved; + ErlOffHeap* off_heap; + ErlOffHeap off_heap_saved; + Uint32 alloc_type; +} ErtsHeapFactory; + +void erts_factory_proc_init(ErtsHeapFactory*, Process*); +void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size); +void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*); +void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*); +void erts_factory_dummy_init(ErtsHeapFactory*); + +Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); +Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need); +void erts_factory_close(ErtsHeapFactory*); +void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size); +void erts_factory_undo(ErtsHeapFactory*); + +#ifdef CHECK_FOR_HOLES +# define ERTS_FACTORY_HOLE_CHECK(f) do { \ + /*if ((f)->p) erts_check_for_holes((f)->p);*/ \ + } while (0) +#else +# define ERTS_FACTORY_HOLE_CHECK(p) +#endif + #include "external.h" #include "erl_process.h" @@ -68,21 +107,6 @@ struct erl_heap_fragment { Eterm mem[1]; /* Data */ }; -typedef struct { - Process* p; - Eterm* hp; -} ErtsHeapFactory; - -Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); -#ifdef CHECK_FOR_HOLES -# define ERTS_FACTORY_HOLE_CHECK(f) do { \ - if ((f)->p) erts_check_for_holes((f)->p); \ - } while (0) -#else -# define ERTS_FACTORY_HOLE_CHECK(p) -#endif - - typedef struct erl_mesg { struct erl_mesg* next; /* Next message */ union { @@ -139,7 +163,7 @@ typedef struct { *(p)->msg.last = (mp); \ (p)->msg.last = &(mp)->next; \ (p)->msg.len++; \ -} while(0) +} while (0) #ifdef ERTS_SMP @@ -212,17 +236,23 @@ do { \ do { \ if ((M)->data.attached) { \ Uint need__ = erts_msg_attached_data_size((M)); \ + { SWPO ; } \ if ((ST) - (HT) >= need__) { \ - Uint *htop__ = (HT); \ - erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\ - ASSERT(htop__ - (HT) <= need__); \ - (HT) = htop__; \ + ErtsHeapFactory factory__; \ + erts_factory_proc_prealloc_init(&factory__, (P), need__); \ + erts_move_msg_attached_data_to_heap(&factory__, (M)); \ + erts_factory_close(&factory__); \ + if ((P)->mbuf != NULL) { \ + /* Heap was exhausted by messages. This is a rare case */ \ + /* that can currently (OTP 18) only happen if hamts are */ \ + /* far exceeding the estimated heap size. Do GC. */ \ + (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ + } \ } \ else { \ - { SWPO ; } \ (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ - { SWPI ; } \ } \ + { SWPI ; } \ ASSERT(!(M)->data.attached); \ } \ } while (0) @@ -266,23 +296,21 @@ void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *); Uint erts_msg_attached_data_size_aux(ErlMessage *msg); -void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *); - +void erts_move_msg_attached_data_to_heap(ErtsHeapFactory*, ErlMessage *); Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, Eterm *, ErtsDistExternal *); void erts_cleanup_offheap(ErlOffHeap *offheap); -ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg); +ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg) +ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp) { - const ErlHeapFragment *bp; Uint sz = 0; - for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) { + for ( ; bp!=NULL; bp=bp->next) { sz += bp->used_size; } return sz; @@ -292,7 +320,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) { ASSERT(msg->data.attached); if (is_value(ERL_MESSAGE_TERM(msg))) - return erts_msg_used_frag_sz(msg); + return erts_used_frag_sz(msg->data.heap_frag); else if (msg->data.dist_ext->heap_size < 0) return erts_msg_attached_data_size_aux(msg); else { diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 45fc949b81..27f6c6f00d 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1208,7 +1208,11 @@ typedef struct enif_resource_t struct enif_resource_type_t* type; #ifdef DEBUG erts_refc_t nif_refc; +# ifdef ARCH_32 + byte align__[4]; +# endif #endif + char data[1]; }ErlNifResource; @@ -1384,7 +1388,7 @@ static void rollback_opened_resource_types(void) static void nif_resource_dtor(Binary* bin) { - ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(bin); + ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ErlNifResourceType* type = resource->type; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); @@ -1405,8 +1409,10 @@ static void nif_resource_dtor(Binary* bin) void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { - Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor); - ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin); + Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size), + &nif_resource_dtor, + 1); /* unaligned */ + ErlNifResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */ resource->type = type; @@ -1421,7 +1427,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size) void enif_release_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); #ifdef DEBUG @@ -1435,7 +1441,7 @@ void enif_release_resource(void* obj) void enif_keep_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); #ifdef DEBUG @@ -1447,7 +1453,7 @@ void enif_keep_resource(void* obj) ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); } @@ -1476,7 +1482,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ return 0; / * Or should we allow "resource binaries" as handles? * / }*/ mbin = pb->val; - resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin); + resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { return 0; @@ -1488,8 +1494,8 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ size_t enif_sizeof_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - Binary* bin = &ERTS_MAGIC_BIN_FROM_DATA(resource)->binary; - return ERTS_MAGIC_BIN_DATA_SIZE(bin) - offsetof(ErlNifResource,data); + Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary; + return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErlNifResource,data); } @@ -2712,6 +2718,8 @@ erts_unload_nif(struct erl_module_nif* lib) void erl_nif_init() { + ERTS_CT_ASSERT((offsetof(ErlNifResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN); + resource_type_list.next = &resource_type_list; resource_type_list.prev = &resource_type_list; resource_type_list.dtor = NULL; diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 6d827c6bda..0950d7e7ef 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -51,6 +51,9 @@ static Uint dist_entries; static int references_atoms_need_init = 1; +static ErtsMonotonicTime orig_node_tab_delete_delay; +static ErtsMonotonicTime node_tab_delete_delay; + /* -- The distribution table ---------------------------------------------- */ #ifdef DEBUG @@ -290,21 +293,46 @@ DistEntry *erts_find_dist_entry(Eterm sysname) return res; } -void erts_delete_dist_entry(DistEntry *dep) +static void try_delete_dist_entry(void *vdep) +{ + DistEntry *dep = (DistEntry *) vdep; + erts_aint_t refc; + + erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + /* + * Another thread might have looked up this dist entry after + * we decided to delete it (refc became zero). If so, the other + * thread incremented refc twice. Once for the new reference + * and once for this thread. + * + * If refc reach -1, noone has used the entry since we + * set up the timer. Delete the entry. + * + * If refc reach 0, the entry is currently not in use + * but has been used since we set up the timer. Set up a + * new timer. + * + * If refc > 0, the entry is in use. Keep the entry. + */ + refc = erts_refc_dectest(&dep->refc, -1); + if (refc == -1) + (void) hash_erase(&erts_dist_table, (void *) dep); + erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + + if (refc == 0) + erts_schedule_delete_dist_entry(dep); +} + +void erts_schedule_delete_dist_entry(DistEntry *dep) { ASSERT(dep != erts_this_dist_entry); - if(dep != erts_this_dist_entry) { - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); - /* - * Another thread might have looked up this dist entry after - * we decided to delete it (refc became zero). If so, the other - * thread incremented refc twice. Once for the new reference - * and once for this thread. Therefore, delete dist entry if - * refc is 0 or -1 after a decrement. - */ - if (erts_refc_dectest(&dep->refc, -1) <= 0) - (void) hash_erase(&erts_dist_table, (void *) dep); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + if (dep != erts_this_dist_entry) { + if (node_tab_delete_delay == 0) + try_delete_dist_entry((void *) dep); + else if (node_tab_delete_delay > 0) + erts_start_timer_callback(node_tab_delete_delay, + try_delete_dist_entry, + (void *) dep); } } @@ -556,14 +584,14 @@ erts_node_table_size(void) #endif int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); #ifdef DEBUG hash_get_info(&hi, &erts_node_table); ASSERT(node_entries == hi.objs); #endif res = hash_table_sz(&erts_node_table) + node_entries*sizeof(ErlNode); if (lock) - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); return res; } @@ -572,10 +600,10 @@ erts_node_table_info(int to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); hash_info(to, to_arg, &erts_node_table); if (lock) - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); } @@ -609,21 +637,46 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) return res; } -void erts_delete_node(ErlNode *enp) +static void try_delete_node(void *venp) +{ + ErlNode *enp = (ErlNode *) venp; + erts_aint_t refc; + + erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + /* + * Another thread might have looked up this node after we + * decided to delete it (refc became zero). If so, the other + * thread incremented refc twice. Once for the new reference + * and once for this thread. + * + * If refc reach -1, noone has used the entry since we + * set up the timer. Delete the entry. + * + * If refc reach 0, the entry is currently not in use + * but has been used since we set up the timer. Set up a + * new timer. + * + * If refc > 0, the entry is in use. Keep the entry. + */ + refc = erts_refc_dectest(&enp->refc, -1); + if (refc == -1) + (void) hash_erase(&erts_node_table, (void *) enp); + erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + + if (refc == 0) + erts_schedule_delete_node(enp); +} + +void erts_schedule_delete_node(ErlNode *enp) { ASSERT(enp != erts_this_node); - if(enp != erts_this_node) { - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); - /* - * Another thread might have looked up this node after we - * decided to delete it (refc became zero). If so, the other - * thread incremented refc twice. Once for the new reference - * and once for this thread. Therefore, delete node if refc - * is 0 or -1 after a decrement. - */ - if (erts_refc_dectest(&enp->refc, -1) <= 0) - (void) hash_erase(&erts_node_table, (void *) enp); - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + if (enp != erts_this_node) { + if (node_tab_delete_delay == 0) + try_delete_node((void *) enp); + else if (node_tab_delete_delay > 0) + erts_start_timer_callback(node_tab_delete_delay, + try_delete_node, + (void *) enp); } } @@ -651,7 +704,7 @@ static void print_node(void *venp, void *vpndp) erts_print(pndp->to, pndp->to_arg, " %d", enp->creation); #ifdef DEBUG erts_print(pndp->to, pndp->to_arg, " (refc=%ld)", - erts_refc_read(&enp->refc, 1)); + erts_refc_read(&enp->refc, 0)); #endif pndp->no_sysname++; } @@ -674,13 +727,13 @@ void erts_print_node_info(int to, pnd.no_total = 0; if (lock) - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); hash_foreach(&erts_node_table, print_node, (void *) &pnd); if (pnd.no_sysname != 0) { erts_print(to, to_arg, "\n"); } if (lock) - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); if(no_sysname) *no_sysname = pnd.no_sysname; @@ -714,11 +767,28 @@ erts_set_this_node(Eterm sysname, Uint creation) } -void erts_init_node_tables(void) +Uint +erts_delayed_node_table_gc(void) +{ + if (node_tab_delete_delay < 0) + return (Uint) ERTS_NODE_TAB_DELAY_GC_INFINITY; + if (node_tab_delete_delay == 0) + return (Uint) 0; + return (Uint) ((node_tab_delete_delay-1)/1000 + 1); +} + +void erts_init_node_tables(int dd_sec) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; HashFunctions f; + if (dd_sec == ERTS_NODE_TAB_DELAY_GC_INFINITY) + node_tab_delete_delay = (ErtsMonotonicTime) -1; + else + node_tab_delete_delay = ((ErtsMonotonicTime) dd_sec)*1000; + + orig_node_tab_delete_delay = node_tab_delete_delay; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; @@ -847,6 +917,7 @@ static Eterm AM_dist_references; static Eterm AM_node_references; static Eterm AM_system; static Eterm AM_timer; +static Eterm AM_delayed_delete_timer; static void setup_reference_table(void); static Eterm reference_table_term(Uint **hpp, Uint *szp); @@ -881,8 +952,10 @@ typedef struct dist_referrer_ { int heap_ref; int node_ref; int ctrl_ref; + int system_ref; Eterm id; Uint creation; + Uint id_heap[ID_HEAP_SIZE]; } DistReferrer; typedef struct { @@ -931,6 +1004,7 @@ erts_get_node_and_dist_references(struct process *proc) INIT_AM(node_references); INIT_AM(timer); INIT_AM(system); + INIT_AM(delayed_delete_timer); references_atoms_need_init = 0; } @@ -988,17 +1062,25 @@ insert_dist_referrer(ReferredDist *referred_dist, sizeof(DistReferrer)); drp->next = referred_dist->referrers; referred_dist->referrers = drp; - drp->id = id; + if(IS_CONST(id)) + drp->id = id; + else { + Uint *hp = &drp->id_heap[0]; + ASSERT(is_tuple(id)); + drp->id = copy_struct(id, size_object(id), &hp, NULL); + } drp->creation = creation; drp->heap_ref = 0; drp->node_ref = 0; drp->ctrl_ref = 0; + drp->system_ref = 0; } switch (type) { case NODE_REF: drp->node_ref++; break; case CTRL_REF: drp->ctrl_ref++; break; case HEAP_REF: drp->heap_ref++; break; + case SYSTEM_REF: drp->system_ref++; break; default: ASSERT(0); } } @@ -1261,6 +1343,33 @@ insert_sys_msg(Eterm from, Eterm to, Eterm msg, ErlHeapFragment *bp) #endif static void +insert_delayed_delete_node(void *state, + ErtsMonotonicTime timeout_pos, + void *vnp) +{ + DeclareTmpHeapNoproc(heap,3); + UseTmpHeapNoproc(3); + insert_node((ErlNode *) vnp, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer)); + UnUseTmpHeapNoproc(3); +} + +static void +insert_delayed_delete_dist_entry(void *state, + ErtsMonotonicTime timeout_pos, + void *vdep) +{ + DeclareTmpHeapNoproc(heap,3); + UseTmpHeapNoproc(3); + insert_dist_entry((DistEntry *) vdep, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer), + 0); + UnUseTmpHeapNoproc(3); +} + +static void setup_reference_table(void) { ErlHeapFragment *hfp; @@ -1288,6 +1397,13 @@ setup_reference_table(void) /* Go through the hole system, and build a table of all references to ErlNode and DistEntry structures */ + erts_debug_callback_timer_foreach(try_delete_node, + insert_delayed_delete_node, + NULL); + erts_debug_callback_timer_foreach(try_delete_dist_entry, + insert_delayed_delete_dist_entry, + NULL); + UseTmpHeapNoproc(3); insert_node(erts_this_node, SYSTEM_REF, @@ -1601,7 +1717,7 @@ reference_table_term(Uint **hpp, Uint *szp) tup = MK_2TUP(referred_nodes[i].node->sysname, MK_UINT(referred_nodes[i].node->creation)); - tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 1)), nril); + tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 0)), nril); nl = MK_CONS(tup, nl); } @@ -1624,6 +1740,10 @@ reference_table_term(Uint **hpp, Uint *szp) tup = MK_2TUP(AM_heap, MK_UINT(drp->heap_ref)); drl = MK_CONS(tup, drl); } + if(drp->system_ref) { + tup = MK_2TUP(AM_system, MK_UINT(drp->system_ref)); + drl = MK_CONS(tup, drl); + } if (is_internal_pid(drp->id)) { ASSERT(!drp->node_ref); @@ -1633,6 +1753,14 @@ reference_table_term(Uint **hpp, Uint *szp) ASSERT(drp->ctrl_ref && !drp->node_ref); tup = MK_2TUP(AM_port, drp->id); } + else if (is_tuple(drp->id)) { + Eterm *t; + ASSERT(drp->system_ref && !drp->node_ref + && !drp->ctrl_ref && !drp->heap_ref); + t = tuple_val(drp->id); + ASSERT(2 == arityval(t[0])); + tup = MK_2TUP(t[1], t[2]); + } else { ASSERT(!drp->ctrl_ref && drp->node_ref); ASSERT(is_atom(drp->id)); @@ -1650,7 +1778,7 @@ reference_table_term(Uint **hpp, Uint *szp) /* DistList = [{Dist, Refc, ReferenceIdList}] */ tup = MK_3TUP(referred_dists[i].dist->sysname, - MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 1)), + MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 0)), dril); dl = MK_CONS(tup, dl); } @@ -1705,3 +1833,15 @@ delete_reference_table(void) } } +void +erts_debug_test_node_tab_delayed_delete(Sint64 millisecs) +{ + erts_smp_thr_progress_block(); + + if (millisecs < 0) + node_tab_delete_delay = orig_node_tab_delete_delay; + else + node_tab_delete_delay = millisecs; + + erts_smp_thr_progress_unblock(); +} diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index af60071ea5..54c5cd1d11 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -46,6 +46,10 @@ #define ERTS_PORT_TASK_ONLY_BASIC_TYPES__ #include "erl_port_task.h" #undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__ + +#define ERTS_NODE_TAB_DELAY_GC_DEFAULT (60) +#define ERTS_NODE_TAB_DELAY_GC_MAX (100*1000*1000) +#define ERTS_NODE_TAB_DELAY_GC_INFINITY (ERTS_NODE_TAB_DELAY_GC_MAX+1) #define ERST_INTERNAL_CHANNEL_NO 0 @@ -166,20 +170,21 @@ extern DistEntry *erts_this_dist_entry; extern ErlNode *erts_this_node; extern char *erts_this_node_sysname; /* must match erl_node_tables.c */ +Uint erts_delayed_node_table_gc(void); DistEntry *erts_channel_no_to_dist_entry(Uint); DistEntry *erts_sysname_to_connected_dist_entry(Eterm); DistEntry *erts_find_or_insert_dist_entry(Eterm); DistEntry *erts_find_dist_entry(Eterm); -void erts_delete_dist_entry(DistEntry *); +void erts_schedule_delete_dist_entry(DistEntry *); Uint erts_dist_table_size(void); void erts_dist_table_info(int, void *); void erts_set_dist_entry_not_connected(DistEntry *); void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint); ErlNode *erts_find_or_insert_node(Eterm, Uint); -void erts_delete_node(ErlNode *); +void erts_schedule_delete_node(ErlNode *); void erts_set_this_node(Eterm, Uint); Uint erts_node_table_size(void); -void erts_init_node_tables(void); +void erts_init_node_tables(int); void erts_node_table_info(int, void *); void erts_print_node_info(int, void *, Eterm, int*, int*); Eterm erts_get_node_and_dist_references(struct process *); @@ -204,7 +209,7 @@ erts_deref_dist_entry(DistEntry *dep) { ASSERT(dep); if (erts_refc_dectest(&dep->refc, 0) == 0) - erts_delete_dist_entry(dep); + erts_schedule_delete_dist_entry(dep); } ERTS_GLB_INLINE void @@ -212,7 +217,7 @@ erts_deref_node_entry(ErlNode *np) { ASSERT(np); if (erts_refc_dectest(&np->refc, 0) == 0) - erts_delete_node(np); + erts_schedule_delete_node(np); } ERTS_GLB_INLINE void @@ -253,5 +258,6 @@ erts_smp_de_links_unlock(DistEntry *dep) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +void erts_debug_test_node_tab_delayed_delete(Sint64 millisecs); #endif diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 3920fae2d9..66cbb67a0d 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -266,8 +266,7 @@ erts_prtsd_set(Port *prt, int ix, void *data) #endif -extern erts_smp_atomic_t erts_bytes_out; /* no bytes written out */ -extern erts_smp_atomic_t erts_bytes_in; /* no bytes sent into the system */ +Eterm erts_request_io_bytes(Process *c_p); /* port status flags */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 4940ffc4a0..88c1b5c121 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -165,6 +165,9 @@ Uint erts_no_dirty_cpu_schedulers; Uint erts_no_dirty_io_schedulers; #endif +static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0}; +int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS; + #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY (4*1024*1024) #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY (512*1024) #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM (64*1024) @@ -508,6 +511,7 @@ dbg_chk_aux_work_val(erts_aint32_t value) #ifdef ERTS_SSI_AUX_WORK_REAP_PORTS valid |= ERTS_SSI_AUX_WORK_REAP_PORTS; #endif + valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; if (~valid & value) erl_exit(ERTS_ABORT_EXIT, @@ -566,6 +570,41 @@ erts_pre_init_process(void) erts_tsd_key_create(&sched_data_key, "erts_sched_data_key"); #endif + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX] + = "DELAYED_AW_WAKEUP"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_IX] + = "DD"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX] + = "DD_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX] + = "FIX_ALLOC_DEALLOC"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX] + = "FIX_ALLOC_LOWER_LIM"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX] + = "THR_PRGR_LATER_OP"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX] + = "CNCLD_TMRS"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX] + = "CNCLD_TMRS_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_IX] + = "ASYNC_READY"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX] + = "ASYNC_READY_CLEAN"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX] + = "MISC_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX] + = "MISC"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX] + = "CHECK_CHILDREN"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX] + = "SET_TMO"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX] + = "MSEG_CACHE_CHECK"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_REAP_PORTS_IX] + = "REAP_PORTS"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX] + = "DEBUG_WAIT_COMPLETED"; + #ifdef ERTS_ENABLE_LOCK_CHECK { int ix; @@ -1193,11 +1232,11 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, ERTS_DBG_CHK_SSI_AUX_WORK(ssi); old_flgs = erts_atomic32_read_nob(&ssi->aux_work); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { #ifdef ERTS_SMP erts_sched_poke(ssi); #else @@ -1217,7 +1256,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { #ifdef ERTS_SMP erts_sched_poke(ssi); #else @@ -1715,11 +1754,6 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin awdp->dd.thr_prgr = wakeup; haw_thr_prgr_soft_wakeup(awdp, wakeup); } - else if (awdp->dd.completed_callback) { - awdp->dd.completed_callback(awdp->dd.completed_arg); - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; - } return aux_work & ~ERTS_SSI_AUX_WORK_DD; } @@ -1761,11 +1795,6 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i } else { unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); - if (awdp->dd.completed_callback) { - awdp->dd.completed_callback(awdp->dd.completed_arg); - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; - } } return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; @@ -1955,78 +1984,142 @@ erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *), #endif } -#ifdef ERTS_SMP +static ERTS_INLINE erts_aint32_t +handle_debug_wait_completed(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + erts_aint32_t saved_aux_work, flags; + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + + flags = awdp->debug.wait_completed.flags; + + if (aux_work & flags) + return aux_work; + + saved_aux_work = erts_atomic32_read_acqb(&ssi->aux_work); -static erts_atomic32_t completed_dealloc_count; + if (saved_aux_work & flags) + return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; + + awdp->debug.wait_completed.callback(awdp->debug.wait_completed.arg); + + awdp->debug.wait_completed.flags = 0; + awdp->debug.wait_completed.callback = NULL; + awdp->debug.wait_completed.arg = NULL; + + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED); + + return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; +} + +static erts_atomic32_t debug_wait_completed_count; +static int debug_wait_completed_flags; static void -completed_dealloc(void *vproc) +thr_debug_wait_completed(void *vproc) { - if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) { + if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 0) { erts_resume((Process *) vproc, (ErtsProcLocks) 0); erts_proc_dec_refc((Process *) vproc); } } static void -setup_completed_dealloc(void *vproc) +setup_thr_debug_wait_completed(void *vproc) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); - ErtsAuxWorkData *awdp = (esdp - ? &esdp->aux_work_data - : aux_thread_aux_work_data); - erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); - set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD); - awdp->dd.completed_callback = completed_dealloc; - awdp->dd.completed_arg = vproc; + ErtsAuxWorkData *awdp; + erts_aint32_t wait_flags, aux_work_flags; +#ifdef ERTS_SMP + awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data; +#else + awdp = &esdp->aux_work_data; +#endif + + wait_flags = 0; + aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; + + if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) { + erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); + wait_flags |= (ERTS_SSI_AUX_WORK_DD + | ERTS_SSI_AUX_WORK_DD_THR_PRGR + | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); +#ifdef ERTS_SMP + aux_work_flags |= ERTS_SSI_AUX_WORK_DD; +#endif + } + + if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) { + wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS + | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR + | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); +#ifdef ERTS_SMP + if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)) + aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; +#endif + } + + set_aux_work_flags_wakeup_nob(awdp->ssi, aux_work_flags); + + awdp->debug.wait_completed.flags = wait_flags; + awdp->debug.wait_completed.callback = thr_debug_wait_completed; + awdp->debug.wait_completed.arg = vproc; } static void -prep_setup_completed_dealloc(void *vproc) +prep_setup_thr_debug_wait_completed(void *vproc) { - erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1); - if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) { + erts_aint32_t count = (erts_aint32_t) erts_no_schedulers; +#ifdef ERTS_SMP + count += 1; /* aux thread */ +#endif + if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) { /* scheduler threads */ erts_schedule_multi_misc_aux_work(0, erts_no_schedulers, - setup_completed_dealloc, + setup_thr_debug_wait_completed, vproc); +#ifdef ERTS_SMP /* aux_thread */ erts_schedule_misc_aux_work(0, - setup_completed_dealloc, + setup_thr_debug_wait_completed, vproc); +#endif } } -#endif /* ERTS_SMP */ int -erts_debug_wait_deallocations(Process *c_p) +erts_debug_wait_completed(Process *c_p, int flags) { -#ifndef ERTS_SMP - erts_alloc_fix_alloc_shrink(1, 0); - return 1; -#else /* Only one process at a time can do this */ - erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1)); - if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count, + erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers); +#ifdef ERTS_SMP + count += 2; /* aux thread */ +#endif + if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count, count, 0)) { + debug_wait_completed_flags = flags; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); erts_proc_inc_refc(c_p); /* scheduler threads */ erts_schedule_multi_misc_aux_work(0, erts_no_schedulers, - prep_setup_completed_dealloc, + prep_setup_thr_debug_wait_completed, (void *) c_p); +#ifdef ERTS_SMP /* aux_thread */ erts_schedule_misc_aux_work(0, - prep_setup_completed_dealloc, + prep_setup_thr_debug_wait_completed, (void *) c_p); +#endif return 1; } return 0; -#endif } @@ -2227,6 +2320,14 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS, handle_reap_ports); + /* + * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED *need* to be + * the last flag checked! + */ + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED, + handle_debug_wait_completed); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); #ifdef ERTS_SMP @@ -5354,8 +5455,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST; awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; awdp->cncld_tmrs.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST; awdp->later_op.size = 0; @@ -5386,6 +5485,9 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) awdp->delayed_wakeup.sched2jix[i] = -1; } #endif + awdp->debug.wait_completed.flags = 0; + awdp->debug.wait_completed.callback = NULL; + awdp->debug.wait_completed.arg = NULL; } static void @@ -5440,6 +5542,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->thr_id = (Uint32) num; erts_sched_bif_unique_init(esdp); + esdp->io.out = (Uint64) 0; + esdp->io.in = (Uint64) 0; + if (daww_ptr) { init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr); #ifdef ERTS_SMP @@ -5694,11 +5799,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online init_swtreq_alloc(); #endif + erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */ + debug_wait_completed_flags = 0; #ifdef ERTS_SMP - erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */ - aux_thread_aux_work_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, sizeof(ErtsAuxWorkData)); @@ -10855,7 +10960,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg_inq.len = 0; #endif p->bif_timers = NULL; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT p->accessor_bif_timers = NULL; +#endif p->mbuf = NULL; p->mbuf_sz = 0; p->psd = NULL; @@ -11035,7 +11142,9 @@ void erts_init_empty_process(Process *p) p->msg.save = &p->msg.first; p->msg.len = 0; p->bif_timers = NULL; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT p->accessor_bif_timers = NULL; +#endif p->dictionary = NULL; p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; @@ -11130,7 +11239,9 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->msg.first == NULL); ASSERT(p->msg.len == 0); ASSERT(p->bif_timers == NULL); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT ASSERT(p->accessor_bif_timers == NULL); +#endif ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); ASSERT(p->cp == NULL); @@ -12086,6 +12197,7 @@ erts_continue_exit_process(Process *p) p->bif_timers = NULL; } +#ifdef ERTS_BTM_ACCESSOR_SUPPORT if (p->accessor_bif_timers) { if (erts_detach_accessor_bif_timers(p, p->accessor_bif_timers, @@ -12096,6 +12208,7 @@ erts_continue_exit_process(Process *p) ASSERT(erts_proc_read_refc(p) > 0); p->accessor_bif_timers = NULL; } +#endif #ifdef ERTS_SMP if (p->flags & F_HAVE_BLCKD_MSCHED) { @@ -12458,45 +12571,13 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) { flg = erts_atomic32_read_dirty(&esdp->ssi->aux_work); erts_print(to, to_arg, "Scheduler Sleep Info Aux Work: "); - for (i = 0; i < ERTS_SSI_AUX_WORK_MAX && flg; i++) { + for (i = 0; i < ERTS_SSI_AUX_WORK_NO_FLAGS && flg; i++) { erts_aint32_t chk = (1 << i); if (flg & chk) { - switch (chk) { - case ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP: - erts_print(to, to_arg, "DELAYED_AW_WAKEUP"); break; - case ERTS_SSI_AUX_WORK_DD: - erts_print(to, to_arg, "DELAYED_DEALLOC"); break; - case ERTS_SSI_AUX_WORK_DD_THR_PRGR: - erts_print(to, to_arg, "DELAYED_DEALLOC_THR_PRGR"); break; - case ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC: - erts_print(to, to_arg, "FIX_ALLOC_DEALLOC"); break; - case ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM: - erts_print(to, to_arg, "FIX_ALLOC_LOWER_LIM"); break; - case ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP: - erts_print(to, to_arg, "THR_PRGR_LATER_OP"); break; - case ERTS_SSI_AUX_WORK_CNCLD_TMRS: - erts_print(to, to_arg, "CANCELED_TIMERS"); break; - case ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR: - erts_print(to, to_arg, "CANCELED_TIMERS_THR_PRGR"); break; - case ERTS_SSI_AUX_WORK_ASYNC_READY: - erts_print(to, to_arg, "ASYNC_READY"); break; - case ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN: - erts_print(to, to_arg, "ASYNC_READY_CLEAN"); break; - case ERTS_SSI_AUX_WORK_MISC_THR_PRGR: - erts_print(to, to_arg, "MISC_THR_PRGR"); break; - case ERTS_SSI_AUX_WORK_MISC: - erts_print(to, to_arg, "MISC"); break; - case ERTS_SSI_AUX_WORK_CHECK_CHILDREN: - erts_print(to, to_arg, "CHECK_CHILDREN"); break; - case ERTS_SSI_AUX_WORK_SET_TMO: - erts_print(to, to_arg, "SET_TMO"); break; - case ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK: - erts_print(to, to_arg, "MSEG_CACHE_CHECK"); break; - case ERTS_SSI_AUX_WORK_REAP_PORTS: - erts_print(to, to_arg, "REAP_PORTS"); break; - default: - erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; - } + if (erts_aux_work_flag_descr[i]) + erts_print(to, to_arg, "%s", erts_aux_work_flag_descr[i]); + else + erts_print(to, to_arg, "1<<%d", i); if (flg > chk) erts_print(to, to_arg, " | "); flg -= chk; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index b1c30e7652..a6c3790991 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -268,28 +268,73 @@ typedef enum { | ERTS_SSI_FLG_SUSPENDED) /* - * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative - * eachother. Most frequent - lowest bit number. + * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency + * order relative eachother. Most frequent at lowest at lowest + * index. + * + * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be + * highest index... + * + * Remember to update description in erts_pre_init_process() + * when adding new flags... */ -#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP (((erts_aint32_t) 1) << 0) -#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 1) -#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 2) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 3) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4) -#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_AUX_WORK_CNCLD_TMRS (((erts_aint32_t) 1) << 6) -#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR (((erts_aint32_t) 1) << 7) -#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 8) -#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 9) -#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 10) -#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 11) -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 12) -#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 13) -#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 14) -#define ERTS_SSI_AUX_WORK_REAP_PORTS (((erts_aint32_t) 1) << 15) - -#define ERTS_SSI_AUX_WORK_MAX 16 +typedef enum { + ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX, + ERTS_SSI_AUX_WORK_DD_IX, + ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX, + ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX, + ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX, + ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX, + ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX, + ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX, + ERTS_SSI_AUX_WORK_ASYNC_READY_IX, + ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX, + ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX, + ERTS_SSI_AUX_WORK_MISC_IX, + ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX, + ERTS_SSI_AUX_WORK_SET_TMO_IX, + ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX, + ERTS_SSI_AUX_WORK_REAP_PORTS_IX, + ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */ + + ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */ +} ErtsSsiAuxWorkFlagIndex; + +#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX) +#define ERTS_SSI_AUX_WORK_DD \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX) +#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX) +#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX) +#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX) +#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX) +#define ERTS_SSI_AUX_WORK_ASYNC_READY \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX) +#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX) +#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX) +#define ERTS_SSI_AUX_WORK_MISC \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX) +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX) +#define ERTS_SSI_AUX_WORK_SET_TMO \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX) +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX) +#define ERTS_SSI_AUX_WORK_REAP_PORTS \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX) +#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX) typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; @@ -515,8 +560,6 @@ typedef struct { #ifdef ERTS_SMP struct { ErtsThrPrgrVal thr_prgr; - void (*completed_callback)(void *); - void (*completed_arg)(void *); } dd; struct { ErtsThrPrgrVal thr_prgr; @@ -545,6 +588,13 @@ typedef struct { ErtsDelayedAuxWorkWakeupJob *job; } delayed_wakeup; #endif + struct { + struct { + erts_aint32_t flags; + void (*callback)(void *); + void *arg; + } wait_completed; + } debug; } ErtsAuxWorkData; #ifdef ERTS_DIRTY_SCHEDULERS @@ -609,6 +659,11 @@ struct ErtsSchedulerData_ { ErtsSchedAllocData alloc_data; + struct { + Uint64 out; + Uint64 in; + } io; + Uint64 reductions; ErtsSchedWallTime sched_wall_time; ErtsGCInfo gc_info; @@ -925,7 +980,9 @@ struct process { ErlMessageQueue msg; /* Message queue */ ErtsBifTimers *bif_timers; /* Bif timers aiming at this process */ +#ifdef ERTS_BTM_ACCESSOR_SUPPORT ErtsBifTimers *accessor_bif_timers; /* Accessor bif timers */ +#endif ProcDict *dictionary; /* Process dictionary, may be NULL */ @@ -1692,7 +1749,11 @@ Eterm erts_get_reader_groups_map(Process *c_p); Eterm erts_debug_reader_groups_map(Process *c_p, int groups); Uint erts_debug_nbalance(void); -int erts_debug_wait_deallocations(Process *c_p); + +#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS (1 << 0) +#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS (1 << 1) + +int erts_debug_wait_completed(Process *c_p, int flags); Uint erts_process_memory(Process *c_p); @@ -2001,14 +2062,10 @@ ERTS_GLB_INLINE Eterm erts_get_current_pid(void); ERTS_GLB_INLINE Uint erts_get_scheduler_id(void); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp); -#ifndef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq); -#endif ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq); ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq); -#ifndef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq); -#endif ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq); ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); @@ -2086,12 +2143,6 @@ erts_smp_runq_lock(ErtsRunQueue *rq) #endif } -#ifdef ERTS_ENABLE_LOCK_COUNT - -#define erts_smp_runq_lock(rq) erts_smp_mtx_lock_x(&(rq)->mtx, __FILE__, __LINE__) - -#endif - ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq) { @@ -2110,31 +2161,6 @@ erts_smp_runq_unlock(ErtsRunQueue *rq) #endif } -#ifdef ERTS_ENABLE_LOCK_COUNT - -#define erts_smp_xrunq_lock(rq, xrq) erts_smp_xrunq_lock_x((rq), (xrq), __FILE__, __LINE__) - -ERTS_GLB_INLINE void -erts_smp_xrunq_lock_x(ErtsRunQueue *rq, ErtsRunQueue *xrq, char* file, int line) -{ -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx)); - if (xrq != rq) { - if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) { - if (rq < xrq) - erts_smp_mtx_lock_x(&xrq->mtx, file, line); - else { - erts_smp_mtx_unlock(&rq->mtx); - erts_smp_mtx_lock_x(&xrq->mtx, file, line); - erts_smp_mtx_lock_x(&rq->mtx, file, line); - } - } - } -#endif -} - -#else - ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq) { @@ -2154,8 +2180,6 @@ erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq) #endif } -#endif - ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq) { diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index fff267ff2a..0548c6137c 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -110,6 +110,9 @@ static struct { erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; +#ifdef ERTS_ENABLE_LOCK_COUNT +static void lcnt_enable_proc_lock_count(Process *proc, int enable); +#endif void erts_init_proc_lock(int cpus) @@ -1003,7 +1006,9 @@ erts_pid2proc_opt(Process *c_p, void erts_proc_lock_init(Process *p) { +#if ERTS_PROC_LOCK_OWN_IMPL || defined(ERTS_PROC_LOCK_DEBUG) int i; +#endif #if ERTS_PROC_LOCK_OWN_IMPL /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL @@ -1081,30 +1086,32 @@ erts_proc_lock_fin(Process *p) /* --- Process lock counting ----------------------------------------------- */ #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) -void erts_lcnt_proc_lock_init(Process *p) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (p->common.id != ERTS_INVALID_PID) { - erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, p->common.id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, p->common.id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_btm), "proc_btm", ERTS_LCNT_LT_PROCLOCK, p->common.id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, p->common.id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK, p->common.id); - } else { - erts_lcnt_init_lock(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK); - erts_lcnt_init_lock(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK); - erts_lcnt_init_lock(&(p->lock.lcnt_btm), "proc_btm", ERTS_LCNT_LT_PROCLOCK); - erts_lcnt_init_lock(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK); - erts_lcnt_init_lock(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK); - } - } else { - sys_memzero(&(p->lock.lcnt_main), sizeof(p->lock.lcnt_main)); - sys_memzero(&(p->lock.lcnt_msgq), sizeof(p->lock.lcnt_msgq)); - sys_memzero(&(p->lock.lcnt_btm), sizeof(p->lock.lcnt_btm)); - sys_memzero(&(p->lock.lcnt_link), sizeof(p->lock.lcnt_link)); - sys_memzero(&(p->lock.lcnt_status), sizeof(p->lock.lcnt_status)); - } + +void erts_lcnt_enable_proc_lock_count(int enable) { + int ix, max = erts_ptab_max(&erts_proc); + Process *proc = NULL; + for (ix = 0; ix < max; ++ix) { + if ((proc = erts_pix2proc(ix)) != NULL) + lcnt_enable_proc_lock_count(proc, enable); + } /* for all processes */ } - + +void erts_lcnt_proc_lock_init(Process *p) { + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) { + erts_lcnt_init_lock_empty(&(p->lock.lcnt_main)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_msgq)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_btm)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_link)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_status)); + } else { /* now the common case */ + Eterm pid = (p->common.id != ERTS_INVALID_PID) ? p->common.id : NIL; + erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_btm), "proc_btm", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_status),"proc_status",ERTS_LCNT_LT_PROCLOCK, pid); + } /* the lock names should really be aligned to four characters */ +} /* logic reversed */ void erts_lcnt_proc_lock_destroy(Process *p) { erts_lcnt_destroy_lock(&(p->lock.lcnt_main)); @@ -1114,28 +1121,31 @@ void erts_lcnt_proc_lock_destroy(Process *p) { erts_lcnt_destroy_lock(&(p->lock.lcnt_status)); } -void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_lock(&(lock->lcnt_main)); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_lock(&(lock->lcnt_msgq)); - } - if (locks & ERTS_PROC_LOCK_BTM) { - erts_lcnt_lock(&(lock->lcnt_btm)); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock(&(lock->lcnt_link)); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_lock(&(lock->lcnt_status)); +static void lcnt_enable_proc_lock_count(Process *proc, int enable) { + if (enable) { + if (!ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) { + erts_lcnt_proc_lock_init(proc); + } } + else { + if (ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) { + erts_lcnt_proc_lock_destroy(proc); + } } } -void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, char *file, unsigned int line) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { +void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock(&(lock->lcnt_msgq)); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock(&(lock->lcnt_btm)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock(&(lock->lcnt_status)); } +} + +void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, + char *file, unsigned int line) { + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_post_x(&(lock->lcnt_main), file, line); } @@ -1151,90 +1161,34 @@ void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, cha if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_post_x(&(lock->lcnt_status), file, line); } - } } void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_lock_unaquire(&(lock->lcnt_main)); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); - } - if (locks & ERTS_PROC_LOCK_BTM) { - erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock_unaquire(&(lock->lcnt_link)); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_lock_unaquire(&(lock->lcnt_status)); - } - } + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unaquire(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_unaquire(&(lock->lcnt_status)); } } void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_unlock(&(lock->lcnt_main)); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_unlock(&(lock->lcnt_msgq)); - } - if (locks & ERTS_PROC_LOCK_BTM) { - erts_lcnt_unlock(&(lock->lcnt_btm)); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_unlock(&(lock->lcnt_link)); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_unlock(&(lock->lcnt_status)); - } - } + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock(&(lock->lcnt_msgq)); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_unlock(&(lock->lcnt_btm)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_unlock(&(lock->lcnt_status)); } } void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_trylock(&(lock->lcnt_main), res); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_trylock(&(lock->lcnt_msgq), res); - } - if (locks & ERTS_PROC_LOCK_BTM) { - erts_lcnt_trylock(&(lock->lcnt_btm), res); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_trylock(&(lock->lcnt_link), res); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_trylock(&(lock->lcnt_status), res); - } - } -} - - -void erts_lcnt_enable_proc_lock_count(int enable) -{ - int i, max = erts_ptab_max(&erts_proc); - - for (i = 0; i < max; ++i) { - Process* p = erts_pix2proc(i); - if (p) { - if (enable) { - if (!ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) { - erts_lcnt_proc_lock_init(p); - } - } else { - if (ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) { - erts_lcnt_proc_lock_destroy(p); - } - } - } - } -} - -#endif /* ifdef ERTS_ENABLE_LOCK_COUNT */ + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock(&(lock->lcnt_main), res); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock(&(lock->lcnt_msgq), res); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_trylock(&(lock->lcnt_btm), res); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_trylock(&(lock->lcnt_status), res); } +} /* reversed logic */ +#endif /* ERTS_ENABLE_LOCK_COUNT */ /* --- Process lock checking ----------------------------------------------- */ diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index bc04d7b78e..565528193e 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -128,10 +128,10 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \ return _unchecked_##FUN(x); \ } -ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer); +ET_DEFINE_CHECKED(Eterm,make_boxed,const Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header); ET_DEFINE_CHECKED(Eterm*,boxed_val,Wterm,_boxed_precond); -ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer); +ET_DEFINE_CHECKED(Eterm,make_list,const Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header); ET_DEFINE_CHECKED(Eterm*,list_val,Wterm,_list_precond); ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small); diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 602aab46dc..7b15b34da1 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -198,7 +198,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #endif #define _is_aligned(x) (((Uint)(x) & 0x3) == 0) #define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED) -_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*) +_ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*) #define make_boxed(x) _ET_APPLY(make_boxed,(x)) #if 1 #define _is_not_boxed(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED)) @@ -214,7 +214,7 @@ _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm) /* cons cell ("list") access methods */ #define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST) -_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*) +_ET_DECLARE_CHECKED(Eterm,make_list,const Eterm*) #define make_list(x) _ET_APPLY(make_list,(x)) #if 1 #define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST)) diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 4560cd23af..e64b86496a 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -453,4 +453,12 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_ #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ +void +erts_twheel_debug_foreach(ErtsTimerWheel *tiw, + void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg); + #endif /* timer wheel api */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index fe48298155..0a69172980 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -94,9 +94,9 @@ static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); struct B2TContext_t; -static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*); +static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); -static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); +static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*); static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*); static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); @@ -930,8 +930,7 @@ Sint erts_decode_ext_size_ets(byte *ext, Uint size) ** on return hpp is updated to point after allocated data */ Eterm -erts_decode_dist_ext(Eterm** hpp, - ErlOffHeap* off_heap, +erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *edep) { Eterm obj; @@ -951,7 +950,7 @@ erts_decode_dist_ext(Eterm** hpp, goto error; ep++; } - ep = dec_term(edep, hpp, ep, off_heap, &obj, NULL); + ep = dec_term(edep, factory, ep, &obj, NULL); if (!ep) goto error; @@ -960,19 +959,22 @@ erts_decode_dist_ext(Eterm** hpp, return obj; error: + erts_factory_undo(factory); bad_dist_ext(edep); return THE_NON_VALUE; } -Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) +Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext) { Eterm obj; byte *ep = *ext; - if (*ep++ != VERSION_MAGIC) + if (*ep++ != VERSION_MAGIC) { + erts_factory_undo(factory); return THE_NON_VALUE; - ep = dec_term(NULL, hpp, ep, off_heap, &obj, NULL); + } + ep = dec_term(NULL, factory, ep, &obj, NULL); if (!ep) { #ifdef DEBUG bin_write(ERTS_PRINT_STDERR,NULL,*ext,500); @@ -983,10 +985,10 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) return obj; } -Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) +Eterm erts_decode_ext_ets(ErtsHeapFactory* factory, byte *ext) { Eterm obj; - ext = dec_term(NULL, hpp, ext, off_heap, &obj, NULL); + ext = dec_term(NULL, factory, ext, &obj, NULL); ASSERT(ext); return obj; } @@ -995,9 +997,8 @@ Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) { + ErtsHeapFactory factory; Eterm res; - Eterm *hp; - Eterm *hendp; Sint hsz; ErtsDistExternal ede; Eterm *tp; @@ -1044,12 +1045,9 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) if (hsz < 0) goto badarg; - hp = HAlloc(BIF_P, (Uint) hsz); - hendp = hp + hsz; - - res = erts_decode_dist_ext(&hp, &MSO(BIF_P), &ede); - - HRelease(BIF_P, hendp, hp); + erts_factory_proc_prealloc_init(&factory, BIF_P, hsz); + res = erts_decode_dist_ext(&factory, &ede); + erts_factory_close(&factory); if (is_value(res)) BIF_RET(res); @@ -1177,13 +1175,11 @@ typedef struct { byte* ep; Eterm res; Eterm* next; - Eterm* hp_start; - Eterm* hp; - Eterm* hp_end; + ErtsHeapFactory factory; int remaining_n; char* remaining_bytes; Eterm* maps_list; - struct dec_term_hamt_placeholder* hamt_list; + ErtsPStack hamt_array; } B2TDecodeContext; typedef struct { @@ -1307,10 +1303,12 @@ binary2term_abort(ErtsBinary2TermState *state) } static ERTS_INLINE Eterm -binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, + ErtsHeapFactory* factory) { Eterm res; - if (!dec_term(edep, hpp, state->extp, ohp, &res, NULL)) + + if (!dec_term(edep, factory, state->extp, &res, NULL)) res = THE_NON_VALUE; if (state->exttmp) { state->exttmp = 0; @@ -1343,9 +1341,9 @@ erts_binary2term_abort(ErtsBinary2TermState *state) } Eterm -erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +erts_binary2term_create(ErtsBinary2TermState *state, ErtsHeapFactory* factory) { - return binary2term_create(NULL,state, hpp, ohp); + return binary2term_create(NULL,state, factory); } static void b2t_destroy_context(B2TContext* context) @@ -1354,8 +1352,21 @@ static void b2t_destroy_context(B2TContext* context) ERTS_ALC_T_EXT_TERM_DATA); context->aligned_alloc = NULL; binary2term_abort(&context->b2ts); - if (context->state == B2TUncompressChunk) { + switch (context->state) { + case B2TUncompressChunk: erl_zlib_inflate_finish(&context->u.uc.stream); + break; + case B2TDecode: + case B2TDecodeList: + case B2TDecodeTuple: + case B2TDecodeString: + case B2TDecodeBinary: + if (context->u.dc.hamt_array.pstart) { + erts_free(context->u.dc.hamt_array.alloc_type, + context->u.dc.hamt_array.pstart); + } + break; + default:; } } @@ -1506,11 +1517,9 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar ctx->u.dc.ep = ctx->b2ts.extp; ctx->u.dc.res = (Eterm) (UWord) NULL; ctx->u.dc.next = &ctx->u.dc.res; - ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size); - ctx->u.dc.hp = ctx->u.dc.hp_start; - ctx->u.dc.hp_end = ctx->u.dc.hp_start + ctx->heap_size; + erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size); ctx->u.dc.maps_list = NULL; - ctx->u.dc.hamt_list = NULL; + ctx->u.dc.hamt_array.pstart = NULL; ctx->state = B2TDecode; /*fall through*/ case B2TDecode: @@ -1520,11 +1529,10 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar case B2TDecodeBinary: { ErtsDistExternal fakedep; fakedep.flags = ctx->flags; - dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx); + dec_term(&fakedep, NULL, NULL, NULL, ctx); break; } case B2TDecodeFail: - HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start); /*fall through*/ case B2TBadArg: BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); @@ -1549,11 +1557,11 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar case B2TDone: b2t_destroy_context(ctx); - if (ctx->u.dc.hp > ctx->u.dc.hp_end) { + if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) { erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", - __FILE__, __LINE__, ctx->u.dc.hp - ctx->u.dc.hp_end); + __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end); } - HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp); + erts_factory_close(&ctx->u.dc.factory); if (!is_first_call) { erts_set_gc_state(p, 1); @@ -2247,7 +2255,7 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) } static byte* -dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) +dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp) { Eterm sysname; Uint data; @@ -2286,15 +2294,15 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete if(node == erts_this_node) { *objp = make_internal_pid(data); } else { - ExternalThing *etp = (ExternalThing *) *hpp; - *hpp += EXTERNAL_THING_HEAD_SIZE + 1; + ExternalThing *etp = (ExternalThing *) factory->hp; + factory->hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_pid_header(1); - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; etp->data.ui[0] = data; - off_heap->first = (struct erl_off_heap_header*) etp; + factory->off_heap->first = (struct erl_off_heap_header*) etp; *objp = make_external_pid(etp); } return ep; @@ -2905,69 +2913,43 @@ is_external_string(Eterm list, int* p_is_string) return len; } -/* Assumes that the ones to undo are preluding the list. */ -static void -undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) -{ - const Uint area_sz = (end - start) * sizeof(Eterm); - struct erl_off_heap_header* hdr; - struct erl_off_heap_header** hdr_nextp = NULL; - - for (hdr = off_heap->first; ; hdr=hdr->next) { - if (!in_area(hdr, start, area_sz)) { - if (hdr_nextp != NULL) { - *hdr_nextp = NULL; - erts_cleanup_offheap(off_heap); - off_heap->first = hdr; - } - break; - } - hdr_nextp = &hdr->next; - } - - /* Assert that the ones to undo were indeed preluding the list. */ -#ifdef DEBUG - for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) { - ASSERT(!in_area(hdr, start, area_sz)); - } -#endif /* DEBUG */ -} -struct dec_term_hamt_placeholder +struct dec_term_hamt { - struct dec_term_hamt_placeholder* next; Eterm* objp; /* write result here */ Uint size; /* nr of leafs */ - Eterm leafs[1]; + Eterm* leaf_array; }; -#define DEC_TERM_HAMT_PLACEHOLDER_SIZE \ - (offsetof(struct dec_term_hamt_placeholder, leafs) / sizeof(Eterm)) /* Decode term from external format into *objp. -** On failure return NULL and *hpp will be unchanged. +** On failure calls erts_factory_undo() and returns NULL */ static byte* -dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, - Eterm* objp, B2TContext* ctx) +dec_term(ErtsDistExternal *edep, + ErtsHeapFactory* factory, + byte* ep, + Eterm* objp, + B2TContext* ctx) { - Eterm* hp_saved; +#define PSTACK_TYPE struct dec_term_hamt + PSTACK_DECLARE(hamt_array, 5); int n; ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ Eterm *maps_list; /* for preprocessing of small maps */ - struct dec_term_hamt_placeholder* hamt_list; /* for preprocessing of big maps */ Eterm* next; SWord reds; +#ifdef DEBUG + Eterm* dbg_resultp = ctx ? &ctx->u.dc.res : objp; +#endif if (ctx) { - hp_saved = ctx->u.dc.hp_start; reds = ctx->reds; next = ctx->u.dc.next; ep = ctx->u.dc.ep; - hpp = &ctx->u.dc.hp; + factory = &ctx->u.dc.factory; maps_list = ctx->u.dc.maps_list; - hamt_list = ctx->u.dc.hamt_list; if (ctx->state != B2TDecode) { int n_limit = reds; @@ -3012,7 +2994,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, break; case B2TDecodeString: - hp = *hpp; + hp = factory->hp; hp[-1] = make_list(hp); /* overwrite the premature NIL */ while (n-- > 0) { hp[0] = make_small(*ep++); @@ -3020,7 +3002,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, hp += 2; } hp[-1] = NIL; - *hpp = hp; + factory->hp = hp; break; case B2TDecodeBinary: @@ -3042,16 +3024,18 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, return NULL; } } + PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK); + if (ctx->u.dc.hamt_array.pstart) { + PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array); + } } else { - hp_saved = *hpp; reds = ERTS_SWORD_MAX; next = objp; *next = (Eterm) (UWord) NULL; maps_list = NULL; - hamt_list = NULL; } - hp = *hpp; + hp = factory->hp; while (next != NULL) { @@ -3288,9 +3272,9 @@ dec_term_atom_common: break; } case PID_EXT: - *hpp = hp; - ep = dec_pid(edep, hpp, ep, off_heap, objp); - hp = *hpp; + factory->hp = hp; + ep = dec_pid(edep, factory, ep, objp); + hp = factory->hp; if (ep == NULL) { goto error; } @@ -3324,11 +3308,11 @@ dec_term_atom_common: hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_port_header(1); - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; etp->data.ui[0] = num; - off_heap->first = (struct erl_off_heap_header*)etp; + factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_port(etp); } @@ -3408,10 +3392,10 @@ dec_term_atom_common: #else etp->header = make_external_ref_header(ref_words); #endif - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; - off_heap->first = (struct erl_off_heap_header*)etp; + factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); ref_num = &(etp->data.ui32[0]); } @@ -3451,9 +3435,9 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3503,9 +3487,9 @@ dec_term_atom_common: pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3557,9 +3541,9 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &name)) == NULL) { goto error; } - *hpp = hp; - ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL); - hp = *hpp; + factory->hp = hp; + ep = dec_term(edep, factory, ep, &temp, NULL); + hp = factory->hp; if (ep == NULL) { goto error; } @@ -3631,15 +3615,11 @@ dec_term_atom_common: } } else { /* Make hamt */ - struct dec_term_hamt_placeholder* holder = - (struct dec_term_hamt_placeholder*) hp; - - holder->next = hamt_list; - hamt_list = holder; - holder->objp = objp; - holder->size = size; + struct dec_term_hamt* hamt = PSTACK_PUSH(hamt_array); - hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE; + hamt->objp = objp; + hamt->size = size; + hamt->leaf_array = hp; for (n = size; n; n--) { CDR(hp) = (Eterm) COMPRESS_POINTER(next); @@ -3681,9 +3661,9 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &module)) == NULL) { goto error; } - *hpp = hp; + factory->hp = hp; /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3692,7 +3672,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3704,8 +3684,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)funp; + funp->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)funp; funp->fe = erts_put_fun_entry2(module, old_uniq, old_index, uniq, index, arity); @@ -3716,7 +3696,7 @@ dec_term_atom_common: } funp->native_address = funp->fe->native_address; #endif - hp = *hpp; + hp = factory->hp; /* Environment */ for (i = num_free-1; i >= 0; i--) { @@ -3742,14 +3722,14 @@ dec_term_atom_common: ep += 4; hp += ERL_FUN_SIZE; hp += num_free; - *hpp = hp; + factory->hp = hp; funp->thing_word = HEADER_FUN; funp->num_free = num_free; *objp = make_fun(funp); /* Creator pid */ if (*ep != PID_EXT - || (ep = dec_pid(edep, hpp, ++ep, off_heap, + || (ep = dec_pid(edep, factory, ++ep, &funp->creator))==NULL) { goto error; } @@ -3760,7 +3740,7 @@ dec_term_atom_common: } /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3769,7 +3749,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3780,8 +3760,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)funp; + funp->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)funp; old_uniq = unsigned_val(temp); funp->fe = erts_put_fun_entry(module, old_uniq, old_index); @@ -3789,7 +3769,7 @@ dec_term_atom_common: #ifdef HIPE funp->native_address = funp->fe->native_address; #endif - hp = *hpp; + hp = factory->hp; /* Environment */ for (i = num_free-1; i >= 0; i--) { @@ -3823,9 +3803,9 @@ dec_term_atom_common: erts_refc_inc(&pb->val->refc, 1); hp += PROC_BIN_SIZE; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; *objp = make_binary(pb); break; @@ -3841,9 +3821,9 @@ dec_term_atom_common: erts_refc_inc(&pb->val->refc, 1); hp += PROC_BIN_SIZE; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; sub = (ErlSubBin*)hp; @@ -3869,9 +3849,11 @@ dec_term_atom_common: if (next || ctx->state != B2TDecode) { ctx->u.dc.ep = ep; ctx->u.dc.next = next; - ctx->u.dc.hp = hp; + ctx->u.dc.factory.hp = hp; ctx->u.dc.maps_list = maps_list; - ctx->u.dc.hamt_list = hamt_list; + if (!PSTACK_IS_EMPTY(hamt_array)) { + PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array); + } ctx->reds = 0; return NULL; } @@ -3894,40 +3876,36 @@ dec_term_atom_common: maps_list = next; } - /* Iterate through all the hamts and build tree nodes. + ASSERT(hp <= factory->hp_end + || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp))); + factory->hp = hp; + /* + * From here on factory may produce (more) heap fragments */ - if (hamt_list) { - ErtsHeapFactory factory; - factory.p = NULL; - factory.hp = hp; - /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */ + if (!PSTACK_IS_EMPTY(hamt_array)) { + do { + struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array); - do { - struct dec_term_hamt_placeholder* hamt = hamt_list; - *hamt->objp = erts_hashmap_from_array(&factory, - hamt->leafs, + *hamt->objp = erts_hashmap_from_array(factory, + hamt->leaf_array, hamt->size, 1); if (is_non_value(*hamt->objp)) - goto error; + goto error_hamt; - hamt_list = hamt->next; - - /* Yes, we waste a couple of heap words per hamt - for the temporary placeholder */ - *(Eterm*)hamt = make_pos_bignum_header(DEC_TERM_HAMT_PLACEHOLDER_SIZE-1); - } while (hamt_list); - - hp = factory.hp; + (void) PSTACK_POP(hamt_array); + } while (!PSTACK_IS_EMPTY(hamt_array)); + PSTACK_DESTROY(hamt_array); } + ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL); + if (ctx) { ctx->state = B2TDone; ctx->reds = reds; } - *hpp = hp; return ep; error: @@ -3935,11 +3913,12 @@ error: * Must unlink all off-heap objects that may have been * linked into the process. */ - if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */ - hp = *hpp; /* the largest must be the freshest */ + if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */ + factory->hp = hp; /* the largest must be the freshest */ } - undo_offheap_in_area(off_heap, hp_saved, hp); - *hpp = hp_saved; +error_hamt: + erts_factory_undo(factory); + PSTACK_DESTROY(hamt_array); if (ctx) { ctx->state = B2TDecodeFail; ctx->reds = reds; @@ -4465,7 +4444,7 @@ init_done: if (n <= MAP_SMALL_MAP_LIMIT) { heap_size += 3 + n + 1 + n; } else { - heap_size += hashmap_over_estimated_heap_size(n); + heap_size += HASHMAP_ESTIMATED_HEAP_SIZE(n); } break; case STRING_EXT: diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index 50fcfa04d6..508ab8dc17 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -148,6 +148,7 @@ typedef struct { byte *extp; int exttmp; Uint extsize; + Uint heap_size; } ErtsBinary2TermState; @@ -185,18 +186,18 @@ void erts_destroy_dist_ext_copy(ErtsDistExternal *); int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, DistEntry *, ErtsAtomCache *); Sint erts_decode_dist_ext_size(ErtsDistExternal *); -Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *); +Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *); Sint erts_decode_ext_size(byte*, Uint); Sint erts_decode_ext_size_ets(byte*, Uint); -Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**); -Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*); +Eterm erts_decode_ext(ErtsHeapFactory*, byte**); +Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*); Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags); Sint erts_binary2term_prepare(ErtsBinary2TermState *, byte *, Sint); void erts_binary2term_abort(ErtsBinary2TermState *); -Eterm erts_binary2term_create(ErtsBinary2TermState *, Eterm **hpp, ErlOffHeap *); +Eterm erts_binary2term_create(ErtsBinary2TermState *, ErtsHeapFactory*); int erts_debug_max_atom_out_cache_index(void); int erts_debug_atom_to_out_cache_index(Eterm); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 340c7033ab..14d42599a1 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -230,9 +230,23 @@ typedef struct { ERTS_INTERNAL_BINARY_FIELDS SWord orig_size; void (*destructor)(Binary *); - char magic_bin_data[1]; + union { + struct { + ERTS_BINARY_STRUCT_ALIGNMENT + char data[1]; + } aligned; + struct { + char data[1]; + } unaligned; + } u; } ErtsMagicBinary; +#ifdef ARCH_32 +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4 +#else +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0 +#endif + typedef union { Binary binary; ErtsMagicBinary magic_binary; @@ -252,15 +266,30 @@ typedef union { #define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \ ((ErtsBinary *) (BP))->magic_binary.destructor #define ERTS_MAGIC_BIN_DATA(BP) \ - ((void *) ((ErtsBinary *) (BP))->magic_binary.magic_bin_data) -#define ERTS_MAGIC_BIN_DATA_SIZE(BP) \ - ((BP)->orig_size - sizeof(void (*)(Binary *))) + ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data) +#define ERTS_MAGIC_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes)) #define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \ - (sizeof(void (*)(Binary *)) + (Sz)) + (ERTS_MAGIC_DATA_OFFSET + (Sz)) #define ERTS_MAGIC_BIN_SIZE(Sz) \ - (offsetof(ErtsMagicBinary,magic_bin_data) + (Sz)) -#define ERTS_MAGIC_BIN_FROM_DATA(DATA) \ - ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,magic_bin_data))) + (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz)) + +/* On 32-bit arch these macro variants will save memory + by not forcing 8-byte alignment for the magic payload. +*/ +#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \ + ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data) +#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes)) +#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \ + ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET) +#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \ + (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz)) +#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \ + (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz)) +#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \ + ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data))) + #define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary) #define ErlDrvBinary2Binary(D) ((Binary *) \ @@ -748,6 +777,15 @@ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ } +#define PSTACK_CHANGE_ALLOCATOR(s,t) \ +do { \ + if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ + erl_exit(1, "Internal error - trying to change allocator " \ + "type of active pstack\n"); \ + } \ + s.alloc_type = (t); \ + } while (0) + #define PSTACK_DESTROY(s) \ do { \ if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ @@ -757,6 +795,8 @@ do { \ #define PSTACK_IS_EMPTY(s) (s.psp < s.pstart) +#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart) + #define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp)) #define PSTACK_PUSH(s) \ @@ -767,6 +807,45 @@ do { \ #define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE))) +/* + * Do not free the stack after this, it may have pointers into what + * was saved in 'dst'. + */ +#define PSTACK_SAVE(s,dst)\ +do {\ + if (s.pstart == (byte*)PSTK_DEF_STACK(s)) {\ + UWord _pbytes = PSTACK_COUNT(s) * sizeof(PSTACK_TYPE);\ + (dst)->pstart = erts_alloc(s.alloc_type,\ + sizeof(PSTK_DEF_STACK(s)));\ + sys_memcpy((dst)->pstart, s.pstart, _pbytes);\ + (dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\ + (dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\ + (dst)->alloc_type = s.alloc_type;\ + } else\ + *(dst) = s;\ + } while (0) + +/* + * Use on empty stack, only the allocator can be changed before this. + * The src stack is reset to NULL. + */ +#define PSTACK_RESTORE(s, src) \ +do { \ + ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s)); \ + s = *(src); /* struct copy */ \ + (src)->pstart = NULL; \ + ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE))); \ + ASSERT(s.psp < s.pend); \ +} while (0) + +#define PSTACK_DESTROY_SAVED(pstack)\ +do {\ + if ((pstack)->pstart) {\ + erts_free((pstack)->alloc_type, (pstack)->pstart);\ + (pstack)->pstart = NULL;\ + }\ +} while(0) + /* binary.c */ @@ -1055,6 +1134,9 @@ Sint erts_binary_set_loop_limit(Sint limit); /* external.c */ void erts_init_external(void); +/* erl_map.c */ +void erts_init_map(void); + /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index ccc7da265e..75d80b1a58 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -65,8 +65,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key; /* Save last DDLL error o per thread basis (for BC interfaces) */ ErtsPTab erts_port erts_align_attribute(ERTS_CACHE_LINE_SIZE); /* The port table */ -erts_smp_atomic_t erts_bytes_out; /* No bytes sent out of the system */ -erts_smp_atomic_t erts_bytes_in; /* No bytes gotten into the system */ const ErlDrvTermData driver_term_nil = (ErlDrvTermData)NIL; @@ -80,6 +78,9 @@ int erts_port_synchronous_ops = 0; int erts_port_schedule_all_ops = 0; int erts_port_parallelism = 0; +static erts_atomic64_t bytes_in; +static erts_atomic64_t bytes_out; + static void deliver_result(Eterm sender, Eterm pid, Eterm res); static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *); static void terminate_port(Port *p); @@ -1395,31 +1396,22 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp) static ERTS_INLINE void queue_port_sched_op_reply(Process *rp, ErtsProcLocks *rp_locksp, - Eterm *hp_start, - Eterm *hp, - Uint h_size, - ErlHeapFragment* bp, + ErtsHeapFactory* factory, Uint32 *ref_num, Eterm msg) { - Eterm ref = make_internal_ref(hp); + Eterm* hp = erts_produce_heap(factory, ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE, 0); + Eterm ref; + + ref= make_internal_ref(hp); write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]); hp += REF_THING_SIZE; msg = TUPLE2(hp, ref, msg); - hp += 3; - if (!bp) { - HRelease(rp, hp_start + h_size, hp); - } - else { - Uint used_h_size = hp - hp_start; - ASSERT(h_size >= used_h_size); - if (h_size > used_h_size) - bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1); - } + erts_factory_trim_and_close(factory, &msg, 1); - erts_queue_message(rp, rp_locksp, bp, msg, NIL); + erts_queue_message(rp, rp_locksp, factory->heap_frags, msg, NIL); } static void @@ -1429,9 +1421,10 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) if (rp) { ErlOffHeap *ohp; ErlHeapFragment* bp; + ErtsHeapFactory factory; Eterm msg_copy; Uint hsz, msg_sz; - Eterm *hp, *hp_start; + Eterm *hp; ErtsProcLocks rp_locks = 0; hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; @@ -1442,22 +1435,22 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) hsz += msg_sz; } - hp_start = hp = erts_alloc_message_heap(hsz, + hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + erts_factory_message_init(&factory, rp, hp, bp); if (is_immed(msg)) msg_copy = msg; - else + else { msg_copy = copy_struct(msg, msg_sz, &hp, ohp); + factory.hp = hp; + } queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, ref_num, msg_copy); @@ -1542,12 +1535,10 @@ erts_schedule_proc2port_signal(Process *c_p, } static ERTS_INLINE void -send_badsig(Port *prt) -{ +send_badsig(Port *prt) { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; Process* rp; Eterm connected = ERTS_PORT_GET_CONNECTED(prt); - ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_LC_ASSERT(erts_get_scheduler_id()); @@ -1567,15 +1558,13 @@ send_badsig(Port *prt) 0); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - } -} + } /* exit sent */ +} /* send_badsig */ static void -badsig_received(int bang_op, - Port *prt, +badsig_received(int bang_op, Port *prt, erts_aint32_t state, - int bad_output_value) -{ + int bad_output_value) { /* * if (bang_op) * we are part of a "Prt ! Something" operation @@ -1591,12 +1580,12 @@ badsig_received(int bang_op, } if (bang_op) send_badsig(prt); - } -} + } /* not invalid */ +} /* behaved accordingly */ static int -port_badsig(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) -{ +port_badsig(Port *prt, erts_aint32_t state, int op, + ErtsProc2PortSigData *sigdp) { if (op == ERTS_PROC2PORT_SIG_EXEC) badsig_received(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP, prt, @@ -1605,16 +1594,14 @@ port_badsig(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); return ERTS_PORT_REDS_BADSIG; -} - - -/* - * bad_port_signal() will +} /* port_badsig */ +/* bad_port_signal() will * - preserve signal order of signals. * - send a 'badsig' exit signal to connected process if 'from' is an * internal pid and the port is alive when the bad signal reaches * it. */ + static ErtsPortOpResult bad_port_signal(Process *c_p, int flags, @@ -1689,6 +1676,7 @@ call_driver_outputv(int bang_op, if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt)) send_badsig(prt); else { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErlDrvSizeT size = evp->size; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) @@ -1706,7 +1694,10 @@ call_driver_outputv(int bang_op, prt->caller = NIL; prt->bytes_out += size; - erts_smp_atomic_add_nob(&erts_bytes_out, size); + if (esdp) + esdp->io.out += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size); } } @@ -1786,7 +1777,7 @@ call_driver_output(int bang_op, if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt)) send_badsig(prt); else { - + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) || ERTS_IS_CRASH_DUMPING); @@ -1802,7 +1793,10 @@ call_driver_output(int bang_op, prt->caller = NIL; prt->bytes_out += size; - erts_smp_atomic_add_nob(&erts_bytes_out, size); + if (esdp) + esdp->io.out += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size); } } @@ -2749,6 +2743,9 @@ void erts_init_io(int port_tab_size, drv_list_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; drv_list_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_atomic64_init_nob(&bytes_in, 0); + erts_atomic64_init_nob(&bytes_out, 0); + common_element_size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); common_element_size += ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErtsPortTaskBusyPortQ)); common_element_size += 10; /* name */ @@ -2790,9 +2787,6 @@ void erts_init_io(int port_tab_size, legacy_port_tab, 1); - erts_smp_atomic_init_nob(&erts_bytes_out, 0); - erts_smp_atomic_init_nob(&erts_bytes_in, 0); - sys_init_io(); erts_smp_tsd_set(driver_list_lock_status_key, (void *) 1); @@ -2812,7 +2806,6 @@ void erts_init_io(int port_tab_size, } #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) - static ERTS_INLINE void lcnt_enable_drv_lock_count(erts_driver_t *dp, int enable) { if (dp->lock) { @@ -2852,25 +2845,26 @@ static ERTS_INLINE void lcnt_enable_port_lock_count(Port *prt, int enable) } } -void erts_lcnt_enable_io_lock_count(int enable) -{ +void erts_lcnt_enable_io_lock_count(int enable) { erts_driver_t *dp; - int i, max = erts_ptab_max(&erts_port); + int ix, max = erts_ptab_max(&erts_port); + Port *prt; - for (i = 0; i < max; i++) { - Port *prt = erts_pix2port(i); - if (prt) + for (ix = 0; ix < max; ix++) { + if ((prt = erts_pix2port(ix)) != NULL) { lcnt_enable_port_lock_count(prt, enable); - } + } + } /* for all ports */ lcnt_enable_drv_lock_count(&vanilla_driver, enable); lcnt_enable_drv_lock_count(&spawn_driver, enable); lcnt_enable_drv_lock_count(&fd_driver, enable); - for (dp = driver_list; dp; dp = dp->next) + /* enable lock counting in all drivers */ + for (dp = driver_list; dp; dp = dp->next) { lcnt_enable_drv_lock_count(dp, enable); -} -#endif - + } +} /* enable/disable lock counting of ports */ +#endif /* defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) */ /* * Buffering of data when using line oriented I/O on ports */ @@ -3896,12 +3890,13 @@ port_sig_control(Port *prt, if (res == ERTS_PORT_OP_DONE) { Eterm msg; - Eterm *hp, *hp_start; + Eterm *hp; ErlHeapFragment *bp; ErlOffHeap *ohp; + ErtsHeapFactory factory; Process *rp; ErtsProcLocks rp_locks = 0; - Uint hsz; + Uint hsz, rsz; int control_flags; rp = erts_proc_lookup_raw(sigdp->caller); @@ -3910,17 +3905,19 @@ port_sig_control(Port *prt, control_flags = prt->control_flags; - hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hsz += port_control_result_size(control_flags, + rsz = port_control_result_size(control_flags, resp_bufp, &resp_size, &resp_buf[0]); + hsz = rsz + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hp_start = hp = erts_alloc_message_heap(hsz, + + hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + erts_factory_message_init(&factory, rp, hp, bp); msg = write_port_control_result(control_flags, resp_bufp, @@ -3929,13 +3926,11 @@ port_sig_control(Port *prt, &hp, bp, ohp); + factory.hp = hp; queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, msg); @@ -4222,8 +4217,6 @@ port_sig_call(Port *prt, if (res == ERTS_PORT_OP_DONE) { Eterm msg; Eterm *hp; - ErlHeapFragment *bp; - ErlOffHeap *ohp; Process *rp; ErtsProcLocks rp_locks = 0; Sint hsz; @@ -4234,29 +4227,31 @@ port_sig_call(Port *prt, hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size); if (hsz >= 0) { - Eterm *hp_start; + ErlHeapFragment* bp; + ErlOffHeap* ohp; + ErtsHeapFactory factory; byte *endp; hsz += 3; /* ok tuple */ hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hp_start = hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); + hp = erts_alloc_message_heap(hsz, + &bp, + &ohp, + rp, + &rp_locks); endp = (byte *) resp_bufp; - msg = erts_decode_ext(&hp, ohp, &endp); + erts_factory_message_init(&factory, rp, hp, bp); + msg = erts_decode_ext(&factory, &endp); if (is_value(msg)) { + hp = erts_produce_heap(&factory, + 3, + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE); msg = TUPLE2(hp, am_ok, msg); - hp += 3; queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, msg); @@ -4264,8 +4259,6 @@ port_sig_call(Port *prt, erts_smp_proc_unlock(rp, rp_locks); goto done; } - if (bp) - free_message_buffer(bp); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); } @@ -4342,10 +4335,11 @@ erts_port_call(Process* c_p, try_call_res = try_imm_drv_call(&try_call_state); switch (try_call_res) { case ERTS_TRY_IMM_DRV_CALL_OK: { - Eterm *hp, *hp_end; + ErtsHeapFactory factory; Sint hsz; unsigned ret_flags = 0U; Eterm term; + Eterm* hp; res = call_driver_call(c_p->common.id, prt, @@ -4365,15 +4359,14 @@ erts_port_call(Process* c_p, if (hsz < 0) return ERTS_PORT_OP_BADARG; hsz += 3; - hp = HAlloc(c_p, hsz); - hp_end = hp + hsz; + erts_factory_proc_prealloc_init(&factory, c_p, hsz); endp = (byte *) resp_bufp; - term = erts_decode_ext(&hp, &MSO(c_p), &endp); + term = erts_decode_ext(&factory, &endp); if (term == THE_NON_VALUE) return ERTS_PORT_OP_BADARG; + hp = erts_produce_heap(&factory,3,0); *retvalp = TUPLE2(hp, am_ok, term); - hp += 3; - HRelease(c_p, hp_end, hp); + erts_factory_close(&factory); if (resp_bufp != &resp_buf[0] && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) driver_free(resp_bufp); @@ -4508,12 +4501,11 @@ port_sig_info(Port *prt, prt, sigdp->u.info.item); if (is_value(value)) { + ErtsHeapFactory factory; + erts_factory_message_init(&factory, NULL, hp, bp); queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, value); } @@ -4584,6 +4576,102 @@ erts_port_info(Process* c_p, } typedef struct { + Uint sched_id; + Eterm pid; + Uint32 refn[ERTS_REF_NUMBERS]; + erts_smp_atomic32_t refc; +} ErtsIOBytesReq; + +static void +reply_io_bytes(void *vreq) +{ + ErtsIOBytesReq *req = (ErtsIOBytesReq *) vreq; + Process *rp; + + rp = erts_proc_lookup(req->pid); + if (rp) { + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + ErtsProcLocks rp_locks; + Eterm ref, msg, ein, eout, *hp; + Uint64 in, out; + Uint hsz; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + Uint sched_id = esdp->no; + in = esdp->io.in; + out = esdp->io.out; + if (req->sched_id != sched_id) + rp_locks = 0; + else { + in += (Uint64) erts_atomic64_read_nob(&bytes_in); + out += (Uint64) erts_atomic64_read_nob(&bytes_out); + rp_locks = ERTS_PROC_LOCK_MAIN; + } + + hsz = 5 /* 4-tuple */ + REF_THING_SIZE; + + erts_bld_uint64(NULL, &hsz, in); + erts_bld_uint64(NULL, &hsz, out); + + hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + + ref = make_internal_ref(hp); + write_ref_thing(hp, req->refn[0], req->refn[1], req->refn[2]); + hp += REF_THING_SIZE; + + ein = erts_bld_uint64(&hp, NULL, in); + eout = erts_bld_uint64(&hp, NULL, out); + + msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout); + erts_queue_message(rp, &rp_locks, bp, msg, NIL); + + if (req->sched_id == sched_id) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + } + + if (erts_smp_atomic32_dec_read_nob(&req->refc) == 0) + erts_free(ERTS_ALC_T_IOB_REQ, req); +} + +Eterm +erts_request_io_bytes(Process *c_p) +{ + Uint *hp; + Eterm ref; + Uint32 *refn; + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsIOBytesReq *req = erts_alloc(ERTS_ALC_T_IOB_REQ, + sizeof(ErtsIOBytesReq)); + + hp = HAlloc(c_p, REF_THING_SIZE); + ref = erts_sched_make_ref_in_buffer(esdp, hp); + refn = internal_ref_numbers(ref); + + req->sched_id = esdp->no; + req->pid = c_p->common.id; + req->refn[0] = refn[0]; + req->refn[1] = refn[1]; + req->refn[2] = refn[2]; + erts_smp_atomic32_init_nob(&req->refc, + (erts_aint32_t) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_io_bytes, + (void *) req); +#endif + + reply_io_bytes((void *) req); + + return ref; +} + + +typedef struct { int to; void *arg; } prt_one_lnk_data; @@ -5106,22 +5194,27 @@ cleanup_b2t_states(struct b2t_states__ *b2tsp) static int driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) { +#define HEAP_EXTRA 200 #define ERTS_DDT_FAIL do { res = -1; goto done; } while (0) Uint need = 0; int depth = 0; int res; - Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL; ErlDrvTermData* ptr; ErlDrvTermData* ptr_end; DECLARE_ESTACK(stack); - Eterm mess = NIL; /* keeps compiler happy */ + Eterm mess; Process* rp = NULL; - ErlHeapFragment *bp = NULL; - ErlOffHeap *ohp; + ErtsHeapFactory factory; ErtsProcLocks rp_locks = 0; struct b2t_states__ b2t; - int scheduler = 1; /* Silence erroneous warning... */ + int scheduler; + int is_heap_need_limited = 1; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ERTS_UNDEF(mess,NIL); + ERTS_UNDEF(scheduler,1); + + factory.mode = FACTORY_CLOSED; init_b2t_states(&b2t); /* @@ -5285,19 +5378,24 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) #ifdef DEBUG b2t.org_ext[b2t.ix] = ext; #endif - hsz = erts_binary2term_prepare(&b2t.state[b2t.ix++], ext, size); + hsz = erts_binary2term_prepare(&b2t.state[b2t.ix], ext, size); if (hsz < 0) ERTS_DDT_FAIL; /* Invalid data */ + b2t.state[b2t.ix++].heap_size = hsz; need += hsz; ptr += 2; depth++; + if (size > MAP_SMALL_MAP_LIMIT*3) { /* may contain big map */ + is_heap_need_limited = 0; + } break; } case ERL_DRV_MAP: { /* int */ ERTS_DDT_CHK_ENOUGH_ARGS(1); if ((int) ptr[0] < 0) ERTS_DDT_FAIL; if (ptr[0] > MAP_SMALL_MAP_LIMIT) { - need += hashmap_over_estimated_heap_size(ptr[0]); + need += HASHMAP_ESTIMATED_HEAP_SIZE(ptr[0]); + is_heap_need_limited = 0; } else { need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0]; } @@ -5336,8 +5434,17 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) goto done; } - hp_start = hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); - hp_end = hp + need; + /* Try copy directly to destination heap if we know there are no big maps */ + if (is_heap_need_limited) { + ErlOffHeap *ohp; + ErlHeapFragment* bp; + Eterm* hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); + erts_factory_message_init(&factory, rp, hp, bp); + } + else { + erts_factory_message_init(&factory, NULL, NULL, + new_message_buffer(need)); + } /* * Interpret the instructions and build the term. @@ -5358,13 +5465,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_INT: /* signed int argument */ #if HALFWORD_HEAP - mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]); + erts_reserve_heap(&factory, BIG_NEED_SIZE(2)); + mess = erts_bld_sint64(&factory.hp, NULL, (Sint64)ptr[0]); #else + erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_SSMALL((Sint)ptr[0])) mess = make_small((Sint)ptr[0]); else { - mess = small_to_big((Sint)ptr[0], hp); - hp += BIG_UINT_HEAP_SIZE; + mess = small_to_big((Sint)ptr[0], factory.hp); + factory.hp += BIG_UINT_HEAP_SIZE; } #endif ptr++; @@ -5372,25 +5481,29 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_UINT: /* unsigned int argument */ #if HALFWORD_HEAP - mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_uint64(&factory.hp, NULL, (Uint64)ptr[0]); #else + erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_USMALL(0, (Uint)ptr[0])) mess = make_small((Uint)ptr[0]); else { - mess = uint_to_big((Uint)ptr[0], hp); - hp += BIG_UINT_HEAP_SIZE; + mess = uint_to_big((Uint)ptr[0], factory.hp); + factory.hp += BIG_UINT_HEAP_SIZE; } #endif ptr++; break; case ERL_DRV_INT64: /* pointer to unsigned 64-bit int argument */ - mess = erts_bld_sint64(&hp, NULL, *((Sint64 *) ptr[0])); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_sint64(&factory.hp, NULL, *((Sint64 *) ptr[0])); ptr++; break; case ERL_DRV_UINT64: /* pointer to unsigned 64-bit int argument */ - mess = erts_bld_uint64(&hp, NULL, *((Uint64 *) ptr[0])); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_uint64(&factory.hp, NULL, *((Uint64 *) ptr[0])); ptr++; break; @@ -5404,11 +5517,14 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) Uint size = ptr[1]; Uint offset = ptr[2]; - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size); + if (esdp) + esdp->io.in += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) size); if (size <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin* hbp = (ErlHeapBin *) hp; - hp += heap_bin_size(size); + ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory, + heap_bin_size(size), HEAP_EXTRA); hbp->thing_word = header_heap_bin(size); hbp->size = size; if (size > 0) { @@ -5417,18 +5533,18 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = make_binary(hbp); } else { - ProcBin* pb = (ProcBin *) hp; + ProcBin* pb = (ProcBin *) erts_produce_heap(&factory, + PROC_BIN_SIZE, HEAP_EXTRA); driver_binary_inc_refc(b); /* caller will free binary */ pb->thing_word = HEADER_PROC_BIN; pb->size = size; - pb->next = ohp->first; - ohp->first = (struct erl_off_heap_header*)pb; + pb->next = factory.off_heap->first; + factory.off_heap->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = ((byte*) b->orig_bytes) + offset; pb->flags = 0; mess = make_binary(pb); - hp += PROC_BIN_SIZE; - OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); + OH_OVERHEAD(factory.off_heap, pb->size / sizeof(Eterm)); } ptr += 3; break; @@ -5438,11 +5554,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) byte *bufp = (byte *) ptr[0]; Uint size = (Uint) ptr[1]; - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size); + if (esdp) + esdp->io.in += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) size); if (size <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin* hbp = (ErlHeapBin *) hp; - hp += heap_bin_size(size); + ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory, + heap_bin_size(size), + HEAP_EXTRA); hbp->thing_word = header_heap_bin(size); hbp->size = size; if (size > 0) { @@ -5457,16 +5577,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) ASSERT(bufp); erts_refc_init(&bp->refc, 1); sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size); - pbp = (ProcBin *) hp; - hp += PROC_BIN_SIZE; + pbp = (ProcBin *) erts_produce_heap(&factory, + PROC_BIN_SIZE, HEAP_EXTRA); pbp->thing_word = HEADER_PROC_BIN; pbp->size = size; - pbp->next = ohp->first; - ohp->first = (struct erl_off_heap_header*)pbp; + pbp->next = factory.off_heap->first; + factory.off_heap->first = (struct erl_off_heap_header*)pbp; pbp->val = bp; pbp->bytes = (byte*) bp->orig_bytes; pbp->flags = 0; - OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm)); + OH_OVERHEAD(factory.off_heap, pbp->size / sizeof(Eterm)); mess = make_binary(pbp); } ptr += 2; @@ -5474,14 +5594,19 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) } case ERL_DRV_STRING: /* char*, length */ - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) ptr[1]); - mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], NIL); + if (esdp) + esdp->io.in += (Uint64) ptr[1]; + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) ptr[1]); + erts_reserve_heap(&factory, 2*ptr[1]); + mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], NIL); ptr += 2; break; case ERL_DRV_STRING_CONS: /* char*, length */ mess = ESTACK_POP(stack); - mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], mess); + erts_reserve_heap(&factory, 2*ptr[1]); + mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], mess); ptr += 2; break; @@ -5490,11 +5615,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = ESTACK_POP(stack); i--; + erts_reserve_heap(&factory, 2*i); while(i > 0) { Eterm hd = ESTACK_POP(stack); - mess = CONS(hp, hd, mess); - hp += 2; + mess = CONS(factory.hp, hd, mess); + factory.hp += 2; i--; } ptr++; @@ -5503,13 +5629,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_TUPLE: { /* int */ int size = (int)ptr[0]; - Eterm* tp = hp; + Eterm* tp = erts_produce_heap(&factory, size+1, HEAP_EXTRA); *tp = make_arityval(size); mess = make_tuple(tp); tp += size; /* point at last element */ - hp = tp+1; /* advance "heap" pointer */ while(size--) { *tp-- = ESTACK_POP(stack); @@ -5525,20 +5650,22 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_FLOAT: { /* double * */ FloatDef f; + Eterm* fp = erts_produce_heap(&factory, FLOAT_SIZE_OBJECT, HEAP_EXTRA); - mess = make_float(hp); + mess = make_float(fp); f.fd = *((double *) ptr[0]); if (!erts_isfinite(f.fd)) ERTS_DDT_FAIL; - PUT_DOUBLE(f, hp); - hp += FLOAT_SIZE_OBJECT; + PUT_DOUBLE(f, fp); ptr++; break; } case ERL_DRV_EXT2TERM: /* char *ext, int size */ ASSERT(b2t.org_ext[b2t.ix] == (byte *) ptr[0]); - mess = erts_binary2term_create(&b2t.state[b2t.ix++], &hp, ohp); + + erts_reserve_heap(&factory, b2t.state[b2t.ix].heap_size); + mess = erts_binary2term_create(&b2t.state[b2t.ix++], &factory); if (mess == THE_NON_VALUE) ERTS_DDT_FAIL; ptr += 2; @@ -5548,41 +5675,32 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) int size = (int)ptr[0]; if (size > MAP_SMALL_MAP_LIMIT) { int ix = 2*size; - ErtsHeapFactory factory; - Eterm* leafs = hp; - - hp += 2*size; - while(ix--) { *--hp = ESTACK_POP(stack); } + Eterm* leafs; - hp += 2*size; - factory.p = NULL; - factory.hp = hp; - /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */ + erts_produce_heap(&factory, ix, HEAP_EXTRA); + leafs = factory.hp; + while(ix--) { *--leafs = ESTACK_POP(stack); } mess = erts_hashmap_from_array(&factory, leafs, size, 1); - if (is_non_value(mess)) ERTS_DDT_FAIL; - - hp = factory.hp; } else { - Eterm* tp = hp; Eterm* vp; flatmap_t *mp; + Eterm* tp = erts_produce_heap(&factory, + 2*size + 1 + MAP_HEADER_FLATMAP_SZ, + HEAP_EXTRA); *tp = make_arityval(size); - hp += 1 + size; - mp = (flatmap_t*)hp; + mp = (flatmap_t*) (tp + 1 + size); mp->thing_word = MAP_HEADER_FLATMAP; mp->size = size; mp->keys = make_tuple(tp); mess = make_flatmap(mp); - hp += MAP_HEADER_FLATMAP_SZ + size; - tp += size; /* point at last key */ - vp = hp - 1; /* point at last value */ + vp = factory.hp - 1; /* point at last value */ while(size--) { *vp-- = ESTACK_POP(stack); @@ -5605,25 +5723,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) if (res > 0) { mess = ESTACK_POP(stack); /* get resulting value */ - if (bp) - bp = erts_resize_message_buffer(bp, hp - hp_start, &mess, 1); - else { - ASSERT(hp); - HRelease(rp, hp_end, hp); - } + erts_factory_close(&factory); /* send message */ - erts_queue_message(rp, &rp_locks, bp, mess, am_undefined); + erts_queue_message(rp, &rp_locks, factory.heap_frags, mess, am_undefined); } else { if (b2t.ix > b2t.used) b2t.used = b2t.ix; for (b2t.ix = 0; b2t.ix < b2t.used; b2t.ix++) erts_binary2term_abort(&b2t.state[b2t.ix]); - if (bp) - free_message_buffer(bp); - else if (hp) { - HRelease(rp, hp_end, hp); - } + erts_factory_undo(&factory); } if (rp) { if (rp_locks) @@ -5635,6 +5744,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) DESTROY_ESTACK(stack); return res; #undef ERTS_DDT_FAIL +#undef HEAP_EXTRA } static ERTS_INLINE int @@ -5763,6 +5873,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, { erts_aint32_t state; Port* prt = erts_drvport2port_state(ix, &state); + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -5773,7 +5884,10 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); + if (esdp) + esdp->io.in += (Uint64) (hlen + len); + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { return erts_net_message(prt, prt->dist_entry, @@ -5798,6 +5912,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, { erts_aint32_t state; Port* prt = erts_drvport2port_state(ix, &state); + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -5809,7 +5924,10 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); + if (esdp) + esdp->io.in += (Uint64) (hlen + len); + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { if (len == 0) return erts_net_message(prt, @@ -5849,6 +5967,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, ErlDrvBinary** binv; Port* prt; erts_aint32_t state; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -5887,7 +6006,10 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, /* XXX handle distribution !!! */ prt->bytes_in += (hlen + size); - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + size)); + if (esdp) + esdp->io.in += (Uint64) (hlen + size); + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + size)); deliver_vec_message(prt, ERTS_PORT_GET_CONNECTED(prt), hbuf, hlen, binv, iov, n, size); return 0; diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index cd53069872..78eb0bfe2b 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -20,6 +20,22 @@ #ifndef __SYS_H__ #define __SYS_H__ +#ifdef ERTS_INLINE +# ifndef ERTS_CAN_INLINE +# define ERTS_CAN_INLINE 1 +# endif +#else +# if defined(__GNUC__) +# define ERTS_CAN_INLINE 1 +# define ERTS_INLINE __inline__ +# elif defined(__WIN32__) +# define ERTS_CAN_INLINE 1 +# define ERTS_INLINE __inline +# else +# define ERTS_CAN_INLINE 0 +# define ERTS_INLINE +# endif +#endif #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) # undef ERTS_CAN_INLINE @@ -94,23 +110,6 @@ typedef int ErtsSysFdType; typedef ERTS_SYS_FD_TYPE ErtsSysFdType; #endif -#ifdef ERTS_INLINE -# ifndef ERTS_CAN_INLINE -# define ERTS_CAN_INLINE 1 -# endif -#else -# if defined(__GNUC__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline__ -# elif defined(__WIN32__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline -# else -# define ERTS_CAN_INLINE 0 -# define ERTS_INLINE -# endif -#endif - #if !defined(__GNUC__) # define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 #elif !defined(__GNUC_MINOR__) diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 8bffdedb2b..fe7c5826f5 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -529,6 +529,9 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; tiw->sentinel.next = &tiw->sentinel; tiw->sentinel.prev = &tiw->sentinel; + tiw->sentinel.u.func.timeout = NULL; + tiw->sentinel.u.func.cancel = NULL; + tiw->sentinel.u.func.arg = NULL; return tiw; } @@ -624,6 +627,41 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) } } +void +erts_twheel_debug_foreach(ErtsTimerWheel *tiw, + void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg) +{ + ErtsTWheelTimer *tmr; + int ix; + + tmr = tiw->sentinel.next; + while (tmr != &tiw->sentinel) { + if (tmr->u.func.timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + tmr = tmr->next; + } + + for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) { + if (tmr->u.func.timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + } + + for (ix = 0; ix < ERTS_TIW_SIZE; ix++) { + tmr = tiw->w[ix]; + if (tmr) { + do { + if (tmr->u.func.timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + tmr = tmr->next; + } while (tmr != tiw->w[ix]); + } + } +} + #ifdef ERTS_TW_DEBUG void erts_p_slpq(void) { diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index e001f31932..119d6c097d 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -835,6 +835,10 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define TCP_ADDF_PENDING_SHUT_RDWR 32 /* Call shutdown(sock, SHUT_RDWR) when queue empties */ #define TCP_ADDF_PENDING_SHUTDOWN \ (TCP_ADDF_PENDING_SHUT_WR | TCP_ADDF_PENDING_SHUT_RDWR) +#define TCP_ADDF_SHOW_ECONNRESET 64 /* Tell user about incoming RST */ +#define TCP_ADDF_DELAYED_ECONNRESET 128 /* An ECONNRESET error occured on send or shutdown */ +#define TCP_ADDF_SHUTDOWN_WR_DONE 256 /* A shutdown(sock, SHUT_WR) or SHUT_RDWR was made */ +#define TCP_ADDF_LINGER_ZERO 512 /* Discard driver queue on port close */ /* *_REQ_* replies */ #define INET_REP_ERROR 0 @@ -879,6 +883,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define INET_LOPT_MSGQ_HIWTRMRK 36 /* set local msgq high watermark */ #define INET_LOPT_MSGQ_LOWTRMRK 37 /* set local msgq low watermark */ #define INET_LOPT_NETNS 38 /* Network namespace pathname */ +#define INET_LOPT_TCP_SHOW_ECONNRESET 39 /* tell user about incoming RST */ /* SCTP options: a separate range, from 100: */ #define SCTP_OPT_RTOINFO 100 #define SCTP_OPT_ASSOCINFO 101 @@ -6138,7 +6143,12 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) desc->active_count = 0; if ((desc->stype == SOCK_STREAM) && (desc->active != INET_PASSIVE) && (desc->state == INET_STATE_CLOSED)) { - tcp_closed_message((tcp_descriptor *) desc); + tcp_descriptor *tdesc = (tcp_descriptor *) desc; + if (tdesc->tcp_add_flags & TCP_ADDF_DELAYED_ECONNRESET) { + tdesc->tcp_add_flags &= ~TCP_ADDF_DELAYED_ECONNRESET; + tcp_error_message(tdesc, ECONNRESET); + } + tcp_closed_message(tdesc); if (desc->exitf) { driver_exit(desc->port, 0); return 0; /* Give up on this socket, descriptor lost */ @@ -6255,6 +6265,16 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) continue; #endif + case INET_LOPT_TCP_SHOW_ECONNRESET: + if (desc->sprotocol == IPPROTO_TCP) { + tcp_descriptor* tdesc = (tcp_descriptor*) desc; + if (ival) + tdesc->tcp_add_flags |= TCP_ADDF_SHOW_ECONNRESET; + else + tdesc->tcp_add_flags &= ~TCP_ADDF_SHOW_ECONNRESET; + } + continue; + case INET_OPT_REUSEADDR: #ifdef __WIN32__ continue; /* Bjorn says */ @@ -6299,6 +6319,13 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) arg_sz = sizeof(li_val); DEBUGF(("inet_set_opts(%ld): s=%d, SO_LINGER=%d,%d", (long)desc->port, desc->s, li_val.l_onoff,li_val.l_linger)); + if (desc->sprotocol == IPPROTO_TCP) { + tcp_descriptor* tdesc = (tcp_descriptor*) desc; + if (li_val.l_onoff && li_val.l_linger == 0) + tdesc->tcp_add_flags |= TCP_ADDF_LINGER_ZERO; + else + tdesc->tcp_add_flags &= ~TCP_ADDF_LINGER_ZERO; + } break; case INET_OPT_PRIORITY: @@ -7234,6 +7261,17 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc, continue; #endif + case INET_LOPT_TCP_SHOW_ECONNRESET: + if (desc->sprotocol == IPPROTO_TCP) { + tcp_descriptor* tdesc = (tcp_descriptor*) desc; + *ptr++ = opt; + ival = !!(tdesc->tcp_add_flags & TCP_ADDF_SHOW_ECONNRESET); + put_int32(ival, ptr); + } else { + TRUNCATE_TO(0,ptr); + } + continue; + case INET_OPT_PRIORITY: #ifdef SO_PRIORITY type = SO_PRIORITY; @@ -9077,6 +9115,11 @@ static tcp_descriptor* tcp_inet_copy(tcp_descriptor* desc,SOCKET s, copy_desc->low = desc->low; copy_desc->send_timeout = desc->send_timeout; copy_desc->send_timeout_close = desc->send_timeout_close; + + if (desc->tcp_add_flags & TCP_ADDF_SHOW_ECONNRESET) + copy_desc->tcp_add_flags |= TCP_ADDF_SHOW_ECONNRESET; + else + copy_desc->tcp_add_flags &= ~TCP_ADDF_SHOW_ECONNRESET; /* The new port will be linked and connected to the original caller */ port = driver_create_port(port, owner, "tcp_inet", (ErlDrvData) copy_desc); @@ -9438,7 +9481,11 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, if (desc->tcp_add_flags & TCP_ADDF_DELAYED_CLOSE_RECV) { desc->tcp_add_flags &= ~(TCP_ADDF_DELAYED_CLOSE_RECV| TCP_ADDF_DELAYED_CLOSE_SEND); - return ctl_reply(INET_REP_ERROR, "closed", 6, rbuf, rsize); + if (desc->tcp_add_flags & TCP_ADDF_DELAYED_ECONNRESET) { + desc->tcp_add_flags &= ~TCP_ADDF_DELAYED_ECONNRESET; + return ctl_reply(INET_REP_ERROR, "econnreset", 10, rbuf, rsize); + } else + return ctl_reply(INET_REP_ERROR, "closed", 6, rbuf, rsize); } return ctl_error(ENOTCONN, rbuf, rsize); } @@ -9499,6 +9546,8 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } if (IS_SOCKET_ERROR(sock_shutdown(INETP(desc)->s, how))) { + if (how != TCP_SHUT_RD) + desc->tcp_add_flags |= TCP_ADDF_SHUTDOWN_WR_DONE; return ctl_error(sock_errno(), rbuf, rsize); } else { return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); @@ -9633,7 +9682,13 @@ static void tcp_inet_commandv(ErlDrvData e, ErlIOVec* ev) if (!IS_CONNECTED(INETP(desc))) { if (desc->tcp_add_flags & TCP_ADDF_DELAYED_CLOSE_SEND) { desc->tcp_add_flags &= ~TCP_ADDF_DELAYED_CLOSE_SEND; - inet_reply_error_am(INETP(desc), am_closed); + if (desc->tcp_add_flags & TCP_ADDF_DELAYED_ECONNRESET) { + /* Don't clear flag. Leave it enabled for the next receive + * operation. + */ + inet_reply_error(INETP(desc), ECONNRESET); + } else + inet_reply_error_am(INETP(desc), am_closed); } else inet_reply_error(INETP(desc), ENOTCONN); @@ -9652,6 +9707,8 @@ static void tcp_inet_flush(ErlDrvData e) /* Discard send queue to avoid hanging port (OTP-7615) */ tcp_clear_output(desc); } + if (desc->tcp_add_flags & TCP_ADDF_LINGER_ZERO) + tcp_clear_output(desc); } static void tcp_inet_process_exit(ErlDrvData e, ErlDrvMonitor *monitorp) @@ -10014,7 +10071,10 @@ static int tcp_recv(tcp_descriptor* desc, int request_len) int err = sock_errno(); if (err == ECONNRESET) { DEBUGF((" => detected close (connreset)\r\n")); - return tcp_recv_closed(desc); + if (desc->tcp_add_flags & TCP_ADDF_SHOW_ECONNRESET) + return tcp_recv_error(desc, err); + else + return tcp_recv_closed(desc); } if (err == ERRNO_BLOCK) { DEBUGF((" => would block\r\n")); @@ -10226,7 +10286,19 @@ static void tcp_inet_event(ErlDrvData e, ErlDrvEvent event) if (netEv.lNetworkEvents & FD_CLOSE) { /* error in err = netEv.iErrorCode[FD_CLOSE_BIT] */ DEBUGF(("Detected close in %s, line %d\r\n", __FILE__, __LINE__)); - tcp_recv_closed(desc); + if (desc->tcp_add_flags & TCP_ADDF_SHOW_ECONNRESET) { + err = netEv.iErrorCode[FD_CLOSE_BIT]; + if (err == ECONNRESET) + tcp_recv_error(desc, err); + else if (err == ECONNABORTED && IS_CONNECTED(INETP(desc))) { + /* translate this error to ECONNRESET */ + tcp_recv_error(desc, ECONNRESET); + } + else + tcp_recv_closed(desc); + } + else + tcp_recv_closed(desc); } DEBUGF(("tcp_inet_event(%ld) }\r\n", (long)desc->inet.port)); return; @@ -10535,6 +10607,9 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event) static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err) { + int show_econnreset = (err == ECONNRESET + && desc->tcp_add_flags & TCP_ADDF_SHOW_ECONNRESET); + /* * If the port is busy, we must do some clean-up before proceeding. */ @@ -10550,14 +10625,21 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err) /* * We used to handle "expected errors" differently from unexpected ones. - * Now we handle all errors in the same way. We just have to distinguish - * between passive and active sockets. + * Now we handle all errors in the same way (unless the show_econnreset + * socket option is enabled). We just have to distinguish between passive + * and active sockets. */ DEBUGF(("driver_failure_eof(%ld) in %s, line %d\r\n", (long)desc->inet.port, __FILE__, __LINE__)); if (desc->inet.active) { - tcp_closed_message(desc); - inet_reply_error_am(INETP(desc), am_closed); + if (show_econnreset) { + tcp_error_message(desc, err); + tcp_closed_message(desc); + inet_reply_error(INETP(desc), err); + } else { + tcp_closed_message(desc); + inet_reply_error_am(INETP(desc), am_closed); + } if (desc->inet.exitf) driver_exit(desc->inet.port, 0); else @@ -10569,7 +10651,10 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err) erl_inet_close(INETP(desc)); if (desc->inet.caller) { - inet_reply_error_am(INETP(desc), am_closed); + if (show_econnreset) + inet_reply_error(INETP(desc), err); + else + inet_reply_error_am(INETP(desc), am_closed); } else { /* No blocking send op to reply to right now. @@ -10586,12 +10671,46 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err) * in the receive operation. */ desc->tcp_add_flags |= TCP_ADDF_DELAYED_CLOSE_RECV; + + if (show_econnreset) { + /* Return {error, econnreset} instead of {error, closed} + * on send or receive operations. + */ + desc->tcp_add_flags |= TCP_ADDF_DELAYED_ECONNRESET; + } } return -1; } static int tcp_send_error(tcp_descriptor* desc, int err) { + /* EPIPE errors usually occur in one of three ways: + * 1. We write to a socket when we've already shutdown() the write side. On + * Windows the error returned for this is ESHUTDOWN rather than EPIPE. + * 2. The TCP peer sends us an RST through no fault of our own (perhaps + * by aborting the connection using SO_LINGER) and we then attempt + * to write to the socket. On Linux and Windows we would actually + * receive an ECONNRESET error for this, but on the BSDs, Darwin, + * Illumos and presumably Solaris, it's an EPIPE. + * 3. We cause the TCP peer to send us an RST by writing to a socket + * after we receive a FIN from them. Our first write will be + * successful, but if the they have closed the connection (rather + * than just shutting down the write side of it) this will cause their + * OS to send us an RST. Then, when we attempt to write to the socket + * a second time, we will get an EPIPE error. On Windows we get an + * ECONNABORTED. + * + * What we are going to do here is to treat all EPIPE messages that aren't + * of type 1 as ECONNRESET errors. This will allow users who have the + * show_econnreset socket option enabled to receive {error, econnreset} on + * both send and recv operations to indicate that an RST has been received. + */ +#ifdef __WIN_32__ + if (err == ECONNABORTED) + err = ECONNRESET; +#endif + if (err == EPIPE && !(desc->tcp_add_flags & TCP_ADDF_SHUTDOWN_WR_DONE)) + err = ECONNRESET; return tcp_send_or_shutdown_error(desc, err); } @@ -10811,6 +10930,8 @@ static void tcp_shutdown_async(tcp_descriptor* desc) TCP_SHUT_WR : TCP_SHUT_RDWR; if (IS_SOCKET_ERROR(sock_shutdown(INETP(desc)->s, how))) tcp_shutdown_error(desc, sock_errno()); + else + desc->tcp_add_flags |= TCP_ADDF_SHUTDOWN_WR_DONE; } #ifdef __OSE__ @@ -12172,6 +12293,8 @@ static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port, void (*timeout_fun)(ErlDrvData drv_data, ErlDrvTermData caller)) { +#define eq_mega(a, b) ((a)->when.megasecs == (b)->when.megasecs) +#define eq_sec(a, b) ((a)->when.secs == (b)->when.secs) MultiTimerData *mtd, *p, *s; mtd = ALLOC(sizeof(MultiTimerData)); absolute_timeout(timeout, &(mtd->when)); @@ -12183,23 +12306,17 @@ static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port, break; } } - if (!p || p->when.megasecs > mtd->when.megasecs) { - goto found; - } - for (; p!= NULL; s = p, p = p->next) { + for (; p!= NULL && eq_mega(p, mtd); s = p, p = p->next) { if (p->when.secs >= mtd->when.secs) { break; } } - if (!p || p->when.secs > mtd->when.secs) { - goto found; - } - for (; p!= NULL; s = p, p = p->next) { + for (; p!= NULL && eq_mega(p, mtd) && eq_sec(p, mtd); s = p, p = p->next) { if (p->when.microsecs >= mtd->when.microsecs) { break; } } - found: + if (!p) { if (!s) { *first = mtd; @@ -12225,6 +12342,8 @@ static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port, } return mtd; } +#undef eq_mega +#undef eq_sec diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c index d050748703..e1d2f66b7e 100644 --- a/erts/emulator/sys/unix/erl_child_setup.c +++ b/erts/emulator/sys/unix/erl_child_setup.c @@ -107,14 +107,16 @@ main(int argc, char *argv[]) if (from <= to) { int spfd = system_properties_fd(); for (i = from; i <= to; i++) { - if (i != spfd) + if (i != spfd) { (void) close(i); + } } } -#else - for (i = from; i <= to; i++) +#else /* !__ANDROID__ */ + for (i = from; i <= to; i++) { (void) close(i); -#endif + } +#endif /* HAVE_CLOSEFROM */ if (!(argv[CS_ARGV_WD_IX][0] == '.' && argv[CS_ARGV_WD_IX][1] == '\0') && chdir(argv[CS_ARGV_WD_IX]) < 0) diff --git a/erts/emulator/test/long_timers_test.erl b/erts/emulator/test/long_timers_test.erl index f381332b51..8dd960ffd4 100644 --- a/erts/emulator/test/long_timers_test.erl +++ b/erts/emulator/test/long_timers_test.erl @@ -28,7 +28,7 @@ -define(MAX_TIMEOUT, 60). % Minutes --define(MAX_LATE_MS, 10*1000). % Milliseconds +-define(MAX_LATE_MS, 15*1000). % Milliseconds -define(REG_NAME, '___LONG___TIMERS___TEST___SERVER___'). -define(DRV_NAME, timer_driver). @@ -196,8 +196,8 @@ driver(Timeout) -> end. bif_timer(Timeout) -> - Tmr = erlang:start_timer(Timeout, self(), ok), Start = erlang:monotonic_time(), + Tmr = erlang:start_timer(Timeout, self(), ok), receive {get_result, ?REG_NAME} -> ?REG_NAME ! #timeout_rec{pid = self(), diff --git a/erts/emulator/test/module_info_SUITE.erl b/erts/emulator/test/module_info_SUITE.erl index 1125cf3072..25ba6d1787 100644 --- a/erts/emulator/test/module_info_SUITE.erl +++ b/erts/emulator/test/module_info_SUITE.erl @@ -24,7 +24,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, - exports/1,functions/1,native/1,info/1]). + exports/1,functions/1,deleted/1,native/1,info/1]). %%-compile(native). @@ -51,9 +51,8 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - modules() -> - [exports, functions, native, info]. + [exports, functions, deleted, native, info]. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:minutes(3)), @@ -81,43 +80,66 @@ all_functions() -> %% Test that the list of exported functions from this module is correct. exports(Config) when is_list(Config) -> - ?line All = all_exported(), - ?line All = lists:sort(?MODULE:module_info(exports)), - ?line (catch ?MODULE:foo()), - ?line All = lists:sort(?MODULE:module_info(exports)), + All = all_exported(), + All = lists:sort(?MODULE:module_info(exports)), + (catch ?MODULE:foo()), + All = lists:sort(?MODULE:module_info(exports)), ok. %% Test that the list of exported functions from this module is correct. functions(Config) when is_list(Config) -> - ?line All = all_functions(), - ?line All = lists:sort(?MODULE:module_info(functions)), + All = all_functions(), + All = lists:sort(?MODULE:module_info(functions)), + ok. + +%% Test that deleted modules cause badarg +deleted(Config) when is_list(Config) -> + Data = ?config(data_dir, Config), + File = filename:join(Data, "module_info_test"), + {ok,module_info_test,Code} = compile:file(File, [binary]), + {module,module_info_test} = erlang:load_module(module_info_test, Code), + 17 = module_info_test:f(), + [_|_] = erlang:get_module_info(module_info_test, attributes), + [_|_] = erlang:get_module_info(module_info_test), + + %% first delete it + true = erlang:delete_module(module_info_test), + {'EXIT',{undef, _}} = (catch module_info_test:f()), + {'EXIT',{badarg, _}} = (catch erlang:get_module_info(module_info_test,attributes)), + {'EXIT',{badarg, _}} = (catch erlang:get_module_info(module_info_test)), + + %% then purge it + true = erlang:purge_module(module_info_test), + {'EXIT',{undef, _}} = (catch module_info_test:f()), + {'EXIT',{badarg, _}} = (catch erlang:get_module_info(module_info_test,attributes)), + {'EXIT',{badarg, _}} = (catch erlang:get_module_info(module_info_test)), ok. %% Test that the list of exported functions from this module is correct. %% Verify that module_info(native) works. native(Config) when is_list(Config) -> - ?line All = all_functions(), - ?line case ?MODULE:module_info(native_addresses) of - [] -> - ?line false = ?MODULE:module_info(native), - {comment,"no native functions"}; - L -> - ?line true = ?MODULE:module_info(native), - %% Verify that all functions have unique addresses. - ?line S0 = sofs:set(L, [{name,arity,addr}]), - ?line S1 = sofs:projection({external,fun ?MODULE:native_proj/1}, S0), - ?line S2 = sofs:relation_to_family(S1), - ?line S3 = sofs:family_specification(fun ?MODULE:native_filter/1, S2), - ?line 0 = sofs:no_elements(S3), - ?line S4 = sofs:range(S1), - - %% Verify that the set of function with native addresses - %% is a subset of all functions in the module. - ?line AllSet = sofs:set(All, [{name,arity}]), - ?line true = sofs:is_subset(S4, AllSet), - - {comment,integer_to_list(sofs:no_elements(S0))++" native functions"} - end. + All = all_functions(), + case ?MODULE:module_info(native_addresses) of + [] -> + false = ?MODULE:module_info(native), + {comment,"no native functions"}; + L -> + true = ?MODULE:module_info(native), + %% Verify that all functions have unique addresses. + S0 = sofs:set(L, [{name,arity,addr}]), + S1 = sofs:projection({external,fun ?MODULE:native_proj/1}, S0), + S2 = sofs:relation_to_family(S1), + S3 = sofs:family_specification(fun ?MODULE:native_filter/1, S2), + 0 = sofs:no_elements(S3), + S4 = sofs:range(S1), + + %% Verify that the set of function with native addresses + %% is a subset of all functions in the module. + AllSet = sofs:set(All, [{name,arity}]), + true = sofs:is_subset(S4, AllSet), + + {comment,integer_to_list(sofs:no_elements(S0))++" native functions"} + end. native_proj({Name,Arity,Addr}) -> {Addr,{Name,Arity}}. diff --git a/erts/emulator/test/module_info_SUITE_data/module_info_test.erl b/erts/emulator/test/module_info_SUITE_data/module_info_test.erl new file mode 100644 index 0000000000..f045f38464 --- /dev/null +++ b/erts/emulator/test/module_info_SUITE_data/module_info_test.erl @@ -0,0 +1,24 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(module_info_test). +-export([f/0]). + +f() -> + 17. diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl index 2f505893b4..479e6f200a 100644 --- a/erts/emulator/test/node_container_SUITE.erl +++ b/erts/emulator/test/node_container_SUITE.erl @@ -73,6 +73,8 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> + erts_debug:set_internal_state(available_internal_state, true), + erts_debug:set_internal_state(node_tab_delayed_delete, -1), %% restore original value available_internal_state(false). init_per_group(_GroupName, Config) -> @@ -419,6 +421,8 @@ node_table_gc(doc) -> ["Tests that node tables are garbage collected."]; node_table_gc(suite) -> []; node_table_gc(Config) when is_list(Config) -> + erts_debug:set_internal_state(available_internal_state, true), + erts_debug:set_internal_state(node_tab_delayed_delete, 0), ?line PreKnown = nodes(known), ?line ?t:format("PreKnown = ~p~n", [PreKnown]), ?line make_node_garbage(0, 200000, 1000, []), @@ -428,6 +432,7 @@ node_table_gc(Config) when is_list(Config) -> ?line ?t:format("PostAreas = ~p~n", [PostAreas]), ?line true = length(PostKnown) =< length(PreKnown), ?line nc_refc_check(node()), + erts_debug:set_internal_state(node_tab_delayed_delete, -1), %% restore original value ?line ok. make_node_garbage(N, L, I, Ps) when N < L -> @@ -579,6 +584,8 @@ node_controller_refc(doc) -> "as they should for entities controlling a connections."]; node_controller_refc(suite) -> []; node_controller_refc(Config) when is_list(Config) -> + erts_debug:set_internal_state(available_internal_state, true), + erts_debug:set_internal_state(node_tab_delayed_delete, 0), ?line NodeFirstName = get_nodefirstname(), ?line ?line {ok, Node} = start_node(NodeFirstName), ?line true = lists:member(Node, nodes()), @@ -606,6 +613,7 @@ node_controller_refc(Config) when is_list(Config) -> ?line false = get_dist_references(Node), ?line false = lists:member(Node, nodes(known)), ?line nc_refc_check(node()), + erts_debug:set_internal_state(node_tab_delayed_delete, -1), %% restore original value ?line ok. %% @@ -686,9 +694,6 @@ timer_refc(doc) -> "as they should for data stored in bif timers."]; timer_refc(suite) -> []; timer_refc(Config) when is_list(Config) -> - {skipped, "Test needs to be UPDATED for new timer implementation"}. - -timer_refc_test(Config) when is_list(Config) -> ?line RNode = {get_nodename(), 1}, ?line RPid = mk_pid(RNode, 4711, 2), ?line RPort = mk_port(RNode, 4711), @@ -992,23 +997,32 @@ check_nd_refc({ThisNodeName, ThisCreation}, NodeRefs, DistRefs, Fail) -> check_refc(ThisNodeName,ThisCreation,Table,EntryList) when is_list(EntryList) -> lists:foreach( fun ({Entry, Refc, ReferrerList}) -> - FoundRefs = + {DelayedDeleteTimer, + FoundRefs} = lists:foldl( - fun ({_Referrer, ReferencesList}, A1) -> - A1 + lists:foldl(fun ({_T,Rs},A2) -> - A2+Rs - end, - 0, - ReferencesList) + fun ({Referrer, ReferencesList}, {DDT, A1}) -> + {case Referrer of + {system,delayed_delete_timer} -> + true; + _ -> + DDT + end, + A1 + lists:foldl(fun ({_T,Rs},A2) -> + A2+Rs + end, + 0, + ReferencesList)} end, - 0, + {false, 0}, ReferrerList), - %% Reference count equals found references ? - case Refc =:= FoundRefs of - true -> + %% Reference count equals found references? + case {Refc, FoundRefs, DelayedDeleteTimer} of + {X, X, _} -> + ok; + {0, 1, true} -> ok; - false -> + _ -> exit({invalid_reference_count, Table, Entry}) end, @@ -1016,7 +1030,8 @@ check_refc(ThisNodeName,ThisCreation,Table,EntryList) when is_list(EntryList) -> case {Entry, Refc} of {ThisNodeName, 0} -> ok; {{ThisNodeName, ThisCreation}, 0} -> ok; - {_, 0} -> exit({not_referred_entry_in_table, Table, Entry}); + {_, 0} when DelayedDeleteTimer == false -> + exit({not_referred_entry_in_table, Table, Entry}); {_, _} -> ok end diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl index 8e1f8df43a..4deae63ce7 100644 --- a/erts/emulator/test/send_term_SUITE.erl +++ b/erts/emulator/test/send_term_SUITE.erl @@ -279,7 +279,10 @@ generate_external_terms_files(BaseDir) -> {4444444444444444,-44444, [[[[[[[[[[[5]]]]]]]]]]], make_ref()}, {444444444444444444444,-44444, {{{{{{{{{{{{6}}}}}}}}}}}}, make_ref()}, {444444444444444,-44444, {{{{{{{{{{{{7}}}}}}}}}}}}, make_ref()}, - {4444444444444444444,-44444, {{{{{{{{{{{{8}}}}}}}}}}}}, make_ref()}], + {4444444444444444444,-44444, {{{{{{{{{{{{8}}}}}}}}}}}}, make_ref()}, + #{}, + #{1 => 11, 2 => 22, 3 => 33}, + maps:from_list([{K,K*11} || K <- lists:seq(1,100)])], ok = file:write_file(filename:join([BaseDir, "send_term_SUITE_data", "ext_terms.bin"]), diff --git a/erts/emulator/test/send_term_SUITE_data/ext_terms.bin b/erts/emulator/test/send_term_SUITE_data/ext_terms.bin Binary files differindex b239284323..5ff0b2ccf1 100644 --- a/erts/emulator/test/send_term_SUITE_data/ext_terms.bin +++ b/erts/emulator/test/send_term_SUITE_data/ext_terms.bin diff --git a/erts/emulator/test/send_term_SUITE_data/ext_terms.h b/erts/emulator/test/send_term_SUITE_data/ext_terms.h index 08134f3b05..5585585ec3 100644 --- a/erts/emulator/test/send_term_SUITE_data/ext_terms.h +++ b/erts/emulator/test/send_term_SUITE_data/ext_terms.h @@ -24,9 +24,9 @@ #ifndef EXT_TERMS_H__ #define EXT_TERMS_H__ static struct { - unsigned char ext[162]; + unsigned char ext[637]; int ext_size; - unsigned char cext[162]; + unsigned char cext[637]; int cext_size; } ext_terms[] = { {{131,104,3,98,0,0,18,103,98,255,255,237,153,108,0,0,0,2,100,0,7,97,110,95,97,116,111,109,107,0,6,97,32,108,105,115,116,106}, @@ -37,26 +37,26 @@ static struct { 46, {131,108,0,0,0,4,110,9,0,0,0,160,222,197,173,201,53,54,110,7,1,199,113,21,183,140,242,3,107,0,6,98,108,117,112,112,33,100,0,5,98,108,105,112,112,106}, 46}, - {{131,104,5,103,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,38,0,0,0,0,3,104,2,114,0,3,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,40,0,0,0,0,0,0,0,0,102,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3,103,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,37,0,0,0,0,3,102,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,59,0,0,0,0,0,0,0,0}, - 162, - {131,80,0,0,0,161,120,156,203,96,77,79,97,224,77,140,207,203,79,73,117,72,207,47,74,74,76,103,96,96,80,3,98,6,230,12,166,34,6,102,116,89,102,160,140,6,3,20,164,97,209,203,200,12,52,145,39,17,85,80,21,108,96,26,166,4,35,51,216,14,20,97,144,21,214,48,43,0,1,209,36,52}, - 82}, - {{131,104,5,104,0,106,106,112,0,0,0,79,0,21,87,190,182,1,38,106,214,65,228,1,52,27,227,2,212,0,0,0,1,0,0,0,0,100,0,15,115,101,110,100,95,116,101,114,109,95,83,85,73,84,69,97,1,98,0,184,11,180,103,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,37,0,0,0,0,3,109,0,0,0,31,104,101,106,32,104,111,112,112,32,116,114,97,108,108,97,108,97,97,97,97,97,97,97,97,97,97,97,97,97,97,97}, - 123, - {131,80,0,0,0,122,120,156,203,96,205,96,200,202,42,96,96,96,240,103,16,13,223,183,141,81,45,235,154,227,19,70,19,233,199,76,87,128,130,140,64,204,144,194,192,95,156,154,151,18,95,146,90,148,27,31,28,234,25,226,154,200,152,196,176,131,123,75,122,10,3,79,98,94,126,74,170,67,122,126,81,82,98,58,80,173,42,72,3,115,46,144,144,207,72,205,82,200,200,47,40,80,40,41,74,204,201,73,204,73,68,5,0,18,237,35,68}, - 117}, + {{131,104,5,103,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,41,0,0,0,0,2,104,2,114,0,3,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,2,0,0,0,42,0,0,0,3,0,0,0,0,102,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,2,103,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,40,0,0,0,0,3,102,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,3,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,56,0,0,0,3,0,0,0,0}, + 204, + {131,80,0,0,0,203,120,156,203,96,77,79,97,16,73,140,207,203,79,73,117,72,205,169,48,54,201,200,171,52,50,210,53,76,98,96,96,208,4,98,6,166,12,166,34,6,102,28,138,152,128,10,180,128,152,25,164,50,13,183,73,12,76,64,107,132,179,19,115,114,48,229,52,64,242,204,105,56,229,25,152,193,246,99,147,5,89,107,1,179,30,0,103,37,46,144}, + 96}, + {{131,104,5,104,0,106,106,112,0,0,0,86,0,123,56,104,225,98,55,108,63,185,201,160,64,191,31,210,203,0,0,0,2,0,0,0,0,100,0,15,115,101,110,100,95,116,101,114,109,95,83,85,73,84,69,97,2,98,3,217,195,71,103,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,40,0,0,0,0,3,109,0,0,0,31,104,101,106,32,104,111,112,112,32,116,114,97,108,108,97,108,97,97,97,97,97,97,97,97,97,97,97,97,97,97,97}, + 130, + {131,80,0,0,0,129,120,156,203,96,205,96,200,202,42,96,96,96,8,99,168,182,200,120,152,100,158,99,191,243,228,2,135,253,242,151,78,3,5,153,128,152,33,133,129,191,56,53,47,37,190,36,181,40,55,62,56,212,51,196,53,145,41,137,249,230,97,247,244,20,6,225,236,196,156,156,84,135,212,156,10,99,147,140,188,74,35,35,93,195,36,160,22,13,144,62,230,92,32,33,159,145,154,165,144,145,95,80,160,80,82,4,84,154,152,147,136,10,0,219,221,39,33}, + 123}, {{131,108,0,0,0,4,110,10,0,28,199,113,166,118,185,145,86,105,9,110,5,1,28,103,24,89,10,107,0,2,98,33,100,0,10,98,108,105,112,112,112,112,112,112,112,106}, 46, {131,108,0,0,0,4,110,10,0,28,199,113,166,118,185,145,86,105,9,110,5,1,28,103,24,89,10,107,0,2,98,33,100,0,10,98,108,105,112,112,112,112,112,112,112,106}, 46}, - {{131,104,5,98,0,0,18,103,103,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,38,0,0,0,0,3,104,2,114,0,3,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,40,0,0,0,0,0,0,0,0,102,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3,98,255,255,237,153,108,0,0,0,2,100,0,7,97,110,95,97,116,111,109,107,0,6,97,32,108,105,115,116,106}, - 120, - {131,80,0,0,0,119,120,156,203,96,77,98,96,16,74,79,79,97,224,77,140,207,203,79,73,117,72,207,47,74,74,76,103,96,96,80,3,98,6,230,12,166,34,6,102,116,89,102,160,140,6,3,20,164,97,209,203,200,156,244,255,255,219,153,57,64,38,83,10,3,123,98,94,124,98,73,126,110,54,3,91,162,66,78,102,113,73,22,0,167,192,30,158}, - 93}, - {{131,104,4,103,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,38,0,0,0,0,3,104,2,114,0,3,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,40,0,0,0,0,0,0,0,0,102,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3,103,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,0,0,0,0,0,3,102,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3}, - 131, - {131,80,0,0,0,130,120,156,203,96,73,79,97,224,77,140,207,203,79,73,117,72,207,47,74,74,76,103,96,96,80,3,98,6,230,12,166,34,6,102,116,89,102,160,140,6,3,20,164,97,209,203,200,12,52,145,39,17,85,16,12,152,211,48,37,24,153,1,215,214,30,50}, - 72}, + {{131,104,5,98,0,0,18,103,103,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,41,0,0,0,0,2,104,2,114,0,3,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,2,0,0,0,42,0,0,0,3,0,0,0,0,102,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,2,98,255,255,237,153,108,0,0,0,2,100,0,7,97,110,95,97,116,111,109,107,0,6,97,32,108,105,115,116,106}, + 141, + {131,80,0,0,0,140,120,156,203,96,77,98,96,16,74,79,79,97,16,73,140,207,203,79,73,117,72,205,169,48,54,201,200,171,52,50,210,53,4,202,49,104,2,49,3,83,6,83,17,3,51,14,69,76,64,5,90,64,204,12,82,153,134,219,36,6,166,164,255,255,223,206,204,1,177,82,24,216,19,243,226,19,75,242,115,179,25,216,18,21,114,50,139,75,178,0,77,99,35,202}, + 100}, + {{131,104,4,103,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,41,0,0,0,0,2,104,2,114,0,3,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,2,0,0,0,42,0,0,0,3,0,0,0,0,102,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,2,103,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,0,0,0,0,3,102,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,3}, + 166, + {131,80,0,0,0,165,120,156,203,96,73,79,97,16,73,140,207,203,79,73,117,72,205,169,48,54,201,200,171,52,50,210,53,76,98,96,96,208,4,98,6,166,12,166,34,6,102,28,138,152,128,10,180,128,152,25,164,50,13,183,73,12,76,64,107,132,179,19,115,114,176,200,129,0,115,26,110,121,102,0,219,33,38,209}, + 84}, {{131,104,3,98,0,0,18,103,98,255,255,237,153,108,0,0,0,2,100,0,7,97,110,95,97,116,111,109,107,0,6,97,32,108,105,115,116,106}, 38, {131,104,3,98,0,0,18,103,98,255,255,237,153,108,0,0,0,2,100,0,7,97,110,95,97,116,111,109,107,0,6,97,32,108,105,115,116,106}, @@ -65,46 +65,58 @@ static struct { 33, {131,104,3,98,0,0,18,103,98,255,255,237,153,108,0,0,0,2,100,0,4,97,116,111,109,107,0,4,108,105,115,116,106}, 33}, - {{131,104,4,103,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,38,0,0,0,0,3,104,2,114,0,3,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,40,0,0,0,0,0,0,0,0,102,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3,103,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,0,0,0,0,0,3,102,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3}, - 131, - {131,80,0,0,0,130,120,156,203,96,73,79,97,224,77,140,207,203,79,73,117,72,207,47,74,74,76,103,96,96,80,3,98,6,230,12,166,34,6,102,116,89,102,160,140,6,3,20,164,97,209,203,200,12,52,145,39,17,85,16,12,152,211,48,37,24,153,1,215,214,30,50}, + {{131,104,4,103,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,41,0,0,0,0,2,104,2,114,0,3,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,2,0,0,0,42,0,0,0,3,0,0,0,0,102,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,2,103,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,0,0,0,0,3,102,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,3}, + 166, + {131,80,0,0,0,165,120,156,203,96,73,79,97,16,73,140,207,203,79,73,117,72,205,169,48,54,201,200,171,52,50,210,53,76,98,96,96,208,4,98,6,166,12,166,34,6,102,28,138,152,128,10,180,128,152,25,164,50,13,183,73,12,76,64,107,132,179,19,115,114,176,200,129,0,115,26,110,121,102,0,219,33,38,209}, + 84}, + {{131,104,4,110,8,0,28,199,17,175,172,214,173,61,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,0,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,57,0,0,0,3,0,0,0,0}, + 81, + {131,80,0,0,0,80,120,156,203,96,201,227,96,144,57,46,184,126,205,181,181,182,73,255,255,7,165,100,48,98,133,12,69,12,204,41,12,194,217,137,57,57,169,14,169,57,21,198,38,25,121,149,70,70,186,134,73,204,12,12,12,150,64,12,162,25,0,231,161,20,138}, + 71}, + {{131,104,4,110,9,0,28,199,241,98,116,219,231,23,24,98,255,255,82,100,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,107,0,1,1,106,106,106,106,106,106,106,106,106,106,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,58,0,0,0,3,0,0,0,0}, + 122, + {131,80,0,0,0,121,120,156,203,96,201,227,100,144,57,254,49,169,228,246,115,113,137,164,255,255,131,82,114,24,24,24,24,73,34,178,25,24,25,179,224,160,136,129,57,133,65,56,59,49,39,39,213,33,53,167,194,216,36,35,175,210,200,72,215,48,137,25,168,214,10,136,65,52,3,0,142,142,25,0}, + 80}, + {{131,104,4,110,8,0,28,199,129,17,222,251,42,6,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,2,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,59,0,0,0,3,0,0,0,0}, + 83, + {131,80,0,0,0,82,120,156,203,96,201,227,96,144,57,222,40,120,239,183,22,91,210,255,255,65,41,25,140,216,97,34,83,17,3,115,10,131,112,118,98,78,78,170,67,106,78,133,177,73,70,94,165,145,145,174,97,18,51,3,3,131,53,16,131,104,6,0,233,167,20,95}, 72}, - {{131,104,4,110,8,0,28,199,17,175,172,214,173,61,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,0,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,60,0,0,0,0,0,0,0,0}, - 74, - {131,80,0,0,0,73,120,156,203,96,201,227,96,144,57,46,184,126,205,181,181,182,73,255,255,7,165,100,48,98,133,12,69,12,204,41,12,60,137,121,249,41,169,14,233,249,69,73,137,233,204,12,12,12,54,12,80,0,0,73,17,18,208}, - 63}, - {{131,104,4,110,9,0,28,199,241,98,116,219,231,23,24,98,255,255,82,100,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,107,0,1,1,106,106,106,106,106,106,106,106,106,106,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,61,0,0,0,0,0,0,0,0}, - 115, - {131,80,0,0,0,114,120,156,203,96,201,227,100,144,57,254,49,169,228,246,115,113,137,164,255,255,131,82,114,24,24,24,24,73,34,178,25,24,25,179,224,160,136,129,57,133,129,39,49,47,63,37,213,33,61,191,40,41,49,157,25,168,200,150,1,10,0,208,188,23,70}, + {{131,104,4,110,9,0,28,199,113,221,139,146,14,239,240,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,3,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,60,0,0,0,3,0,0,0,0}, + 84, + {131,80,0,0,0,83,120,156,203,96,201,227,100,144,57,94,120,183,123,18,223,251,15,73,255,255,7,165,100,48,98,135,137,204,69,12,204,41,12,194,217,137,57,57,169,14,169,57,21,198,38,25,121,149,70,70,186,134,73,204,12,12,12,54,64,12,162,25,0,106,11,22,31}, + 73}, + {{131,104,4,110,9,0,28,199,177,214,190,98,202,104,2,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,4,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,61,0,0,0,3,0,0,0,0}, + 84, + {131,80,0,0,0,83,120,156,203,96,201,227,100,144,57,190,241,218,190,164,83,25,76,73,255,255,7,165,100,48,98,135,137,44,69,12,204,41,12,194,217,137,57,57,169,14,169,57,21,198,38,25,121,149,70,70,186,134,73,204,12,12,12,182,64,12,162,25,0,74,151,21,164}, + 73}, + {{131,104,4,110,7,0,28,199,85,220,50,202,15,98,255,255,82,100,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,107,0,1,5,106,106,106,106,106,106,106,106,106,106,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,62,0,0,0,3,0,0,0,0}, + 120, + {131,80,0,0,0,119,120,156,203,96,201,99,103,144,57,30,122,199,232,20,127,210,255,255,65,41,57,12,12,12,140,36,17,217,12,140,172,89,112,80,196,192,156,194,32,156,157,152,147,147,234,144,154,83,97,108,146,145,87,105,100,164,107,152,196,12,84,107,7,196,32,154,1,0,225,225,23,138}, + 78}, + {{131,104,4,110,9,0,28,199,241,98,116,219,231,23,24,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,6,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,63,0,0,0,3,0,0,0,0}, + 84, + {131,80,0,0,0,83,120,156,203,96,201,227,100,144,57,254,49,169,228,246,115,113,137,164,255,255,131,82,50,24,177,195,68,182,34,6,230,20,6,225,236,196,156,156,84,135,212,156,10,99,147,140,188,74,35,35,93,195,36,102,6,6,6,123,32,6,209,12,0,64,205,21,133}, + 73}, + {{131,104,4,110,7,0,28,199,59,73,56,148,1,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,7,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,64,0,0,0,3,0,0,0,0}, + 82, + {131,80,0,0,0,81,120,156,203,96,201,99,103,144,57,110,237,105,49,133,49,233,255,255,160,148,12,70,236,48,145,189,136,129,57,133,65,56,59,49,39,39,213,33,53,167,194,216,36,35,175,210,200,72,215,48,137,153,129,129,193,1,136,65,52,3,0,137,143,19,30}, 71}, - {{131,104,4,110,8,0,28,199,129,17,222,251,42,6,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,2,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,62,0,0,0,0,0,0,0,0}, - 76, - {131,80,0,0,0,75,120,156,203,96,201,227,96,144,57,222,40,120,239,183,22,91,210,255,255,65,41,25,140,216,97,34,83,17,3,115,10,3,79,98,94,126,74,170,67,122,126,81,82,98,58,51,3,3,131,29,3,20,0,0,76,82,18,165}, - 64}, - {{131,104,4,110,9,0,28,199,113,221,139,146,14,239,240,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,3,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,63,0,0,0,0,0,0,0,0}, - 77, - {131,80,0,0,0,76,120,156,203,96,201,227,100,144,57,94,120,183,123,18,223,251,15,73,255,255,7,165,100,48,98,135,137,204,69,12,204,41,12,60,137,121,249,41,169,14,233,249,69,73,137,233,204,12,12,12,246,12,80,0,0,192,110,20,101}, - 65}, - {{131,104,4,110,9,0,28,199,177,214,190,98,202,104,2,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,4,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,64,0,0,0,0,0,0,0,0}, - 77, - {131,80,0,0,0,76,120,156,203,96,201,227,100,144,57,190,241,218,190,164,83,25,76,73,255,255,7,165,100,48,98,135,137,44,69,12,204,41,12,60,137,121,249,41,169,14,233,249,69,73,137,233,204,12,12,12,14,12,80,0,0,164,94,19,234}, - 65}, - {{131,104,4,110,7,0,28,199,85,220,50,202,15,98,255,255,82,100,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,107,0,1,5,106,106,106,106,106,106,106,106,106,106,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,65,0,0,0,0,0,0,0,0}, - 113, - {131,80,0,0,0,112,120,156,203,96,201,99,103,144,57,30,122,199,232,20,127,210,255,255,65,41,57,12,12,12,140,36,17,217,12,140,172,89,112,80,196,192,156,194,192,147,152,151,159,146,234,144,158,95,148,148,152,206,12,84,228,200,0,5,0,46,116,21,208}, - 69}, - {{131,104,4,110,9,0,28,199,241,98,116,219,231,23,24,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,6,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,66,0,0,0,0,0,0,0,0}, - 77, - {131,80,0,0,0,76,120,156,203,96,201,227,100,144,57,254,49,169,228,246,115,113,137,164,255,255,131,82,50,24,177,195,68,182,34,6,230,20,6,158,196,188,252,148,84,135,244,252,162,164,196,116,102,6,6,6,39,6,40,0,0,155,123,19,203}, - 65}, - {{131,104,4,110,7,0,28,199,59,73,56,148,1,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,7,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,67,0,0,0,0,0,0,0,0}, - 75, - {131,80,0,0,0,74,120,156,203,96,201,99,103,144,57,110,237,105,49,133,49,233,255,255,160,148,12,70,236,48,145,189,136,129,57,133,129,39,49,47,63,37,213,33,61,191,40,41,49,157,153,129,129,193,153,1,10,0,245,21,17,100}, - 62}, - {{131,104,4,110,8,0,28,199,17,175,172,214,173,61,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,8,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,68,0,0,0,0,0,0,0,0}, - 76, - {131,80,0,0,0,75,120,156,203,96,201,227,96,144,57,46,184,126,205,181,181,182,73,255,255,7,165,100,48,98,135,137,28,69,12,204,41,12,60,137,121,249,41,169,14,233,249,69,73,137,233,204,12,12,12,46,12,80,0,0,112,226,19,66}, - 64} + {{131,104,4,110,8,0,28,199,17,175,172,214,173,61,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,8,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,65,0,0,0,3,0,0,0,0}, + 83, + {131,80,0,0,0,82,120,156,203,96,201,227,96,144,57,46,184,126,205,181,181,182,73,255,255,7,165,100,48,98,135,137,28,69,12,204,41,12,194,217,137,57,57,169,14,169,57,21,198,38,25,121,149,70,70,186,134,73,204,12,12,12,142,64,12,162,25,0,18,103,20,252}, + 72}, + {{131,116,0,0,0,0}, + 6, + {131,116,0,0,0,0}, + 6}, + {{131,116,0,0,0,3,97,1,97,11,97,2,97,22,97,3,97,33}, + 18, + {131,116,0,0,0,3,97,1,97,11,97,2,97,22,97,3,97,33}, + 18}, + {{131,116,0,0,0,100,97,48,98,0,0,2,16,97,62,98,0,0,2,170,97,11,97,121,97,39,98,0,0,1,173,97,83,98,0,0,3,145,97,63,98,0,0,2,181,97,34,98,0,0,1,118,97,68,98,0,0,2,236,97,26,98,0,0,1,30,97,78,98,0,0,3,90,97,52,98,0,0,2,60,97,15,97,165,97,64,98,0,0,2,192,97,75,98,0,0,3,57,97,81,98,0,0,3,123,97,71,98,0,0,3,13,97,20,97,220,97,50,98,0,0,2,38,97,17,97,187,97,25,98,0,0,1,19,97,65,98,0,0,2,203,97,98,98,0,0,4,54,97,79,98,0,0,3,101,97,13,97,143,97,44,98,0,0,1,228,97,8,97,88,97,99,98,0,0,4,65,97,36,98,0,0,1,140,97,67,98,0,0,2,225,97,7,97,77,97,66,98,0,0,2,214,97,85,98,0,0,3,167,97,76,98,0,0,3,68,97,1,97,11,97,32,98,0,0,1,96,97,69,98,0,0,2,247,97,37,98,0,0,1,151,97,35,98,0,0,1,129,97,84,98,0,0,3,156,97,3,97,33,97,82,98,0,0,3,134,97,45,98,0,0,1,239,97,55,98,0,0,2,93,97,6,97,66,97,2,97,22,97,94,98,0,0,4,10,97,49,98,0,0,2,27,97,41,98,0,0,1,195,97,91,98,0,0,3,233,97,87,98,0,0,3,189,97,33,98,0,0,1,107,97,42,98,0,0,1,206,97,74,98,0,0,3,46,97,60,98,0,0,2,148,97,43,98,0,0,1,217,97,10,97,110,97,70,98,0,0,3,2,97,9,97,99,97,72,98,0,0,3,24,97,86,98,0,0,3,178,97,19,97,209,97,56,98,0,0,2,104,97,95,98,0,0,4,21,97,57,98,0,0,2,115,97,51,98,0,0,2,49,97,14,97,154,97,5,97,55,97,54,98,0,0,2,82,97,18,97,198,97,61,98,0,0,2,159,97,31,98,0,0,1,85,97,22,97,242,97,29,98,0,0,1,63,97,97,98,0,0,4,43,97,21,97,231,97,89,98,0,0,3,211,97,27,98,0,0,1,41,97,24,98,0,0,1,8,97,47,98,0,0,2,5,97,100,98,0,0,4,76,97,40,98,0,0,1,184,97,96,98,0,0,4,32,97,73,98,0,0,3,35,97,90,98,0,0,3,222,97,30,98,0,0,1,74,97,58,98,0,0,2,126,97,80,98,0,0,3,112,97,88,98,0,0,3,200,97,59,98,0,0,2,137,97,77,98,0,0,3,79,97,23,97,253,97,28,98,0,0,1,52,97,46,98,0,0,1,250,97,92,98,0,0,3,244,97,53,98,0,0,2,71,97,93,98,0,0,3,255,97,16,97,176,97,38,98,0,0,1,162,97,4,97,44,97,12,97,132}, + 637, + {131,80,0,0,2,124,120,156,21,143,123,100,87,113,24,135,207,126,219,218,165,86,171,109,181,182,118,107,181,181,123,171,181,181,218,90,91,91,171,93,90,91,173,221,219,158,93,24,145,40,145,49,70,98,140,68,68,70,196,196,136,137,24,137,136,68,68,70,140,68,68,34,34,35,34,117,158,191,158,247,115,121,223,239,57,55,130,32,152,228,224,120,16,68,146,57,33,150,217,204,45,10,195,49,234,41,23,66,68,223,163,193,224,57,123,53,111,210,172,250,65,134,42,155,115,86,6,169,210,172,99,27,75,156,116,124,69,187,65,45,221,98,134,86,145,68,42,159,56,100,94,192,118,94,176,219,27,41,52,234,188,99,60,68,76,53,93,86,167,72,226,46,165,230,95,137,167,159,9,195,70,246,233,44,112,202,141,47,196,209,73,147,227,71,122,221,122,66,135,104,38,42,252,139,92,171,99,180,152,255,102,191,234,1,249,98,142,139,214,22,137,38,143,30,199,59,148,25,252,164,198,246,8,155,104,34,194,78,46,251,106,34,149,186,153,20,217,121,205,144,27,223,233,19,47,201,211,188,66,177,120,79,155,102,57,117,46,220,167,68,115,157,68,174,114,218,32,66,2,19,156,113,76,231,146,120,70,10,31,56,106,125,154,81,95,75,163,86,117,157,195,162,146,173,60,36,150,26,170,149,61,236,224,13,245,142,143,200,241,122,111,248,149,191,200,114,108,0,15,148,144,198,55,6,188,190,70,166,65,17,233,34,158,10,23,99,153,180,214,193,1,205,85,198,84,185,156,117,33,159,65,241,153,108,179,54,142,185,48,203,121,205,107,244,139,183,28,215,156,167,83,213,197,46,254,178,199,118,21,229,226,15,195,6,27,28,177,214,202,136,234,31,201,172,80,96,254,152,24,74,217,194,237,255,248,120,145,79}, + 418} }; -#define NO_OF_EXT_TERMS 19 +#define NO_OF_EXT_TERMS 22 #endif diff --git a/erts/emulator/test/timer_bif_SUITE.erl b/erts/emulator/test/timer_bif_SUITE.erl index d406456f98..f41fc7552e 100644 --- a/erts/emulator/test/timer_bif_SUITE.erl +++ b/erts/emulator/test/timer_bif_SUITE.erl @@ -29,7 +29,8 @@ read_timer_trivial/1, read_timer/1, read_timer_async/1, cleanup/1, evil_timers/1, registered_process/1, same_time_yielding/1, same_time_yielding_with_cancel/1, same_time_yielding_with_cancel_other/1, - same_time_yielding_with_cancel_other_accessor/1, auto_cancel_yielding/1]). +% same_time_yielding_with_cancel_other_accessor/1, + auto_cancel_yielding/1]). -include_lib("test_server/include/test_server.hrl"). @@ -67,7 +68,7 @@ all() -> cleanup, evil_timers, registered_process, same_time_yielding, same_time_yielding_with_cancel, same_time_yielding_with_cancel_other, - same_time_yielding_with_cancel_other_accessor, +% same_time_yielding_with_cancel_other_accessor, auto_cancel_yielding]. groups() -> @@ -532,8 +533,8 @@ same_time_yielding_with_cancel(Config) when is_list(Config) -> same_time_yielding_with_cancel_other(Config) when is_list(Config) -> same_time_yielding_with_cancel_test(true, false). -same_time_yielding_with_cancel_other_accessor(Config) when is_list(Config) -> - same_time_yielding_with_cancel_test(true, true). +%same_time_yielding_with_cancel_other_accessor(Config) when is_list(Config) -> +% same_time_yielding_with_cancel_test(true, true). do_cancel_tmrs(Tmo, Tmrs, Tester) -> BeginCancel = erlang:convert_time_unit(Tmo, @@ -631,7 +632,6 @@ auto_cancel_yielding(Config) when is_list(Config) -> true = mem_larger_than(Mem), exit(P, bang), wait_until(fun () -> process_is_cleaned_up(P) end), - receive after 1000 -> ok end, Mem = mem(), ok. @@ -747,7 +747,7 @@ mem_larger_than(Mem) -> mem() > Mem. mem() -> - erts_debug:set_internal_state(wait, deallocations), + erts_debug:set_internal_state(wait, timer_cancellations), erts_debug:set_internal_state(wait, deallocations), case mem_get() of {-1, -1} -> no_fix_alloc; diff --git a/erts/epmd/src/epmd.c b/erts/epmd/src/epmd.c index 447aae47aa..2fd9845d1a 100644 --- a/erts/epmd/src/epmd.c +++ b/erts/epmd/src/epmd.c @@ -498,11 +498,7 @@ static void dbg_gen_printf(int onsyslog,int perr,int from_level, #ifdef HAVE_SYSLOG_H if (onsyslog) { - int len; - len = erts_vsnprintf(buf, DEBUG_BUFFER_SIZE, format, args); - if (perr != 0 && len < sizeof(buf)) { - erts_snprintf(buf+len, sizeof(buf)-len, ": %s", strerror(perr)); - } + erts_vsnprintf(buf, DEBUG_BUFFER_SIZE, format, args); syslog(LOG_ERR,"epmd: %s",buf); } #endif diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index d6544a2829..50f4f0e8a6 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -157,6 +157,8 @@ static char *plusr_val_switches[] = { /* +z arguments with values */ static char *plusz_val_switches[] = { "dbbl", + "dntgc", + "ebwt", NULL }; diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 3ee092418e..c51b9e94ed 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -3634,7 +3634,7 @@ define etp-carrier-blocks printf "Free blocks: %u\n", $etp_fblk_cnt end if $etp_error_cnt - printf "%u ERRORs reported above\n", $etp-error-cnt + printf "%u ERRORs reported above\n", $etp_error_cnt end end diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c index 1ba51882c3..5b82a081ad 100644 --- a/erts/lib_src/common/ethr_aux.c +++ b/erts/lib_src/common/ethr_aux.c @@ -40,8 +40,8 @@ #include <unistd.h> #endif -#define ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE 4000 -#define ERTS_TS_EV_ALLOC_POOL_SIZE 25 +#define ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE 2048 +#define ERTS_TS_EV_ALLOC_POOL_SIZE 32 erts_cpu_info_t *ethr_cpu_info__; diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex c0fca6aafa..863a5e61ef 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex 0e0811af3f..dc8c711e1a 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex 6729f06b79..5a188be3ba 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index ea8a911a2c..cf941ea6ca 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -1557,7 +1557,7 @@ send_after(_Time, _Dest, _Msg) -> Msg :: term(), Options :: [Option], Abs :: boolean(), - Option :: {abs, Abs}, %% | {accessor, Accessor} undocumented feature for now, + Option :: {abs, Abs}, TimerRef :: reference(). send_after(_Time, _Dest, _Msg, _Options) -> @@ -1645,7 +1645,7 @@ start_timer(_Time, _Dest, _Msg) -> Msg :: term(), Options :: [Option], Abs :: boolean(), - Option :: {abs, Abs}, %% | {accessor, Accessor} undocumented feature for now, + Option :: {abs, Abs}, TimerRef :: reference(). start_timer(_Time, _Dest, _Msg, _Options) -> @@ -2392,6 +2392,7 @@ tuple_to_list(_Tuple) -> CpuTopology :: cpu_topology(); (creation) -> integer(); (debug_compiled) -> boolean(); + (delayed_node_table_gc) -> infinity | non_neg_integer(); (dirty_cpu_schedulers) -> non_neg_integer(); (dirty_cpu_schedulers_online) -> non_neg_integer(); (dirty_io_schedulers) -> non_neg_integer(); diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl index 65a1f1ed3a..cf8edefd7d 100644 --- a/erts/preloaded/src/erts_internal.erl +++ b/erts/preloaded/src/erts_internal.erl @@ -40,7 +40,7 @@ -export([flush_monitor_messages/3]). --export([await_result/1]). +-export([await_result/1, gather_io_bytes/2]). -export([time_unit/0]). @@ -67,6 +67,23 @@ await_result(Ref) when is_reference(Ref) -> end. %% +%% statistics(io) end up in gather_io_bytes/2 +%% + +gather_io_bytes(Ref, No) when is_reference(Ref), + is_integer(No), + No > 0 -> + gather_io_bytes(Ref, No, 0, 0). + +gather_io_bytes(_Ref, 0, InAcc, OutAcc) -> + {{input, InAcc}, {output, OutAcc}}; +gather_io_bytes(Ref, No, InAcc, OutAcc) -> + receive + {Ref, _SchedId, In, Out} -> + gather_io_bytes(Ref, No-1, InAcc + In, OutAcc + Out) + end. + +%% %% Statically linked port NIFs %% diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl index 622e1be869..5e0b38aa68 100644 --- a/erts/preloaded/src/prim_inet.erl +++ b/erts/preloaded/src/prim_inet.erl @@ -146,11 +146,16 @@ shutdown_1(S, How) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% close(S) when is_port(S) -> - case subscribe(S, [subs_empty_out_q]) of - {ok, [{subs_empty_out_q,N}]} when N > 0 -> - close_pend_loop(S, N); %% wait for pending output to be sent + case getopt(S, linger) of + {ok,{true,0}} -> + close_port(S); _ -> - close_port(S) + case subscribe(S, [subs_empty_out_q]) of + {ok, [{subs_empty_out_q,N}]} when N > 0 -> + close_pend_loop(S, N); %% wait for pending output to be sent + _ -> + close_port(S) + end end. close_pend_loop(S, N) -> @@ -1140,6 +1145,7 @@ enc_opt(delay_send) -> ?INET_LOPT_TCP_DELAY_SEND; enc_opt(packet_size) -> ?INET_LOPT_PACKET_SIZE; enc_opt(read_packets) -> ?INET_LOPT_READ_PACKETS; enc_opt(netns) -> ?INET_LOPT_NETNS; +enc_opt(show_econnreset) -> ?INET_LOPT_TCP_SHOW_ECONNRESET; enc_opt(raw) -> ?INET_OPT_RAW; % Names of SCTP opts: enc_opt(sctp_rtoinfo) -> ?SCTP_OPT_RTOINFO; @@ -1197,6 +1203,7 @@ dec_opt(?INET_LOPT_TCP_DELAY_SEND) -> delay_send; dec_opt(?INET_LOPT_PACKET_SIZE) -> packet_size; dec_opt(?INET_LOPT_READ_PACKETS) -> read_packets; dec_opt(?INET_LOPT_NETNS) -> netns; +dec_opt(?INET_LOPT_TCP_SHOW_ECONNRESET) -> show_econnreset; dec_opt(?INET_OPT_RAW) -> raw; dec_opt(I) when is_integer(I) -> undefined. @@ -1296,6 +1303,7 @@ type_opt_1(delay_send) -> bool; type_opt_1(packet_size) -> uint; type_opt_1(read_packets) -> uint; type_opt_1(netns) -> binary; +type_opt_1(show_econnreset) -> bool; %% %% SCTP options (to be set). If the type is a record type, the corresponding %% record signature is returned, otherwise, an "elementary" type tag diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index ea4522c40b..f63f9581a6 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -252,7 +252,7 @@ </func> <func> - <name>Module:end_per_suite(Config) -> void() | + <name>Module:end_per_suite(Config) -> term() | {save_config,SaveConfig}</name> <fsummary>Test suite finalization. </fsummary> <type> @@ -385,7 +385,7 @@ </func> <func> - <name>Module:end_per_group(GroupName, Config) -> void() | + <name>Module:end_per_group(GroupName, Config) -> term() | {return_group_result,Status}</name> <fsummary>Test case group finalization.</fsummary> <type> @@ -440,7 +440,7 @@ </func> <func> - <name>Module:end_per_testcase(TestCase, Config) -> void() | {fail,Reason} | {save_config,SaveConfig}</name> + <name>Module:end_per_testcase(TestCase, Config) -> term() | {fail,Reason} | {save_config,SaveConfig}</name> <fsummary>Test case finalization.</fsummary> <type> <v> TestCase = atom()</v> @@ -538,7 +538,7 @@ <func> - <name>Module:Testcase(Config) -> void() | {skip,Reason} | {comment,Comment} | {save_config,SaveConfig} | {skip_and_save,Reason,SaveConfig} | exit() </name> + <name>Module:Testcase(Config) -> term() | {skip,Reason} | {comment,Comment} | {save_config,SaveConfig} | {skip_and_save,Reason,SaveConfig} | exit() </name> <fsummary>A test case</fsummary> <type> <v> Config = SaveConfig = [{Key,Value}]</v> diff --git a/lib/common_test/doc/src/ct_run.xml b/lib/common_test/doc/src/ct_run.xml index d8e79ca80e..3ac8691fb5 100644 --- a/lib/common_test/doc/src/ct_run.xml +++ b/lib/common_test/doc/src/ct_run.xml @@ -86,98 +86,99 @@ <marker id="ct_run"></marker> <title>Run tests from command line</title> <pre> - ct_run [-dir TestDir1 TestDir2 .. TestDirN] | - [[-dir TestDir] -suite Suite1 Suite2 .. SuiteN - [[-group Groups1 Groups2 .. GroupsN] [-case Case1 Case2 .. CaseN]]] - [-step [config | keep_inactive]] - [-config ConfigFile1 ConfigFile2 .. ConfigFileN] - [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 - ConfigString2 and .. CallbackModuleN ConfigStringN] - [-decrypt_key Key] | [-decrypt_file KeyFile] - [-label Label] - [-logdir LogDir] - [-logopts LogOpts] - [-verbosity GenVLevel | [Category1 VLevel1 and - Category2 VLevel2 and .. CategoryN VLevelN]] - [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] - [-stylesheet CSSFile] - [-cover CoverCfgFile] - [-cover_stop Bool] - [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] | - [-event_handler_init EvHandler1 InitArg1 and - EvHandler2 InitArg2 and .. EvHandlerN InitArgN] - [-include InclDir1 InclDir2 .. InclDirN] - [-no_auto_compile] - [-abort_if_missing_suites] - [-muliply_timetraps Multiplier] - [-scale_timetraps] - [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc] - [-repeat N] | - [-duration HHMMSS [-force_stop [skip_rest]]] | - [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]] - [-basic_html] - [-ct_hooks CTHModule1 CTHOpts1 and CTHModule2 CTHOpts2 and .. - CTHModuleN CTHOptsN] - [-exit_status ignore_config] + ct_run -dir TestDir1 TestDir2 .. TestDirN | + [-dir TestDir] -suite Suite1 Suite2 .. SuiteN + [-group Groups1 Groups2 .. GroupsN] [-case Case1 Case2 .. CaseN] + [-step [config | keep_inactive]] + [-config ConfigFile1 ConfigFile2 .. ConfigFileN] + [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 + ConfigString2 and .. CallbackModuleN ConfigStringN] + [-decrypt_key Key] | [-decrypt_file KeyFile] + [-label Label] + [-logdir LogDir] + [-logopts LogOpts] + [-verbosity GenVLevel | [Category1 VLevel1 and + Category2 VLevel2 and .. CategoryN VLevelN]] + [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] + [-stylesheet CSSFile] + [-cover CoverCfgFile] + [-cover_stop Bool] + [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] | + [-event_handler_init EvHandler1 InitArg1 and + EvHandler2 InitArg2 and .. EvHandlerN InitArgN] + [-include InclDir1 InclDir2 .. InclDirN] + [-no_auto_compile] + [-abort_if_missing_suites] + [-muliply_timetraps Multiplier] + [-scale_timetraps] + [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc] + [-repeat N] | + [-duration HHMMSS [-force_stop [skip_rest]]] | + [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]] + [-basic_html] + [-ct_hooks CTHModule1 CTHOpts1 and CTHModule2 CTHOpts2 and .. + CTHModuleN CTHOptsN] + [-exit_status ignore_config] + [-help] </pre> </section> <section> <title>Run tests using test specification</title> <pre> ct_run -spec TestSpec1 TestSpec2 .. TestSpecN - [-join_specs] - [-config ConfigFile1 ConfigFile2 .. ConfigFileN] - [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 - ConfigString2 and .. and CallbackModuleN ConfigStringN] - [-decrypt_key Key] | [-decrypt_file KeyFile] - [-label Label] - [-logdir LogDir] - [-logopts LogOpts] - [-verbosity GenVLevel | [Category1 VLevel1 and - Category2 VLevel2 and .. CategoryN VLevelN]] - [-allow_user_terms] - [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] - [-stylesheet CSSFile] - [-cover CoverCfgFile] - [-cover_stop Bool] - [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] | - [-event_handler_init EvHandler1 InitArg1 and - EvHandler2 InitArg2 and .. EvHandlerN InitArgN] - [-include InclDir1 InclDir2 .. InclDirN] - [-no_auto_compile] - [-abort_if_missing_suites] - [-muliply_timetraps Multiplier] - [-scale_timetraps] - [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc] - [-repeat N] | - [-duration HHMMSS [-force_stop [skip_rest]]] | - [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]] - [-basic_html] - [-ct_hooks CTHModule1 CTHOpts1 and CTHModule2 CTHOpts2 and .. - CTHModuleN CTHOptsN] - [-exit_status ignore_config] + [-join_specs] + [-config ConfigFile1 ConfigFile2 .. ConfigFileN] + [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 + ConfigString2 and .. and CallbackModuleN ConfigStringN] + [-decrypt_key Key] | [-decrypt_file KeyFile] + [-label Label] + [-logdir LogDir] + [-logopts LogOpts] + [-verbosity GenVLevel | [Category1 VLevel1 and + Category2 VLevel2 and .. CategoryN VLevelN]] + [-allow_user_terms] + [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] + [-stylesheet CSSFile] + [-cover CoverCfgFile] + [-cover_stop Bool] + [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] | + [-event_handler_init EvHandler1 InitArg1 and + EvHandler2 InitArg2 and .. EvHandlerN InitArgN] + [-include InclDir1 InclDir2 .. InclDirN] + [-no_auto_compile] + [-abort_if_missing_suites] + [-muliply_timetraps Multiplier] + [-scale_timetraps] + [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc] + [-repeat N] | + [-duration HHMMSS [-force_stop [skip_rest]]] | + [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]] + [-basic_html] + [-ct_hooks CTHModule1 CTHOpts1 and CTHModule2 CTHOpts2 and .. + CTHModuleN CTHOptsN] + [-exit_status ignore_config] </pre> </section> <section> <title>Run tests in web based GUI</title> <pre> ct_run -vts [-browser Browser] - [-dir TestDir1 TestDir2 .. TestDirN] | - [[dir TestDir] -suite Suite [[-group Group] [-case Case]]] - [-config ConfigFile1 ConfigFile2 .. ConfigFileN] - [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 - ConfigString2 and .. and CallbackModuleN ConfigStringN] - [-logopts LogOpts] - [-verbosity GenVLevel | [Category1 VLevel1 and - Category2 VLevel2 and .. CategoryN VLevelN]] - [-decrypt_key Key] | [-decrypt_file KeyFile] - [-include InclDir1 InclDir2 .. InclDirN] - [-no_auto_compile] - [-abort_if_missing_suites] - [-muliply_timetraps Multiplier] - [-scale_timetraps] - [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc] - [-basic_html]</pre> + [-dir TestDir1 TestDir2 .. TestDirN] | + [[dir TestDir] -suite Suite [[-group Group] [-case Case]]] + [-config ConfigFile1 ConfigFile2 .. ConfigFileN] + [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 + ConfigString2 and .. and CallbackModuleN ConfigStringN] + [-logopts LogOpts] + [-verbosity GenVLevel | [Category1 VLevel1 and + Category2 VLevel2 and .. CategoryN VLevelN]] + [-decrypt_key Key] | [-decrypt_file KeyFile] + [-include InclDir1 InclDir2 .. InclDirN] + [-no_auto_compile] + [-abort_if_missing_suites] + [-muliply_timetraps Multiplier] + [-scale_timetraps] + [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc] + [-basic_html]</pre> </section> <section> <title>Refresh the HTML index files</title> @@ -188,10 +189,10 @@ <title>Run CT in interactive mode</title> <pre> ct_run -shell - [-config ConfigFile1 ConfigFile2 ... ConfigFileN] - [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 - ConfigString2 and .. and CallbackModuleN ConfigStringN] - [-decrypt_key Key] | [-decrypt_file KeyFile]</pre> + [-config ConfigFile1 ConfigFile2 ... ConfigFileN] + [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 + ConfigString2 and .. and CallbackModuleN ConfigStringN] + [-decrypt_key Key] | [-decrypt_file KeyFile]</pre> </section> <section> <title>Start a Common Test Master node</title> diff --git a/lib/common_test/doc/src/example_chapter.xml b/lib/common_test/doc/src/example_chapter.xml index 36781d152c..2bc8cfdbcc 100644 --- a/lib/common_test/doc/src/example_chapter.xml +++ b/lib/common_test/doc/src/example_chapter.xml @@ -81,7 +81,7 @@ init_per_suite(Config) -> [{con_ref, Ref },{table_name, TableName}| Config]. %%-------------------------------------------------------------------- -%% Function: end_per_suite(Config) -> void() +%% Function: end_per_suite(Config) -> term() %% %% Config = [tuple()] %% A list of key/value pairs, holding the test case configuration. @@ -110,7 +110,7 @@ init_per_testcase(Case, Config) -> Config. %%-------------------------------------------------------------------- -%% Function: end_per_testcase(TestCase, Config) -> void() +%% Function: end_per_testcase(TestCase, Config) -> term() %% %% TestCase = atom() %% Name of the test case that is finished. @@ -226,7 +226,7 @@ init_per_suite(Config) -> Config. %%-------------------------------------------------------------------- -%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Function: end_per_suite(Config0) -> term() | {save_config,Config1} %% %% Config0 = Config1 = [tuple()] %% A list of key/value pairs, holding the test case configuration. @@ -254,7 +254,7 @@ init_per_group(_GroupName, Config) -> %%-------------------------------------------------------------------- %% Function: end_per_group(GroupName, Config0) -> -%% void() | {save_config,Config1} +%% term() | {save_config,Config1} %% %% GroupName = atom() %% Name of the test case group that is finished. @@ -287,7 +287,7 @@ init_per_testcase(_TestCase, Config) -> %%-------------------------------------------------------------------- %% Function: end_per_testcase(TestCase, Config0) -> -%% void() | {save_config,Config1} | {fail,Reason} +%% term() | {save_config,Config1} | {fail,Reason} %% %% TestCase = atom() %% Name of the test case that is finished. @@ -414,7 +414,7 @@ init_per_suite(Config) -> Config. %%-------------------------------------------------------------------- -%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Function: end_per_suite(Config0) -> term() | {save_config,Config1} %% Config0 = Config1 = [tuple()] %%-------------------------------------------------------------------- end_per_suite(_Config) -> @@ -432,7 +432,7 @@ init_per_group(_GroupName, Config) -> %%-------------------------------------------------------------------- %% Function: end_per_group(GroupName, Config0) -> -%% void() | {save_config,Config1} +%% term() | {save_config,Config1} %% GroupName = atom() %% Config0 = Config1 = [tuple()] %%-------------------------------------------------------------------- @@ -451,7 +451,7 @@ init_per_testcase(_TestCase, Config) -> %%-------------------------------------------------------------------- %% Function: end_per_testcase(TestCase, Config0) -> -%% void() | {save_config,Config1} | {fail,Reason} +%% term() | {save_config,Config1} | {fail,Reason} %% TestCase = atom() %% Config0 = Config1 = [tuple()] %% Reason = term() diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index df60e5f7f2..fc8d82c2c3 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -149,6 +149,7 @@ <p>Other flags that may be used with <c>ct_run</c>:</p> <list> + <item><c><![CDATA[-help]]></c>, lists all available start flags.</item> <item><c><![CDATA[-logdir <dir>]]></c>, specifies where the HTML log files are to be written.</item> <item><c><![CDATA[-label <name_of_test_run>]]></c>, associates the test run with a name that gets printed in the overview HTML log files.</item> @@ -223,6 +224,9 @@ behaviour using start flag:</p> <pre>-exit_status ignore_config</pre> + <note><p>Executing <c>ct_run</c> without start flags, is equal to the command: + <c>ct_run -dir ./</c></p></note> + <p>For more information about the <c>ct_run</c> program, see the <seealso marker="ct_run">Reference Manual</seealso> and the <seealso marker="install_chapter#general">Installation</seealso> chapter. @@ -251,6 +255,10 @@ <c>{error,Reason}</c>, where the term <c>Reason</c> explains the failure.</p> + <p>The default start option <c>{dir,Cwd}</c> (run all suites in the current + working directory) is used if the function is called with an empty + list of options.</p> + <section> <title>Releasing the Erlang shell</title> <p>During execution of tests, started with diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 5ed1346f1e..208632e2dc 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -729,7 +729,7 @@ capture_get([]) -> test_server:capture_get(). %%%----------------------------------------------------------------- -%%% @spec fail(Reason) -> void() +%%% @spec fail(Reason) -> ok %%% Reason = term() %%% %%% @doc Terminate a test case with the given error @@ -747,7 +747,7 @@ fail(Reason) -> end. %%%----------------------------------------------------------------- -%%% @spec fail(Format, Args) -> void() +%%% @spec fail(Format, Args) -> ok %%% Format = string() %%% Args = list() %%% @@ -773,7 +773,7 @@ fail(Format, Args) -> end. %%%----------------------------------------------------------------- -%%% @spec comment(Comment) -> void() +%%% @spec comment(Comment) -> ok %%% Comment = term() %%% %%% @doc Print the given <c>Comment</c> in the comment field in @@ -796,7 +796,7 @@ comment(Comment) -> send_html_comment(lists:flatten(Formatted)). %%%----------------------------------------------------------------- -%%% @spec comment(Format, Args) -> void() +%%% @spec comment(Format, Args) -> ok %%% Format = string() %%% Args = list() %%% diff --git a/lib/common_test/src/ct_event.erl b/lib/common_test/src/ct_event.erl index c1c1d943b9..729d3fbfac 100644 --- a/lib/common_test/src/ct_event.erl +++ b/lib/common_test/src/ct_event.erl @@ -259,8 +259,8 @@ handle_info(_Info, State) -> {ok, State}. %%-------------------------------------------------------------------- -%% Function: terminate(Reason, State) -> void() -%% Description:Whenever an event handler is deleted from an event manager, +%% Function: terminate(Reason, State) -> ok +%% Description: Whenever an event handler is deleted from an event manager, %% this function is called. It should be the opposite of Module:init/1 and %% do any necessary cleaning up. %%-------------------------------------------------------------------- diff --git a/lib/common_test/src/ct_master_event.erl b/lib/common_test/src/ct_master_event.erl index fd97ab16f7..d127b98afe 100644 --- a/lib/common_test/src/ct_master_event.erl +++ b/lib/common_test/src/ct_master_event.erl @@ -168,7 +168,7 @@ handle_info(_Info,State) -> {ok,State}. %%-------------------------------------------------------------------- -%% Function: terminate(Reason, State) -> void() +%% Function: terminate(Reason, State) -> ok %% Description:Whenever an event handler is deleted from an event manager, %% this function is called. It should be the opposite of Module:init/1 and %% do any necessary cleaning up. diff --git a/lib/common_test/src/ct_master_status.erl b/lib/common_test/src/ct_master_status.erl index f9f511ecca..b49a906236 100644 --- a/lib/common_test/src/ct_master_status.erl +++ b/lib/common_test/src/ct_master_status.erl @@ -100,8 +100,8 @@ handle_info(_Info, State) -> {ok, State}. %%-------------------------------------------------------------------- -%% Function: terminate(Reason, State) -> void() -%% Description:Whenever an event handler is deleted from an event manager, +%% Function: terminate(Reason, State) -> ok +%% Description: Whenever an event handler is deleted from an event manager, %% this function is called. It should be the opposite of Module:init/1 and %% do any necessary cleaning up. %%-------------------------------------------------------------------- diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 0eafe72020..e9f685c685 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -83,7 +83,7 @@ starter}). %%%----------------------------------------------------------------- -%%% @spec script_start() -> void() +%%% @spec script_start() -> term() %%% %%% @doc Start tests via the ct_run program or script. %%% @@ -687,8 +687,10 @@ script_start3(Opts, Args) -> if Opts#opts.vts ; Opts#opts.shell -> script_start4(Opts#opts{tests = []}, Args); true -> - script_usage(), - {error,missing_start_options} + %% no start options, use default "-dir ./" + {ok,Dir} = file:get_cwd(), + io:format("ct_run -dir ~ts~n~n", [Dir]), + script_start4(Opts#opts{tests = tests([Dir])}, Args) end end. @@ -767,82 +769,84 @@ script_start4(Opts = #opts{tests = Tests}, Args) -> %%% @spec script_usage() -> ok %%% @doc Print usage information for <code>ct_run</code>. script_usage() -> - io:format("\n\nUsage:\n\n"), + io:format("\nUsage:\n\n"), io:format("Run tests from command line:\n\n" - "\tct_run [-dir TestDir1 TestDir2 .. TestDirN] |" - "\n\t[[-dir TestDir] -suite Suite1 Suite2 .. SuiteN" - "\n\t [[-group Groups1 Groups2 .. GroupsN] [-case Case1 Case2 .. CaseN]]]" - "\n\t[-step [config | keep_inactive]]" - "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]" - "\n\t[-userconfig CallbackModule ConfigFile1 .. ConfigFileN]" - "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]" - "\n\t[-logdir LogDir]" - "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" - "\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" - "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" - "\n\t[-stylesheet CSSFile]" - "\n\t[-cover CoverCfgFile]" - "\n\t[-cover_stop Bool]" - "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" - "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" - "\n\t[-include InclDir1 InclDir2 .. InclDirN]" - "\n\t[-no_auto_compile]" - "\n\t[-abort_if_missing_suites]" - "\n\t[-multiply_timetraps N]" - "\n\t[-scale_timetraps]" - "\n\t[-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]" - "\n\t[-basic_html]" - "\n\t[-repeat N] |" - "\n\t[-duration HHMMSS [-force_stop [skip_rest]]] |" - "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]\n\n"), + "\tct_run -dir TestDir1 TestDir2 .. TestDirN |" + "\n\t [-dir TestDir] -suite Suite1 Suite2 .. SuiteN" + "\n\t [-group Group1 Group2 .. GroupN] [-case Case1 Case2 .. CaseN]" + "\n\t [-step [config | keep_inactive]]" + "\n\t [-config ConfigFile1 ConfigFile2 .. ConfigFileN]" + "\n\t [-userconfig CallbackModule ConfigFile1 .. ConfigFileN]" + "\n\t [-decrypt_key Key] | [-decrypt_file KeyFile]" + "\n\t [-logdir LogDir]" + "\n\t [-logopts LogOpt1 LogOpt2 .. LogOptN]" + "\n\t [-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" + "\n\t [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" + "\n\t [-stylesheet CSSFile]" + "\n\t [-cover CoverCfgFile]" + "\n\t [-cover_stop Bool]" + "\n\t [-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" + "\n\t [-ct_hooks CTHook1 CTHook2 .. CTHookN]" + "\n\t [-include InclDir1 InclDir2 .. InclDirN]" + "\n\t [-no_auto_compile]" + "\n\t [-abort_if_missing_suites]" + "\n\t [-multiply_timetraps N]" + "\n\t [-scale_timetraps]" + "\n\t [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]" + "\n\t [-basic_html]" + "\n\t [-repeat N] |" + "\n\t [-duration HHMMSS [-force_stop [skip_rest]]] |" + "\n\t [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]" + "\n\t [-exit_status ignore_config]" + "\n\t [-help]\n\n"), io:format("Run tests using test specification:\n\n" "\tct_run -spec TestSpec1 TestSpec2 .. TestSpecN" - "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]" - "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]" - "\n\t[-logdir LogDir]" - "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" - "\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" - "\n\t[-allow_user_terms]" - "\n\t[-join_specs]" - "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" - "\n\t[-stylesheet CSSFile]" - "\n\t[-cover CoverCfgFile]" - "\n\t[-cover_stop Bool]" - "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" - "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" - "\n\t[-include InclDir1 InclDir2 .. InclDirN]" - "\n\t[-no_auto_compile]" - "\n\t[-abort_if_missing_suites]" - "\n\t[-multiply_timetraps N]" - "\n\t[-scale_timetraps]" - "\n\t[-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]" - "\n\t[-basic_html]" - "\n\t[-repeat N] |" - "\n\t[-duration HHMMSS [-force_stop [skip_rest]]] |" - "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]\n\n"), + "\n\t [-config ConfigFile1 ConfigFile2 .. ConfigFileN]" + "\n\t [-decrypt_key Key] | [-decrypt_file KeyFile]" + "\n\t [-logdir LogDir]" + "\n\t [-logopts LogOpt1 LogOpt2 .. LogOptN]" + "\n\t [-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" + "\n\t [-allow_user_terms]" + "\n\t [-join_specs]" + "\n\t [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" + "\n\t [-stylesheet CSSFile]" + "\n\t [-cover CoverCfgFile]" + "\n\t [-cover_stop Bool]" + "\n\t [-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" + "\n\t [-ct_hooks CTHook1 CTHook2 .. CTHookN]" + "\n\t [-include InclDir1 InclDir2 .. InclDirN]" + "\n\t [-no_auto_compile]" + "\n\t [-abort_if_missing_suites]" + "\n\t [-multiply_timetraps N]" + "\n\t [-scale_timetraps]" + "\n\t [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]" + "\n\t [-basic_html]" + "\n\t [-repeat N] |" + "\n\t [-duration HHMMSS [-force_stop [skip_rest]]] |" + "\n\t [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]\n\n"), io:format("Refresh the HTML index files:\n\n" "\tct_run -refresh_logs [LogDir]" - "[-logdir LogDir] " - "[-basic_html]\n\n"), + " [-logdir LogDir] " + " [-basic_html]\n\n"), io:format("Run CT in interactive mode:\n\n" "\tct_run -shell" - "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]" - "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]\n\n"), + "\n\t [-config ConfigFile1 ConfigFile2 .. ConfigFileN]" + "\n\t [-decrypt_key Key] | [-decrypt_file KeyFile]\n\n"), io:format("Run tests in web based GUI:\n\n" "\tct_run -vts [-browser Browser]" - "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]" - "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]" - "\n\t[-dir TestDir1 TestDir2 .. TestDirN] |" - "\n\t[-suite Suite [-case Case]]" - "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" - "\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" - "\n\t[-include InclDir1 InclDir2 .. InclDirN]" - "\n\t[-no_auto_compile]" - "\n\t[-abort_if_missing_suites]" - "\n\t[-multiply_timetraps N]" - "\n\t[-scale_timetraps]" - "\n\t[-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]" - "\n\t[-basic_html]\n\n"). + "\n\t [-config ConfigFile1 ConfigFile2 .. ConfigFileN]" + "\n\t [-decrypt_key Key] | [-decrypt_file KeyFile]" + "\n\t [-dir TestDir1 TestDir2 .. TestDirN] |" + "\n\t [-suite Suite [-case Case]]" + "\n\t [-logopts LogOpt1 LogOpt2 .. LogOptN]" + "\n\t [-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" + "\n\t [-include InclDir1 InclDir2 .. InclDirN]" + "\n\t [-no_auto_compile]" + "\n\t [-abort_if_missing_suites]" + "\n\t [-multiply_timetraps N]" + "\n\t [-scale_timetraps]" + "\n\t [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]" + "\n\t [-basic_html]\n\n"). %%%----------------------------------------------------------------- %%% @hidden @@ -1347,7 +1351,9 @@ run_dir(Opts = #opts{logdir = LogDir, end; {undefined,undefined,[]} -> - exit({error,no_test_specified}); + {ok,Dir} = file:get_cwd(), + %% No start options, use default {dir,CWD} + reformat_result(catch do_run(tests(Dir), [], Opts1, StartOpts)); {Dir,Suite,GsAndCs} -> exit({error,{incorrect_start_options,{Dir,Suite,GsAndCs}}}) diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 80b2998ddc..5bfaa41b7d 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -490,16 +490,17 @@ is_label_used_in_1([], _, _) -> false. is_label_used_in_block({set,_,_,Info}, Lbl) -> case Info of - {bif,_,{f,F}} -> F =:= Lbl; - {alloc,_,{gc_bif,_,{f,F}}} -> F =:= Lbl; + {bif,_,{f,F}} -> F =:= Lbl; + {alloc,_,{gc_bif,_,{f,F}}} -> F =:= Lbl; {alloc,_,{put_map,_,{f,F}}} -> F =:= Lbl; - {'catch',{f,F}} -> F =:= Lbl; - {alloc,_,_} -> false; - {put_tuple,_} -> false; - {get_tuple_element,_} -> false; - {set_tuple_element,_} -> false; - {line,_} -> false; - _ when is_atom(Info) -> false + {get_map_elements,{f,F}} -> F =:= Lbl; + {'catch',{f,F}} -> F =:= Lbl; + {alloc,_,_} -> false; + {put_tuple,_} -> false; + {get_tuple_element,_} -> false; + {set_tuple_element,_} -> false; + {line,_} -> false; + _ when is_atom(Info) -> false end. %% remove_unused_labels(Instructions0) -> Instructions diff --git a/lib/compiler/test/Makefile b/lib/compiler/test/Makefile index 98125fc84e..6553d10077 100644 --- a/lib/compiler/test/Makefile +++ b/lib/compiler/test/Makefile @@ -33,6 +33,7 @@ MODULES= \ num_bif_SUITE \ receive_SUITE \ record_SUITE \ + regressions_SUITE \ trycatch_SUITE \ warnings_SUITE \ z_SUITE \ diff --git a/lib/compiler/test/regressions_SUITE.erl b/lib/compiler/test/regressions_SUITE.erl new file mode 100644 index 0000000000..51bcc5737f --- /dev/null +++ b/lib/compiler/test/regressions_SUITE.erl @@ -0,0 +1,98 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% Test specific code snippets that has crashed the compiler in the past. +-module(regressions_SUITE). +-include_lib("test_server/include/test_server.hrl"). + +-export([all/0, groups/0, init_per_testcase/2,end_per_testcase/2]). + +-export([maps/1]). + +groups() -> + [{p,test_lib:parallel(), + [maps]}]. + +% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(2)). + +init_per_testcase(_Case, Config) -> + ?line Dog = ?t:timetrap(?default_timeout), + [{watchdog, Dog} | Config]. + +end_per_testcase(_Case, Config) -> + Dog = ?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +all() -> + test_lib:recompile(?MODULE), + [{group,p}]. + +%%% test cases + +maps(Config) when is_list(Config) -> + Ts = [{beam_bool_get_elements, + <<"century(#{ron := operator}, _century) -> + if 0.0; _century, _century, _century -> _century end. + ">>}], + ok = run(Config, Ts), + ok. + +%% aux + +run(Config, Tests) -> + F = fun({N,P}) -> + io:format("Compiling test for: ~w~n", [N]), + case catch run_test(Config, P) of + {'EXIT', Reason} -> + ?t:format("~nTest ~p failed.~nReason: ~p~n", [N, Reason]), + fail(); + _ -> ok + end + end, + lists:foreach(F, Tests). + + +run_test(Conf, Test0) -> + Module = "regressions_"++test_lib:uniq(), + Filename = Module ++ ".erl", + DataDir = ?config(priv_dir, Conf), + Test = ["-module(", Module, "). ", Test0], + File = filename:join(DataDir, Filename), + Def = [binary,export_all,return], + Opts = [ Opt ++ Def || + Opt <- [ [no_postopt], + [no_copt], + [no_postopt,no_copt], + [inline], + [inline,no_postopt], + [] + ]], + ok = file:write_file(File, Test), + lists:foreach(fun(Opt) -> + io:format(" - compiling with ~p~n", [Opt]), + {ok,_M,_Bin,_} = compile:file(File,Opt) + end, Opts), + file:delete(File), + ok. + +fail() -> + io:format("failed~n"), + ?t:fail(). diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index 4a1ba9c539..914a4c6d8f 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -409,7 +409,7 @@ contract_from_form([{type, _, 'fun', [_, _]} = Form | Left], Module, RecDict, fun(ExpTypes, AllRecords) -> NewType = try - erl_types:t_from_form(Form, ExpTypes, Module, AllRecords) + from_form_with_check(Form, ExpTypes, Module, AllRecords) catch throw:{error, Msg} -> {File, Line} = FileLine, @@ -430,8 +430,8 @@ contract_from_form([{type, _L1, bounded_fun, fun(ExpTypes, AllRecords) -> {Constr1, VarDict} = process_constraints(Constr, Module, RecDict, ExpTypes, AllRecords), - NewType = erl_types:t_from_form(Form, ExpTypes, Module, AllRecords, - VarDict), + NewType = from_form_with_check(Form, ExpTypes, Module, AllRecords, + VarDict), NewTypeNoVars = erl_types:subst_all_vars_to_any(NewType), {NewTypeNoVars, Constr1} end, @@ -454,7 +454,7 @@ initialize_constraints([], _Module, _RecDict, _ExpTypes, _AllRecords, Acc) -> initialize_constraints([Constr|Rest], Module, RecDict, ExpTypes, AllRecords, Acc) -> case Constr of {type, _, constraint, [{atom, _, is_subtype}, [Type1, Type2]]} -> - T1 = final_form(Type1, Module, ExpTypes, AllRecords, dict:new()), + T1 = final_form(Type1, ExpTypes, Module, AllRecords, dict:new()), Entry = {T1, Type2}, initialize_constraints(Rest, Module, RecDict, ExpTypes, AllRecords, [Entry|Acc]); {type, _, constraint, [{atom,_,Name}, List]} -> @@ -483,7 +483,16 @@ constraints_fixpoint(OldVarDict, Module, Constrs, RecDict, ExpTypes, AllRecords) constraints_fixpoint(NewVarDict, Module, Constrs, RecDict, ExpTypes, AllRecords) end. -final_form(Form, Module, ExpTypes, AllRecords, VarDict) -> +final_form(Form, ExpTypes, Module, AllRecords, VarDict) -> + from_form_with_check(Form, ExpTypes, Module, AllRecords, VarDict). + +from_form_with_check(Form, ExpTypes, Module, AllRecords) -> + erl_types:t_check_record_fields(Form, ExpTypes, Module, AllRecords), + erl_types:t_from_form(Form, ExpTypes, Module, AllRecords). + +from_form_with_check(Form, ExpTypes, Module, AllRecords, VarDict) -> + erl_types:t_check_record_fields(Form, ExpTypes, Module, AllRecords, + VarDict), erl_types:t_from_form(Form, ExpTypes, Module, AllRecords, VarDict). constraints_to_dict(Constrs, Module, RecDict, ExpTypes, AllRecords, VarDict) -> @@ -495,7 +504,7 @@ constraints_to_subs([], _Module, _RecDict, _ExpTypes, _AllRecords, _VarDict, Acc Acc; constraints_to_subs([C|Rest], Module, RecDict, ExpTypes, AllRecords, VarDict, Acc) -> {T1, Form2} = C, - T2 = final_form(Form2, Module, ExpTypes, AllRecords, VarDict), + T2 = final_form(Form2, ExpTypes, Module, AllRecords, VarDict), NewAcc = [{subtype, T1, T2}|Acc], constraints_to_subs(Rest, Module, RecDict, ExpTypes, AllRecords, VarDict, NewAcc). diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index e29fc3ba8b..dbfd20014c 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -64,15 +64,15 @@ print_types(RecDict) -> print_types1([], _) -> ok; print_types1([{type, _Name, _NArgs} = Key|T], RecDict) -> - {ok, {{_Mod, _Form, _Args}, Type}} = dict:find(Key, RecDict), + {ok, {{_Mod, _FileLine, _Form, _Args}, Type}} = dict:find(Key, RecDict), io:format("\n~w: ~w\n", [Key, Type]), print_types1(T, RecDict); print_types1([{opaque, _Name, _NArgs} = Key|T], RecDict) -> - {ok, {{_Mod, _Form, _Args}, Type}} = dict:find(Key, RecDict), + {ok, {{_Mod, _FileLine, _Form, _Args}, Type}} = dict:find(Key, RecDict), io:format("\n~w: ~w\n", [Key, Type]), print_types1(T, RecDict); print_types1([{record, _Name} = Key|T], RecDict) -> - {ok, [{_Arity, _Fields} = AF]} = dict:find(Key, RecDict), + {ok, {_FileLine, [{_Arity, _Fields} = AF]}} = dict:find(Key, RecDict), io:format("~w: ~w\n\n", [Key, AF]), print_types1(T, RecDict). -define(debug(D_), print_types(D_)). @@ -203,52 +203,53 @@ get_record_and_type_info(AbstractCode) -> {'ok', dict:dict()} | {'error', string()}. get_record_and_type_info(AbstractCode, Module, RecDict) -> - get_record_and_type_info(AbstractCode, Module, [], RecDict). + get_record_and_type_info(AbstractCode, Module, RecDict, "nofile"). -get_record_and_type_info([{attribute, _, record, {Name, Fields0}}|Left], - Module, Records, RecDict) -> +get_record_and_type_info([{attribute, A, record, {Name, Fields0}}|Left], + Module, RecDict, File) -> {ok, Fields} = get_record_fields(Fields0, RecDict), Arity = length(Fields), - NewRecDict = dict:store({record, Name}, [{Arity, Fields}], RecDict), - get_record_and_type_info(Left, Module, [{record, Name}|Records], NewRecDict); -get_record_and_type_info([{attribute, _, type, {{record, Name}, Fields0, []}} - |Left], Module, Records, RecDict) -> + FN = {File, erl_anno:line(A)}, + NewRecDict = dict:store({record, Name}, {FN, [{Arity,Fields}]}, RecDict), + get_record_and_type_info(Left, Module, NewRecDict, File); +get_record_and_type_info([{attribute, A, type, {{record, Name}, Fields0, []}} + |Left], Module, RecDict, File) -> %% This overrides the original record declaration. {ok, Fields} = get_record_fields(Fields0, RecDict), Arity = length(Fields), - NewRecDict = dict:store({record, Name}, [{Arity, Fields}], RecDict), - get_record_and_type_info(Left, Module, Records, NewRecDict); -get_record_and_type_info([{attribute, _, Attr, {Name, TypeForm}}|Left], - Module, Records, RecDict) when Attr =:= 'type'; - Attr =:= 'opaque' -> - try add_new_type(Attr, Name, TypeForm, [], Module, RecDict) of + FN = {File, erl_anno:line(A)}, + NewRecDict = dict:store({record, Name}, {FN, [{Arity, Fields}]}, RecDict), + get_record_and_type_info(Left, Module, NewRecDict, File); +get_record_and_type_info([{attribute, A, Attr, {Name, TypeForm}}|Left], + Module, RecDict, File) + when Attr =:= 'type'; Attr =:= 'opaque' -> + FN = {File, erl_anno:line(A)}, + try add_new_type(Attr, Name, TypeForm, [], Module, FN, RecDict) of NewRecDict -> - get_record_and_type_info(Left, Module, Records, NewRecDict) + get_record_and_type_info(Left, Module, NewRecDict, File) catch throw:{error, _} = Error -> Error end; -get_record_and_type_info([{attribute, _, Attr, {Name, TypeForm, Args}}|Left], - Module, Records, RecDict) when Attr =:= 'type'; - Attr =:= 'opaque' -> - try add_new_type(Attr, Name, TypeForm, Args, Module, RecDict) of +get_record_and_type_info([{attribute, A, Attr, {Name, TypeForm, Args}}|Left], + Module, RecDict, File) + when Attr =:= 'type'; Attr =:= 'opaque' -> + FN = {File, erl_anno:line(A)}, + try add_new_type(Attr, Name, TypeForm, Args, Module, FN, RecDict) of NewRecDict -> - get_record_and_type_info(Left, Module, Records, NewRecDict) + get_record_and_type_info(Left, Module, NewRecDict, File) catch throw:{error, _} = Error -> Error end; -get_record_and_type_info([_Other|Left], Module, Records, RecDict) -> - get_record_and_type_info(Left, Module, Records, RecDict); -get_record_and_type_info([], _Module, Records, RecDict) -> - case - check_type_of_record_fields(lists:reverse(Records), RecDict) - of - ok -> - {ok, RecDict}; - {error, Name, Error} -> - {error, flat_format(" Error while parsing #~w{}: ~s\n", [Name, Error])} - end. - -add_new_type(TypeOrOpaque, Name, TypeForm, ArgForms, Module, RecDict) -> +get_record_and_type_info([{attribute, _, file, {IncludeFile, _}}|Left], + Module, RecDict, _File) -> + get_record_and_type_info(Left, Module, RecDict, IncludeFile); +get_record_and_type_info([_Other|Left], Module, RecDict, File) -> + get_record_and_type_info(Left, Module, RecDict, File); +get_record_and_type_info([], _Module, RecDict, _File) -> + {ok, RecDict}. + +add_new_type(TypeOrOpaque, Name, TypeForm, ArgForms, Module, FN, + RecDict) -> Arity = length(ArgForms), case erl_types:type_is_defined(TypeOrOpaque, Name, Arity, RecDict) of true -> @@ -258,7 +259,7 @@ add_new_type(TypeOrOpaque, Name, TypeForm, ArgForms, Module, RecDict) -> try erl_types:t_var_names(ArgForms) of ArgNames -> dict:store({TypeOrOpaque, Name, Arity}, - {{Module, TypeForm, ArgNames}, + {{Module, FN, TypeForm, ArgNames}, erl_types:t_any()}, RecDict) catch _:_ -> @@ -280,32 +281,16 @@ get_record_fields([{typed_record_field, OrdRecField, TypeForm}|Left], end, get_record_fields(Left, RecDict, [{Name, TypeForm}|Acc]); get_record_fields([{record_field, _Line, Name}|Left], RecDict, Acc) -> - NewAcc = [{erl_parse:normalise(Name), {var, -1, '_'}}|Acc], + A = erl_anno:set_generated(true, erl_anno:new(1)), + NewAcc = [{erl_parse:normalise(Name), {var, A, '_'}}|Acc], get_record_fields(Left, RecDict, NewAcc); get_record_fields([{record_field, _Line, Name, _Init}|Left], RecDict, Acc) -> - NewAcc = [{erl_parse:normalise(Name), {var, -1, '_'}}|Acc], + A = erl_anno:set_generated(true, erl_anno:new(1)), + NewAcc = [{erl_parse:normalise(Name), {var, A, '_'}}|Acc], get_record_fields(Left, RecDict, NewAcc); get_record_fields([], _RecDict, Acc) -> lists:reverse(Acc). -%% Just check the local types. process_record_remote_types will add -%% the types later. -check_type_of_record_fields([], _RecDict) -> - ok; -check_type_of_record_fields([RecKey|Recs], RecDict) -> - {ok, [{_Arity, Fields}]} = dict:find(RecKey, RecDict), - try - [erl_types:t_from_form_without_remote(FieldTypeForm, RecDict) - || {_FieldName, FieldTypeForm, _} <- Fields] - of - L when is_list(L) -> - check_type_of_record_fields(Recs, RecDict) - catch - throw:{error, Error} -> - {record, Name} = RecKey, - {error, Name, Error} - end. - -spec process_record_remote_types(codeserver()) -> codeserver(). %% The field types are cached. Used during analysis when handling records. @@ -327,9 +312,10 @@ process_record_remote_types(CServer) -> TempRecords)} || {Name, Field, _} <- Fields] end, - orddict:map(FieldFun, Value); + {FileLine, Fields} = Value, + {FileLine, orddict:map(FieldFun, Fields)}; {opaque, _, _} -> - {{_Module, Form, _ArgNames}=F, _Type} = Value, + {{_Module, _FileLine, Form, _ArgNames}=F, _Type} = Value, Type = erl_types:t_from_form(Form, TempExpTypes, Module, TempRecords), {F, Type}; @@ -340,13 +326,53 @@ process_record_remote_types(CServer) -> end, try dict:map(ModuleFun, TempRecords) of NewRecords -> + ok = check_record_fields(NewRecords, TempExpTypes), CServer1 = dialyzer_codeserver:finalize_records(NewRecords, CServer), dialyzer_codeserver:finalize_exported_types(TempExpTypes, CServer1) catch - throw:{error, _RecName, _Error} = Error-> + throw:{error, _RecName, _Error} = Error -> Error end. +check_record_fields(Records, TempExpTypes) -> + CheckFun = + fun({Module, Element}) -> + CheckForm = fun(F) -> + erl_types:t_check_record_fields(F, TempExpTypes, + Module, Records) + end, + ElemFun = + fun({Key, Value}) -> + case Key of + {record, _Name} -> + FieldFun = + fun({_Arity, Fields}) -> + _ = [ok = CheckForm(Field) || {_, Field, _} <- Fields], + ok + end, + {FileLine, Fields} = Value, + Fun = fun() -> lists:foreach(FieldFun, Fields) end, + msg_with_position(Fun, FileLine); + {_OpaqueOrType, _Name, _} -> + {{_Module, FileLine, Form, _ArgNames}, _Type} = Value, + Fun = fun() -> ok = CheckForm(Form) end, + msg_with_position(Fun, FileLine) + end + end, + lists:foreach(ElemFun, dict:to_list(Element)) + end, + lists:foreach(CheckFun, dict:to_list(Records)). + +msg_with_position(Fun, FileLine) -> + try Fun() + catch + throw:{error, Msg} -> + {File, Line} = FileLine, + BaseName = filename:basename(File), + NewMsg = io_lib:format("~s:~p: ~s", [BaseName, Line, Msg]), + throw({error, NewMsg}) + end. + -spec merge_records(dict:dict(), dict:dict()) -> dict:dict(). merge_records(NewRecords, OldRecords) -> @@ -385,11 +411,11 @@ get_optional_callbacks(Abs) -> %% - Constraint is of the form {subtype, T1, T2} where T1 and T2 %% are erl_types:erl_type() -get_spec_info([{attribute, Attr, Contract, {Id, TypeSpec}}|Left], +get_spec_info([{attribute, Anno, Contract, {Id, TypeSpec}}|Left], SpecDict, CallbackDict, RecordsDict, ModName, OptCb, File) when ((Contract =:= 'spec') or (Contract =:= 'callback')), is_list(TypeSpec) -> - Ln = erl_anno:line(Attr), + Ln = erl_anno:line(Anno), MFA = case Id of {_, _, _} = T -> T; {F, A} -> {ModName, F, A} diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/para b/lib/dialyzer/test/opaque_SUITE_data/results/para index 3aaa238de6..8fe67e39ad 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/para +++ b/lib/dialyzer/test/opaque_SUITE_data/results/para @@ -7,15 +7,27 @@ para1.erl:38: Attempt to test for equality between a term of type para1_adt:t(in para1.erl:43: Attempt to test for equality between a term of type para1_adt:t() and a term of opaque type para1_adt:t(atom()) para1.erl:48: Attempt to test for equality between a term of type para1_adt:t(integer()) and a term of opaque type para1_adt:t() para1.erl:53: The test {3,2} =:= {'a','b'} can never evaluate to 'true' -para2.erl:103: Attempt to test for equality between a term of type para2_adt:circ({{integer(),integer()},{integer(),integer()}},{{integer(),integer()},{integer(),integer()}}) and a term of opaque type para2_adt:circ({{integer(),integer()},{integer(),integer()}}) +para2.erl:103: Attempt to test for equality between a term of type para2_adt:circ(integer(),integer()) and a term of opaque type para2_adt:circ(integer()) para2.erl:117: Attempt to test for equality between a term of type para2_adt:un(atom(),integer()) and a term of opaque type para2_adt:un(integer(),atom()) para2.erl:31: The test 'a' =:= 'b' can never evaluate to 'true' para2.erl:61: Attempt to test for equality between a term of type para2_adt:c2() and a term of opaque type para2_adt:c1() para2.erl:66: The test 'a' =:= 'b' can never evaluate to 'true' -para2.erl:88: The test para2:circ({{integer(),integer()},{integer(),integer()}}) =:= para2:circ({{integer(),integer()},{integer(),integer()}},{{integer(),integer()},{integer(),integer()}}) can never evaluate to 'true' +para2.erl:88: The test para2:circ(integer()) =:= para2:circ(integer(),integer()) can never evaluate to 'true' para3.erl:28: Invalid type specification for function para3:ot2/0. The success typing is () -> 'foo' para3.erl:36: The pattern {{{17}}} can never match the type {{{{{{_,_,_,_,_}}}}}} para3.erl:55: Invalid type specification for function para3:t2/0. The success typing is () -> 'foo' para3.erl:65: The attempt to match a term of type {{{{{para3_adt:ot1(_,_,_,_,_)}}}}} against the pattern {{{{{17}}}}} breaks the opaqueness of para3_adt:ot1(_,_,_,_,_) para3.erl:68: The pattern {{{{17}}}} can never match the type {{{{{para3_adt:ot1(_,_,_,_,_)}}}}} para3.erl:74: Invalid type specification for function para3:exp_adt/0. The success typing is () -> 3 +para4.erl:21: Invalid type specification for function para4:a/1. The success typing is (dict:dict(atom() | integer(),atom() | integer()) | para4:d_all()) -> [{atom() | integer(),atom() | integer()}] +para4.erl:26: Invalid type specification for function para4:i/1. The success typing is (dict:dict(atom() | integer(),atom() | integer()) | para4:d_all()) -> [{atom() | integer(),atom() | integer()}] +para4.erl:31: Invalid type specification for function para4:t/1. The success typing is (dict:dict(atom() | integer(),atom() | integer()) | para4:d_all()) -> [{atom() | integer(),atom() | integer()}] +para4.erl:59: Attempt to test for equality between a term of type para4_adt:t(atom() | integer()) and a term of opaque type para4_adt:t(integer()) +para4.erl:64: Attempt to test for equality between a term of type para4_adt:t(atom() | integer()) and a term of opaque type para4_adt:t(atom()) +para4.erl:69: Attempt to test for equality between a term of type para4_adt:int(1 | 2 | 3 | 4) and a term of opaque type para4_adt:int(1 | 2) +para4.erl:74: Attempt to test for equality between a term of type para4_adt:int(2 | 3 | 4) and a term of opaque type para4_adt:int(1 | 2) +para4.erl:79: Attempt to test for equality between a term of type para4_adt:int(2 | 3 | 4) and a term of opaque type para4_adt:int(5 | 6 | 7) +para4.erl:84: Attempt to test for equality between a term of type para4_adt:un(3 | 4) and a term of opaque type para4_adt:un(1 | 2) +para4.erl:89: Attempt to test for equality between a term of type para4_adt:tup({_,_}) and a term of opaque type para4_adt:tup(tuple()) +para5.erl:13: Attempt to test for inequality between a term of type para5_adt:dd(atom()) and a term of opaque type para5_adt:d() +para5.erl:8: The test para5_adt:d() =:= para5_adt:d() can never evaluate to 'true' diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/simple b/lib/dialyzer/test/opaque_SUITE_data/results/simple index 29864d6065..1a7a139d6e 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/simple +++ b/lib/dialyzer/test/opaque_SUITE_data/results/simple @@ -28,11 +28,11 @@ rec_api.erl:99: Record construction #r2{f1::10} violates the declared type of fi simple1_api.erl:113: The test simple1_api:d1() =:= simple1_api:d2() can never evaluate to 'true' simple1_api.erl:118: Guard test simple1_api:d2() =:= A::simple1_api:d1() can never succeed simple1_api.erl:142: Attempt to test for equality between a term of type simple1_adt:o2() and a term of opaque type simple1_adt:o1() -simple1_api.erl:148: Guard test simple1_adt:o2() =:= A::simple1_adt:o1() contains an opaque term as 1st argument +simple1_api.erl:148: Guard test simple1_adt:o2() =:= A::simple1_adt:o1() contains opaque terms as 1st and 2nd arguments simple1_api.erl:154: Attempt to test for inequality between a term of type simple1_adt:o2() and a term of opaque type simple1_adt:o1() simple1_api.erl:160: Attempt to test for inequality between a term of type simple1_adt:o2() and a term of opaque type simple1_adt:o1() simple1_api.erl:165: Attempt to test for equality between a term of type simple1_adt:c2() and a term of opaque type simple1_adt:c1() -simple1_api.erl:181: Guard test A::simple1_adt:d1() =< B::simple1_adt:d2() contains an opaque term as 1st argument +simple1_api.erl:181: Guard test A::simple1_adt:d1() =< B::simple1_adt:d2() contains opaque terms as 1st and 2nd arguments simple1_api.erl:185: Guard test 'a' =< B::simple1_adt:d2() contains an opaque term as 2nd argument simple1_api.erl:189: Guard test A::simple1_adt:d1() =< 'd' contains an opaque term as 1st argument simple1_api.erl:197: The type test is_integer(A::simple1_adt:d1()) breaks the opaqueness of the term A::simple1_adt:d1() @@ -72,12 +72,12 @@ simple1_api.erl:499: The call 'foo':A(A::simple1_api:i()) requires that A is of simple1_api.erl:503: The call 'foo':A(A::simple1_adt:i()) requires that A is of type atom() not simple1_adt:i() simple1_api.erl:507: The call A:'foo'(A::simple1_api:i()) requires that A is of type atom() | tuple() not simple1_api:i() simple1_api.erl:511: The call A:'foo'(A::simple1_adt:i()) requires that A is of type atom() | tuple() not simple1_adt:i() -simple1_api.erl:519: Guard test A::simple1_adt:d2() == B::simple1_adt:d1() contains an opaque term as 1st argument +simple1_api.erl:519: Guard test A::simple1_adt:d2() == B::simple1_adt:d1() contains opaque terms as 1st and 2nd arguments simple1_api.erl:534: Guard test A::simple1_adt:d1() >= 3 contains an opaque term as 1st argument simple1_api.erl:536: Guard test A::simple1_adt:d1() == 3 contains an opaque term as 1st argument simple1_api.erl:538: Guard test A::simple1_adt:d1() =:= 3 contains an opaque term as 1st argument simple1_api.erl:548: The call erlang:'<'(A::simple1_adt:d1(),3) contains an opaque term as 1st argument when terms of different types are expected in these positions -simple1_api.erl:558: The call erlang:'=<'(A::simple1_adt:d1(),B::simple1_adt:d2()) contains an opaque term as 1st argument when terms of different types are expected in these positions +simple1_api.erl:558: The call erlang:'=<'(A::simple1_adt:d1(),B::simple1_adt:d2()) contains opaque terms as 1st and 2nd arguments when terms of different types are expected in these positions simple1_api.erl:565: Guard test {digraph:graph(),3} > {digraph:graph(),atom() | ets:tid()} contains an opaque term as 2nd argument simple1_api.erl:91: Invalid type specification for function simple1_api:tup/0. The success typing is () -> {'a','b'} simple2_api.erl:100: The call lists:flatten(A::simple1_adt:tuple1()) contains an opaque term as 1st argument when a structured term of type [any()] is expected diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_macros.hrl b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_macros.hrl new file mode 100644 index 0000000000..07243f8d23 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_macros.hrl @@ -0,0 +1,215 @@ +%% -*- erlang-indent-level: 2 -*- +%%------------------------------------------------------------------------------ + +%%==================================================================== +%% Types +%%==================================================================== + +%% Code and Monitor servers' info. +-record(svs, { + code :: pid(), + monitor :: pid() +}). + +%% Tags of an AST's node. +-record(tags, { + this = undefined :: cuter_cerl:tag() | undefined, + next = undefined :: cuter_cerl:tag() | undefined +}). + +-type loaded_ret_atoms() :: cover_compiled | preloaded | non_existing. +-type servers() :: #svs{}. +-type ast_tags() :: #tags{}. + +%%==================================================================== +%% Directories +%%==================================================================== + +-define(RELATIVE_TMP_DIR, "temp"). +-define(PYTHON_CALL, ?PYTHON_PATH ++ " -u " ++ ?PRIV ++ "/cuter_interface.py"). + +%%==================================================================== +%% Prefixes +%%==================================================================== + +-define(DEPTH_PREFIX, '__conc_depth'). +-define(EXECUTION_PREFIX, '__conc_prefix'). +-define(SYMBOLIC_PREFIX, '__s'). +-define(CONCOLIC_PREFIX_MSG, '__concm'). +-define(ZIPPED_VALUE_PREFIX, '__czip'). +-define(CONCOLIC_PREFIX_PDICT, '__concp'). +-define(FUNCTION_PREFIX, '__cfunc'). +-define(UNBOUND_VAR_PREFIX, '__uboundvar'). +-define(BRANCH_TAG_PREFIX, '__branch_tag'). +-define(VISITED_TAGS_PREFIX, '__visited_tags'). +-define(EXECUTION_COUNTER_PREFIX, '__exec_counter'). + +%%==================================================================== +%% Flags & Default Values +%%==================================================================== + +-define(LOGGING_FLAG, ok). +-define(DELETE_TRACE, ok). +-define(LOG_UNSUPPORTED_MFAS, ok). +%%-define(VERBOSE_SCHEDULER, ok). +%%-define(VERBOSE_FILE_DELETION, ok). +%%-define(VERBOSE_SOLVING, ok). +%%-define(VERBOSE_MERGING, ok). +%%-define(VERBOSE_REPORTING, ok). +-define(USE_SPECS, ok). + +%%==================================================================== +%% Solver Responses +%%==================================================================== + +-define(RSP_MODEL_DELIMITER_START, <<"model_start">>). +-define(RSP_MODEL_DELIMITER_END, <<"model_end">>). + +%%==================================================================== +%% OpCodes for types in JSON objects +%%==================================================================== + +-define(JSON_TYPE_ANY, 0). +-define(JSON_TYPE_INT, 1). +-define(JSON_TYPE_FLOAT, 2). +-define(JSON_TYPE_ATOM, 3). +-define(JSON_TYPE_LIST, 4). +-define(JSON_TYPE_TUPLE, 5). +-define(JSON_TYPE_PID, 6). +-define(JSON_TYPE_REF, 7). + +%%==================================================================== +%% OpCodes for the commands to the solver +%%==================================================================== + +-define(JSON_CMD_LOAD_TRACE_FILE, 1). +-define(JSON_CMD_SOLVE, 2). +-define(JSON_CMD_GET_MODEL, 3). +-define(JSON_CMD_ADD_AXIOMS, 4). +-define(JSON_CMD_FIX_VARIABLE, 5). +-define(JSON_CMD_RESET_SOLVER, 6). +-define(JSON_CMD_STOP, 42). + +%%==================================================================== +%% OpCodes for constraint types +%%==================================================================== + +-define(CONSTRAINT_TRUE, 1). +-define(CONSTRAINT_FALSE, 2). +-define(NOT_CONSTRAINT, 3). + +-define(CONSTRAINT_TRUE_REPR, 84). %% $T +-define(CONSTRAINT_FALSE_REPR, 70). %% $F + +%%==================================================================== +%% OpCodes of constraints & built-in operations +%%==================================================================== + +%% Empty tag ID +-define(EMPTY_TAG_ID, 0). + +%% MFA's Parameters & Spec definitions. +-define(OP_PARAMS, 1). +-define(OP_SPEC, 2). +%% Constraints. +-define(OP_GUARD_TRUE, 3). +-define(OP_GUARD_FALSE, 4). +-define(OP_MATCH_EQUAL_TRUE, 5). +-define(OP_MATCH_EQUAL_FALSE, 6). +-define(OP_TUPLE_SZ, 7). +-define(OP_TUPLE_NOT_SZ, 8). +-define(OP_TUPLE_NOT_TPL, 9). +-define(OP_LIST_NON_EMPTY, 10). +-define(OP_LIST_EMPTY, 11). +-define(OP_LIST_NOT_LST, 12). +%% Information used for syncing & merging the traces of many processes. +-define(OP_SPAWN, 13). +-define(OP_SPAWNED, 14). +-define(OP_MSG_SEND, 15). +-define(OP_MSG_RECEIVE, 16). +-define(OP_MSG_CONSUME, 17). +%% Necessary operations for the evaluation of Core Erlang. +-define(OP_UNFOLD_TUPLE, 18). +-define(OP_UNFOLD_LIST, 19). +%% Bogus operation (operations interpreted as the identity function). +-define(OP_BOGUS, 48). +%% Type conversions. +-define(OP_FLOAT, 47). +-define(OP_LIST_TO_TUPLE, 52). +-define(OP_TUPLE_TO_LIST, 53). +%% Query types. +-define(OP_IS_INTEGER, 27). +-define(OP_IS_ATOM, 28). +-define(OP_IS_FLOAT, 29). +-define(OP_IS_LIST, 30). +-define(OP_IS_TUPLE, 31). +-define(OP_IS_BOOLEAN, 32). +-define(OP_IS_NUMBER, 33). +%% Arithmetic operations. +-define(OP_PLUS, 34). +-define(OP_MINUS, 35). +-define(OP_TIMES, 36). +-define(OP_RDIV, 37). +-define(OP_IDIV_NAT, 38). +-define(OP_REM_NAT, 39). +-define(OP_UNARY, 40). +%% Operations on atoms. +-define(OP_ATOM_NIL, 49). +-define(OP_ATOM_HEAD, 50). +-define(OP_ATOM_TAIL, 51). +%% Operations on lists. +-define(OP_HD, 25). +-define(OP_TL, 26). +-define(OP_CONS, 56). +%% Operations on tuples. +-define(OP_TCONS, 57). +%% Comparisons. +-define(OP_EQUAL, 41). +-define(OP_UNEQUAL, 42). +-define(OP_LT_INT, 54). +-define(OP_LT_FLOAT, 55). + +%% Maps MFAs to their JSON Opcodes +-define(OPCODE_MAPPING, + dict:from_list([ %% Simulated built-in operations + { {cuter_erlang, atom_to_list_bogus, 1}, ?OP_BOGUS } + , { {cuter_erlang, is_atom_nil, 1}, ?OP_ATOM_NIL } + , { {cuter_erlang, safe_atom_head, 1}, ?OP_ATOM_HEAD } + , { {cuter_erlang, safe_atom_tail, 1}, ?OP_ATOM_TAIL } + , { {cuter_erlang, safe_pos_div, 2}, ?OP_IDIV_NAT } + , { {cuter_erlang, safe_pos_rem, 2}, ?OP_REM_NAT } + , { {cuter_erlang, lt_int, 2}, ?OP_LT_INT } + , { {cuter_erlang, lt_float, 2}, ?OP_LT_FLOAT } + , { {cuter_erlang, safe_plus, 2}, ?OP_PLUS } + , { {cuter_erlang, safe_minus, 2}, ?OP_MINUS } + , { {cuter_erlang, safe_times, 2}, ?OP_TIMES } + , { {cuter_erlang, safe_rdiv, 2}, ?OP_RDIV } + , { {cuter_erlang, safe_float, 1}, ?OP_FLOAT } + , { {cuter_erlang, safe_list_to_tuple, 1}, ?OP_LIST_TO_TUPLE } + , { {cuter_erlang, safe_tuple_to_list, 1}, ?OP_TUPLE_TO_LIST } + , { {bogus_erlang, cons, 2}, ?OP_CONS } + %% Actual erlang BIFs + , { {erlang, hd, 1}, ?OP_HD } + , { {erlang, tl, 1}, ?OP_TL } + , { {erlang, is_integer, 1}, ?OP_IS_INTEGER } + , { {erlang, is_atom, 1}, ?OP_IS_ATOM } + , { {erlang, is_boolean, 1}, ?OP_IS_BOOLEAN } + , { {erlang, is_float, 1}, ?OP_IS_FLOAT } + , { {erlang, is_list, 1}, ?OP_IS_LIST } + , { {erlang, is_tuple, 1}, ?OP_IS_TUPLE } + , { {erlang, is_number, 1}, ?OP_IS_NUMBER } + , { {erlang, '-', 1}, ?OP_UNARY } + , { {erlang, '=:=', 2}, ?OP_EQUAL } + , { {erlang, '=/=', 2}, ?OP_UNEQUAL } + ])). + +%% All the MFAs that are supported for symbolic evaluation. +-define(SUPPORTED_MFAS, gb_sets:from_list(dict:fetch_keys(?OPCODE_MAPPING))). + +-define(UNSUPPORTED_MFAS, + gb_sets:from_list([ {cuter_erlang, unsupported_lt, 2} ])). + +%% The set of all the built-in operations that the solver can try to reverse. +-define (REVERSIBLE_OPERATIONS, + gb_sets:from_list([ ?OP_HD, ?OP_TL + ])). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.erl b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.erl new file mode 100644 index 0000000000..e9561374cc --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.erl @@ -0,0 +1,607 @@ +%% -*- erlang-indent-level: 2 -*- +%%------------------------------------------------------------------------------ +-module(cuter_types). + +-export([parse_spec/3, retrieve_types/1, retrieve_specs/1, find_spec/2, get_kind/1]). + +-export([params_of_t_function_det/1, ret_of_t_function_det/1, atom_of_t_atom_lit/1, integer_of_t_integer_lit/1, + elements_type_of_t_list/1, elements_type_of_t_nonempty_list/1, elements_types_of_t_tuple/1, + elements_types_of_t_union/1, bounds_of_t_range/1, segment_size_of_bitstring/1]). + +-export_type([erl_type/0, erl_spec_clause/0, erl_spec/0, stored_specs/0, stored_types/0, stored_spec_value/0, t_range_limit/0]). + +-include("cuter_macros.hrl"). +-include("cuter_types.hrl"). + + +%% Define tags +-define(type_variable, vart). +-define(type_var, tvar). +-define(max_char, 16#10ffff). + +%% Pre-processed types. + +-type type_name() :: atom(). +-type type_arity() :: byte(). +-type type_var() :: {?type_var, atom()}. +-type remote_type() :: {module(), type_name(), type_arity()}. +-type record_name() :: atom(). +-type record_field_name() :: atom(). +-type record_field_type() :: {record_field_name(), raw_type()}. +-type dep() :: remote_type(). +-type deps() :: ordsets:ordset(remote_type()). +-record(t, { + kind, + rep, + deps = ordsets:new() :: deps() +}). +-type erl_type() :: t_any() % any() + | t_nil() % [] + | t_atom() % atom() + | t_atom_lit() % Erlang_Atom + | t_integer() % integer(), +infinity, -inifinity + | t_integer_lit() % Erlang_Integer + | t_float() % float() + | t_tuple() % tuple(), {TList} + | t_list() % list(Type) + | t_nonempty_list() % nonempty_list(Type) + | t_union() % Type1 | ... | TypeN + | t_range() % Erlang_Integer..Erlang_Integer + | t_bitstring() % <<_:M>> + | t_function() % function() | Fun | BoundedFun + . +-type raw_type() :: erl_type() + | t_local() % Local Type Usage + | t_remote() % Remote Type Usage + | t_record() % Record Usage + | t_type_var() % Type Variable + . + +-type t_any() :: #t{kind :: ?any_tag}. +-type t_nil() :: #t{kind :: ?nil_tag}. +-type t_atom() :: #t{kind :: ?atom_tag}. +-type t_atom_lit() :: #t{kind :: ?atom_lit_tag, rep :: atom()}. +-type t_integer() :: #t{kind :: ?integer_tag}. +-type t_integer_lit() :: #t{kind :: ?integer_lit_tag, rep :: integer()}. +-type t_float() :: #t{kind :: ?float_tag}. +-type t_tuple() :: #t{kind :: ?tuple_tag, rep :: [raw_type()]}. +-type t_list() :: #t{kind :: ?list_tag, rep :: raw_type()}. +-type t_nonempty_list() :: #t{kind :: ?nonempty_list_tag, rep :: raw_type()}. +-type t_union() :: #t{kind :: ?union_tag, rep :: [raw_type()]}. +-type t_range() :: #t{kind :: ?range_tag, rep :: {t_range_limit(), t_range_limit()}}. +-type t_range_limit() :: t_integer_lit() | t_integer_inf(). +-type t_integer_inf() :: t_integer_pos_inf() | t_integer_neg_inf(). +-type t_integer_pos_inf() :: #t{kind :: ?pos_inf}. +-type t_integer_neg_inf() :: #t{kind :: ?neg_inf}. +-type t_bitstring() :: #t{kind :: ?bitstring_tag, rep :: 1|8}. +-type t_function() :: #t{kind :: ?function_tag} | t_function_det(). +-type t_function_det() :: #t{kind :: ?function_tag, rep :: {[raw_type()], raw_type(), [t_constraint()]}, deps :: deps()}. +-type t_constraint() :: {t_type_var(), raw_type()}. +-type t_local() :: #t{kind :: ?local_tag, rep :: {type_name(), [raw_type()]}}. +-type t_remote() :: #t{kind :: ?remote_tag, rep :: {module(), type_name(), [raw_type()]}}. +-type t_record() :: #t{kind :: ?record_tag, rep :: {record_name(), [record_field_type()]}}. +-type t_type_var() :: #t{kind :: ?type_variable, rep :: type_var()}. + +%% How pre-processed types are stored. +-type stored_type_key() :: {record, record_name()} | {type, type_name(), type_arity()}. +-type stored_type_value() :: [record_field_type()] | {any(), [type_var()]}. % raw_type() +-type stored_types() :: dict:dict(stored_type_key(), stored_type_value()). + +-type stored_spec_key() :: {type_name(), type_arity()}. +-type stored_spec_value() :: [t_function_det()]. +-type stored_specs() :: dict:dict(stored_spec_key(), stored_spec_value()). + +-type type_var_env() :: dict:dict(type_var(), raw_type()). +-type erl_spec_clause() :: t_function_det(). +-type erl_spec() :: [erl_spec_clause()]. + +%% Pre-process the type & record declarations of a module. +-spec retrieve_types([cuter_cerl:cerl_attr_type()]) -> stored_types(). +retrieve_types(TypeAttrs) -> + lists:foldl(fun process_type_attr/2, dict:new(), TypeAttrs). + +-spec process_type_attr(cuter_cerl:cerl_recdef() | cuter_cerl:cerl_typedef(), stored_types()) -> stored_types(). +%% Declaration of a record. +process_type_attr({{record, Name}, Fields, []}, Processed) -> + Fs = [t_field_from_form(Field) || Field <- Fields], + Record = t_record(Name, Fs), + dict:store({record, Name}, Record, Processed); +%% Declaration of a type. +process_type_attr({Name, Repr, Vars}, Processed) -> + Type = safe_t_from_form(Repr), + Vs = [{?type_var, Var} || {var, _, Var} <- Vars], + dict:store({type, Name, length(Vs)}, {Type, Vs}, Processed). + +%% The fields of a declared record. +-spec t_field_from_form(cuter_cerl:cerl_record_field()) -> record_field_type(). +t_field_from_form({record_field, _, {atom, _, Name}}) -> + {Name, t_any()}; +t_field_from_form({record_field, _, {atom, _, Name}, _Default}) -> + {Name, t_any()}; +t_field_from_form({typed_record_field, {record_field, _, {atom, _, Name}}, Type}) -> + {Name, safe_t_from_form(Type)}; +t_field_from_form({typed_record_field, {record_field, _, {atom, _, Name}, _Default}, Type}) -> + {Name, safe_t_from_form(Type)}. + +%% Provision for unsupported types. +safe_t_from_form(Form) -> + try t_from_form(Form) + catch throw:{unsupported, Info} -> + cuter_pp:form_has_unsupported_type(Info), + t_any() + end. + +%% Parse a type. + +-spec t_from_form(cuter_cerl:cerl_type()) -> raw_type(). +%% Erlang_Atom +t_from_form({atom, _, Atom}) -> + t_atom_lit(Atom); +%% Erlang_Integer +t_from_form({integer, _, Integer}) -> + t_integer_lit(Integer); +%% integer() +t_from_form({type, _, integer, []}) -> + t_integer(); +%% nil +t_from_form({type, _, nil, []}) -> + t_nil(); +%% any() +t_from_form({type, _, any, []}) -> + t_any(); +%% term() +t_from_form({type, _, term, []}) -> + t_any(); +%% atom() +t_from_form({type, _, atom, []}) -> + t_atom(); +%% module() +t_from_form({type, _, module, []}) -> + t_module(); +%% float() +t_from_form({type, _, float, []}) -> + t_float(); +%% tuple() +t_from_form({type, _, tuple, any}) -> + t_tuple(); +%% {TList} +t_from_form({type, _, tuple, Types}) -> + Ts = [t_from_form(T) || T <- Types], + t_tuple(Ts); +%% list() +t_from_form({type, _, list, []}) -> + t_list(); +%% list(Type) +t_from_form({type, _, list, [Type]}) -> + T = t_from_form(Type), + t_list(T); +%% Type1 | ... | TypeN +t_from_form({type, _, union, Types}) -> + Ts = [t_from_form(T) || T <- Types], + t_union(Ts); +%% boolean() +t_from_form({type, _, boolean, []}) -> + t_union([t_atom_lit(true), t_atom_lit(false)]); +%% number() +t_from_form({type, _, number, []}) -> + t_union([t_integer(), t_float()]); +%% Erlang_Integer..Erlang_Integer +t_from_form({type, _, range, [{integer, _, I1}, {integer, _, I2}]}) -> + t_range(t_integer_lit(I1), t_integer_lit(I2)); +%% non_neg_integer() +t_from_form({type, _, non_neg_integer, []}) -> + t_range(t_integer_lit(0), t_pos_inf()); +%% pos_integer() +t_from_form({type, _, pos_integer, []}) -> + t_range(t_integer_lit(1), t_pos_inf()); +%% neg_integer() +t_from_form({type, _, neg_integer, []}) -> + t_range(t_neg_inf(), t_integer_lit(-1)); +%% char() +t_from_form({type, _, char, []}) -> + t_char(); +%% byte() +t_from_form({type, _, byte, []}) -> + t_byte(); +%% mfa() +t_from_form({type, _, mfa, []}) -> + t_tuple([t_module(), t_atom(), t_byte()]); +%% string() +t_from_form({type, _, string, []}) -> + t_list(t_char()); +%% nonempty_list() +t_from_form({type, _, nonempty_list, []}) -> + t_nonempty_list(); +%% nonempty_list(Type) +t_from_form({type, _, nonempty_list, [Type]}) -> + T = t_from_form(Type), + t_nonempty_list(T); +%% binary() +t_from_form({type, _, binary, []}) -> + t_bitstring(8); +%% bitstring() +t_from_form({type, _, bitstring, []}) -> + t_bitstring(1); +%% function() +t_from_form({type, _, function, []}) -> + t_function(); +%% fun((TList) -> Type) +t_from_form({type, _, 'fun', [_Product, _RetType]}=Fun) -> + t_function_from_form(Fun); +%% fun((TList) -> Type) (bounded_fun) +t_from_form({type, _, 'bounded_fun', [_Fun, _Cs]}=BoundedFun) -> + t_bounded_function_from_form(BoundedFun); +%% ann_type +t_from_form({ann_type, _, [_Var, Type]}) -> + t_from_form(Type); +%% paren_type +t_from_form({paren_type, _, [Type]}) -> + t_from_form(Type); +%% remote_type +t_from_form({remote_type, _, [{atom, _, M}, {atom, _, Name}, Types]}) -> + Ts = [t_from_form(T) || T <- Types], + t_remote(M, Name, Ts); +%% Record +t_from_form({type, _, record, [{atom, _, Name} | FieldTypes]}) -> + Fields = [t_bound_field_from_form(F) || F <- FieldTypes], + t_record(Name, Fields); +%% Map +t_from_form({type, _, map, _}=X) -> + throw({unsupported, X}); +%% local type +t_from_form({type, _, Name, Types}) -> + Ts = [t_from_form(T) || T <- Types], + t_local(Name, Ts); +%% Type Variable +t_from_form({var, _, Var}) -> + t_var(Var); +%% Unsupported forms +t_from_form(Type) -> + throw({unsupported, Type}). + +-spec t_bound_field_from_form(cuter_cerl:cerl_type_record_field()) -> record_field_type(). +%% Record Field. +t_bound_field_from_form({type, _, field_type, [{atom, _, Name}, Type]}) -> + {Name, t_from_form(Type)}. + +-spec t_function_from_form(cuter_cerl:cerl_func()) -> t_function_det(). +t_function_from_form({type, _, 'fun', [{type, _, 'product', Types}, RetType]}) -> + Ret = t_from_form(RetType), + Ts = [t_from_form(T) || T <- Types], + t_function(Ts, Ret). + +-spec t_bounded_function_from_form(cuter_cerl:cerl_bounded_func()) -> t_function_det(). +t_bounded_function_from_form({type, _, 'bounded_fun', [Fun, Constraints]}) -> + {type, _, 'fun', [{type, _, 'product', Types}, RetType]} = Fun, + Ret = t_from_form(RetType), + Ts = [t_from_form(T) || T <- Types], + Cs = [t_constraint_from_form(C) || C <- Constraints], + t_function(Ts, Ret, Cs). + +-spec t_constraint_from_form(cuter_cerl:cerl_constraint()) -> t_constraint(). +t_constraint_from_form({type, _, constraint, [{atom, _, is_subtype}, [{var, _, Var}, Type]]}) -> + {t_var(Var), t_from_form(Type)}. + + +%% Type constructors. + +-spec t_any() -> t_any(). +t_any() -> + #t{kind = ?any_tag}. + +-spec t_atom_lit(atom()) -> t_atom_lit(). +t_atom_lit(Atom) -> + #t{kind = ?atom_lit_tag, rep = Atom}. + +-spec t_atom() -> t_atom(). +t_atom() -> + #t{kind = ?atom_tag}. + +-spec t_module() -> t_atom(). +t_module() -> t_atom(). + +-spec t_integer_lit(integer()) -> t_integer_lit(). +t_integer_lit(Integer) -> + #t{kind = ?integer_lit_tag, rep = Integer}. + +-spec t_integer() -> t_integer(). +t_integer() -> + #t{kind = ?integer_tag}. + +-spec t_range(t_range_limit(), t_range_limit()) -> t_range(). +t_range(Int1, Int2) -> + #t{kind = ?range_tag, rep = {Int1, Int2}}. + +-spec t_pos_inf() -> t_integer_pos_inf(). +t_pos_inf() -> + #t{kind = ?pos_inf}. + +-spec t_neg_inf() -> t_integer_neg_inf(). +t_neg_inf() -> + #t{kind = ?neg_inf}. + +-spec t_char() -> t_range(). +t_char() -> + t_range(t_integer_lit(0), t_integer_lit(?max_char)). + +-spec t_nil() -> t_nil(). +t_nil() -> + #t{kind = ?nil_tag}. + +-spec t_float() -> t_float(). +t_float() -> + #t{kind = ?float_tag}. + +-spec t_list() -> t_list(). +t_list() -> + #t{kind = ?list_tag, rep = t_any()}. + +-spec t_list(raw_type()) -> t_list(). +t_list(Type) -> + #t{kind = ?list_tag, rep = Type, deps = get_deps(Type)}. + +-spec t_nonempty_list() -> t_nonempty_list(). +t_nonempty_list() -> + #t{kind = ?nonempty_list_tag, rep = t_any()}. + +-spec t_nonempty_list(raw_type()) -> t_nonempty_list(). +t_nonempty_list(Type) -> + #t{kind = ?nonempty_list_tag, rep = Type, deps = get_deps(Type)}. + +-spec t_tuple() -> t_tuple(). +t_tuple() -> + #t{kind = ?tuple_tag, rep = []}. + +-spec t_tuple([raw_type()]) -> t_tuple(). +t_tuple(Types) -> + #t{kind = ?tuple_tag, rep = Types, deps = unify_deps(Types)}. + +-spec t_union([raw_type()]) -> t_union(). +t_union(Types) -> + #t{kind = ?union_tag, rep = Types, deps = unify_deps(Types)}. + +-spec t_byte() -> t_range(). +t_byte() -> + t_range(t_integer_lit(0), t_integer_lit(255)). + +-spec t_local(type_name(), [raw_type()]) -> t_local(). +t_local(Name, Types) -> + Rep = {Name, Types}, + #t{kind = ?local_tag, rep = Rep, deps = unify_deps(Types)}. + +-spec t_remote(module(), type_name(), [raw_type()]) -> t_remote(). +t_remote(Mod, Name, Types) -> + Rep = {Mod, Name, Types}, + Dep = {Mod, Name, length(Types)}, + #t{kind = ?remote_tag, rep = Rep, deps = add_dep(Dep, unify_deps(Types))}. + +-spec t_var(atom()) -> t_type_var(). +t_var(Var) -> + #t{kind = ?type_variable, rep = {?type_var, Var}}. + +-spec t_record(record_name(), [record_field_type()]) -> t_record(). +t_record(Name, Fields) -> + Rep = {Name, Fields}, + Ts = [T || {_, T} <- Fields], + #t{kind = ?record_tag, rep = Rep, deps = unify_deps(Ts)}. + +-spec fields_of_t_record(t_record()) -> [record_field_type()]. +fields_of_t_record(Record) -> + Rep = Record#t.rep, + element(2, Rep). + +-spec t_bitstring(1 | 8) -> t_bitstring(). +t_bitstring(N) -> + #t{kind = ?bitstring_tag, rep = N}. + +-spec t_function() -> t_function(). +t_function() -> + #t{kind = ?function_tag}. + +-spec t_function([raw_type()], raw_type()) -> t_function_det(). +t_function(Types, Ret) -> + Rep = {Types, Ret, []}, + #t{kind = ?function_tag, rep = Rep, deps = unify_deps([Ret|Types])}. + +-spec t_function([raw_type()], raw_type(), [t_constraint()]) -> t_function_det(). +t_function(Types, Ret, Constraints) -> + Rep = {Types, Ret, Constraints}, + Ts = [T || {_V, T} <- Constraints], + #t{kind = ?function_tag, rep = Rep, deps = unify_deps([Ret|Types] ++ Ts)}. + +%% Accessors of representations. + +-spec params_of_t_function_det(t_function_det()) -> [raw_type()]. +params_of_t_function_det(#t{kind = ?function_tag, rep = {Params, _Ret, _Constraints}}) -> + Params. + +-spec ret_of_t_function_det(t_function_det()) -> raw_type(). +ret_of_t_function_det(#t{kind = ?function_tag, rep = {_Params, Ret, _Constraints}}) -> + Ret. + +-spec atom_of_t_atom_lit(t_atom_lit()) -> atom(). +atom_of_t_atom_lit(#t{kind = ?atom_lit_tag, rep = Atom}) -> + Atom. + +-spec integer_of_t_integer_lit(t_integer_lit()) -> integer(). +integer_of_t_integer_lit(#t{kind = ?integer_lit_tag, rep = Integer}) -> + Integer. + +-spec elements_type_of_t_list(t_list()) -> raw_type(). +elements_type_of_t_list(#t{kind = ?list_tag, rep = Type}) -> + Type. + +-spec elements_type_of_t_nonempty_list(t_nonempty_list()) -> raw_type(). +elements_type_of_t_nonempty_list(#t{kind = ?nonempty_list_tag, rep = Type}) -> + Type. + +-spec elements_types_of_t_tuple(t_tuple()) -> [raw_type()]. +elements_types_of_t_tuple(#t{kind = ?tuple_tag, rep = Types}) -> + Types. + +-spec elements_types_of_t_union(t_union()) -> [raw_type()]. +elements_types_of_t_union(#t{kind = ?union_tag, rep = Types}) -> + Types. + +-spec bounds_of_t_range(t_range()) -> {t_range_limit(), t_range_limit()}. +bounds_of_t_range(#t{kind = ?range_tag, rep = Limits}) -> + Limits. + +-spec segment_size_of_bitstring(t_bitstring()) -> integer(). +segment_size_of_bitstring(#t{kind = ?bitstring_tag, rep = Sz}) -> + Sz. + +-spec is_tvar_wild_card(t_type_var()) -> boolean(). +is_tvar_wild_card(#t{kind = ?type_variable, rep = {?type_var, Var}}) -> + Var =:= '_'. + +%% Helper functions for kinds. + +-spec get_kind(raw_type()) -> atom(). +get_kind(Type) -> + Type#t.kind. + +%% Helper functions for dependencies. + +-spec get_deps(raw_type()) -> deps(). +get_deps(Type) -> + Type#t.deps. + +-spec has_deps(raw_type()) -> boolean(). +has_deps(Type) -> + get_deps(Type) =/= ordsets:new(). + +-spec add_dep(dep(), deps()) -> deps(). +add_dep(Dep, Deps) -> + ordsets:add_element(Dep, Deps). + +-spec unify_deps([raw_type()]) -> deps(). +unify_deps(Types) -> + ordsets:union([T#t.deps || T <- Types]). + +%% Deal with specs. + +-spec retrieve_specs([cuter_cerl:cerl_attr_spec()]) -> stored_specs(). +retrieve_specs(SpecAttrs) -> + lists:foldl(fun process_spec_attr/2, dict:new(), SpecAttrs). + +-spec process_spec_attr(cuter_cerl:cerl_attr_spec(), stored_specs()) -> stored_specs(). +process_spec_attr({FA, Specs}, Processed) -> + Xs = [t_spec_from_form(Spec) || Spec <- Specs], + dict:store(FA, Xs, Processed). + +-spec t_spec_from_form(cuter_cerl:cerl_spec_func()) -> t_function_det(). +t_spec_from_form({type, _, 'fun', _}=Fun) -> + t_function_from_form(Fun); +t_spec_from_form({type, _, 'bounded_fun', _}=Fun) -> + t_bounded_function_from_form(Fun). + +-spec find_spec(stored_spec_key(), stored_specs()) -> {'ok', stored_spec_value()} | 'error'. +find_spec(FA, Specs) -> + dict:find(FA, Specs). + +%% Parse the spec of an MFA. + +-type spec_parse_reply() :: {error, has_remote_types | recursive_type} + | {error, unsupported_type, type_name()} + | {ok, erl_spec()}. + +-spec parse_spec(stored_spec_key(), stored_spec_value(), stored_types()) -> spec_parse_reply(). +parse_spec(FA, Spec, Types) -> + try parse_spec_clauses(FA, Spec, Types, []) of + {error, has_remote_types}=E -> E; + Parsed -> {ok, Parsed} + catch + throw:remote_type -> {error, has_remote_types}; + throw:recursive_type -> {error, recursive_type}; + throw:{unsupported, Name} -> {error, unsupported_type, Name} + end. + + +parse_spec_clauses(_FA, [], _Types, Acc) -> + lists:reverse(Acc); +parse_spec_clauses(FA, [Clause|Clauses], Types, Acc) -> + case has_deps(Clause) of + true -> {error, has_remote_types}; + false -> + Visited = ordsets:add_element(FA, ordsets:new()), + Simplified = simplify(Clause, Types, dict:new(), Visited), + parse_spec_clauses(FA, Clauses, Types, [Simplified|Acc]) + end. + +add_constraints_to_env([], Env) -> + Env; +add_constraints_to_env([{Var, Type}|Cs], Env) -> + F = fun(StoredTypes, E, Visited) -> simplify(Type, StoredTypes, E, Visited) end, + Env1 = dict:store(Var#t.rep, F, Env), + add_constraints_to_env(Cs, Env1). + +bind_parameters([], [], Env) -> + Env; +bind_parameters([P|Ps], [A|As], Env) -> + F = fun(StoredTypes, E, Visited) -> simplify(A, StoredTypes, E, Visited) end, + Env1 = dict:store(P, F, Env), + bind_parameters(Ps, As, Env1). + +-spec simplify(raw_type(), stored_types(), type_var_env(), ordsets:ordset(stored_spec_key())) -> raw_type(). +%% fun +simplify(#t{kind = ?function_tag, rep = {Params, Ret, Constraints}}=Raw, StoredTypes, Env, Visited) -> + Env1 = add_constraints_to_env(Constraints, Env), + ParamsSimplified = [simplify(P, StoredTypes, Env1, Visited) || P <- Params], + RetSimplified = simplify(Ret, StoredTypes, Env1, Visited), + Rep = {ParamsSimplified, RetSimplified, []}, + Raw#t{rep = Rep}; +%% tuple +simplify(#t{kind = ?tuple_tag, rep = Types}=Raw, StoredTypes, Env, Visited) -> + Rep = [simplify(T, StoredTypes, Env, Visited) || T <- Types], + Raw#t{rep = Rep}; +%% list / nonempty_list +simplify(#t{kind = Tag, rep = Type}=Raw, StoredTypes, Env, Visited) when Tag =:= ?list_tag; Tag =:= ?nonempty_list_tag -> + Rep = simplify(Type, StoredTypes, Env, Visited), + Raw#t{rep = Rep}; +%% union +simplify(#t{kind = ?union_tag, rep = Types}=Raw, StoredTypes, Env, Visited) -> + Rep = [simplify(T, StoredTypes, Env, Visited) || T <- Types], + Raw#t{rep = Rep}; +%% local type +simplify(#t{kind = ?local_tag, rep = {Name, Args}}, StoredTypes, Env, Visited) -> + Arity = length(Args), + TA = {Name, Arity}, + case ordsets:is_element(TA, Visited) of + true -> throw(recursive_type); + false -> + case dict:find({type, Name, Arity}, StoredTypes) of + error -> throw({unsupported, Name}); + {ok, {Type, Params}} -> + Env1 = bind_parameters(Params, Args, Env), + simplify(Type, StoredTypes, Env1, [TA|Visited]) + end + end; +%% type variable +simplify(#t{kind = ?type_variable, rep = TVar}=T, StoredTypes, Env, Visited) -> + case is_tvar_wild_card(T) of + true -> t_any(); + false -> + V = dict:fetch(TVar, Env), + V(StoredTypes, Env, Visited) + end; +simplify(#t{kind = ?remote_tag}, _StoredTypes, _Env, _Visited) -> + throw(remote_type); +%% record +simplify(#t{kind = ?record_tag, rep = {Name, OverridenFields}}, StoredTypes, Env, Visited) -> + RecordDecl = dict:fetch({record, Name}, StoredTypes), + Fields = fields_of_t_record(RecordDecl), + ActualFields = replace_record_fields(Fields, OverridenFields), + FinalFields = [{N, simplify(T, StoredTypes, Env, Visited)} || {N, T} <- ActualFields], + Simplified = [T || {_, T} <- FinalFields], + t_tuple([t_atom_lit(Name)|Simplified]); +%% all others +simplify(Raw, _StoredTypes, _Env, _Visited) -> + Raw. + +-spec replace_record_fields([record_field_type()], [record_field_type()]) -> [record_field_type()]. +replace_record_fields(Fields, []) -> + Fields; +replace_record_fields(Fields, [{Name, Type}|Rest]) -> + Replaced = lists:keyreplace(Name, 1, Fields, {Name, Type}), + replace_record_fields(Replaced, Rest). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.hrl b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.hrl new file mode 100644 index 0000000000..4172184709 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.hrl @@ -0,0 +1,26 @@ +%% -*- erlang-indent-level: 2 -*- +%%------------------------------------------------------------------------------ + +%%==================================================================== +%% Tags for the kind of encoded types. +%%==================================================================== + +-define(atom_lit_tag, atom_lit). +-define(integer_lit_tag, integer_lit). +-define(integer_tag, integer). +-define(nil_tag, nil). +-define(any_tag, any). +-define(atom_tag, atom). +-define(float_tag, float). +-define(tuple_tag, tuple). +-define(list_tag, list). +-define(nonempty_list_tag, nonempty_list). +-define(union_tag, union). +-define(range_tag, range). +-define(bitstring_tag, bitstring). +-define(neg_inf, neg_inf). +-define(pos_inf, pos_inf). +-define(remote_tag, remote). +-define(local_tag, local). +-define(record_tag, record). +-define(function_tag, function). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para2.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para2.erl index 09b2235fa5..4461ff291c 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/para/para2.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para2.erl @@ -85,7 +85,7 @@ ct2_adt() -> tcirc() -> A = circ1(), B = circ2(), - A =:= B. % can never evaluate to 'true' (but the types are not OK, or?) + A =:= B. % can never evaluate to 'true' -spec circ1() -> circ(integer()). @@ -100,7 +100,7 @@ circ2() -> tcirc_adt() -> A = circ1_adt(), B = circ2_adt(), - A =:= B. % opaque attempt (one would expect them to be the same...) + A =:= B. % opaque attempt (number of parameters differs) circ1_adt() -> para2_adt:circ1(). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para3.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para3.erl index 792ae40d39..102215b28d 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/para/para3.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para3.erl @@ -71,7 +71,7 @@ t2_adt() -> -type exp() :: para3_adt:exp1(para3_adt:exp2()). --spec exp_adt() -> exp(). +-spec exp_adt() -> exp(). % invalid type spec exp_adt() -> 3. diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para4.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para4.erl new file mode 100644 index 0000000000..b9794672a9 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para4.erl @@ -0,0 +1,134 @@ +-module(para4). + +-compile(export_all). + +-export_type([d_atom/0, d_integer/0, d_tuple/0, d_all/0]). + +-export_type([t/1]). + +-type ai() :: atom() | integer(). + +-type d(T) :: dict:dict(T, T). + +-opaque d_atom() :: d(atom()). +-opaque d_integer() :: d(integer()). +-opaque d_tuple() :: d(tuple()). +-opaque d_all() :: d(ai()). + +b(D) -> + a(D) ++ i(D). + +-spec a(d_atom()) -> [{atom(), atom()}]. % Invalid type spec + +a(D) -> + c(D). + +-spec i(d_integer()) -> [{integer(), integer()}]. % Invalid type spec + +i(D) -> + c(D). + +-spec t(d_tuple()) -> [{tuple(), tuple()}]. % Invalid type spec. + +t(D) -> + c(D). + +-spec c(d_all()) -> [{ai(), ai()}]. + +c(D) -> + dict:to_list(D). + + + + +-opaque t(A) :: {A, A}. + +adt_tt5() -> + I1 = adt_y1(), + I2 = adt_y3(), + I1 =:= I2. + +adt_tt6() -> + I1 = adt_y2(), + I2 = adt_y3(), + I1 =:= I2. + +adt_tt7() -> + I1 = adt_t1(), + I2 = adt_t3(), + I1 =:= I2. % opaque attempt + +adt_tt8() -> + I1 = adt_t2(), + I2 = adt_t3(), + I1 =:= I2. % opaque attempt + +adt_tt9() -> + I1 = adt_int2(), + I2 = adt_int4(), + I1 =:= I2. % opaque attempt + +adt_tt10() -> + I1 = adt_int2(), + I2 = adt_int2_4(), + I1 =:= I2. % opaque attempt + +adt_tt11() -> + I1 = adt_int5_7(), + I2 = adt_int2_4(), + I1 =:= I2. % opaque attempt + +adt_tt12() -> + I1 = adt_un1_2(), + I2 = adt_un3_4(), + I1 =:= I2. % opaque attempt + +adt_tt13() -> + I1 = adt_tup(), + I2 = adt_tup2(), + I1 =:= I2. % opaque attempt + +y3() -> + {a, 3}. + +adt_t1() -> + para4_adt:t1(). + +adt_t2() -> + para4_adt:t2(). + +adt_t3() -> + para4_adt:t3(). + +adt_y1() -> + para4_adt:y1(). + +adt_y2() -> + para4_adt:y2(). + +adt_y3() -> + para4_adt:y3(). + +adt_int2() -> + para4_adt:int2(). + +adt_int4() -> + para4_adt:int4(). + +adt_int2_4() -> + para4_adt:int2_4(). + +adt_int5_7() -> + para4_adt:int5_7(). + +adt_un1_2() -> + para4_adt:un1_2(). + +adt_un3_4() -> + para4_adt:un3_4(). + +adt_tup() -> + para4_adt:tup(). + +adt_tup2() -> + para4_adt:tup2(). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para4_adt.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para4_adt.erl new file mode 100644 index 0000000000..407dd198a7 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para4_adt.erl @@ -0,0 +1,108 @@ +-module(para4_adt). + +-export([t1/0, t2/0, t3/0, y1/0, y2/0, y3/0]). + +-export([int2/0, int4/0, int2_4/0, int5_7/0]). + +-export([un1_2/0, un3_4/0]). + +-export([tup/0, tup2/0]). + +-export_type([t/1, y/1, int/1, tup/1, un/1]). + +-type ai() :: atom() | integer(). + +-opaque t(A) :: {A, A}. + +-type y(A) :: {A, A}. + +-opaque int(I) :: I. + +-opaque un(I) :: atom() | I. + +-opaque tup(T) :: T. + +-spec t1() -> t(integer()). + +t1() -> + {i(), i()}. + +-spec t2() -> t(atom()). + +t2() -> + {a(), a()}. + +-spec t3() -> t(ai()). + +t3() -> + {ai(), ai()}. + +-spec y1() -> y(integer()). + +y1() -> + {i(), i()}. + +-spec y2() -> y(atom()). + +y2() -> + {a(), a()}. + +-spec y3() -> y(ai()). + +y3() -> + {ai(), ai()}. + +-spec a() -> atom(). + +a() -> + foo:a(). + +-spec i() -> integer(). + +i() -> + foo:i(). + +-spec ai() -> ai(). + +ai() -> + foo:ai(). + +-spec int2() -> int(1..2). + +int2() -> + foo:int2(). + +-spec int4() -> int(1..4). + +int4() -> + foo:int4(). + +-spec int2_4() -> int(2..4). + +int2_4() -> + foo:int2_4(). + +-spec int5_7() -> int(5..7). + +int5_7() -> + foo:int5_7(). + +-spec un1_2() -> un(1..2). + +un1_2() -> + foo:un1_2(). + +-spec un3_4() -> un(3..4). + +un3_4() -> + foo:un3_4(). + +-spec tup() -> tup(tuple()). + +tup() -> + foo:tup(). + +-spec tup2() -> tup({_, _}). + +tup2() -> + foo:tup2(). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para5.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para5.erl new file mode 100644 index 0000000000..76ea3e76b5 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para5.erl @@ -0,0 +1,33 @@ +-module(para5). + +-export([d/0, dd/0, da1/0]). + +d() -> + I1 = adt_d1(), + I2 = adt_d2(), + I1 =:= I2. % can never evaluate to true + +dd() -> + I1 = adt_d1(), + I2 = adt_dd(), + I1 =/= I2. % incompatible opaque types + +da1() -> + I1 = adt_da1(), + I2 = adt_da2(), + I1 =:= I2. + +adt_d1() -> + para5_adt:d1(). + +adt_d2() -> + para5_adt:d2(). + +adt_dd() -> + para5_adt:dd(). + +adt_da1() -> + para5_adt:da1(). + +adt_da2() -> + para5_adt:da2(). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para5_adt.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para5_adt.erl new file mode 100644 index 0000000000..a62e0488e0 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para5_adt.erl @@ -0,0 +1,36 @@ +-module(para5_adt). + +-export([d1/0, d2/0, dd/0, da1/0, da2/0]). + +-export_type([d/0, dd/1, da/2]). + +-opaque d() :: 1 | 2. + +-spec d1() -> d(). + +d1() -> + 1. + +-spec d2() -> d(). + +d2() -> + 2. + +-opaque dd(A) :: A. + +-spec dd() -> dd(atom()). + +dd() -> + foo:atom(). + +-opaque da(A, B) :: {A, B}. + +-spec da1() -> da(any(), atom()). + +da1() -> + {3, a}. + +-spec da2() -> da(integer(), any()). + +da2() -> + {3, a}. diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl b/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl index eef2074e0c..7db1100597 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl @@ -145,7 +145,7 @@ adt_t4() -> A = simple1_adt:n1(), B = simple1_adt:n2(), A = A, % OK - A = B. % opaque term + A = B. % opaque terms adt_t7() -> A = simple1_adt:n1(), @@ -178,7 +178,7 @@ c2(A, B) -> c2() -> A = simple1_adt:d1(), B = simple1_adt:d2(), - if A =< B -> ok end. % opaque term + if A =< B -> ok end. % opaque terms c3() -> B = simple1_adt:d2(), @@ -516,7 +516,7 @@ eq1() -> A = simple1_adt:d2(), B = simple1_adt:d1(), if - A == B -> % opaque term + A == B -> % opaque terms 0; A == A -> 1; @@ -555,7 +555,7 @@ c6(A, B) -> c7(A, B) -> A = simple1_adt:d1(), B = simple1_adt:d2(), - A =< B. % opaque term + A =< B. % opaque terms c8() -> D = digraph:new(), diff --git a/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl b/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl index c34fa1b9dd..d83f2e3234 100644 --- a/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl +++ b/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl @@ -3,7 +3,7 @@ -type f(T) :: {ssl:sslsocket(), T}. -record(r1, { f1 :: f(_) }). --type r1(T) :: #r1{ f1 :: fun((ssl:sslsocket(), T) -> any()) }. +-type r1(T) :: #r1{ f1 :: {ssl:sslsocket(), T} }. -record(state, { r :: r1(T), diff --git a/lib/edoc/src/edoc_layout.erl b/lib/edoc/src/edoc_layout.erl index 62d5eb9a18..b67ec31ae3 100644 --- a/lib/edoc/src/edoc_layout.erl +++ b/lib/edoc/src/edoc_layout.erl @@ -520,7 +520,7 @@ format_spec(Name, Type, Defs, #opts{pretty_printer = erl_pp}=Opts) -> {R, ".\n"} = etypef(L, O), [{pre, R}] catch _:_ -> - %% Example: "@spec ... -> record(a)" + %% Should not happen. format_spec(Name, Type, Defs, Opts#opts{pretty_printer=''}) end; format_spec(Sep, Type, Defs, _Opts) -> diff --git a/lib/edoc/src/edoc_specs.erl b/lib/edoc/src/edoc_specs.erl index 59f6cb8ddf..eb69058148 100644 --- a/lib/edoc/src/edoc_specs.erl +++ b/lib/edoc/src/edoc_specs.erl @@ -295,47 +295,54 @@ arg_name([A | As], Default) -> is_name(A) -> is_atom(A). -d2e({ann_type,_,[V, T0]}) -> +d2e(T) -> + d2e(T, 0). + +d2e({ann_type,_,[V, T0]}, Prec) -> %% Note: the -spec/-type syntax allows annotations everywhere, but %% EDoc does not. The fact that the annotation is added to the %% type here does not necessarily mean that it will be used by the %% layout module. - T = d2e(T0), - ?add_t_ann(T, element(3, V)); -d2e({remote_type,_,[{atom,_,M},{atom,_,F},Ts0]}) -> + {_L,P,R} = erl_parse:type_inop_prec('::'), + T1 = d2e(T0, R), + T = ?add_t_ann(T1, element(3, V)), + maybe_paren(P, Prec, T); % the only necessary call to maybe_paren() +d2e({remote_type,_,[{atom,_,M},{atom,_,F},Ts0]}, _Prec) -> Ts = d2e(Ts0), typevar_anno(#t_type{name = #t_name{module = M, name = F}, args = Ts}, Ts); -d2e({type,_,'fun',[{type,_,product,As0},Ran0]}) -> +d2e({type,_,'fun',[{type,_,product,As0},Ran0]}, _Prec) -> Ts = [Ran|As] = d2e([Ran0|As0]), %% Assume that the linter has checked type variables. typevar_anno(#t_fun{args = As, range = Ran}, Ts); -d2e({type,_,'fun',[A0={type,_,any},Ran0]}) -> +d2e({type,_,'fun',[A0={type,_,any},Ran0]}, _Prec) -> Ts = [A, Ran] = d2e([A0, Ran0]), typevar_anno(#t_fun{args = [A], range = Ran}, Ts); -d2e({type,_,'fun',[]}) -> +d2e({type,_,'fun',[]}, _Prec) -> #t_type{name = #t_name{name = function}, args = []}; -d2e({type,_,any}) -> +d2e({type,_,any}, _Prec) -> #t_var{name = '...'}; % Kludge... not a type variable! -d2e({type,_,nil,[]}) -> +d2e({type,_,nil,[]}, _Prec) -> #t_nil{}; -d2e({paren_type,_,[T]}) -> - #t_paren{type = d2e(T)}; -d2e({type,_,list,[T0]}) -> +d2e({paren_type,_,[T]}, Prec) -> + d2e(T, Prec); +d2e({type,_,list,[T0]}, _Prec) -> T = d2e(T0), typevar_anno(#t_list{type = T}, [T]); -d2e({type,_,nonempty_list,[T0]}) -> +d2e({type,_,nonempty_list,[T0]}, _Prec) -> T = d2e(T0), typevar_anno(#t_nonempty_list{type = T}, [T]); -d2e({type,_,bounded_fun,[T,Gs]}) -> +d2e({type,_,bounded_fun,[T,Gs]}, _Prec) -> [F0|Defs] = d2e([T|Gs]), F = ?set_t_ann(F0, lists:keydelete(type_variables, 1, ?t_ann(F0))), %% Assume that the linter has checked type variables. #t_spec{type = typevar_anno(F, [F0]), defs = Defs}; -d2e({type,_,range,[V1,V2]}) -> +d2e({type,_,range,[V1,V2]}, Prec) -> + {_L,P,_R} = erl_parse:type_inop_prec('..'), {integer,_,I1} = erl_eval:partial_eval(V1), {integer,_,I2} = erl_eval:partial_eval(V2), - #t_integer_range{from = I1, to = I2}; -d2e({type,_,constraint,[Sub,Ts0]}) -> + T0 = #t_integer_range{from = I1, to = I2}, + maybe_paren(P, Prec, T0); +d2e({type,_,constraint,[Sub,Ts0]}, _Prec) -> case {Sub,Ts0} of {{atom,_,is_subtype},[{var,_,N},T0]} -> Ts = [T] = d2e([T0]), @@ -347,50 +354,60 @@ d2e({type,_,constraint,[Sub,Ts0]}) -> _ -> throw_error(get_line(element(2, Sub)), "cannot handle guard", []) end; -d2e({type,_,union,Ts0}) -> - Ts = d2e(Ts0), - typevar_anno(#t_union{types = Ts}, Ts); -d2e({type,_,tuple,any}) -> +d2e({type,_,union,Ts0}, Prec) -> + {_L,P,R} = erl_parse:type_inop_prec('|'), + Ts = d2e(Ts0, R), + T = maybe_paren(P, Prec, #t_union{types = Ts}), + typevar_anno(T, Ts); +d2e({type,_,tuple,any}, _Prec) -> #t_type{name = #t_name{name = tuple}, args = []}; -d2e({type,_,binary,[Base,Unit]}) -> - #t_binary{base_size = element(3, Base), - unit_size = element(3, Unit)}; -d2e({type,_,map,any}) -> - #t_map{ types = []}; -d2e({type,_,map,Es}) -> - #t_map{ types = d2e(Es) }; -d2e({type,_,map_field_assoc,[K,V]}) -> - #t_map_field{ k_type = d2e(K), v_type=d2e(V) }; -d2e({type,_,map_field_exact,K,V}) -> - #t_map_field{ k_type = d2e(K), v_type=d2e(V) }; -d2e({type,_,tuple,Ts0}) -> +d2e({type,_,binary,[Base,Unit]}, _Prec) -> + {integer,_,B} = erl_eval:partial_eval(Base), + {integer,_,U} = erl_eval:partial_eval(Unit), + #t_binary{base_size = B, unit_size = U}; +d2e({type,_,map,any}, _Prec) -> + #t_map{types = []}; +d2e({type,_,map,Es}, _Prec) -> + #t_map{types = d2e(Es) }; +d2e({type,_,map_field_assoc,[K,V]}, Prec) -> + T = #t_map_field{k_type = d2e(K), v_type=d2e(V) }, + {P,_R} = erl_parse:type_preop_prec('#'), + maybe_paren(P, Prec, T); +d2e({type,_,map_field_exact,K,V}, Prec) -> + T = #t_map_field{k_type = d2e(K), v_type=d2e(V) }, + {P,_R} = erl_parse:type_preop_prec('#'), + maybe_paren(P, Prec, T); +d2e({type,_,tuple,Ts0}, _Prec) -> Ts = d2e(Ts0), typevar_anno(#t_tuple{types = Ts}, Ts); -d2e({type,_,record,[Name|Fs0]}) -> +d2e({type,_,record,[Name|Fs0]}, Prec) -> Atom = #t_atom{val = element(3, Name)}, Fs = d2e(Fs0), - typevar_anno(#t_record{name = Atom, fields = Fs}, Fs); -d2e({type,_,field_type,[Name,Type0]}) -> - Type = d2e(Type0), - typevar_anno(#t_field{name = #t_atom{val = element(3, Name)}, type = Type}, - [Type]); -d2e({typed_record_field,{record_field,L,Name},Type}) -> - d2e({type,L,field_type,[Name,Type]}); -d2e({typed_record_field,{record_field,L,Name,_E},Type}) -> - d2e({type,L,field_type,[Name,Type]}); -d2e({record_field,L,_Name,_E}=F) -> - d2e({typed_record_field,F,{type,L,any,[]}}); % Maybe skip... -d2e({record_field,L,_Name}=F) -> - d2e({typed_record_field,F,{type,L,any,[]}}); % Maybe skip... -d2e({type,_,Name,Types0}) -> + {P,_R} = erl_parse:type_preop_prec('#'), + T = maybe_paren(P, Prec, #t_record{name = Atom, fields = Fs}), + typevar_anno(T, Fs); +d2e({type,_,field_type,[Name,Type0]}, Prec) -> + {_L,P,R} = erl_parse:type_inop_prec('::'), + Type = maybe_paren(P, Prec, d2e(Type0, R)), + T = #t_field{name = #t_atom{val = element(3, Name)}, type = Type}, + typevar_anno(T, [Type]); +d2e({typed_record_field,{record_field,L,Name},Type}, Prec) -> + d2e({type,L,field_type,[Name,Type]}, Prec); +d2e({typed_record_field,{record_field,L,Name,_E},Type}, Prec) -> + d2e({type,L,field_type,[Name,Type]}, Prec); +d2e({record_field,L,_Name,_E}=F, Prec) -> + d2e({typed_record_field,F,{type,L,any,[]}}, Prec); % Maybe skip... +d2e({record_field,L,_Name}=F, Prec) -> + d2e({typed_record_field,F,{type,L,any,[]}}, Prec); % Maybe skip... +d2e({type,_,Name,Types0}, _Prec) -> Types = d2e(Types0), typevar_anno(#t_type{name = #t_name{name = Name}, args = Types}, Types); -d2e({user_type,_,Name,Types0}) -> +d2e({user_type,_,Name,Types0}, _Prec) -> Types = d2e(Types0), typevar_anno(#t_type{name = #t_name{name = Name}, args = Types}, Types); -d2e({var,_,'_'}) -> +d2e({var,_,'_'}, _Prec) -> #t_type{name = #t_name{name = ?TOP_TYPE}}; -d2e({var,_,TypeName}) -> +d2e({var,_,TypeName}, _Prec) -> TypeVar = ordsets:from_list([TypeName]), T = #t_var{name = TypeName}, %% Annotate type variables with the name of the variable. @@ -398,13 +415,13 @@ d2e({var,_,TypeName}) -> %% from using the argument name from the source or to invent a new name. T1 = ?add_t_ann(T, {type_variables, TypeVar}), ?add_t_ann(T1, TypeName); -d2e(L) when is_list(L) -> - [d2e(T) || T <- L]; -d2e({atom,_,A}) -> +d2e(L, Prec) when is_list(L) -> + [d2e(T, Prec) || T <- L]; +d2e({atom,_,A}, _Prec) -> #t_atom{val = A}; -d2e(undefined = U) -> % opaque +d2e(undefined = U, _Prec) -> % opaque U; -d2e(Expr) -> +d2e(Expr, _Prec) -> {integer,_,I} = erl_eval:partial_eval(Expr), #t_integer{val = I}. @@ -422,6 +439,11 @@ typevars(Ts) -> get_typevars(Ts) -> [Vs || T <- Ts, T =/= undefined, {type_variables, Vs} <- ?t_ann(T)]. +maybe_paren(P, Prec, T) when P < Prec -> + #t_paren{type = T}; +maybe_paren(_P, _Prec, T) -> + T. + -record(parms, {tab, warn, file, line}). %% Expands record references. Explicitly given record fields are kept, @@ -484,11 +506,11 @@ xrecs(#t_fun{args = Args0, range = Range0}=T, P) -> Args = xrecs(Args0, P), Range = xrecs(Range0, P), T#t_fun{args = Args, range = Range}; -xrecs(#t_map{ types = Ts0 }=T,P) -> +xrecs(#t_map{types = Ts0 }=T,P) -> Ts = xrecs(Ts0, P), - T#t_map{ types = Ts }; -xrecs(#t_map_field{ k_type=Kt, v_type=Vt}=T, P) -> - T#t_map_field{ k_type=xrecs(Kt,P), v_type=xrecs(Vt,P)}; + T#t_map{types = Ts }; +xrecs(#t_map_field{k_type=Kt, v_type=Vt}=T, P) -> + T#t_map_field{k_type=xrecs(Kt,P), v_type=xrecs(Vt,P)}; xrecs(#t_tuple{types = Types0}=T, P) -> Types = xrecs(Types0, P), T#t_tuple{types = Types}; diff --git a/lib/eunit/include/eunit.hrl b/lib/eunit/include/eunit.hrl index 53d291430d..88e9d6c19b 100644 --- a/lib/eunit/include/eunit.hrl +++ b/lib/eunit/include/eunit.hrl @@ -15,11 +15,14 @@ %% %% Copyright (C) 2004-2006 Mickaël Rémond, Richard Carlsson +-ifndef(EUNIT_HRL). +-define(EUNIT_HRL, true). + %% Including this file turns on testing and defines TEST, unless NOTEST %% is defined before the file is included. If both NOTEST and TEST are %% already defined, then TEST takes precedence, and NOTEST will become %% undefined. -%% +%% %% If NODEBUG is defined before this file is included, the debug macros %% are disabled, unless DEBUG is also defined, in which case NODEBUG %% will become undefined. NODEBUG also implies NOASSERT, unless testing @@ -31,14 +34,10 @@ %% even if NODEBUG is defined. If both ASSERT and NOASSERT are defined %% before the file is included, then ASSERT takes precedence, and NOASSERT %% will become undefined regardless of TEST. -%% +%% %% After including this file, EUNIT will be defined if and only if TEST %% is defined. --ifndef(EUNIT_HRL). --define(EUNIT_HRL, true). - - %% allow defining TEST to override NOTEST -ifdef(TEST). -undef(NOTEST). @@ -49,13 +48,6 @@ -undef(NODEBUG). -endif. -%% allow NODEBUG to imply NOASSERT, unless overridden below --ifdef(NODEBUG). --ifndef(NOASSERT). --define(NOASSERT, true). --endif. --endif. - %% note that the main switch used within this file is NOTEST; however, %% both TEST and EUNIT may be used to check whether testing is enabled -ifndef(NOTEST). @@ -70,10 +62,8 @@ -undef(EUNIT). -endif. -%% allow ASSERT to override NOASSERT (regardless of TEST/NOTEST) --ifdef(ASSERT). --undef(NOASSERT). --endif. +%% include the assert macros; ASSERT overrides NOASSERT if defined +-include_lib("stdlib/include/assert.hrl"). %% Parse transforms for automatic exporting/stripping of test functions. %% (Note that although automatic stripping is convenient, it will make @@ -91,7 +81,7 @@ %% All macros should be available even if testing is turned off, and %% should preferably not require EUnit to be present at runtime. -%% +%% %% We must use fun-call wrappers ((fun () -> ... end)()) to avoid %% exporting local variables, and furthermore we only use variable names %% prefixed with "__", that hopefully will not be bound outside the fun. @@ -128,211 +118,24 @@ current_function)))). -endif. -%% The plain assert macro should be defined to do nothing if this file -%% is included when debugging/testing is turned off. --ifdef(NOASSERT). --ifndef(assert). --define(assert(BoolExpr),ok). --endif. --else. -%% The assert macro is written the way it is so as not to cause warnings -%% for clauses that cannot match, even if the expression is a constant. --undef(assert). --define(assert(BoolExpr), - begin - ((fun () -> - case (BoolExpr) of - true -> ok; - __V -> erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??BoolExpr)}, - {expected, true}, - {value, case __V of false -> __V; - _ -> {not_a_boolean,__V} - end}]}) - end - end)()) - end). --endif. --define(assertNot(BoolExpr), ?assert(not (BoolExpr))). +%% General test macros -define(_test(Expr), {?LINE, fun () -> (Expr) end}). - -define(_assert(BoolExpr), ?_test(?assert(BoolExpr))). - -define(_assertNot(BoolExpr), ?_assert(not (BoolExpr))). - -%% This is mostly a convenience which gives more detailed reports. -%% Note: Guard is a guarded pattern, and can not be used for value. --ifdef(NOASSERT). --define(assertMatch(Guard, Expr), ok). --else. --define(assertMatch(Guard, Expr), - begin - ((fun () -> - case (Expr) of - Guard -> ok; - __V -> erlang:error({assertMatch_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {pattern, (??Guard)}, - {value, __V}]}) - end - end)()) - end). --endif. -define(_assertMatch(Guard, Expr), ?_test(?assertMatch(Guard, Expr))). - -%% This is the inverse case of assertMatch, for convenience. --ifdef(NOASSERT). --define(assertNotMatch(Guard, Expr), ok). --else. --define(assertNotMatch(Guard, Expr), - begin - ((fun () -> - __V = (Expr), - case __V of - Guard -> erlang:error({assertNotMatch_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {pattern, (??Guard)}, - {value, __V}]}); - _ -> ok - end - end)()) - end). --endif. -define(_assertNotMatch(Guard, Expr), ?_test(?assertNotMatch(Guard, Expr))). - -%% This is a convenience macro which gives more detailed reports when -%% the expected LHS value is not a pattern, but a computed value --ifdef(NOASSERT). --define(assertEqual(Expect, Expr), ok). --else. --define(assertEqual(Expect, Expr), - begin - ((fun (__X) -> - case (Expr) of - __X -> ok; - __V -> erlang:error({assertEqual_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {expected, __X}, - {value, __V}]}) - end - end)(Expect)) - end). --endif. -define(_assertEqual(Expect, Expr), ?_test(?assertEqual(Expect, Expr))). - -%% This is the inverse case of assertEqual, for convenience. --ifdef(NOASSERT). --define(assertNotEqual(Unexpected, Expr), ok). --else. --define(assertNotEqual(Unexpected, Expr), - begin - ((fun (__X) -> - case (Expr) of - __X -> erlang:error({assertNotEqual_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {value, __X}]}); - _ -> ok - end - end)(Unexpected)) - end). --endif. -define(_assertNotEqual(Unexpected, Expr), ?_test(?assertNotEqual(Unexpected, Expr))). - -%% Note: Class and Term are patterns, and can not be used for value. -%% Term can be a guarded pattern, but Class cannot. --ifdef(NOASSERT). --define(assertException(Class, Term, Expr), ok). --else. --define(assertException(Class, Term, Expr), - begin - ((fun () -> - try (Expr) of - __V -> erlang:error({assertException_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {pattern, - "{ "++(??Class)++" , "++(??Term) - ++" , [...] }"}, - {unexpected_success, __V}]}) - catch - Class:Term -> ok; - __C:__T -> - erlang:error({assertException_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {pattern, - "{ "++(??Class)++" , "++(??Term) - ++" , [...] }"}, - {unexpected_exception, - {__C, __T, - erlang:get_stacktrace()}}]}) - end - end)()) - end). --endif. - --define(assertError(Term, Expr), ?assertException(error, Term, Expr)). --define(assertExit(Term, Expr), ?assertException(exit, Term, Expr)). --define(assertThrow(Term, Expr), ?assertException(throw, Term, Expr)). - -define(_assertException(Class, Term, Expr), ?_test(?assertException(Class, Term, Expr))). -define(_assertError(Term, Expr), ?_assertException(error, Term, Expr)). -define(_assertExit(Term, Expr), ?_assertException(exit, Term, Expr)). -define(_assertThrow(Term, Expr), ?_assertException(throw, Term, Expr)). - -%% This is the inverse case of assertException, for convenience. -%% Note: Class and Term are patterns, and can not be used for value. -%% Both Class and Term can be guarded patterns. --ifdef(NOASSERT). --define(assertNotException(Class, Term, Expr), ok). --else. --define(assertNotException(Class, Term, Expr), - begin - ((fun () -> - try (Expr) of - _ -> ok - catch - __C:__T -> - case __C of - Class -> - case __T of - Term -> - erlang:error({assertNotException_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {pattern, - "{ "++(??Class)++" , " - ++(??Term)++" , [...] }"}, - {unexpected_exception, - {__C, __T, - erlang:get_stacktrace() - }}]}); - _ -> ok - end; - _ -> ok - end - end - end)()) - end). --endif. -define(_assertNotException(Class, Term, Expr), ?_test(?assertNotException(Class, Term, Expr))). +-define(_assertReceive(Guard, Expr), ?_test(?assertReceive(Guard, Expr))). %% Macros for running operating system commands. (Note that these %% require EUnit to be present at runtime, or at least eunit_lib.) @@ -364,18 +167,18 @@ -else. -define(assertCmdStatus(N, Cmd), begin - ((fun () -> - case ?_cmd_(Cmd) of - {(N), _} -> ok; - {__N, _} -> erlang:error({assertCmd_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {command, (Cmd)}, - {expected_status,(N)}, - {status,__N}]}) - end - end)()) - end). + ((fun () -> + case ?_cmd_(Cmd) of + {(N), _} -> ok; + {__N, _} -> erlang:error({assertCmd_failed, + [{module, ?MODULE}, + {line, ?LINE}, + {command, (Cmd)}, + {expected_status,(N)}, + {status,__N}]}) + end + end)()) + end). -endif. -define(assertCmd(Cmd), ?assertCmdStatus(0, Cmd)). @@ -384,17 +187,17 @@ -else. -define(assertCmdOutput(T, Cmd), begin - ((fun () -> - case ?_cmd_(Cmd) of - {_, (T)} -> ok; - {_, __T} -> erlang:error({assertCmdOutput_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {command,(Cmd)}, - {expected_output,(T)}, - {output,__T}]}) - end - end)()) + ((fun () -> + case ?_cmd_(Cmd) of + {_, (T)} -> ok; + {_, __T} -> erlang:error({assertCmdOutput_failed, + [{module, ?MODULE}, + {line, ?LINE}, + {command,(Cmd)}, + {expected_output,(T)}, + {output,__T}]}) + end + end)()) end). -endif. @@ -439,5 +242,4 @@ end). -endif. - -endif. % EUNIT_HRL diff --git a/lib/eunit/src/Makefile b/lib/eunit/src/Makefile index 47aef104ff..86a6d8831e 100644 --- a/lib/eunit/src/Makefile +++ b/lib/eunit/src/Makefile @@ -24,7 +24,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/eunit-$(VSN) EBIN = ../ebin INCLUDE=../include -ERL_COMPILE_FLAGS += -pa $(EBIN) -I$(INCLUDE) +warn_unused_vars +nowarn_shadow_vars +warn_unused_import +warn_obsolete_guard +ERL_COMPILE_FLAGS += -pa $(EBIN) -pa ../../stdlib/ebin -I$(INCLUDE) +warn_unused_vars +nowarn_shadow_vars +warn_unused_import +warn_obsolete_guard PARSE_TRANSFORM = eunit_autoexport.erl diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl index 14335cf635..d39f350c0f 100644 --- a/lib/hipe/cerl/erl_types.erl +++ b/lib/hipe/cerl/erl_types.erl @@ -82,6 +82,8 @@ t_from_form/4, t_from_form/5, t_from_form_without_remote/2, + t_check_record_fields/4, + t_check_record_fields/5, t_from_range/2, t_from_range_unsafe/2, t_from_term/1, @@ -581,13 +583,11 @@ t_find_opaque_mismatch_list([H|T]) -> %% calling t_contains_opaque/2 is that the traversal stops when %% there is a mismatch which means that unknown opaque types "below" %% the mismatch are not found. -%% XXX. Returns one element even if both oparands contain opaque types. -%% XXX. Slow since t_inf() is called but the results are ignored. t_find_unknown_opaque(_T1, _T2, 'universe') -> []; t_find_unknown_opaque(T1, T2, Opaques) -> try t_inf(T1, T2, {match, Opaques}) of _ -> [] - catch throw:N when is_integer(N) -> [N] + catch throw:{pos, Ns} -> Ns end. -spec t_decorate_with_opaque(erl_type(), erl_type(), [erl_type()]) -> erl_type(). @@ -747,14 +747,15 @@ t_opaque_from_records(RecDict) -> end end, RecDict), OpaqueTypeDict = - dict:map(fun({opaque, Name, _Arity}, {{Module, _Form, ArgNames}, _Type}) -> + dict:map(fun({opaque, Name, _Arity}, + {{Module, _FileLine, _Form, ArgNames}, _Type}) -> %% Args = args_to_types(ArgNames), %% List = lists:zip(ArgNames, Args), %% TmpVarDict = dict:from_list(List), %% Rep = t_from_form(Type, RecDict, TmpVarDict), - Rep = t_none(), % not used for anything right now + Rep = t_any(), % not used for anything right now Args = [t_any() || _ <- ArgNames], - skip_opaque_alias(Rep, Module, Name, Args) + t_opaque(Module, Name, Args, Rep) end, OpaqueRecDict), [OpaqueType || {_Key, OpaqueType} <- dict:to_list(OpaqueTypeDict)]. @@ -2605,7 +2606,7 @@ inf_opaque1(T1, ?opaque(Set2)=T2, Pos, Opaques) -> end. inf_is_opaque_type(T, Pos, {match, Opaques}) -> - is_opaque_type(T, Opaques) orelse throw(Pos); + is_opaque_type(T, Opaques) orelse throw({pos, [Pos]}); inf_is_opaque_type(T, _Pos, Opaques) -> is_opaque_type(T, Opaques). @@ -2624,13 +2625,13 @@ combine(S, T1, T2) -> #opaque{mod = Mod1, name = Name1, args = Args1} = T1, #opaque{mod = Mod2, name = Name2, args = Args2} = T2, Comb1 = comb(Mod1, Name1, Args1, S, T1), - case is_same_type_name({Mod1, Name1, Args1}, {Mod2, Name2, Args2}) of + case is_compat_opaque_names({Mod1, Name1, Args1}, {Mod2, Name2, Args2}) of true -> Comb1; false -> Comb1 ++ comb(Mod2, Name2, Args2, S, T2) end. comb(Mod, Name, Args, S, T) -> - case is_same_name(Mod, Name, Args, S) of + case can_combine_opaque_names(Mod, Name, Args, S) of true -> ?opaque(Set) = S, Set; @@ -2638,17 +2639,17 @@ comb(Mod, Name, Args, S, T) -> [T#opaque{struct = S}] end. -is_same_name(Mod1, Name1, Args1, - ?opaque([#opaque{mod = Mod2, name = Name2, args = Args2}])) -> - is_same_type_name({Mod1, Name1, Args1}, {Mod2, Name2, Args2}); -is_same_name(_, _, _, _) -> false. +can_combine_opaque_names(Mod1, Name1, Args1, + ?opaque([#opaque{mod = Mod2, name = Name2, args = Args2}])) -> + is_compat_opaque_names({Mod1, Name1, Args1}, {Mod2, Name2, Args2}); +can_combine_opaque_names(_, _, _, _) -> false. %% Combining two lists this way can be very time consuming... %% Note: two parameterized opaque types are not the same if their %% actual parameters differ inf_opaque(Set1, Set2, Opaques) -> - List1 = inf_look_up(Set1, 1, Opaques), - List2 = inf_look_up(Set2, 2, Opaques), + List1 = inf_look_up(Set1, Opaques), + List2 = inf_look_up(Set2, Opaques), List0 = [combine(Inf, T1, T2) || {Is1, ModNameArgs1, T1} <- List1, {Is2, ModNameArgs2, T2} <- List2, @@ -2659,14 +2660,14 @@ inf_opaque(Set1, Set2, Opaques) -> sup_opaque(List). %% Optimization: do just one lookup. -inf_look_up(Set, Pos, Opaques) -> - [{Opaques =:= 'universe' orelse inf_is_opaque_type2(T, Pos, Opaques), +inf_look_up(Set, Opaques) -> + [{Opaques =:= 'universe' orelse inf_is_opaque_type2(T, Opaques), {M, N, Args}, T} || #opaque{mod = M, name = N, args = Args} = T <- set_to_list(Set)]. -inf_is_opaque_type2(T, Pos, {match, Opaques}) -> - is_opaque_type2(T, Opaques) orelse throw(Pos); -inf_is_opaque_type2(T, _Pos, Opaques) -> +inf_is_opaque_type2(T, {match, Opaques}) -> + is_opaque_type2(T, Opaques); +inf_is_opaque_type2(T, Opaques) -> is_opaque_type2(T, Opaques). inf_opaque_types(IsOpaque1, ModNameArgs1, T1, @@ -2675,18 +2676,33 @@ inf_opaque_types(IsOpaque1, ModNameArgs1, T1, #opaque{struct = S2}=T2, case Opaques =:= 'universe' orelse - is_same_type_name(ModNameArgs1, ModNameArgs2) + is_compat_opaque_names(ModNameArgs1, ModNameArgs2) of true -> t_inf(S1, S2, Opaques); false -> case {IsOpaque1, IsOpaque2} of - {true, true} -> t_inf(S1, S2, Opaques); - {true, false} -> t_inf(S1, ?opaque(set_singleton(T2)), Opaques); - {false, true} -> t_inf(?opaque(set_singleton(T1)), S2, Opaques); + {true, true} -> t_inf(S1, S2, Opaques); + {true, false} -> t_inf(S1, ?opaque(set_singleton(T2)), Opaques); + {false, true} -> t_inf(?opaque(set_singleton(T1)), S2, Opaques); + {false, false} when element(1, Opaques) =:= match -> + throw({pos, [1, 2]}); {false, false} -> t_none() end end. +is_compat_opaque_names(ModNameArgs, ModNameArgs) -> true; +is_compat_opaque_names({Mod,Name,Args1}, {Mod,Name,Args2}) -> + is_compat_args(Args1, Args2); +is_compat_opaque_names(_, _) -> false. + +is_compat_args([A1|Args1], [A2|Args2]) -> + is_compat_arg(A1, A2) andalso is_compat_args(Args1, Args2); +is_compat_args([], []) -> true; +is_compat_args(_, _) -> false. + +is_compat_arg(A, A) -> true; +is_compat_arg(A1, A2) -> t_is_any(A1) orelse t_is_any(A2). + -spec t_inf_lists([erl_type()], [erl_type()]) -> [erl_type()]. t_inf_lists(L1, L2) -> @@ -2785,7 +2801,7 @@ inf_union(U1, U2, Opaques) -> {Union, ThrowList3} = inf_union(U1, U2, 0, [], [], Opaques), ThrowList = lists:merge3(ThrowList1, ThrowList2, ThrowList3), case t_sup([O1, O2, Union]) of - ?none when ThrowList =/= [] -> throw(hd(ThrowList)); + ?none when ThrowList =/= [] -> throw({pos, lists:usort(ThrowList)}); Sup -> Sup end. @@ -2797,8 +2813,8 @@ inf_union_collect([E|L], Opaque, InfFun, InfList, ThrowList) -> try InfFun(E, Opaque)of Inf -> inf_union_collect(L, Opaque, InfFun, [Inf|InfList], ThrowList) - catch throw:N when is_integer(N) -> - inf_union_collect(L, Opaque, InfFun, InfList, [N|ThrowList]) + catch throw:{pos, Ns} -> + inf_union_collect(L, Opaque, InfFun, InfList, Ns ++ ThrowList) end. inf_union([?none|Left1], [?none|Left2], N, Acc, ThrowList, Opaques) -> @@ -2807,8 +2823,8 @@ inf_union([T1|Left1], [T2|Left2], N, Acc, ThrowList, Opaques) -> try t_inf(T1, T2, Opaques) of ?none -> inf_union(Left1, Left2, N, [?none|Acc], ThrowList, Opaques); T -> inf_union(Left1, Left2, N+1, [T|Acc], ThrowList, Opaques) - catch throw:N when is_integer(N) -> - inf_union(Left1, Left2, N, [?none|Acc], [N|ThrowList], Opaques) + catch throw:{pos, Ns} -> + inf_union(Left1, Left2, N, [?none|Acc], Ns ++ ThrowList, Opaques) end; inf_union([], [], N, Acc, ThrowList, _Opaques) -> if N =:= 0 -> {?none, ThrowList}; @@ -4189,7 +4205,7 @@ builtin_type(Name, Type, TypeNames, ET, M, MR, V, D, L) -> case dict:find(M, MR) of {ok, R} -> case lookup_type(Name, 0, R) of - {_, {{_M, _F, _A}, _T}} -> + {_, {{_M, _FL, _F, _A}, _T}} -> type_from_form(Name, [], TypeNames, ET, M, MR, V, D, L); error -> {Type, L} @@ -4203,7 +4219,7 @@ type_from_form(Name, Args, TypeNames, ET, M, MR, V, D, L) -> {ArgTypes, L1} = list_from_form(Args, TypeNames, ET, M, MR, V, D, L), {ok, R} = dict:find(M, MR), case lookup_type(Name, ArgsLen, R) of - {type, {{Module, Form, ArgNames}, _Type}} -> + {type, {{Module, _FileName, Form, ArgNames}, _Type}} -> TypeName = {type, Module, Name, ArgsLen}, case can_unfold_more(TypeName, TypeNames) of true -> @@ -4213,7 +4229,7 @@ type_from_form(Name, Args, TypeNames, ET, M, MR, V, D, L) -> false -> {t_any(), L1} end; - {opaque, {{Module, Form, ArgNames}, Type}} -> + {opaque, {{Module, _FileName, Form, ArgNames}, Type}} -> TypeName = {opaque, Module, Name, ArgsLen}, {Rep, L2} = case can_unfold_more(TypeName, TypeNames) of @@ -4224,17 +4240,19 @@ type_from_form(Name, Args, TypeNames, ET, M, MR, V, D, L) -> false -> {t_any(), L1} end, Rep1 = choose_opaque_type(Rep, Type), - Args2 = [subst_all_vars_to_any(ArgType) || ArgType <- ArgTypes], - {skip_opaque_alias(Rep1, Module, Name, Args2), L2}; + Rep2 = case t_is_none(Rep1) of + true -> Rep1; + false -> + Args2 = [subst_all_vars_to_any(ArgType) || + ArgType <- ArgTypes], + t_opaque(Module, Name, Args2, Rep1) + end, + {Rep2, L2}; error -> Msg = io_lib:format("Unable to find type ~w/~w\n", [Name, ArgsLen]), throw({error, Msg}) end. -skip_opaque_alias(?opaque(_) = T, _Mod, _Name, _Args) -> T; -skip_opaque_alias(T, Module, Name, Args) -> - t_opaque(Module, Name, Args, T). - remote_from_form(RemMod, Name, Args, TypeNames, ET, M, MR, V, D, L) -> {ArgTypes, L1} = list_from_form(Args, TypeNames, ET, M, MR, V, D, L), if @@ -4251,7 +4269,7 @@ remote_from_form(RemMod, Name, Args, TypeNames, ET, M, MR, V, D, L) -> case sets:is_element(MFA, ET) of true -> case lookup_type(Name, ArgsLen, RemDict) of - {type, {{_Mod, Form, ArgNames}, _Type}} -> + {type, {{_Mod, _FileLine, Form, ArgNames}, _Type}} -> RemType = {type, RemMod, Name, ArgsLen}, case can_unfold_more(RemType, TypeNames) of true -> @@ -4263,7 +4281,7 @@ remote_from_form(RemMod, Name, Args, TypeNames, ET, M, MR, V, D, L) -> false -> {t_any(), L1} end; - {opaque, {{Mod, Form, ArgNames}, Type}} -> + {opaque, {{Mod, _FileLine, Form, ArgNames}, Type}} -> RemType = {opaque, RemMod, Name, ArgsLen}, List = lists:zip(ArgNames, ArgTypes), TmpVarDict = dict:from_list(List), @@ -4277,7 +4295,11 @@ remote_from_form(RemMod, Name, Args, TypeNames, ET, M, MR, V, D, L) -> {t_any(), L1} end, NewRep1 = choose_opaque_type(NewRep, Type), - {skip_opaque_alias(NewRep1, Mod, Name, ArgTypes), L2}; + NewRep2 = case t_is_none(NewRep1) of + true -> NewRep1; + false -> t_opaque(Mod, Name, ArgTypes, NewRep1) + end, + {NewRep2, L2}; error -> Msg = io_lib:format("Unable to find remote type ~w:~w()\n", [RemMod, Name]), @@ -4358,34 +4380,24 @@ build_field_dict(FieldTypes, TypeNames, ET, M, MR, V, D, L) -> build_field_dict([{type, _, field_type, [{atom, _, Name}, Type]}|Left], TypeNames, ET, M, MR, V, D, L, Acc) -> {T, L1} = t_from_form(Type, TypeNames, ET, M, MR, V, D, L - 1), - %% The cached record field type (DeclType) in - %% get_mod_record_types()), was created with a similar call as TT. - %% Using T for the subtype test does not work since any() is not - %% always a subset of the field type. - TT = t_from_form(Type, ET, M, MR, V), - NewAcc = [{Name, Type, T, TT}|Acc], + NewAcc = [{Name, Type, T}|Acc], {Dict, L2} = build_field_dict(Left, TypeNames, ET, M, MR, V, D, L1, NewAcc), {Dict, L2}; build_field_dict([], _TypeNames, _ET, _M, _MR, _V, _D, L, Acc) -> {lists:keysort(1, Acc), L}. -get_mod_record_types([{FieldName, _Abstr, DeclType}|Left1], - [{FieldName, TypeForm, ModType, ModTypeTest}|Left2], +get_mod_record_types([{FieldName, _Abstr, _DeclType}|Left1], + [{FieldName, TypeForm, ModType}|Left2], Acc) -> - ModTypeNoVars = subst_all_vars_to_any(ModTypeTest), - case t_is_subtype(ModTypeNoVars, DeclType) of - false -> {error, FieldName}; - true -> get_mod_record_types(Left1, Left2, - [{FieldName, TypeForm, ModType}|Acc]) - end; + get_mod_record_types(Left1, Left2, [{FieldName, TypeForm, ModType}|Acc]); get_mod_record_types([{FieldName1, _Abstr, _DeclType} = DT|Left1], - [{FieldName2, _FormType, _ModType, _TT}|_] = List2, + [{FieldName2, _FormType, _ModType}|_] = List2, Acc) when FieldName1 < FieldName2 -> get_mod_record_types(Left1, List2, [DT|Acc]); get_mod_record_types(Left1, [], Acc) -> {ok, lists:keysort(1, Left1++Acc)}; -get_mod_record_types(_, [{FieldName2, _FormType, _ModType, _TT}|_], _Acc) -> +get_mod_record_types(_, [{FieldName2, _FormType, _ModType}|_], _Acc) -> {error, FieldName2}. %% It is important to create a limited version of the record type @@ -4406,6 +4418,74 @@ list_from_form([H|Tail], TypeNames, ET, M, MR, V, D, L) -> {T1, L2} = list_from_form(Tail, TypeNames, ET, M, MR, V, D, L1), {[H1|T1], L2}. +-spec t_check_record_fields(parse_form(), sets:set(mfa()), module(), + mod_records()) -> ok. + +t_check_record_fields(Form, ExpTypes, Module, RecDict) -> + t_check_record_fields(Form, ExpTypes, Module, RecDict, dict:new()). + +-spec t_check_record_fields(parse_form(), sets:set(mfa()), module(), + mod_records(), var_table()) -> ok. + +%% If there is something wrong with parse_form() +%% throw({error, io_lib:chars()} is called. + +t_check_record_fields({var, _L, _}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({ann_type, _L, [_Var, Type]}, ET, M, MR, V) -> + t_check_record_fields(Type, ET, M, MR, V); +t_check_record_fields({paren_type, _L, [Type]}, ET, M, MR, V) -> + t_check_record_fields(Type, ET, M, MR, V); +t_check_record_fields({remote_type, _L, [{atom, _, _}, {atom, _, _}, Args]}, + ET, M, MR, V) -> + list_check_record_fields(Args, ET, M, MR, V); +t_check_record_fields({atom, _L, _}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({integer, _L, _}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({op, _L, _Op, _Arg}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({op, _L, _Op, _Arg1, _Arg2}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({type, _L, tuple, any}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({type, _L, map, any}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({type, _L, binary, [_Base, _Unit]}, _ET, _M, _MR, _V) -> + ok; +t_check_record_fields({type, _L, 'fun', [{type, _, any}, Range]}, + ET, M, MR, V) -> + t_check_record_fields(Range, ET, M, MR, V); +t_check_record_fields({type, _L, range, [_From, _To]}, _ET, _M, _MR, _V) -> + ok; +t_check_record_fields({type, _L, record, [Name|Fields]}, ET, M, MR, V) -> + check_record(Name, Fields, ET, M, MR, V); +t_check_record_fields({type, _L, _, Args}, ET, M, MR, V) -> + list_check_record_fields(Args, ET, M, MR, V); +t_check_record_fields({user_type, _L, _Name, Args}, ET, M, MR, V) -> + list_check_record_fields(Args, ET, M, MR, V). + +check_record({atom, _, Name}, ModFields, ET, M, MR, V) -> + {ok, R} = dict:find(M, MR), + {ok, DeclFields} = lookup_record(Name, R), + case check_fields(ModFields, DeclFields, ET, M, MR, V) of + {error, FieldName} -> + throw({error, io_lib:format("Illegal declaration of #~w{~w}\n", + [Name, FieldName])}); + ok -> ok + end. + +check_fields([{type, _, field_type, [{atom, _, Name}, Abstr]}|Left], + DeclFields, ET, M, MR, V) -> + Type = t_from_form(Abstr, ET, M, MR, V), + {Name, _, DeclType} = lists:keyfind(Name, 1, DeclFields), + TypeNoVars = subst_all_vars_to_any(Type), + case t_is_subtype(TypeNoVars, DeclType) of + false -> {error, Name}; + true -> check_fields(Left, DeclFields, ET, M, MR, V) + end; +check_fields([], _Decl, _ET, _M, _MR, _V) -> + ok. + +list_check_record_fields([], _ET, _M, _MR, _V) -> + ok; +list_check_record_fields([H|Tail], ET, M, MR, V) -> + ok = t_check_record_fields(H, ET, M, MR, V), + list_check_record_fields(Tail, ET, M, MR, V). + -spec t_var_names([erl_type()]) -> [atom()]. t_var_names([{var, _, Name}|L]) when L =/= '_' -> @@ -4556,9 +4636,9 @@ is_erl_type(_) -> false. lookup_record(Tag, RecDict) when is_atom(Tag) -> case dict:find({record, Tag}, RecDict) of - {ok, [{_Arity, Fields}]} -> + {ok, {_FileLine, [{_Arity, Fields}]}} -> {ok, Fields}; - {ok, List} when is_list(List) -> + {ok, {_FileLine, List}} when is_list(List) -> %% This will have to do, since we do not know which record we %% are looking for. error; @@ -4571,8 +4651,8 @@ lookup_record(Tag, RecDict) when is_atom(Tag) -> lookup_record(Tag, Arity, RecDict) when is_atom(Tag) -> case dict:find({record, Tag}, RecDict) of - {ok, [{Arity, Fields}]} -> {ok, Fields}; - {ok, OrdDict} -> orddict:find(Arity, OrdDict); + {ok, {_FileLine, [{Arity, Fields}]}} -> {ok, Fields}; + {ok, {_FileLine, OrdDict}} -> orddict:find(Arity, OrdDict); error -> error end. @@ -4619,17 +4699,6 @@ do_opaque(?union(List) = Type, Opaques, Pred) -> do_opaque(Type, _Opaques, Pred) -> Pred(Type). -is_same_type_name(ModNameArgs, ModNameArgs) -> true; -is_same_type_name({Mod, Name, Args1}, {Mod, Name, Args2}) -> - all_any(Args1) orelse all_any(Args2); -is_same_type_name(_ModNameArgs1, _ModNameArgs2) -> - false. - -all_any([]) -> true; -all_any([T|L]) -> - t_is_any(T) andalso all_any(L); -all_any(_) -> false. - map_keys(?map(Pairs)) -> [K || {K, _} <- Pairs]. diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index 539ce883c0..b614f5f1ab 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -649,8 +649,9 @@ run_compiler_1(DisasmFun, IcodeFun, Options) -> %% The full option expansion is not done %% until the DisasmFun returns. {Code, CompOpts} = DisasmFun(Options), - Opts0 = expand_options(Options ++ CompOpts), - Opts = + Opts0 = expand_options(Options ++ CompOpts, + get(hipe_target_arch)), + Opts = case proplists:get_bool(to_llvm, Opts0) andalso not llvm_support_available() of true -> @@ -895,8 +896,7 @@ do_load(Mod, Bin, BeamBinOrPath) when is_binary(BeamBinOrPath); code:load_native_sticky(Mod, Bin, Beam); false -> %% Normal loading of a whole module - Architecture = erlang:system_info(hipe_architecture), - ChunkName = hipe_unified_loader:chunk_name(Architecture), + ChunkName = hipe_unified_loader:chunk_name(HostArch), {ok, _, Chunks0} = beam_lib:all_chunks(BeamBinOrPath), Chunks = [{ChunkName, Bin}|lists:keydelete(ChunkName, 1, Chunks0)], {ok, BeamPlusNative} = beam_lib:build_module(Chunks), @@ -933,9 +933,9 @@ assemble(CompiledCode, Closures, Exports, Options) -> %% but can be overridden by passing an option {target, Target}. set_architecture(Options) -> - put(hipe_host_arch, erlang:system_info(hipe_architecture)), - put(hipe_target_arch, - proplists:get_value(target, Options, get(hipe_host_arch))), + HostArch = erlang:system_info(hipe_architecture), + put(hipe_host_arch, HostArch), + put(hipe_target_arch, proplists:get_value(target, Options, HostArch)), ok. %% This sets up some globally accessed stuff that are needed by the @@ -943,7 +943,7 @@ set_architecture(Options) -> %% Therefore, this expands the current set of options for local use. pre_init(Opts) -> - Options = expand_options(Opts), + Options = expand_options(Opts, get(hipe_target_arch)), %% Initialise some counters used for measurements and benchmarking. If %% the option 'measure_regalloc' is given the compilation will return %% a keylist with the counter values. @@ -1105,10 +1105,10 @@ help_hiper() -> -spec help_options() -> 'ok'. help_options() -> - set_architecture([]), %% needed for target-specific option expansion - O1 = expand_options([o1]), - O2 = expand_options([o2]), - O3 = expand_options([o3]), + HostArch = erlang:system_info(hipe_architecture), + O1 = expand_options([o1], HostArch), + O2 = expand_options([o2], HostArch), + O3 = expand_options([o3], HostArch), io:format("HiPE Compiler Options\n" ++ " Boolean-valued options generally have corresponding " ++ "aliases `no_...',\n" ++ @@ -1134,7 +1134,7 @@ help_options() -> [ordsets:from_list([verbose, debug, time, load, pp_beam, pp_icode, pp_rtl, pp_native, pp_asm, timeout]), - expand_options([pp_all]), + expand_options([pp_all], HostArch), O1 -- [o1], (O2 -- O1) -- [o2], (O3 -- O2) -- [o3]]), @@ -1232,8 +1232,8 @@ option_text(Opt) when is_atom(Opt) -> -spec help_option(comp_option()) -> 'ok'. help_option(Opt) -> - set_architecture([]), %% needed for target-specific option expansion - case expand_options([Opt]) of + HostArch = erlang:system_info(hipe_architecture), + case expand_options([Opt], HostArch) of [Opt] -> Name = if is_atom(Opt) -> Opt; tuple_size(Opt) =:= 2 -> element(1, Opt) @@ -1364,11 +1364,11 @@ opt_keys() -> %% verbose_spills, x87]. -%% Definitions: +%% Definitions: -o1_opts() -> +o1_opts(TargetArch) -> Common = [inline_fp, pmatch, peephole], - case get(hipe_target_arch) of + case TargetArch of ultrasparc -> Common; powerpc -> @@ -1385,13 +1385,13 @@ o1_opts() -> ?EXIT({executing_on_an_unsupported_architecture,Arch}) end. -o2_opts() -> +o2_opts(TargetArch) -> Common = [icode_ssa_const_prop, icode_ssa_copy_prop, % icode_ssa_struct_reuse, icode_type, icode_inline_bifs, rtl_lcm, rtl_ssa, rtl_ssa_const_prop, - spillmin_color, use_indexing, remove_comments, - concurrent_comp, binary_opt | o1_opts()], - case get(hipe_target_arch) of + spillmin_color, use_indexing, remove_comments, + concurrent_comp, binary_opt | o1_opts(TargetArch)], + case TargetArch of ultrasparc -> Common; powerpc -> @@ -1409,9 +1409,9 @@ o2_opts() -> ?EXIT({executing_on_an_unsupported_architecture,Arch}) end. -o3_opts() -> - Common = [icode_range, {regalloc,coalescing} | o2_opts()], - case get(hipe_target_arch) of +o3_opts(TargetArch) -> + Common = [icode_range, {regalloc,coalescing} | o2_opts(TargetArch)], + case TargetArch of ultrasparc -> Common; powerpc -> @@ -1489,18 +1489,18 @@ opt_aliases() -> opt_basic_expansions() -> [{pp_all, [pp_beam, pp_icode, pp_rtl, pp_native]}]. -opt_expansions() -> - [{o1, o1_opts()}, - {o2, o2_opts()}, - {o3, o3_opts()}, +opt_expansions(TargetArch) -> + [{o1, o1_opts(TargetArch)}, + {o2, o2_opts(TargetArch)}, + {o3, o3_opts(TargetArch)}, {to_llvm, llvm_opts(o3)}, {{to_llvm, o0}, llvm_opts(o0)}, {{to_llvm, o1}, llvm_opts(o1)}, {{to_llvm, o2}, llvm_opts(o2)}, {{to_llvm, o3}, llvm_opts(o3)}, {x87, [x87, inline_fp]}, - {inline_fp, case get(hipe_target_arch) of %% XXX: Temporary until x86 - x86 -> [x87, inline_fp]; %% has sse2 + {inline_fp, case TargetArch of %% XXX: Temporary until x86 has sse2 + x86 -> [x87, inline_fp]; _ -> [inline_fp] end}]. llvm_opts(O) -> @@ -1523,18 +1523,18 @@ expand_kt2(Opts) -> [{use_callgraph, fixpoint}, core, {core_transform, cerl_typean}]}]}]). -%% Note that set_architecture/1 must be called first, and that the given +%% Note that the given %% list should contain the total set of options, since things like 'o2' %% are expanded here. Basic expansions are processed here also, since %% this function is called from the help functions. --spec expand_options(comp_options()) -> comp_options(). +-spec expand_options(comp_options(), hipe_architecture()) -> comp_options(). -expand_options(Opts) -> +expand_options(Opts, TargetArch) -> proplists:normalize(Opts, [{negations, opt_negations()}, {aliases, opt_aliases()}, {expand, opt_basic_expansions()}, - {expand, opt_expansions()}]). + {expand, opt_expansions(TargetArch)}]). -spec check_options(comp_options()) -> 'ok'. diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 9d832ef18b..ce9a0e3d26 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1302,7 +1302,8 @@ handle_pipeline(#state{status = pipeline, handle_keep_alive_queue(#state{status = keep_alive, session = Session, profile_name = ProfileName, - options = #options{keep_alive_timeout = TimeOut}} = State, + options = #options{keep_alive_timeout = TimeOut, + proxy = Proxy}} = State, Data) -> ?hcrd("handle keep_alive", [{profile, ProfileName}, @@ -1323,7 +1324,8 @@ handle_keep_alive_queue(#state{status = keep_alive, State#state{keep_alive = KeepAlive}, Data); false -> ?hcrv("next request", [{request, NextRequest}]), - #request{address = Address} = NextRequest, + #request{address = Addr} = NextRequest, + Address = handle_proxy(Addr, Proxy), case httpc_request:send(Address, Session, NextRequest) of ok -> receive_response(NextRequest, @@ -1388,6 +1390,8 @@ case_insensitive_header(Str) -> activate_once(#session{socket = Socket, socket_type = SocketType}) -> http_transport:setopts(SocketType, Socket, [{active, once}]). +close_socket(#session{socket = {remote_close,_}}) -> + ok; close_socket(#session{socket = Socket, socket_type = SocketType}) -> http_transport:close(SocketType, Socket). diff --git a/lib/inets/src/http_server/httpd_conf.erl b/lib/inets/src/http_server/httpd_conf.erl index 9c70f8d1b8..89a7e96f3e 100644 --- a/lib/inets/src/http_server/httpd_conf.erl +++ b/lib/inets/src/http_server/httpd_conf.erl @@ -19,141 +19,29 @@ %% -module(httpd_conf). -%% EWSAPI --export([is_directory/1, is_file/1, make_integer/1, clean/1, - custom_clean/3, check_enum/2]). - %% Application internal API -export([load/1, load/2, load_mime_types/1, store/1, store/2, remove/1, remove_all/1, get_config/3, get_config/4, lookup_socket_type/1, lookup/2, lookup/3, lookup/4, - validate_properties/1]). + validate_properties/1, white_space_clean/1]). + +%% Deprecated +-export([is_directory/1, is_file/1, make_integer/1, clean/1, + custom_clean/3, check_enum/2]). + +-deprecated({is_directory, 1, next_major_release}). +-deprecated({is_file, 1, next_major_release}). +-deprecated({make_integer, 1, next_major_release}). +-deprecated({clean, 1, next_major_release}). +-deprecated({custom_clean, 3, next_major_release}). +-deprecated({check_enum, 2, next_major_release}). -define(VMODULE,"CONF"). -include("httpd_internal.hrl"). -include("httpd.hrl"). -include_lib("inets/src/http_lib/http_internal.hrl"). - -%%%========================================================================= -%%% EWSAPI -%%%========================================================================= -%%------------------------------------------------------------------------- -%% is_directory(FilePath) -> Result -%% FilePath = string() -%% Result = {ok,Directory} | {error,Reason} -%% Directory = string() -%% Reason = string() | enoent | eacces | enotdir | FileInfo -%% FileInfo = File info record -%% -%% Description: Checks if FilePath is a directory in which case it is -%% returned. -%%------------------------------------------------------------------------- -is_directory(Directory) -> - case file:read_file_info(Directory) of - {ok,FileInfo} -> - #file_info{type = Type, access = Access} = FileInfo, - is_directory(Type,Access,FileInfo,Directory); - {error,Reason} -> - {error,Reason} - end. -is_directory(directory,read,_FileInfo,Directory) -> - {ok,Directory}; -is_directory(directory,read_write,_FileInfo,Directory) -> - {ok,Directory}; -is_directory(_Type,_Access,FileInfo,_Directory) -> - {error,FileInfo}. - - -%%------------------------------------------------------------------------- -%% is_file(FilePath) -> Result -%% FilePath = string() -%% Result = {ok,File} | {error,Reason} -%% File = string() -%% Reason = string() | enoent | eacces | enotdir | FileInfo -%% FileInfo = File info record -%% -%% Description: Checks if FilePath is a regular file in which case it -%% is returned. -%%------------------------------------------------------------------------- -is_file(File) -> - case file:read_file_info(File) of - {ok,FileInfo} -> - #file_info{type = Type, access = Access} = FileInfo, - is_file(Type,Access,FileInfo,File); - {error,Reason} -> - {error,Reason} - end. -is_file(regular,read,_FileInfo,File) -> - {ok,File}; -is_file(regular,read_write,_FileInfo,File) -> - {ok,File}; -is_file(_Type,_Access,FileInfo,_File) -> - {error,FileInfo}. - - -%%------------------------------------------------------------------------- -%% make_integer(String) -> Result -%% String = string() -%% Result = {ok,integer()} | {error,nomatch} -%% -%% Description: make_integer/1 returns an integer representation of String. -%%------------------------------------------------------------------------- -make_integer(String) -> - case inets_regexp:match(clean(String),"[0-9]+") of - {match, _, _} -> - {ok, list_to_integer(clean(String))}; - nomatch -> - {error, nomatch} - end. - - -%%------------------------------------------------------------------------- -%% clean(String) -> Stripped -%% String = Stripped = string() -%% -%% Description:clean/1 removes leading and/or trailing white spaces -%% from String. -%%------------------------------------------------------------------------- -clean(String) -> - {ok,CleanedString,_} = - inets_regexp:gsub(String, "^[ \t\n\r\f]*|[ \t\n\r\f]*\$",""), - CleanedString. - - -%%------------------------------------------------------------------------- -%% custom_clean(String,Before,After) -> Stripped -%% Before = After = regexp() -%% String = Stripped = string() -%% -%% Description: custom_clean/3 removes leading and/or trailing white -%% spaces and custom characters from String. -%%------------------------------------------------------------------------- -custom_clean(String,MoreBefore,MoreAfter) -> - {ok,CleanedString,_} = inets_regexp:gsub(String,"^[ \t\n\r\f"++MoreBefore++ - "]*|[ \t\n\r\f"++MoreAfter++"]*\$",""), - CleanedString. - - -%%------------------------------------------------------------------------- -%% check_enum(EnumString,ValidEnumStrings) -> Result -%% EnumString = string() -%% ValidEnumStrings = [string()] -%% Result = {ok,atom()} | {error,not_valid} -%% -%% Description: check_enum/2 checks if EnumString is a valid -%% enumeration of ValidEnumStrings in which case it is returned as an -%% atom. -%%------------------------------------------------------------------------- -check_enum(_Enum,[]) -> - {error, not_valid}; -check_enum(Enum,[Enum|_Rest]) -> - {ok, list_to_atom(Enum)}; -check_enum(Enum, [_NotValid|Rest]) -> - check_enum(Enum, Rest). - - %%%========================================================================= %%% Application internal API %%%========================================================================= @@ -192,7 +80,7 @@ load("MaxHeaderSize " ++ MaxHeaderSize, []) -> {ok, Integer} -> {ok, [], {max_header_size,Integer}}; {error, _} -> - {error, ?NICE(clean(MaxHeaderSize)++ + {error, ?NICE(string:strip(MaxHeaderSize)++ " is an invalid number of MaxHeaderSize")} end; @@ -201,7 +89,7 @@ load("MaxURISize " ++ MaxHeaderSize, []) -> {ok, Integer} -> {ok, [], {max_uri_size, Integer}}; {error, _} -> - {error, ?NICE(clean(MaxHeaderSize)++ + {error, ?NICE(string:strip(MaxHeaderSize)++ " is an invalid number of MaxHeaderSize")} end; @@ -210,12 +98,12 @@ load("MaxContentLength " ++ Max, []) -> {ok, Integer} -> {ok, [], {max_content_length, Integer}}; {error, _} -> - {error, ?NICE(clean(Max) ++ + {error, ?NICE(string:strip(Max) ++ " is an invalid number of MaxContentLength")} end; load("ServerName " ++ ServerName, []) -> - {ok,[], {server_name, clean(ServerName)}}; + {ok,[], {server_name, string:strip(ServerName)}}; load("ServerTokens " ++ ServerTokens, []) -> %% These are the valid *plain* server tokens: @@ -223,28 +111,28 @@ load("ServerTokens " ++ ServerTokens, []) -> %% It can also be a "private" server token: private:<any string> case string:tokens(ServerTokens, [$:]) of ["private", Private] -> - {ok,[], {server_tokens, clean(Private)}}; + {ok,[], {server_tokens, string:strip(Private)}}; [TokStr] -> - Tok = list_to_atom(clean(TokStr)), + Tok = list_to_atom(string:strip(TokStr)), case lists:member(Tok, [none, prod, major, minor, minimum, os, full]) of true -> {ok,[], {server_tokens, Tok}}; false -> - {error, ?NICE(clean(ServerTokens) ++ + {error, ?NICE(string:strip(ServerTokens) ++ " is an invalid ServerTokens")} end; _ -> - {error, ?NICE(clean(ServerTokens) ++ " is an invalid ServerTokens")} + {error, ?NICE(string:strip(ServerTokens) ++ " is an invalid ServerTokens")} end; load("SocketType " ++ SocketType, []) -> %% ssl is the same as HTTP_DEFAULT_SSL_KIND %% essl is the pure Erlang-based ssl (the "new" ssl) - case check_enum(clean(SocketType), ["ssl", "essl", "ip_comm"]) of + case check_enum(string:strip(SocketType), ["ssl", "essl", "ip_comm"]) of {ok, ValidSocketType} -> {ok, [], {socket_type, ValidSocketType}}; {error,_} -> - {error, ?NICE(clean(SocketType) ++ " is an invalid SocketType")} + {error, ?NICE(string:strip(SocketType) ++ " is an invalid SocketType")} end; load("Port " ++ Port, []) -> @@ -252,7 +140,7 @@ load("Port " ++ Port, []) -> {ok, Integer} -> {ok, [], {port, Integer}}; {error, _} -> - {error, ?NICE(clean(Port)++" is an invalid Port")} + {error, ?NICE(string:strip(Port)++" is an invalid Port")} end; load("BindAddress " ++ Address0, []) -> @@ -308,7 +196,7 @@ load("BindAddress " ++ Address0, []) -> end; load("KeepAlive " ++ OnorOff, []) -> - case list_to_atom(clean(OnorOff)) of + case list_to_atom(string:strip(OnorOff)) of off -> {ok, [], {keep_alive, false}}; _ -> @@ -320,7 +208,7 @@ load("MaxKeepAliveRequests " ++ MaxRequests, []) -> {ok, Integer} -> {ok, [], {max_keep_alive_request, Integer}}; {error, _} -> - {error, ?NICE(clean(MaxRequests) ++ + {error, ?NICE(string:strip(MaxRequests) ++ " is an invalid MaxKeepAliveRequests")} end; @@ -330,7 +218,7 @@ load("MaxKeepAliveRequest " ++ MaxRequests, []) -> {ok, Integer} -> {ok, [], {max_keep_alive_request, Integer}}; {error, _} -> - {error, ?NICE(clean(MaxRequests) ++ + {error, ?NICE(string:strip(MaxRequests) ++ " is an invalid MaxKeepAliveRequest")} end; @@ -339,7 +227,7 @@ load("KeepAliveTimeout " ++ Timeout, []) -> {ok, Integer} -> {ok, [], {keep_alive_timeout, Integer}}; {error, _} -> - {error, ?NICE(clean(Timeout)++" is an invalid KeepAliveTimeout")} + {error, ?NICE(string:strip(Timeout)++" is an invalid KeepAliveTimeout")} end; load("Modules " ++ Modules, []) -> @@ -347,18 +235,18 @@ load("Modules " ++ Modules, []) -> {ok, [], {modules,[list_to_atom(X) || X <- ModuleList]}}; load("ServerAdmin " ++ ServerAdmin, []) -> - {ok, [], {server_admin,clean(ServerAdmin)}}; + {ok, [], {server_admin,string:strip(ServerAdmin)}}; load("ServerRoot " ++ ServerRoot, []) -> - case is_directory(clean(ServerRoot)) of + case is_directory(string:strip(ServerRoot)) of {ok, Directory} -> {ok, [], [{server_root,string:strip(Directory,right,$/)}]}; {error, _} -> - {error, ?NICE(clean(ServerRoot)++" is an invalid ServerRoot")} + {error, ?NICE(string:strip(ServerRoot)++" is an invalid ServerRoot")} end; load("MimeTypes " ++ MimeTypes, []) -> - case load_mime_types(clean(MimeTypes)) of + case load_mime_types(white_space_clean(MimeTypes)) of {ok, MimeTypesList} -> {ok, [], [{mime_types, MimeTypesList}]}; {error, Reason} -> @@ -370,24 +258,24 @@ load("MaxClients " ++ MaxClients, []) -> {ok, Integer} -> {ok, [], {max_clients,Integer}}; {error, _} -> - {error, ?NICE(clean(MaxClients) ++ + {error, ?NICE(string:strip(MaxClients) ++ " is an invalid number of MaxClients")} end; load("DocumentRoot " ++ DocumentRoot,[]) -> - case is_directory(clean(DocumentRoot)) of + case is_directory(string:strip(DocumentRoot)) of {ok, Directory} -> {ok, [], {document_root,string:strip(Directory,right,$/)}}; {error, _} -> - {error, ?NICE(clean(DocumentRoot)++" is an invalid DocumentRoot")} + {error, ?NICE(string:strip(DocumentRoot)++" is an invalid DocumentRoot")} end; load("DefaultType " ++ DefaultType, []) -> - {ok, [], {default_type,clean(DefaultType)}}; + {ok, [], {default_type,string:strip(DefaultType)}}; load("SSLCertificateFile " ++ SSLCertificateFile, []) -> - case is_file(clean(SSLCertificateFile)) of + case is_file(string:strip(SSLCertificateFile)) of {ok, File} -> {ok, [], {ssl_certificate_file,File}}; {error, _} -> - {error, ?NICE(clean(SSLCertificateFile)++ + {error, ?NICE(string:strip(SSLCertificateFile)++ " is an invalid SSLCertificateFile")} end; load("SSLLogLevel " ++ SSLLogAlert, []) -> @@ -398,69 +286,69 @@ load("SSLLogLevel " ++ SSLLogAlert, []) -> {ok, [], {ssl_log_alert, true}} end; load("SSLCertificateKeyFile " ++ SSLCertificateKeyFile, []) -> - case is_file(clean(SSLCertificateKeyFile)) of + case is_file(string:strip(SSLCertificateKeyFile)) of {ok, File} -> {ok, [], {ssl_certificate_key_file,File}}; {error, _} -> - {error, ?NICE(clean(SSLCertificateKeyFile)++ + {error, ?NICE(string:strip(SSLCertificateKeyFile)++ " is an invalid SSLCertificateKeyFile")} end; load("SSLVerifyClient " ++ SSLVerifyClient, []) -> - case make_integer(clean(SSLVerifyClient)) of + case make_integer(string:strip(SSLVerifyClient)) of {ok, Integer} when (Integer >=0) andalso (Integer =< 2) -> {ok, [], {ssl_verify_client,Integer}}; {ok, _Integer} -> - {error,?NICE(clean(SSLVerifyClient) ++ + {error,?NICE(string:strip(SSLVerifyClient) ++ " is an invalid SSLVerifyClient")}; {error, nomatch} -> - {error,?NICE(clean(SSLVerifyClient) ++ + {error,?NICE(string:strip(SSLVerifyClient) ++ " is an invalid SSLVerifyClient")} end; load("SSLVerifyDepth " ++ SSLVerifyDepth, []) -> - case make_integer(clean(SSLVerifyDepth)) of + case make_integer(string:strip(SSLVerifyDepth)) of {ok, Integer} when Integer > 0 -> {ok, [], {ssl_verify_client_depth,Integer}}; {ok, _Integer} -> - {error,?NICE(clean(SSLVerifyDepth) ++ + {error,?NICE(string:strip(SSLVerifyDepth) ++ " is an invalid SSLVerifyDepth")}; {error, nomatch} -> - {error,?NICE(clean(SSLVerifyDepth) ++ + {error,?NICE(string:strip(SSLVerifyDepth) ++ " is an invalid SSLVerifyDepth")} end; load("SSLCiphers " ++ SSLCiphers, []) -> - {ok, [], {ssl_ciphers, clean(SSLCiphers)}}; + {ok, [], {ssl_ciphers, string:strip(SSLCiphers)}}; load("SSLCACertificateFile " ++ SSLCACertificateFile, []) -> - case is_file(clean(SSLCACertificateFile)) of + case is_file(string:strip(SSLCACertificateFile)) of {ok, File} -> {ok, [], {ssl_ca_certificate_file,File}}; {error, _} -> - {error, ?NICE(clean(SSLCACertificateFile)++ + {error, ?NICE(string:strip(SSLCACertificateFile)++ " is an invalid SSLCACertificateFile")} end; load("SSLPasswordCallbackModule " ++ SSLPasswordCallbackModule, []) -> {ok, [], {ssl_password_callback_module, - list_to_atom(clean(SSLPasswordCallbackModule))}}; + list_to_atom(string:strip(SSLPasswordCallbackModule))}}; load("SSLPasswordCallbackFunction " ++ SSLPasswordCallbackFunction, []) -> {ok, [], {ssl_password_callback_function, - list_to_atom(clean(SSLPasswordCallbackFunction))}}; + list_to_atom(string:strip(SSLPasswordCallbackFunction))}}; load("SSLPasswordCallbackArguments " ++ SSLPasswordCallbackArguments, []) -> {ok, [], {ssl_password_callback_arguments, SSLPasswordCallbackArguments}}; load("DisableChunkedTransferEncodingSend " ++ TrueOrFalse, []) -> - case list_to_atom(clean(TrueOrFalse)) of + case list_to_atom(string:strip(TrueOrFalse)) of true -> {ok, [], {disable_chunked_transfer_encoding_send, true}}; _ -> {ok, [], {disable_chunked_transfer_encoding_send, false}} end; load("LogFormat " ++ LogFormat, []) -> - {ok,[],{log_format, list_to_atom(httpd_conf:clean(LogFormat))}}; + {ok,[],{log_format, list_to_atom(string:strip(LogFormat))}}; load("ErrorLogFormat " ++ LogFormat, []) -> - {ok,[],{error_log_format, list_to_atom(httpd_conf:clean(LogFormat))}}. + {ok,[],{error_log_format, list_to_atom(string:strip(LogFormat))}}. clean_address(Addr) -> - string:strip(string:strip(clean(Addr), left, $[), right, $]). + string:strip(string:strip(string:strip(Addr), left, $[), right, $]). make_ipfamily(IpFamilyStr) -> @@ -1086,7 +974,7 @@ verify_modules([]) -> verify_modules([Mod|Rest]) -> case code:which(Mod) of non_existing -> - {error, ?NICE(atom_to_list(Mod)++" does not exist")}; + {error, ?NICE(string:strip(atom_to_list(Mod), right, $\n) ++" does not exist")}; _Path -> verify_modules(Rest) end. @@ -1128,7 +1016,7 @@ parse_mime_types(Stream,MimeTypesList) -> eof -> eof; String -> - clean(String) + white_space_clean(String) end, parse_mime_types(Stream, MimeTypesList, Line). parse_mime_types(Stream, MimeTypesList, eof) -> @@ -1314,4 +1202,68 @@ plain_server_tokens() -> error_report(Where,M,F,Error) -> error_logger:error_report([{?MODULE, Where}, {apply, {M, F, []}}, Error]). +white_space_clean(String) -> + {ok,CleanedString,_} = + inets_regexp:gsub(String, "^[ \t\n\r\f]*|[ \t\n\r\f]*\$",""), + CleanedString. + + +%%%========================================================================= +%%% Deprecated remove in 19 +%%%========================================================================= +is_directory(Directory) -> + case file:read_file_info(Directory) of + {ok,FileInfo} -> + #file_info{type = Type, access = Access} = FileInfo, + is_directory(Type,Access,FileInfo,Directory); + {error,Reason} -> + {error,Reason} + end. +is_directory(directory,read,_FileInfo,Directory) -> + {ok,Directory}; +is_directory(directory,read_write,_FileInfo,Directory) -> + {ok,Directory}; +is_directory(_Type,_Access,FileInfo,_Directory) -> + {error,FileInfo}. + +is_file(File) -> + case file:read_file_info(File) of + {ok,FileInfo} -> + #file_info{type = Type, access = Access} = FileInfo, + is_file(Type,Access,FileInfo,File); + {error,Reason} -> + {error,Reason} + end. +is_file(regular,read,_FileInfo,File) -> + {ok,File}; +is_file(regular,read_write,_FileInfo,File) -> + {ok,File}; +is_file(_Type,_Access,FileInfo,_File) -> + {error,FileInfo}. + +make_integer(String) -> + case inets_regexp:match(string:strip(String),"[0-9]+") of + {match, _, _} -> + {ok, list_to_integer(string:strip(String))}; + nomatch -> + {error, nomatch} + end. + +clean(String) -> + {ok,CleanedString,_} = + inets_regexp:gsub(String, "^[ \t\n\r\f]*|[ \t\n\r\f]*\$",""), + CleanedString. + +custom_clean(String,MoreBefore,MoreAfter) -> + {ok,CleanedString,_} = inets_regexp:gsub(String,"^[ \t\n\r\f"++MoreBefore++ + "]*|[ \t\n\r\f"++MoreAfter++"]*\$",""), + CleanedString. + +check_enum(_Enum,[]) -> + {error, not_valid}; +check_enum(Enum,[Enum|_Rest]) -> + {ok, list_to_atom(Enum)}; +check_enum(Enum, [_NotValid|Rest]) -> + check_enum(Enum, Rest). + diff --git a/lib/inets/src/http_server/httpd_example.erl b/lib/inets/src/http_server/httpd_example.erl index 6fc07f033c..b2c2ea7480 100644 --- a/lib/inets/src/http_server/httpd_example.erl +++ b/lib/inets/src/http_server/httpd_example.erl @@ -126,9 +126,7 @@ delay(Time) when is_integer(Time) -> i("httpd_example:delay(~p) -> done, now reply",[Time]), delay_reply("delay ok"); delay(Time) when is_list(Time) -> - delay(httpd_conf:make_integer(Time)); -delay({ok,Time}) when is_integer(Time) -> - delay(Time); + delay(list_to_integer(Time)); delay({error,_Reason}) -> i("delay -> called with invalid time"), delay_reply("delay failed: invalid delay time"). diff --git a/lib/inets/src/http_server/mod_actions.erl b/lib/inets/src/http_server/mod_actions.erl index c3946ff9b4..e493ddbf93 100644 --- a/lib/inets/src/http_server/mod_actions.erl +++ b/lib/inets/src/http_server/mod_actions.erl @@ -84,14 +84,14 @@ load("Action "++ Action, []) -> {ok,[MimeType, CGIScript]} -> {ok,[],{action, {MimeType, CGIScript}}}; {ok,_} -> - {error,?NICE(httpd_conf:clean(Action)++" is an invalid Action")} + {error,?NICE(string:strip(Action)++" is an invalid Action")} end; load("Script " ++ Script,[]) -> case inets_regexp:split(Script, " ") of {ok,[Method, CGIScript]} -> {ok,[],{script, {Method, CGIScript}}}; {ok,_} -> - {error,?NICE(httpd_conf:clean(Script)++" is an invalid Script")} + {error,?NICE(string:strip(Script)++" is an invalid Script")} end. store({action, {MimeType, CGIScript}} = Conf, _) when is_list(MimeType), diff --git a/lib/inets/src/http_server/mod_alias.erl b/lib/inets/src/http_server/mod_alias.erl index 5039cd56b5..e0d34288cd 100644 --- a/lib/inets/src/http_server/mod_alias.erl +++ b/lib/inets/src/http_server/mod_alias.erl @@ -212,7 +212,7 @@ load("Alias " ++ Alias, []) -> {ok, [FakeName, RealName]} -> {ok,[],{alias,{FakeName,RealName}}}; {ok, _} -> - {error,?NICE(httpd_conf:clean(Alias)++" is an invalid Alias")} + {error,?NICE(string:strip(Alias)++" is an invalid Alias")} end; load("ReWrite " ++ Rule, Acc) -> load_re_write(Rule, Acc, "ReWrite", re_write); @@ -223,7 +223,7 @@ load("ScriptAlias " ++ ScriptAlias, []) -> RealName1 = filename:join(filename:split(RealName)), {ok, [], {script_alias, {FakeName, RealName1++"/"}}}; {ok, _} -> - {error, ?NICE(httpd_conf:clean(ScriptAlias)++ + {error, ?NICE(string:strip(ScriptAlias)++ " is an invalid ScriptAlias")} end; load("ScriptReWrite " ++ Rule, Acc) -> @@ -234,7 +234,7 @@ load_re_write(Rule0, Acc, Type, Tag) -> fun ($\s) -> true; ($\t) -> true; (_) -> false end, Rule0) of "" -> - {error, ?NICE(httpd_conf:clean(Rule0)++" is an invalid "++Type)}; + {error, ?NICE(string:strip(Rule0)++" is an invalid "++Type)}; Rule -> case string:chr(Rule, $\s) of 0 -> diff --git a/lib/inets/src/http_server/mod_auth.erl b/lib/inets/src/http_server/mod_auth.erl index 1f4470622d..ae716a6cff 100644 --- a/lib/inets/src/http_server/mod_auth.erl +++ b/lib/inets/src/http_server/mod_auth.erl @@ -127,35 +127,35 @@ do(Info) -> %% state it was previously. load("<Directory " ++ Directory,[]) -> - Dir = httpd_conf:custom_clean(Directory,"",">"), + Dir = string:strip(string:strip(Directory),right, $>), {ok,[{directory, {Dir, [{path, Dir}]}}]}; load(eof,[{directory, {Directory, _DirData}}|_]) -> {error, ?NICE("Premature end-of-file in "++ Directory)}; load("AuthName " ++ AuthName, [{directory, {Directory, DirData}}|Rest]) -> {ok, [{directory, {Directory, - [{auth_name, httpd_conf:clean(AuthName)} | DirData]}} + [{auth_name, string:strip(AuthName)} | DirData]}} | Rest ]}; load("AuthUserFile " ++ AuthUserFile0, [{directory, {Directory, DirData}}|Rest]) -> - AuthUserFile = httpd_conf:clean(AuthUserFile0), + AuthUserFile = string:strip(AuthUserFile0), {ok, [{directory, {Directory, [{auth_user_file, AuthUserFile}|DirData]}} | Rest ]}; load("AuthGroupFile " ++ AuthGroupFile0, [{directory, {Directory, DirData}}|Rest]) -> - AuthGroupFile = httpd_conf:clean(AuthGroupFile0), + AuthGroupFile = string:strip(AuthGroupFile0), {ok,[{directory, {Directory, [{auth_group_file, AuthGroupFile}|DirData]}} | Rest]}; load("AuthAccessPassword " ++ AuthAccessPassword0, [{directory, {Directory, DirData}}|Rest]) -> - AuthAccessPassword = httpd_conf:clean(AuthAccessPassword0), + AuthAccessPassword = string:strip(AuthAccessPassword0), {ok,[{directory, {Directory, [{auth_access_password, AuthAccessPassword}|DirData]}} | Rest]}; load("AuthDBType " ++ Type, [{directory, {Dir, DirData}}|Rest]) -> - case httpd_conf:clean(Type) of + case string:strip(Type) of "plain" -> {ok, [{directory, {Dir, [{auth_type, plain}|DirData]}} | Rest ]}; "mnesia" -> @@ -163,7 +163,7 @@ load("AuthDBType " ++ Type, "dets" -> {ok, [{directory, {Dir, [{auth_type, dets}|DirData]}} | Rest ]}; _ -> - {error, ?NICE(httpd_conf:clean(Type)++" is an invalid AuthDBType")} + {error, ?NICE(string:strip(Type)++" is an invalid AuthDBType")} end; load("require " ++ Require,[{directory, {Directory, DirData}}|Rest]) -> @@ -175,7 +175,7 @@ load("require " ++ Require,[{directory, {Directory, DirData}}|Rest]) -> {ok,[{directory, {Directory, [{require_group,Groups}|DirData]}} | Rest]}; {ok,_} -> - {error,?NICE(httpd_conf:clean(Require) ++" is an invalid require")} + {error,?NICE(string:strip(Require) ++" is an invalid require")} end; load("allow " ++ Allow,[{directory, {Directory, DirData}}|Rest]) -> @@ -187,7 +187,7 @@ load("allow " ++ Allow,[{directory, {Directory, DirData}}|Rest]) -> {ok,[{directory, {Directory, [{allow_from,Hosts}|DirData]}} | Rest]}; {ok,_} -> - {error,?NICE(httpd_conf:clean(Allow) ++" is an invalid allow")} + {error,?NICE(string:strip(Allow) ++" is an invalid allow")} end; load("deny " ++ Deny,[{directory, {Directory, DirData}}|Rest]) -> @@ -199,7 +199,7 @@ load("deny " ++ Deny,[{directory, {Directory, DirData}}|Rest]) -> {ok,[{{directory, Directory, [{deny_from, Hosts}|DirData]}} | Rest]}; {ok, _} -> - {error,?NICE(httpd_conf:clean(Deny) ++" is an invalid deny")} + {error,?NICE(string:strip(Deny) ++" is an invalid deny")} end; load("</Directory>",[{directory, {Directory, DirData}}|Rest]) -> @@ -207,14 +207,14 @@ load("</Directory>",[{directory, {Directory, DirData}}|Rest]) -> load("AuthMnesiaDB " ++ AuthMnesiaDB, [{directory, {Dir, DirData}}|Rest]) -> - case httpd_conf:clean(AuthMnesiaDB) of + case string:strip(AuthMnesiaDB) of "On" -> {ok,[{directory, {Dir,[{auth_type,mnesia}|DirData]}}|Rest]}; "Off" -> {ok,[{directory, {Dir,[{auth_type,plain}|DirData]}}|Rest]}; _ -> - {error, ?NICE(httpd_conf:clean(AuthMnesiaDB) ++ - " is an invalid AuthMnesiaDB")} + {error, ?NICE(string:strip(AuthMnesiaDB) ++ + " is an invalid AuthMnesiaDB")} end. store({directory, {Directory, DirData}}, ConfigList) diff --git a/lib/inets/src/http_server/mod_auth_plain.erl b/lib/inets/src/http_server/mod_auth_plain.erl index 7bb86fc812..b411202efc 100644 --- a/lib/inets/src/http_server/mod_auth_plain.erl +++ b/lib/inets/src/http_server/mod_auth_plain.erl @@ -231,7 +231,7 @@ parse_group(Stream, GroupList) -> eof -> eof; String -> - httpd_conf:clean(String) + httpd_conf:white_space_clean(String) end, parse_group(Stream, GroupList, Line). @@ -265,7 +265,7 @@ parse_passwd(Stream, PasswdList) -> eof -> eof; String -> - httpd_conf:clean(String) + httpd_conf:white_space_clean(String) end, parse_passwd(Stream, PasswdList, Line). diff --git a/lib/inets/src/http_server/mod_cgi.erl b/lib/inets/src/http_server/mod_cgi.erl index d933b0a4ba..fbcfa78e5f 100644 --- a/lib/inets/src/http_server/mod_cgi.erl +++ b/lib/inets/src/http_server/mod_cgi.erl @@ -95,24 +95,24 @@ do(ModData) -> %% or cache %% load("ScriptNoCache " ++ CacheArg, [])-> - case catch list_to_atom(httpd_conf:clean(CacheArg)) of + case catch list_to_atom(string:strip(CacheArg)) of true -> {ok, [], {script_nocache, true}}; false -> {ok, [], {script_nocache, false}}; _ -> - {error, ?NICE(httpd_conf:clean(CacheArg)++ + {error, ?NICE(string:strip(CacheArg)++ " is an invalid ScriptNoCache directive")} end; %% ScriptTimeout Seconds, The number of seconds that the server %% maximum will wait for the script to %% generate a part of the document load("ScriptTimeout " ++ Timeout, [])-> - case catch list_to_integer(httpd_conf:clean(Timeout)) of + case catch list_to_integer(string:strip(Timeout)) of TimeoutSec when is_integer(TimeoutSec) -> {ok, [], {script_timeout,TimeoutSec*1000}}; _ -> - {error, ?NICE(httpd_conf:clean(Timeout)++ + {error, ?NICE(string:strip(Timeout)++ " is an invalid ScriptTimeout")} end. diff --git a/lib/inets/src/http_server/mod_disk_log.erl b/lib/inets/src/http_server/mod_disk_log.erl index 5a3766de66..89a3193aa3 100644 --- a/lib/inets/src/http_server/mod_disk_log.erl +++ b/lib/inets/src/http_server/mod_disk_log.erl @@ -139,70 +139,70 @@ do(Info) -> load("TransferDiskLogSize " ++ TransferDiskLogSize, []) -> case inets_regexp:split(TransferDiskLogSize," ") of {ok,[MaxBytes,MaxFiles]} -> - case httpd_conf:make_integer(MaxBytes) of + case make_integer(MaxBytes) of {ok,MaxBytesInteger} -> - case httpd_conf:make_integer(MaxFiles) of + case make_integer(MaxFiles) of {ok,MaxFilesInteger} -> {ok,[],{transfer_disk_log_size, {MaxBytesInteger,MaxFilesInteger}}}; {error,_} -> {error, - ?NICE(httpd_conf:clean(TransferDiskLogSize)++ + ?NICE(string:strip(TransferDiskLogSize)++ " is an invalid TransferDiskLogSize")} end; {error,_} -> - {error,?NICE(httpd_conf:clean(TransferDiskLogSize)++ + {error,?NICE(string:strip(TransferDiskLogSize)++ " is an invalid TransferDiskLogSize")} end end; load("TransferDiskLog " ++ TransferDiskLog,[]) -> - {ok,[],{transfer_disk_log,httpd_conf:clean(TransferDiskLog)}}; + {ok,[],{transfer_disk_log,string:strip(TransferDiskLog)}}; load("ErrorDiskLogSize " ++ ErrorDiskLogSize, []) -> case inets_regexp:split(ErrorDiskLogSize," ") of {ok,[MaxBytes,MaxFiles]} -> - case httpd_conf:make_integer(MaxBytes) of + case make_integer(MaxBytes) of {ok,MaxBytesInteger} -> - case httpd_conf:make_integer(MaxFiles) of + case make_integer(MaxFiles) of {ok,MaxFilesInteger} -> {ok,[],{error_disk_log_size, {MaxBytesInteger,MaxFilesInteger}}}; {error,_} -> - {error,?NICE(httpd_conf:clean(ErrorDiskLogSize)++ + {error,?NICE(string:strip(ErrorDiskLogSize)++ " is an invalid ErrorDiskLogSize")} end; {error,_} -> - {error,?NICE(httpd_conf:clean(ErrorDiskLogSize)++ + {error,?NICE(string:strip(ErrorDiskLogSize)++ " is an invalid ErrorDiskLogSize")} end end; load("ErrorDiskLog " ++ ErrorDiskLog, []) -> - {ok, [], {error_disk_log, httpd_conf:clean(ErrorDiskLog)}}; + {ok, [], {error_disk_log, string:strip(ErrorDiskLog)}}; load("SecurityDiskLogSize " ++ SecurityDiskLogSize, []) -> case inets_regexp:split(SecurityDiskLogSize, " ") of {ok, [MaxBytes, MaxFiles]} -> - case httpd_conf:make_integer(MaxBytes) of + case make_integer(MaxBytes) of {ok, MaxBytesInteger} -> - case httpd_conf:make_integer(MaxFiles) of + case make_integer(MaxFiles) of {ok, MaxFilesInteger} -> {ok, [], {security_disk_log_size, {MaxBytesInteger, MaxFilesInteger}}}; {error,_} -> {error, - ?NICE(httpd_conf:clean(SecurityDiskLogSize) ++ + ?NICE(string:strip(SecurityDiskLogSize) ++ " is an invalid SecurityDiskLogSize")} end; {error, _} -> - {error, ?NICE(httpd_conf:clean(SecurityDiskLogSize) ++ + {error, ?NICE(string:strip(SecurityDiskLogSize) ++ " is an invalid SecurityDiskLogSize")} end end; load("SecurityDiskLog " ++ SecurityDiskLog, []) -> - {ok, [], {security_disk_log, httpd_conf:clean(SecurityDiskLog)}}; + {ok, [], {security_disk_log, string:strip(SecurityDiskLog)}}; load("DiskLogFormat " ++ Format, []) -> - case httpd_conf:clean(Format) of + case string:strip(Format) of "internal" -> {ok, [], {disk_log_format,internal}}; "external" -> @@ -314,7 +314,7 @@ log_size(ConfigList, Tag) -> proplists:get_value(Tag, ConfigList, {500*1024,8}). create_disk_log(LogFile, SizeTag, ConfigList) -> - Filename = httpd_conf:clean(LogFile), + Filename = string:strip(LogFile), {MaxBytes, MaxFiles} = log_size(ConfigList, SizeTag), case filename:pathtype(Filename) of absolute -> @@ -413,3 +413,11 @@ log_internal_info(Info,Date,[{internal_info,Reason}|Rest]) -> log_internal_info(Info,Date,[_|Rest]) -> log_internal_info(Info,Date,Rest). +make_integer(List) -> + try list_to_integer(List) of + N -> + {ok, N} + catch + _:_ -> + {error, {badarg, list_to_integer, List}} + end. diff --git a/lib/inets/src/http_server/mod_esi.erl b/lib/inets/src/http_server/mod_esi.erl index b11df34f9e..d385afbd70 100644 --- a/lib/inets/src/http_server/mod_esi.erl +++ b/lib/inets/src/http_server/mod_esi.erl @@ -98,40 +98,40 @@ load("ErlScriptAlias " ++ ErlScriptAlias, []) -> case inets_regexp:split(ErlScriptAlias," ") of {ok, [ErlName | StrModules]} -> Modules = lists:map(fun(Str) -> - list_to_atom(httpd_conf:clean(Str)) + list_to_atom(string:strip(Str)) end, StrModules), {ok, [], {erl_script_alias, {ErlName, Modules}}}; {ok, _} -> - {error, ?NICE(httpd_conf:clean(ErlScriptAlias) ++ + {error, ?NICE(string:strip(ErlScriptAlias) ++ " is an invalid ErlScriptAlias")} end; load("EvalScriptAlias " ++ EvalScriptAlias, []) -> case inets_regexp:split(EvalScriptAlias, " ") of {ok, [EvalName | StrModules]} -> Modules = lists:map(fun(Str) -> - list_to_atom(httpd_conf:clean(Str)) + list_to_atom(string:strip(Str)) end, StrModules), {ok, [], {eval_script_alias, {EvalName, Modules}}}; {ok, _} -> - {error, ?NICE(httpd_conf:clean(EvalScriptAlias) ++ + {error, ?NICE(string:strip(EvalScriptAlias) ++ " is an invalid EvalScriptAlias")} end; load("ErlScriptTimeout " ++ Timeout, [])-> - case catch list_to_integer(httpd_conf:clean(Timeout)) of + case catch list_to_integer(string:strip(Timeout)) of TimeoutSec when is_integer(TimeoutSec) -> {ok, [], {erl_script_timeout, TimeoutSec * 1000}}; _ -> - {error, ?NICE(httpd_conf:clean(Timeout) ++ + {error, ?NICE(string:strip(Timeout) ++ " is an invalid ErlScriptTimeout")} end; load("ErlScriptNoCache " ++ CacheArg, [])-> - case catch list_to_atom(httpd_conf:clean(CacheArg)) of + case catch list_to_atom(string:strip(CacheArg)) of true -> {ok, [], {erl_script_nocache, true}}; false -> {ok, [], {erl_script_nocache, false}}; _ -> - {error, ?NICE(httpd_conf:clean(CacheArg)++ + {error, ?NICE(string:strip(CacheArg)++ " is an invalid ErlScriptNoCache directive")} end. diff --git a/lib/inets/src/http_server/mod_htaccess.erl b/lib/inets/src/http_server/mod_htaccess.erl index e1f66d01c8..d526fbe156 100644 --- a/lib/inets/src/http_server/mod_htaccess.erl +++ b/lib/inets/src/http_server/mod_htaccess.erl @@ -34,7 +34,7 @@ % Names on accessfiles %---------------------------------------------------------------------- load("AccessFileName" ++ FileNames, _Context)-> - CleanFileNames=httpd_conf:clean(FileNames), + CleanFileNames=string:strip(FileNames), {ok,[],{access_files,string:tokens(CleanFileNames," ")}}. store({access_files, Files} = Conf, _) when is_list(Files)-> diff --git a/lib/inets/src/http_server/mod_log.erl b/lib/inets/src/http_server/mod_log.erl index a912f5616c..2367d9cfeb 100644 --- a/lib/inets/src/http_server/mod_log.erl +++ b/lib/inets/src/http_server/mod_log.erl @@ -127,11 +127,11 @@ do(Info) -> %% Description: See httpd(3) ESWAPI CALLBACK FUNCTIONS %%------------------------------------------------------------------------- load("TransferLog " ++ TransferLog, []) -> - {ok,[],{transfer_log,httpd_conf:clean(TransferLog)}}; + {ok,[],{transfer_log,string:strip(TransferLog)}}; load("ErrorLog " ++ ErrorLog, []) -> - {ok,[],{error_log,httpd_conf:clean(ErrorLog)}}; + {ok,[],{error_log,string:strip(ErrorLog)}}; load("SecurityLog " ++ SecurityLog, []) -> - {ok, [], {security_log, httpd_conf:clean(SecurityLog)}}. + {ok, [], {security_log, string:strip(SecurityLog)}}. %%-------------------------------------------------------------------------- %% store(Directive, DirectiveList) -> {ok, NewDirective} | @@ -200,7 +200,7 @@ transfer_log(Info,RFC931,AuthUser,Date,StatusCode,Bytes) -> end. create_log(LogFile, ConfigList) -> - Filename = httpd_conf:clean(LogFile), + Filename = string:strip(LogFile), case filename:pathtype(Filename) of absolute -> case file:open(Filename, [read, write]) of diff --git a/lib/inets/src/http_server/mod_security.erl b/lib/inets/src/http_server/mod_security.erl index a85383a921..b09bd5e008 100644 --- a/lib/inets/src/http_server/mod_security.erl +++ b/lib/inets/src/http_server/mod_security.erl @@ -101,38 +101,38 @@ do(Info) -> end. load("<Directory " ++ Directory, []) -> - Dir = httpd_conf:custom_clean(Directory,"",">"), + Dir = string:strip(string:strip(Directory),right, $>), {ok, [{security_directory, {Dir, [{path, Dir}]}}]}; load(eof,[{security_directory, {Directory, _DirData}}|_]) -> {error, ?NICE("Premature end-of-file in "++Directory)}; load("SecurityDataFile " ++ FileName, [{security_directory, {Dir, DirData}}]) -> - File = httpd_conf:clean(FileName), + File = string:strip(FileName), {ok, [{security_directory, {Dir, [{data_file, File}|DirData]}}]}; load("SecurityCallbackModule " ++ ModuleName, [{security_directory, {Dir, DirData}}]) -> - Mod = list_to_atom(httpd_conf:clean(ModuleName)), + Mod = list_to_atom(string:strip(ModuleName)), {ok, [{security_directory, {Dir, [{callback_module, Mod}|DirData]}}]}; load("SecurityMaxRetries " ++ Retries, [{security_directory, {Dir, DirData}}]) -> load_return_int_tag("SecurityMaxRetries", max_retries, - httpd_conf:clean(Retries), Dir, DirData); + string:strip(Retries), Dir, DirData); load("SecurityBlockTime " ++ Time, [{security_directory, {Dir, DirData}}]) -> load_return_int_tag("SecurityBlockTime", block_time, - httpd_conf:clean(Time), Dir, DirData); + string:strip(Time), Dir, DirData); load("SecurityFailExpireTime " ++ Time, [{security_directory, {Dir, DirData}}]) -> load_return_int_tag("SecurityFailExpireTime", fail_expire_time, - httpd_conf:clean(Time), Dir, DirData); + string:strip(Time), Dir, DirData); load("SecurityAuthTimeout " ++ Time0, [{security_directory, {Dir, DirData}}]) -> - Time = httpd_conf:clean(Time0), + Time = string:strip(Time0), load_return_int_tag("SecurityAuthTimeout", auth_timeout, - httpd_conf:clean(Time), Dir, DirData); + string:strip(Time), Dir, DirData); load("AuthName " ++ Name0, [{security_directory, {Dir, DirData}}]) -> - Name = httpd_conf:clean(Name0), + Name = string:strip(Name0), {ok, [{security_directory, {Dir, [{auth_name, Name}|DirData]}}]}; load("</Directory>",[{security_directory, {Dir, DirData}}]) -> {ok, [], {security_directory, {Dir, DirData}}}. diff --git a/lib/kernel/doc/src/inet.xml b/lib/kernel/doc/src/inet.xml index 77a8caaaf6..4dd9e0e221 100644 --- a/lib/kernel/doc/src/inet.xml +++ b/lib/kernel/doc/src/inet.xml @@ -1037,6 +1037,36 @@ setcap cap_sys_admin,cap_sys_ptrace,cap_dac_read_search+epi beam.smp <marker id="option-sndbuf"></marker> </item> + <tag><c>{show_econnreset, Boolean}</c>(TCP/IP sockets)</tag> + <item> + <p>When this option is set to <c>false</c>, as it is by + default, an RST that is received from the TCP peer is treated + as a normal close (as though a FIN was sent). A caller + to <seealso marker="gen_tcp#recv/2">gen_tcp:recv/2</seealso> + will get <c>{error, closed}</c>. In active + mode the controlling process will receive a + <c>{tcp_close, Socket}</c> message, indicating that the + peer has closed the connection.</p> + <p>Setting this option to <c>true</c> will allow you to + distinguish between a connection that was closed normally, + and one which was aborted (intentionally or unintentionally) + by the TCP peer. A call to + <seealso marker="gen_tcp#recv/2">gen_tcp:recv/2</seealso> + will return <c>{error, econnreset}</c>. In + active mode, the controlling process will receive a + <c>{tcp_error, Socket, econnreset}</c> message + before the usual <c>{tcp_closed, Socket}</c>, as is + the case for any other socket error. Calls to + <seealso marker="gen_tcp#send/2">gen_tcp:send/2</seealso> + will also return <c>{error, econnreset}</c> when it + is detected that a TCP peer has sent an RST.</p> + <p>A connected socket returned from + <seealso marker="gen_tcp#accept/1">gen_tcp:accept/1</seealso> + will inherit the <c>show_econnreset</c> setting from the + listening socket.</p> + <marker id="option-show_econnreset"></marker> + </item> + <tag><c>{sndbuf, Size}</c></tag> <item> <p>The minimum size of the send buffer to use for the socket. diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index 65045666ec..580c070389 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -339,7 +339,8 @@ do_start(Flags) -> ok end, %% Quietly load native code for all modules loaded so far - load_native_code_for_all_loaded(), + Architecture = erlang:system_info(hipe_architecture), + load_native_code_for_all_loaded(Architecture), Ok2; Other -> Other @@ -554,9 +555,9 @@ has_ext(Ext, Extlen, File) -> %%% Silently load native code for all modules loaded so far. %%% --spec load_native_code_for_all_loaded() -> ok. -load_native_code_for_all_loaded() -> - Architecture = erlang:system_info(hipe_architecture), +load_native_code_for_all_loaded(undefined) -> + ok; +load_native_code_for_all_loaded(Architecture) -> try hipe_unified_loader:chunk_name(Architecture) of ChunkTag -> Loaded = all_loaded(), diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index 819554ce74..eecd26863a 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -324,12 +324,15 @@ handle_call({load_binary,Mod,File,Bin}, Caller, S) -> do_load_binary(Mod, File, Bin, Caller, S); handle_call({load_native_partial,Mod,Bin}, {_From,_Tag}, S) -> - Result = (catch hipe_unified_loader:load(Mod, Bin)), + Architecture = erlang:system_info(hipe_architecture), + Result = (catch hipe_unified_loader:load(Mod, Bin, Architecture)), Status = hipe_result_to_status(Result), {reply,Status,S}; handle_call({load_native_sticky,Mod,Bin,WholeModule}, {_From,_Tag}, S) -> - Result = (catch hipe_unified_loader:load_module(Mod, Bin, WholeModule)), + Architecture = erlang:system_info(hipe_architecture), + Result = (catch hipe_unified_loader:load_module(Mod, Bin, WholeModule, + Architecture)), Status = hipe_result_to_status(Result), {reply,Status,S}; @@ -1256,33 +1259,43 @@ try_load_module(File, Mod, Bin, {From,_}=Caller, St0) -> try_load_module_1(File, Mod, Bin, Caller, #state{moddb=Db}=St) -> case is_sticky(Mod, Db) of true -> %% Sticky file reject the load - error_msg("Can't load module that resides in sticky dir\n",[]), + error_msg("Can't load module '~w' that resides in sticky dir\n",[Mod]), {reply,{error,sticky_directory},St}; false -> - case catch load_native_code(Mod, Bin) of - {module,Mod} = Module -> - ets:insert(Db, {Mod,File}), - {reply,Module,St}; - no_native -> - case erlang:load_module(Mod, Bin) of - {module,Mod} = Module -> - ets:insert(Db, {Mod,File}), - post_beam_load(Mod), - {reply,Module,St}; - {error,on_load} -> - handle_on_load(Mod, File, Caller, St); - {error,What} = Error -> - error_msg("Loading of ~ts failed: ~p\n", [File, What]), - {reply,Error,St} - end; - Error -> - error_msg("Native loading of ~ts failed: ~p\n", - [File,Error]), - {reply,ok,St} - end + Architecture = erlang:system_info(hipe_architecture), + try_load_module_2(File, Mod, Bin, Caller, Architecture, St) + end. + +try_load_module_2(File, Mod, Bin, Caller, undefined, St) -> + try_load_module_3(File, Mod, Bin, Caller, undefined, St); +try_load_module_2(File, Mod, Bin, Caller, Architecture, + #state{moddb=Db}=St) -> + case catch load_native_code(Mod, Bin, Architecture) of + {module,Mod} = Module -> + ets:insert(Db, {Mod,File}), + {reply,Module,St}; + no_native -> + try_load_module_3(File, Mod, Bin, Caller, Architecture, St); + Error -> + error_msg("Native loading of ~ts failed: ~p\n", [File,Error]), + {reply,ok,St} + end. + +try_load_module_3(File, Mod, Bin, Caller, Architecture, + #state{moddb=Db}=St) -> + case erlang:load_module(Mod, Bin) of + {module,Mod} = Module -> + ets:insert(Db, {Mod,File}), + post_beam_load(Mod, Architecture), + {reply,Module,St}; + {error,on_load} -> + handle_on_load(Mod, File, Caller, St); + {error,What} = Error -> + error_msg("Loading of ~ts failed: ~p\n", [File, What]), + {reply,Error,St} end. -load_native_code(Mod, Bin) -> +load_native_code(Mod, Bin, Architecture) -> %% During bootstrapping of Open Source Erlang, we don't have any hipe %% loader modules, but the Erlang emulator might be hipe enabled. %% Therefore we must test for that the loader modules are available @@ -1291,7 +1304,8 @@ load_native_code(Mod, Bin) -> false -> no_native; true -> - Result = hipe_unified_loader:load_native_code(Mod, Bin), + Result = hipe_unified_loader:load_native_code(Mod, Bin, + Architecture), case Result of {module,_} -> put(?ANY_NATIVE_CODE_LOADED, true); @@ -1310,12 +1324,12 @@ hipe_result_to_status(Result) -> {error,Result} end. -post_beam_load(Mod) -> - %% post_beam_load/1 can potentially be very expensive because it +post_beam_load(Mod, Architecture) -> + %% post_beam_load/2 can potentially be very expensive because it %% blocks multi-scheduling; thus we want to avoid the call if we %% know that it is not needed. case get(?ANY_NATIVE_CODE_LOADED) of - true -> hipe_unified_loader:post_beam_load(Mod); + true -> hipe_unified_loader:post_beam_load(Mod, Architecture); false -> ok end. diff --git a/lib/kernel/src/gen_tcp.erl b/lib/kernel/src/gen_tcp.erl index bc8ffbe5e3..86251fc8f1 100644 --- a/lib/kernel/src/gen_tcp.erl +++ b/lib/kernel/src/gen_tcp.erl @@ -58,6 +58,7 @@ {reuseaddr, boolean()} | {send_timeout, non_neg_integer() | infinity} | {send_timeout_close, boolean()} | + {show_econnreset, boolean()} | {sndbuf, non_neg_integer()} | {tos, non_neg_integer()} | {ipv6_v6only, boolean()}. @@ -89,6 +90,7 @@ reuseaddr | send_timeout | send_timeout_close | + show_econnreset | sndbuf | tos | ipv6_v6only. diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl index 49d4a8fe54..ddbbc548dd 100644 --- a/lib/kernel/src/hipe_unified_loader.erl +++ b/lib/kernel/src/hipe_unified_loader.erl @@ -43,10 +43,10 @@ -export([chunk_name/1, %% Only the code and code_server modules may call the entries below! - load_native_code/2, - post_beam_load/1, - load_module/3, - load/2]). + load_native_code/3, + post_beam_load/2, + load_module/4, + load/3]). %%-define(DEBUG,true). -define(DO_ASSERT,true). @@ -82,58 +82,57 @@ chunk_name(Architecture) -> %% HW32 %% HiPE, x86, Win32 end. +word_size(Architecture) -> + case Architecture of + amd64 -> 8; + ppc64 -> 8; + _ -> 4 + end. + %%======================================================================== --spec load_native_code(Mod, binary()) -> 'no_native' | {'module', Mod} - when Mod :: atom(). +-spec load_native_code(Mod, binary(), hipe_architecture()) -> + 'no_native' | {'module', Mod} when Mod :: atom(). %% @doc %% Loads the native code of a module Mod. %% Returns {module,Mod} on success (for compatibility with %% code:load_file/1) and the atom `no_native' on failure. -load_native_code(Mod, Bin) when is_atom(Mod), is_binary(Bin) -> - Architecture = erlang:system_info(hipe_architecture), - try chunk_name(Architecture) of - ChunkTag -> - %% patch_to_emu(Mod), - case code:get_chunk(Bin, ChunkTag) of - undefined -> no_native; - NativeCode when is_binary(NativeCode) -> - erlang:system_flag(multi_scheduling, block), - try - OldReferencesToPatch = patch_to_emu_step1(Mod), - case load_module(Mod, NativeCode, Bin, OldReferencesToPatch) of - bad_crc -> no_native; - Result -> Result - end - after - erlang:system_flag(multi_scheduling, unblock) - end +load_native_code(_Mod, _Bin, undefined) -> + no_native; +load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) -> + %% patch_to_emu(Mod), + case code:get_chunk(Bin, chunk_name(Architecture)) of + undefined -> no_native; + NativeCode when is_binary(NativeCode) -> + erlang:system_flag(multi_scheduling, block), + try + OldReferencesToPatch = patch_to_emu_step1(Mod), + case load_module(Mod, NativeCode, Bin, OldReferencesToPatch, + Architecture) of + bad_crc -> no_native; + Result -> Result + end + after + erlang:system_flag(multi_scheduling, unblock) end - catch - _:_ -> - %% Unknown HiPE architecture. Can't happen (in principle). - no_native end. %%======================================================================== --spec post_beam_load(atom()) -> 'ok'. +-spec post_beam_load(atom(), hipe_architecture()) -> 'ok'. -post_beam_load(Mod) when is_atom(Mod) -> - Architecture = erlang:system_info(hipe_architecture), - try chunk_name(Architecture) of - _ChunkTag -> - erlang:system_flag(multi_scheduling, block), - try - patch_to_emu(Mod) - after - erlang:system_flag(multi_scheduling, unblock) - end - catch - _:_ -> - ok - end. +%% does nothing on a hipe-disabled system +post_beam_load(_Mod, undefined) -> + ok; +post_beam_load(Mod, _) when is_atom(Mod) -> + erlang:system_flag(multi_scheduling, block), + try + patch_to_emu(Mod) + after + erlang:system_flag(multi_scheduling, unblock) + end, + ok. %%======================================================================== @@ -148,46 +147,48 @@ version_check(Version, Mod) when is_atom(Mod) -> %%======================================================================== --spec load_module(Mod, binary(), _) -> 'bad_crc' | {'module', Mod} - when Mod :: atom(). -load_module(Mod, Bin, Beam) -> +-spec load_module(Mod, binary(), _, hipe_architecture()) -> + 'bad_crc' | {'module', Mod} when Mod :: atom(). + +load_module(Mod, Bin, Beam, Architecture) -> erlang:system_flag(multi_scheduling, block), try - load_module_nosmp(Mod, Bin, Beam) + load_module_nosmp(Mod, Bin, Beam, Architecture) after erlang:system_flag(multi_scheduling, unblock) end. -load_module_nosmp(Mod, Bin, Beam) -> - load_module(Mod, Bin, Beam, []). +load_module_nosmp(Mod, Bin, Beam, Architecture) -> + load_module(Mod, Bin, Beam, [], Architecture). -load_module(Mod, Bin, Beam, OldReferencesToPatch) -> +load_module(Mod, Bin, Beam, OldReferencesToPatch, Architecture) -> ?debug_msg("************ Loading Module ~w ************\n",[Mod]), %% Loading a whole module, let the BEAM loader patch closures. put(hipe_patch_closures, false), - load_common(Mod, Bin, Beam, OldReferencesToPatch). + load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture). %%======================================================================== --spec load(Mod, binary()) -> 'bad_crc' | {'module', Mod} when Mod :: atom(). +-spec load(Mod, binary(), hipe_architecture()) -> + 'bad_crc' | {'module', Mod} when Mod :: atom(). -load(Mod, Bin) -> +load(Mod, Bin, Architecture) -> erlang:system_flag(multi_scheduling, block), try - load_nosmp(Mod, Bin) + load_nosmp(Mod, Bin, Architecture) after erlang:system_flag(multi_scheduling, unblock) end. -load_nosmp(Mod, Bin) -> +load_nosmp(Mod, Bin, Architecture) -> ?debug_msg("********* Loading funs in module ~w *********\n",[Mod]), %% Loading just some functions in a module; patch closures separately. put(hipe_patch_closures, true), - load_common(Mod, Bin, [], []). + load_common(Mod, Bin, [], [], Architecture). %%------------------------------------------------------------------------ -load_common(Mod, Bin, Beam, OldReferencesToPatch) -> +load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) -> %% Unpack the binary. [{Version, CheckSum}, ConstAlign, ConstSize, ConstMap, LabelMap, ExportMap, @@ -212,18 +213,21 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> bad_crc; true -> put(closures_to_patch, []), + WordSize = word_size(Architecture), + WriteWord = write_word_fun(WordSize), %% Create data segment {ConstAddr,ConstMap2} = - create_data_segment(ConstAlign, ConstSize, ConstMap), + create_data_segment(ConstAlign, ConstSize, ConstMap, WriteWord), %% Find callees for which we may need trampolines. - CalleeMFAs = find_callee_mfas(Refs), + CalleeMFAs = find_callee_mfas(Refs, Architecture), %% Write the code to memory. {CodeAddress,Trampolines} = enter_code(CodeSize, CodeBinary, CalleeMFAs, Mod, Beam), %% Construct CalleeMFA-to-trampoline mapping. - TrampolineMap = mk_trampoline_map(CalleeMFAs, Trampolines), + TrampolineMap = mk_trampoline_map(CalleeMFAs, Trampolines, + Architecture), %% Patch references to code labels in data seg. - ok = patch_consts(LabelMap, ConstAddr, CodeAddress), + ok = patch_consts(LabelMap, ConstAddr, CodeAddress, WriteWord), %% Find out which functions are being loaded (and where). %% Note: Addresses are sorted descending. {MFAs,Addresses} = exports(ExportMap, CodeAddress), @@ -275,14 +279,26 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> %% Scan the list of patches and build a set (returned as a tuple) %% of the callees for which we may need trampolines. %% -find_callee_mfas(Patches) when is_list(Patches) -> - case erlang:system_info(hipe_architecture) of - amd64 -> []; - arm -> find_callee_mfas(Patches, gb_sets:empty(), false); - powerpc -> find_callee_mfas(Patches, gb_sets:empty(), true); - ppc64 -> find_callee_mfas(Patches, gb_sets:empty(), true); - ultrasparc -> []; - x86 -> [] +find_callee_mfas(Patches, Architecture) when is_list(Patches) -> + case needs_trampolines(Architecture) of + true -> find_callee_mfas(Patches, gb_sets:empty(), + no_erts_trampolines(Architecture)); + _ -> [] + end. + +needs_trampolines(Architecture) -> + case Architecture of + arm -> true; + powerpc -> true; + ppc64 -> true; + _ -> false + end. + +no_erts_trampolines(Architecture) -> + case Architecture of + powerpc -> true; + ppc64 -> true; + _ -> false end. find_callee_mfas([{Type,Data}|Patches], MFAs, SkipErtsSyms) -> @@ -318,14 +334,9 @@ add_callee_mfas([], MFAs, _SkipErtsSyms) -> MFAs. %%---------------------------------------------------------------- %% -mk_trampoline_map([], []) -> []; % archs not using trampolines -mk_trampoline_map(CalleeMFAs, Trampolines) -> - SizeofLong = - case erlang:system_info(hipe_architecture) of - amd64 -> 8; - ppc64 -> 8; - _ -> 4 - end, +mk_trampoline_map([], [], _) -> []; % archs not using trampolines +mk_trampoline_map(CalleeMFAs, Trampolines, Architecture) -> + SizeofLong = word_size(Architecture), mk_trampoline_map(tuple_size(CalleeMFAs), CalleeMFAs, Trampolines, SizeofLong, gb_trees:empty()). @@ -621,22 +632,24 @@ patch_load_mfa(CodeAddress, DestMFA, Addresses, RemoteOrLocal) -> %%---------------------------------------------------------------- %% Patch references to code labels in the data segment. %% -patch_consts(Labels, DataAddress, CodeAddress) -> +patch_consts(Labels, DataAddress, CodeAddress, WriteWord) -> lists:foreach(fun (L) -> - patch_label_or_labels(L, DataAddress, CodeAddress) + patch_label_or_labels(L, DataAddress, CodeAddress, + WriteWord) end, Labels). -patch_label_or_labels({Pos,Offset}, DataAddress, CodeAddress) -> +patch_label_or_labels({Pos,Offset}, DataAddress, CodeAddress, WriteWord) -> ?ASSERT(assert_local_patch(CodeAddress+Offset)), - write_word(DataAddress+Pos, CodeAddress+Offset); -patch_label_or_labels({sorted,Base,UnOrderdList}, DataAddress, CodeAddress) -> - sort_and_write(UnOrderdList, Base, DataAddress, CodeAddress). + WriteWord(DataAddress+Pos, CodeAddress+Offset); +patch_label_or_labels({sorted,Base,UnOrderdList}, DataAddress, CodeAddress, + WriteWord) -> + sort_and_write(UnOrderdList, Base, DataAddress, CodeAddress, WriteWord). -sort_and_write(UnOrderdList, Base, DataAddress, CodeAddress) -> +sort_and_write(UnOrderdList, Base, DataAddress, CodeAddress, WriteWord) -> WriteAndInc = fun ({_, Offset}, DataPos) -> ?ASSERT(assert_local_patch(CodeAddress+Offset)), - write_word(DataPos, CodeAddress+Offset) + WriteWord(DataPos, CodeAddress+Offset) end, lists:foldl(WriteAndInc, DataAddress+Base, sort_on_representation(UnOrderdList)). @@ -662,17 +675,18 @@ patch_instr(Address, Value, Type) -> %% XXX: It appears this is used for inserting both code addresses %% and other data. In HiPE, code addresses are still 32-bit on %% some 64-bit machines. -write_word(DataAddress, DataWord) -> - case erlang:system_info(hipe_architecture) of - amd64 -> - hipe_bifs:write_u64(DataAddress, DataWord), - DataAddress+8; - ppc64 -> - hipe_bifs:write_u64(DataAddress, DataWord), - DataAddress+8; - _ -> - hipe_bifs:write_u32(DataAddress, DataWord), - DataAddress+4 +write_word_fun(WordSize) -> + case WordSize of + 8 -> + fun (DataAddress, DataWord) -> + hipe_bifs:write_u64(DataAddress, DataWord), + DataAddress+8 + end; + 4 -> + fun (DataAddress, DataWord) -> + hipe_bifs:write_u32(DataAddress, DataWord), + DataAddress+4 + end end. %%-------------------------------------------------------------------- @@ -688,30 +702,31 @@ bif_address(Name) when is_atom(Name) -> %% memory, and produces a ConstMap2 mapping each constant's ConstNo to %% its runtime address, tagged if the constant is a term. %% -create_data_segment(DataAlign, DataSize, DataList) -> +create_data_segment(DataAlign, DataSize, DataList, WriteWord) -> %%io:format("create_data_segment: \nDataAlign: ~p\nDataSize: ~p\nDataList: ~p\n",[DataAlign,DataSize,DataList]), DataAddress = hipe_bifs:alloc_data(DataAlign, DataSize), - enter_data(DataList, [], DataAddress, DataSize). + enter_data(DataList, [], DataAddress, DataSize, WriteWord). -enter_data(List, ConstMap2, DataAddress, DataSize) -> +enter_data(List, ConstMap2, DataAddress, DataSize, WriteWord) -> case List of [ConstNo,Offset,Type,Data|Rest] when is_integer(Offset) -> %%?msg("Const ~w\n",[[ConstNo,Offset,Type,Data]]), ?ASSERT((Offset >= 0) and (Offset =< DataSize)), - Res = enter_datum(Type, Data, DataAddress+Offset), - enter_data(Rest, [{ConstNo,Res}|ConstMap2], DataAddress, DataSize); + Res = enter_datum(Type, Data, DataAddress+Offset, WriteWord), + enter_data(Rest, [{ConstNo,Res}|ConstMap2], DataAddress, DataSize, + WriteWord); [] -> {DataAddress, ConstMap2} end. -enter_datum(Type, Data, Address) -> +enter_datum(Type, Data, Address, WriteWord) -> case ?EXT2CONST_TYPE(Type) of term -> %% Address is unused for terms hipe_bifs:term_to_word(hipe_bifs:merge_term(Data)); sorted_block -> L = lists:sort([hipe_bifs:term_to_word(Term) || Term <- Data]), - write_words(L, Address), + write_words(L, Address, WriteWord), Address; block -> case Data of @@ -719,7 +734,7 @@ enter_datum(Type, Data, Address) -> write_bytes(Lbls, Address); {Lbls, SortOrder} -> SortedLbls = [Lbl || {_,Lbl} <- lists:sort(group(Lbls, SortOrder))], - write_words(SortedLbls, Address); + write_words(SortedLbls, Address, WriteWord); Lbls -> write_bytes(Lbls, Address) end, @@ -734,9 +749,9 @@ group([B1,B2,B3,B4|Ls], [O|Os]) -> bytes_to_32(B4,B3,B2,B1) -> (B4 bsl 24) bor (B3 bsl 16) bor (B2 bsl 8) bor B1. -write_words([W|Rest], Addr) -> - write_words(Rest, write_word(Addr, W)); -write_words([], Addr) when is_integer(Addr) -> true. +write_words([W|Rest], Addr, WriteWord) -> + write_words(Rest, WriteWord(Addr, W), WriteWord); +write_words([], Addr, _) when is_integer(Addr) -> true. write_bytes([B|Rest], Addr) -> hipe_bifs:write_u8(Addr, B), @@ -812,7 +827,7 @@ address_to_mfa_lth(_Address, [], Prev) -> %%---------------------------------------------------------------- %% Change callers of the given module to instead trap to BEAM. -%% load_native_code/2 calls this just before loading native code. +%% load_native_code/3 calls this just before loading native code. %% patch_to_emu(Mod) -> patch_to_emu_step2(patch_to_emu_step1(Mod)). diff --git a/lib/kernel/src/inet.erl b/lib/kernel/src/inet.erl index d668738109..1ae90aaf0c 100644 --- a/lib/kernel/src/inet.erl +++ b/lib/kernel/src/inet.erl @@ -654,7 +654,7 @@ options() -> multicast_if, multicast_ttl, multicast_loop, exit_on_close, high_watermark, low_watermark, high_msgq_watermark, low_msgq_watermark, - send_timeout, send_timeout_close + send_timeout, send_timeout_close, show_econnreset ]. %% Return a list of statistics options @@ -672,7 +672,8 @@ connect_options() -> [tos, priority, reuseaddr, keepalive, linger, sndbuf, recbuf, nodelay, header, active, packet, packet_size, buffer, mode, deliver, exit_on_close, high_watermark, low_watermark, high_msgq_watermark, - low_msgq_watermark, send_timeout, send_timeout_close, delay_send, raw]. + low_msgq_watermark, send_timeout, send_timeout_close, delay_send, raw, + show_econnreset]. connect_options(Opts, Family) -> BaseOpts = @@ -740,7 +741,7 @@ listen_options() -> header, active, packet, buffer, mode, deliver, backlog, ipv6_v6only, exit_on_close, high_watermark, low_watermark, high_msgq_watermark, low_msgq_watermark, send_timeout, send_timeout_close, delay_send, - packet_size, raw]. + packet_size, raw, show_econnreset]. listen_options(Opts, Family) -> BaseOpts = diff --git a/lib/kernel/src/inet_int.hrl b/lib/kernel/src/inet_int.hrl index 889b596a22..7bd973af99 100644 --- a/lib/kernel/src/inet_int.hrl +++ b/lib/kernel/src/inet_int.hrl @@ -147,6 +147,7 @@ -define(INET_LOPT_MSGQ_HIWTRMRK, 36). -define(INET_LOPT_MSGQ_LOWTRMRK, 37). -define(INET_LOPT_NETNS, 38). +-define(INET_LOPT_TCP_SHOW_ECONNRESET, 39). % Specific SCTP options: separate range: -define(SCTP_OPT_RTOINFO, 100). -define(SCTP_OPT_ASSOCINFO, 101). diff --git a/lib/kernel/src/rpc.erl b/lib/kernel/src/rpc.erl index 2300b7e901..d8250418f9 100644 --- a/lib/kernel/src/rpc.erl +++ b/lib/kernel/src/rpc.erl @@ -22,7 +22,7 @@ %% facility %% This code used to reside in net.erl, but has now been moved to -%% a searate module. +%% a separate module. -define(NAME, rex). diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index c82aaf0582..be55e25811 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -810,14 +810,6 @@ check_funs({'$M_EXPR','$F_EXPR',_}, {unicode,characters_to_binary,3}, {filename,filename_string_to_binary,1}|_]) -> 0; check_funs({'$M_EXPR','$F_EXPR',_}, - [{code_server,load_native_code,4}, - {code_server,load_native_code_1,2}, - {code_server,load_native_code,2}, - {code_server,try_load_module,4}, - {code_server,do_load_binary,4}, - {code_server,handle_call,3}, - {code_server,loop,1}|_]) -> 0; -check_funs({'$M_EXPR','$F_EXPR',_}, [{code_server,do_mod_call,4}, {code_server,handle_call,3}|_]) -> 0; check_funs({'$M_EXPR','$F_EXPR',_}, @@ -866,8 +858,14 @@ check_funs({'$M_EXPR','$F_EXPR',_}, check_funs({'$M_EXPR',module_info,1}, [{hipe_unified_loader,patch_to_emu_step1,1} | _]) -> 0; check_funs({'$M_EXPR','$F_EXPR',2}, + [{hipe_unified_loader,write_words,3} | _]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',2}, + [{hipe_unified_loader,patch_label_or_labels,4} | _]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',2}, + [{hipe_unified_loader,sort_and_write,5} | _]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',2}, [{lists,foldl,3}, - {hipe_unified_loader,sort_and_write,4} | _]) -> 0; + {hipe_unified_loader,sort_and_write,5} | _]) -> 0; check_funs({'$M_EXPR','$F_EXPR',1}, [{lists,foreach,2}, {hipe_unified_loader,patch_consts,3} | _]) -> 0; diff --git a/lib/kernel/test/error_logger_warn_SUITE.erl b/lib/kernel/test/error_logger_warn_SUITE.erl index fb576d77a3..16e0286593 100644 --- a/lib/kernel/test/error_logger_warn_SUITE.erl +++ b/lib/kernel/test/error_logger_warn_SUITE.erl @@ -69,7 +69,7 @@ end_per_group(_GroupName, Config) -> init_per_testcase(_Case, Config) -> - ?line Dog = ?t:timetrap(?default_timeout), + Dog = ?t:timetrap(?default_timeout), [{watchdog, Dog} | Config]. end_per_testcase(_Case, Config) -> Dog = ?config(watchdog, Config), @@ -184,83 +184,81 @@ nn() -> basic() -> - ?line Node = start_node(nn(),[]), - ?line ok = install_relay(Node), - ?line Self = self(), - ?line GL = group_leader(), - ?line warning = warning_map(Node), - ?line format(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), - ?line error_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), - ?line warning_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{warning_msg,GL,{_,"~p~n",[Self]}}}), - ?line info_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{info_msg,GL,{_,"~p~n",[Self]}}}), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line error_report(Node,Report), - ?line ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), - ?line warning_report(Node,Report), - ?line ?EXPECT({handle_event,{warning_report,GL,{_,std_warning,Report}}}), - ?line info_report(Node,Report), - ?line ?EXPECT({handle_event,{info_report,GL,{_,std_info,Report}}}), - - ?line stop_node(Node), + Node = start_node(nn(),[]), + ok = install_relay(Node), + Self = self(), + GL = group_leader(), + warning = warning_map(Node), + format(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), + error_msg(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), + warning_msg(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{warning_msg,GL,{_,"~p~n",[Self]}}}), + info_msg(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{info_msg,GL,{_,"~p~n",[Self]}}}), + Report = [{self,Self},{gl,GL},make_ref()], + error_report(Node,Report), + ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), + warning_report(Node,Report), + ?EXPECT({handle_event,{warning_report,GL,{_,std_warning,Report}}}), + info_report(Node,Report), + ?EXPECT({handle_event,{info_report,GL,{_,std_info,Report}}}), + stop_node(Node), ok. warnings_info() -> - ?line Node = start_node(nn(),"+Wi"), - ?line ok = install_relay(Node), - ?line Self = self(), - ?line GL = group_leader(), - ?line info = warning_map(Node), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line warning_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{info_msg,GL,{_,"~p~n",[Self]}}}), - ?line warning_report(Node,Report), - ?line ?EXPECT({handle_event,{info_report,GL,{_,std_info,Report}}}), - ?line stop_node(Node), + Node = start_node(nn(),"+Wi"), + ok = install_relay(Node), + Self = self(), + GL = group_leader(), + info = warning_map(Node), + Report = [{self,Self},{gl,GL},make_ref()], + warning_msg(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{info_msg,GL,{_,"~p~n",[Self]}}}), + warning_report(Node,Report), + ?EXPECT({handle_event,{info_report,GL,{_,std_info,Report}}}), + stop_node(Node), ok. warnings_errors() -> - ?line Node = start_node(nn(),"+We"), - ?line ok = install_relay(Node), - ?line Self = self(), - ?line GL = group_leader(), - ?line error = warning_map(Node), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line warning_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), - ?line warning_report(Node,Report), - ?line ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), - ?line stop_node(Node), + Node = start_node(nn(),"+We"), + ok = install_relay(Node), + Self = self(), + GL = group_leader(), + error = warning_map(Node), + Report = [{self,Self},{gl,GL},make_ref()], + warning_msg(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), + warning_report(Node,Report), + ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), + stop_node(Node), ok. - - + % RB... quote(String) -> case os:type() of - {win32,_} -> - "\\\""++String++"\\\""; - _ -> - "'\""++String++"\"'" + {win32,_} -> + "\\\""++String++"\\\""; + _ -> + "'\""++String++"\"'" end. iquote(String) -> case os:type() of - {win32,_} -> - "\\\""++String++"\\\""; - _ -> - "\""++String++"\"" + {win32,_} -> + "\\\""++String++"\\\""; + _ -> + "\""++String++"\"" end. oquote(String) -> case os:type() of - {win32,_} -> - "\""++String++"\""; - _ -> - "'"++String++"'" + {win32,_} -> + "\""++String++"\""; + _ -> + "'"++String++"'" end. @@ -270,18 +268,18 @@ findstr(String,FileName) -> findstrc(String,File) -> case string:str(File,String) of - N when is_integer(N), - N > 0 -> - S2 = lists:sublist(File,N,length(File)), - case string:str(S2,"\n") of - 0 -> - 1; - M -> - S3 = lists:sublist(S2,M,length(S2)), - 1 + findstrc(String,S3) - end; - _ -> - 0 + N when is_integer(N), + N > 0 -> + S2 = lists:sublist(File,N,length(File)), + case string:str(S2,"\n") of + 0 -> + 1; + M -> + S3 = lists:sublist(S2,M,length(S2)), + 1 + findstrc(String,S3) + end; + _ -> + 0 end. % Doesn't count empty lines @@ -355,182 +353,182 @@ one_rb_findstr(Param,String) -> % Tests rb_basic() -> - ?line clean_rd(), + clean_rd(), % Behold, the magic parameters to activate rb logging... - ?line Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ - quote(rd())++" error_logger_mf_maxbytes 5000 " - "error_logger_mf_maxfiles 5"), - ?line Self = self(), - ?line GL = group_leader(), - ?line warning = warning_map(Node), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_report,Report), - ?line nice_stop_node(Node), - ?line application:start(sasl), - ?line rb:start([{report_dir, rd()}]), - ?line rb:list(), - ?line true = (one_rb_lines([error]) =:= 0), - ?line true = (one_rb_lines([error_report]) =:= 0), - ?line 0 = one_rb_findstr([error],pid_to_list(Self)), - ?line 0 = one_rb_findstr([error_report],pid_to_list(Self)), - ?line 1 = one_rb_findstr([warning_msg],pid_to_list(Self)), - ?line 1 = one_rb_findstr([warning_report],pid_to_list(Self)), - ?line 0 = one_rb_findstr([info_msg],pid_to_list(Self)), - ?line 0 = one_rb_findstr([info_report],pid_to_list(Self)), - ?line 2 = one_rb_findstr([],pid_to_list(Self)), - ?line true = (one_rb_findstr([progress],"===") > 4), - ?line rb:stop(), - ?line application:stop(sasl), - ?line stop_node(Node), + Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ + quote(rd())++" error_logger_mf_maxbytes 5000 " + "error_logger_mf_maxfiles 5"), + Self = self(), + GL = group_leader(), + warning = warning_map(Node), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,warning_report,Report), + nice_stop_node(Node), + application:start(sasl), + rb:start([{report_dir, rd()}]), + rb:list(), + true = (one_rb_lines([error]) =:= 0), + true = (one_rb_lines([error_report]) =:= 0), + 0 = one_rb_findstr([error],pid_to_list(Self)), + 0 = one_rb_findstr([error_report],pid_to_list(Self)), + 1 = one_rb_findstr([warning_msg],pid_to_list(Self)), + 1 = one_rb_findstr([warning_report],pid_to_list(Self)), + 0 = one_rb_findstr([info_msg],pid_to_list(Self)), + 0 = one_rb_findstr([info_report],pid_to_list(Self)), + 2 = one_rb_findstr([],pid_to_list(Self)), + true = (one_rb_findstr([progress],"===") > 4), + rb:stop(), + application:stop(sasl), + stop_node(Node), ok. rb_warnings_info() -> - ?line clean_rd(), - ?line Node = start_node(nn(),"+W i -boot start_sasl -sasl error_logger_mf_dir "++ - quote(rd())++" error_logger_mf_maxbytes 5000 " - "error_logger_mf_maxfiles 5"), - ?line Self = self(), - ?line GL = group_leader(), - ?line info = warning_map(Node), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_report,Report), - ?line nice_stop_node(Node), - ?line application:start(sasl), - ?line rb:start([{report_dir, rd()}]), - ?line rb:list(), - ?line true = (one_rb_lines([error]) =:= 0), - ?line true = (one_rb_lines([error_report]) =:= 0), - ?line 0 = one_rb_findstr([error],pid_to_list(Self)), - ?line 0 = one_rb_findstr([error_report],pid_to_list(Self)), - ?line 0 = one_rb_findstr([warning_msg],pid_to_list(Self)), - ?line 0 = one_rb_findstr([warning_report],pid_to_list(Self)), - ?line 1 = one_rb_findstr([info_msg],pid_to_list(Self)), - ?line 1 = one_rb_findstr([info_report],pid_to_list(Self)), - ?line 2 = one_rb_findstr([],pid_to_list(Self)), - ?line true = (one_rb_findstr([progress],"===") > 4), - ?line rb:stop(), - ?line application:stop(sasl), - ?line stop_node(Node), + clean_rd(), + Node = start_node(nn(),"+W i -boot start_sasl -sasl error_logger_mf_dir "++ + quote(rd())++" error_logger_mf_maxbytes 5000 " + "error_logger_mf_maxfiles 5"), + Self = self(), + GL = group_leader(), + info = warning_map(Node), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,warning_report,Report), + nice_stop_node(Node), + application:start(sasl), + rb:start([{report_dir, rd()}]), + rb:list(), + true = (one_rb_lines([error]) =:= 0), + true = (one_rb_lines([error_report]) =:= 0), + 0 = one_rb_findstr([error],pid_to_list(Self)), + 0 = one_rb_findstr([error_report],pid_to_list(Self)), + 0 = one_rb_findstr([warning_msg],pid_to_list(Self)), + 0 = one_rb_findstr([warning_report],pid_to_list(Self)), + 1 = one_rb_findstr([info_msg],pid_to_list(Self)), + 1 = one_rb_findstr([info_report],pid_to_list(Self)), + 2 = one_rb_findstr([],pid_to_list(Self)), + true = (one_rb_findstr([progress],"===") > 4), + rb:stop(), + application:stop(sasl), + stop_node(Node), ok. rb_warnings_errors() -> - ?line clean_rd(), - ?line Node = start_node(nn(),"+W e -boot start_sasl -sasl error_logger_mf_dir "++ - quote(rd())++" error_logger_mf_maxbytes 5000 " - "error_logger_mf_maxfiles 5"), - ?line Self = self(), - ?line GL = group_leader(), - ?line error = warning_map(Node), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_report,Report), - ?line nice_stop_node(Node), - ?line application:start(sasl), - ?line rb:start([{report_dir, rd()}]), - ?line rb:list(), - ?line true = (one_rb_lines([error]) > 1), - ?line true = (one_rb_lines([error_report]) > 1), - ?line 1 = one_rb_findstr([error],pid_to_list(Self)), - ?line 1 = one_rb_findstr([error_report],pid_to_list(Self)), - ?line 0 = one_rb_findstr([warning_msg],pid_to_list(Self)), - ?line 0 = one_rb_findstr([warning_report],pid_to_list(Self)), - ?line 0 = one_rb_findstr([info_msg],pid_to_list(Self)), - ?line 0 = one_rb_findstr([info_report],pid_to_list(Self)), - ?line 2 = one_rb_findstr([],pid_to_list(Self)), - ?line true = (one_rb_findstr([progress],"===") > 4), - ?line rb:stop(), - ?line application:stop(sasl), - ?line stop_node(Node), + clean_rd(), + Node = start_node(nn(),"+W e -boot start_sasl -sasl error_logger_mf_dir "++ + quote(rd())++" error_logger_mf_maxbytes 5000 " + "error_logger_mf_maxfiles 5"), + Self = self(), + GL = group_leader(), + error = warning_map(Node), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,warning_report,Report), + nice_stop_node(Node), + application:start(sasl), + rb:start([{report_dir, rd()}]), + rb:list(), + true = (one_rb_lines([error]) > 1), + true = (one_rb_lines([error_report]) > 1), + 1 = one_rb_findstr([error],pid_to_list(Self)), + 1 = one_rb_findstr([error_report],pid_to_list(Self)), + 0 = one_rb_findstr([warning_msg],pid_to_list(Self)), + 0 = one_rb_findstr([warning_report],pid_to_list(Self)), + 0 = one_rb_findstr([info_msg],pid_to_list(Self)), + 0 = one_rb_findstr([info_report],pid_to_list(Self)), + 2 = one_rb_findstr([],pid_to_list(Self)), + true = (one_rb_findstr([progress],"===") > 4), + rb:stop(), + application:stop(sasl), + stop_node(Node), ok. rb_trunc() -> - ?line clean_rd(), - ?line Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ - quote(rd())++" error_logger_mf_maxbytes 5000 " - "error_logger_mf_maxfiles 5"), - ?line Self = self(), - ?line GL = group_leader(), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_report,Report), - ?line nice_stop_node(Node), - ?line application:start(sasl), - ?line {ok,File} = file:read_file(rf()), - ?line S=byte_size(File)-2, - ?line <<TFile:S/binary,_/binary>>=File, - ?line file:write_file(rf(),TFile), - ?line rb:start([{report_dir, rd()}]), - ?line rb:list(), - ?line true = (one_rb_lines([error]) =:= 0), - ?line true = (one_rb_lines([error_report]) =:= 0), - ?line 0 = one_rb_findstr([error],pid_to_list(Self)), - ?line 0 = one_rb_findstr([error_report],pid_to_list(Self)), - ?line 1 = one_rb_findstr([warning_msg],pid_to_list(Self)), - ?line 0 = one_rb_findstr([warning_report],pid_to_list(Self)), - ?line 0 = one_rb_findstr([info_msg],pid_to_list(Self)), - ?line 0 = one_rb_findstr([info_report],pid_to_list(Self)), - ?line 1 = one_rb_findstr([],pid_to_list(Self)), - ?line true = (one_rb_findstr([progress],"===") > 4), - ?line rb:stop(), - ?line application:stop(sasl), - ?line stop_node(Node), + clean_rd(), + Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ + quote(rd())++" error_logger_mf_maxbytes 5000 " + "error_logger_mf_maxfiles 5"), + Self = self(), + GL = group_leader(), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,warning_report,Report), + nice_stop_node(Node), + application:start(sasl), + {ok,File} = file:read_file(rf()), + S=byte_size(File)-2, + <<TFile:S/binary,_/binary>>=File, + file:write_file(rf(),TFile), + rb:start([{report_dir, rd()}]), + rb:list(), + true = (one_rb_lines([error]) =:= 0), + true = (one_rb_lines([error_report]) =:= 0), + 0 = one_rb_findstr([error],pid_to_list(Self)), + 0 = one_rb_findstr([error_report],pid_to_list(Self)), + 1 = one_rb_findstr([warning_msg],pid_to_list(Self)), + 0 = one_rb_findstr([warning_report],pid_to_list(Self)), + 0 = one_rb_findstr([info_msg],pid_to_list(Self)), + 0 = one_rb_findstr([info_report],pid_to_list(Self)), + 1 = one_rb_findstr([],pid_to_list(Self)), + true = (one_rb_findstr([progress],"===") > 4), + rb:stop(), + application:stop(sasl), + stop_node(Node), ok. rb_utc() -> - ?line clean_rd(), - ?line Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ - quote(rd())++" error_logger_mf_maxbytes 5000 " - "error_logger_mf_maxfiles 5 -sasl utc_log true"), - ?line Self = self(), - ?line GL = group_leader(), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_report,Report), - ?line nice_stop_node(Node), - ?line application:stop(sasl), - ?line UtcLog=case application:get_env(sasl,utc_log) of - {ok,true} -> - true; - _AllOthers -> - application:set_env(sasl,utc_log,true), - false - end, - ?line application:start(sasl), - ?line rb:start([{report_dir, rd()}]), - ?line rb:list(), - ?line Pr=one_rb_findstr([progress],"==="), - ?line Wm=one_rb_findstr([warning_msg],"==="), - ?line Wr=one_rb_findstr([warning_report],"==="), - ?line Sum=Pr+Wm+Wr, - ?line Sum=one_rb_findstr([],"UTC"), - ?line rb:stop(), - ?line application:stop(sasl), - ?line application:set_env(sasl,utc_log,UtcLog), - ?line stop_node(Node), + clean_rd(), + Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ + quote(rd())++" error_logger_mf_maxbytes 5000 " + "error_logger_mf_maxfiles 5 -sasl utc_log true"), + Self = self(), + GL = group_leader(), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,warning_report,Report), + nice_stop_node(Node), + application:stop(sasl), + UtcLog=case application:get_env(sasl,utc_log) of + {ok,true} -> + true; + _AllOthers -> + application:set_env(sasl,utc_log,true), + false + end, + application:start(sasl), + rb:start([{report_dir, rd()}]), + rb:list(), + Pr=one_rb_findstr([progress],"==="), + Wm=one_rb_findstr([warning_msg],"==="), + Wr=one_rb_findstr([warning_report],"==="), + Sum=Pr+Wm+Wr, + Sum=one_rb_findstr([],"UTC"), + rb:stop(), + application:stop(sasl), + application:set_env(sasl,utc_log,UtcLog), + stop_node(Node), ok. file_utc() -> - ?line file:delete(lf()), - ?line SS="-stdlib utc_log true -kernel error_logger "++ oquote("{file,"++iquote(lf())++"}"), + file:delete(lf()), + SS="-stdlib utc_log true -kernel error_logger "++ oquote("{file,"++iquote(lf())++"}"), %erlang:display(SS), - ?line Node = start_node(nn(),SS), + Node = start_node(nn(),SS), %erlang:display(rpc:call(Node,application,get_env,[kernel,error_logger])), - ?line Self = self(), - ?line GL = group_leader(), - ?line fake_gl(Node,error_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,info_msg,"~p~n",[Self]), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,error_report,Report), - ?line fake_gl(Node,warning_report,Report), - ?line fake_gl(Node,info_report,Report), - ?line nice_stop_node(Node), - ?line receive after 5000 -> ok end, % Let the node die, needed - ?line 6 = findstr("UTC",lf()), - ?line 2 = findstr("WARNING",lf()), - ?line 2 = findstr("ERROR",lf()), - ?line 2 = findstr("INFO",lf()), - ?line stop_node(Node), + Self = self(), + GL = group_leader(), + fake_gl(Node,error_msg,"~p~n",[Self]), + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,info_msg,"~p~n",[Self]), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,error_report,Report), + fake_gl(Node,warning_report,Report), + fake_gl(Node,info_report,Report), + nice_stop_node(Node), + receive after 5000 -> ok end, % Let the node die, needed + 6 = findstr("UTC",lf()), + 2 = findstr("WARNING",lf()), + 2 = findstr("ERROR",lf()), + 2 = findstr("INFO",lf()), + stop_node(Node), ok. diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index 4f0d7a7d50..44e7984401 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -31,6 +31,11 @@ init_per_testcase/2, end_per_testcase/2, otp_3924/1, otp_3924_sender/4, closed_socket/1, shutdown_active/1, shutdown_passive/1, shutdown_pending/1, + show_econnreset_active/1, show_econnreset_active_once/1, + show_econnreset_passive/1, econnreset_after_sync_send/1, + econnreset_after_async_send_active/1, + econnreset_after_async_send_active_once/1, + econnreset_after_async_send_passive/1, linger_zero/1, default_options/1, http_bad_packet/1, busy_send/1, busy_disconnect_passive/1, busy_disconnect_active/1, fill_sendq/1, partial_recv_and_close/1, @@ -38,7 +43,9 @@ % Accept tests primitive_accept/1,multi_accept_close_listen/1,accept_timeout/1, accept_timeouts_in_order/1,accept_timeouts_in_order2/1, - accept_timeouts_in_order3/1,accept_timeouts_mixed/1, + accept_timeouts_in_order3/1,accept_timeouts_in_order4/1, + accept_timeouts_in_order5/1,accept_timeouts_in_order6/1, + accept_timeouts_in_order7/1,accept_timeouts_mixed/1, killing_acceptor/1,killing_multi_acceptors/1,killing_multi_acceptors2/1, several_accepts_in_one_go/1, accept_system_limit/1, active_once_closed/1, send_timeout/1, send_timeout_active/1, @@ -92,6 +99,11 @@ all() -> iter_max_socks, passive_sockets, active_n, accept_closed_by_other_process, otp_3924, closed_socket, shutdown_active, shutdown_passive, shutdown_pending, + show_econnreset_active, show_econnreset_active_once, + show_econnreset_passive, econnreset_after_sync_send, + econnreset_after_async_send_active, + econnreset_after_async_send_active_once, + econnreset_after_async_send_passive, linger_zero, default_options, http_bad_packet, busy_send, busy_disconnect_passive, busy_disconnect_active, fill_sendq, partial_recv_and_close, @@ -99,7 +111,9 @@ all() -> so_priority, primitive_accept, multi_accept_close_listen, accept_timeout, accept_timeouts_in_order, accept_timeouts_in_order2, - accept_timeouts_in_order3, accept_timeouts_mixed, + accept_timeouts_in_order3, accept_timeouts_in_order4, + accept_timeouts_in_order5, accept_timeouts_in_order6, + accept_timeouts_in_order7, accept_timeouts_mixed, killing_acceptor, killing_multi_acceptors, killing_multi_acceptors2, several_accepts_in_one_go, accept_system_limit, active_once_closed, send_timeout, send_timeout_active, otp_7731, @@ -1075,6 +1089,311 @@ shutdown_pending(Config) when is_list(Config) -> gen_tcp:close(S) end. +%% +%% Test 'show_econnreset' option +%% + +show_econnreset_active(Config) when is_list(Config) -> + %% First confirm everything works with option turned off. + {ok, L} = gen_tcp:listen(0, []), + {ok, Port} = inet:port(L), + {ok, Client} = gen_tcp:connect(localhost, Port, [{active, false}]), + {ok, S} = gen_tcp:accept(L), + ok = gen_tcp:close(L), + ok = inet:setopts(Client, [{linger, {true, 0}}]), + ok = gen_tcp:close(Client), + receive + {tcp_closed, S} -> + ok; + Other -> + ?t:fail({unexpected1, Other}) + after 1000 -> + ?t:fail({timeout, {server, no_tcp_closed}}) + end, + + %% Now test with option switched on. + %% Note: We are also testing that the show_econnreset option is + %% inherited from the listening socket by the accepting socket. + {ok, L1} = gen_tcp:listen(0, [{show_econnreset, true}]), + {ok, Port1} = inet:port(L1), + {ok, Client1} = gen_tcp:connect(localhost, Port1, [{active, false}]), + {ok, S1} = gen_tcp:accept(L1), + ok = gen_tcp:close(L1), + ok = inet:setopts(Client1, [{linger, {true, 0}}]), + ok = gen_tcp:close(Client1), + receive + {tcp_error, S1, econnreset} -> + receive + {tcp_closed, S1} -> + ok; + Other1 -> + ?t:fail({unexpected2, Other1}) + after 1 -> + ?t:fail({timeout, {server, no_tcp_closed}}) + end; + Other2 -> + ?t:fail({unexpected3, Other2}) + after 1000 -> + ?t:fail({timeout, {server, no_tcp_error}}) + end. + +show_econnreset_active_once(Config) when is_list(Config) -> + %% Now test using {active, once} + {ok, L} = gen_tcp:listen(0, + [{active, false}, + {show_econnreset, true}]), + {ok, Port} = inet:port(L), + {ok, Client} = gen_tcp:connect(localhost, Port, [{active, false}]), + {ok, S} = gen_tcp:accept(L), + ok = gen_tcp:close(L), + ok = inet:setopts(Client, [{linger, {true, 0}}]), + ok = gen_tcp:close(Client), + ok = ?t:sleep(20), + ok = receive Msg -> {unexpected_msg, Msg} after 0 -> ok end, + ok = inet:setopts(S, [{active, once}]), + receive + {tcp_error, S, econnreset} -> + receive + {tcp_closed, S} -> + ok; + Other1 -> + ?t:fail({unexpected1, Other1}) + after 1 -> + ?t:fail({timeout, {server, no_tcp_closed}}) + end; + Other2 -> + ?t:fail({unexpected2, Other2}) + after 1000 -> + ?t:fail({timeout, {server, no_tcp_error}}) + end. + +show_econnreset_passive(Config) when is_list(Config) -> + %% First confirm everything works with option turned off. + {ok, L} = gen_tcp:listen(0, [{active, false}]), + {ok, Port} = inet:port(L), + {ok, Client} = gen_tcp:connect(localhost, Port, [{active, false}]), + {ok, S} = gen_tcp:accept(L), + ok = gen_tcp:close(L), + ok = inet:setopts(S, [{linger, {true, 0}}]), + ok = gen_tcp:close(S), + ok = ?t:sleep(1), + {error, closed} = gen_tcp:recv(Client, 0), + + %% Now test with option switched on. + {ok, L1} = gen_tcp:listen(0, [{active, false}]), + {ok, Port1} = inet:port(L1), + {ok, Client1} = gen_tcp:connect(localhost, Port1, + [{active, false}, + {show_econnreset, true}]), + {ok, S1} = gen_tcp:accept(L1), + ok = gen_tcp:close(L1), + ok = inet:setopts(S1, [{linger, {true, 0}}]), + ok = gen_tcp:close(S1), + ok = ?t:sleep(1), + {error, econnreset} = gen_tcp:recv(Client1, 0). + +econnreset_after_sync_send(Config) when is_list(Config) -> + %% First confirm everything works with option turned off. + {ok, L} = gen_tcp:listen(0, [{active, false}]), + {ok, Port} = inet:port(L), + {ok, Client} = gen_tcp:connect(localhost, Port, [{active, false}]), + {ok, S} = gen_tcp:accept(L), + ok = gen_tcp:close(L), + ok = inet:setopts(S, [{linger, {true, 0}}]), + ok = gen_tcp:close(S), + ok = ?t:sleep(20), + {error, closed} = gen_tcp:send(Client, "Whatever"), + + %% Now test with option switched on. + {ok, L1} = gen_tcp:listen(0, [{active, false}]), + {ok, Port1} = inet:port(L1), + {ok, Client1} = gen_tcp:connect(localhost, Port1, + [{active, false}, + {show_econnreset, true}]), + {ok, S1} = gen_tcp:accept(L1), + ok = gen_tcp:close(L1), + ok = inet:setopts(S1, [{linger, {true, 0}}]), + ok = gen_tcp:close(S1), + ok = ?t:sleep(20), + {error, econnreset} = gen_tcp:send(Client1, "Whatever"). + +econnreset_after_async_send_active(Config) when is_list(Config) -> + {OS, _} = os:type(), + Payload = lists:duplicate(1024 * 1024, $.), + + %% First confirm everything works with option turned off. + {ok, L} = gen_tcp:listen(0, [{active, false}, {recbuf, 4096}]), + {ok, Port} = inet:port(L), + {ok, Client} = gen_tcp:connect(localhost, Port, [{sndbuf, 4096}]), + {ok, S} = gen_tcp:accept(L), + ok = gen_tcp:close(L), + ok = gen_tcp:send(Client, Payload), + case erlang:port_info(Client, queue_size) of + {queue_size, N} when N > 0 -> ok; + {queue_size, 0} when OS =:= win32 -> ok; + {queue_size, 0} = T -> ?t:fail(T) + end, + ok = gen_tcp:send(S, "Whatever"), + ok = ?t:sleep(20), + ok = inet:setopts(S, [{linger, {true, 0}}]), + ok = gen_tcp:close(S), + ok = ?t:sleep(20), + receive + {tcp, Client, "Whatever"} -> + receive + {tcp_closed, Client} -> + ok; + Other1 -> + ?t:fail({unexpected1, Other1}) + end; + Other2 -> + ?t:fail({unexpected2, Other2}) + end, + + %% Now test with option switched on. + {ok, L1} = gen_tcp:listen(0, [{active, false}, {recbuf, 4096}]), + {ok, Port1} = inet:port(L1), + {ok, Client1} = gen_tcp:connect(localhost, Port1, + [{sndbuf, 4096}, + {show_econnreset, true}]), + {ok, S1} = gen_tcp:accept(L1), + ok = gen_tcp:close(L1), + ok = gen_tcp:send(Client1, Payload), + case erlang:port_info(Client1, queue_size) of + {queue_size, N1} when N1 > 0 -> ok; + {queue_size, 0} when OS =:= win32 -> ok; + {queue_size, 0} = T1 -> ?t:fail(T1) + end, + ok = gen_tcp:send(S1, "Whatever"), + ok = ?t:sleep(20), + ok = inet:setopts(S1, [{linger, {true, 0}}]), + ok = gen_tcp:close(S1), + ok = ?t:sleep(20), + receive + {tcp, Client1, "Whatever"} -> + receive + {tcp_error, Client1, econnreset} -> + receive + {tcp_closed, Client1} -> + ok; + Other3 -> + ?t:fail({unexpected3, Other3}) + end; + Other4 -> + ?t:fail({unexpected4, Other4}) + end; + Other5 -> + ?t:fail({unexpected5, Other5}) + end. + +econnreset_after_async_send_active_once(Config) when is_list(Config) -> + {OS, _} = os:type(), + {ok, L} = gen_tcp:listen(0, [{active, false}, {recbuf, 4096}]), + {ok, Port} = inet:port(L), + {ok, Client} = gen_tcp:connect(localhost, Port, + [{active, false}, + {sndbuf, 4096}, + {show_econnreset, true}]), + {ok,S} = gen_tcp:accept(L), + ok = gen_tcp:close(L), + Payload = lists:duplicate(1024 * 1024, $.), + ok = gen_tcp:send(Client, Payload), + case erlang:port_info(Client, queue_size) of + {queue_size, N} when N > 0 -> ok; + {queue_size, 0} when OS =:= win32 -> ok; + {queue_size, 0} = T -> ?t:fail(T) + end, + ok = gen_tcp:send(S, "Whatever"), + ok = ?t:sleep(20), + ok = inet:setopts(S, [{linger, {true, 0}}]), + ok = gen_tcp:close(S), + ok = ?t:sleep(20), + ok = receive Msg -> {unexpected_msg, Msg} after 0 -> ok end, + ok = inet:setopts(Client, [{active, once}]), + receive + {tcp_error, Client, econnreset} -> + receive + {tcp_closed, Client} -> + ok; + Other -> + ?t:fail({unexpected1, Other}) + end; + Other -> + ?t:fail({unexpected2, Other}) + end. + +econnreset_after_async_send_passive(Config) when is_list(Config) -> + {OS, _} = os:type(), + Payload = lists:duplicate(1024 * 1024, $.), + + %% First confirm everything works with option turned off. + {ok, L} = gen_tcp:listen(0, [{active, false}, {recbuf, 4096}]), + {ok, Port} = inet:port(L), + {ok, Client} = gen_tcp:connect(localhost, Port, + [{active, false}, + {sndbuf, 4096}]), + {ok, S} = gen_tcp:accept(L), + ok = gen_tcp:close(L), + ok = inet:setopts(S, [{linger, {true, 0}}]), + ok = gen_tcp:send(S, "Whatever"), + ok = gen_tcp:send(Client, Payload), + case erlang:port_info(Client, queue_size) of + {queue_size, N} when N > 0 -> ok; + {queue_size, 0} when OS =:= win32 -> ok; + {queue_size, 0} = T -> ?t:fail(T) + end, + ok = gen_tcp:close(S), + ok = ?t:sleep(20), + {error, closed} = gen_tcp:recv(Client, 0), + + %% Now test with option switched on. + {ok, L1} = gen_tcp:listen(0, [{active, false}, {recbuf, 4096}]), + {ok, Port1} = inet:port(L1), + {ok, Client1} = gen_tcp:connect(localhost, Port1, + [{active, false}, + {sndbuf, 4096}, + {show_econnreset, true}]), + {ok, S1} = gen_tcp:accept(L1), + ok = gen_tcp:close(L1), + ok = inet:setopts(S1, [{linger, {true, 0}}]), + ok = gen_tcp:send(S1, "Whatever"), + ok = gen_tcp:send(Client1, Payload), + ok = gen_tcp:close(S1), + ok = ?t:sleep(20), + {error, econnreset} = gen_tcp:recv(Client1, 0). + +%% +%% Test {linger {true, 0}} aborts a connection +%% + +linger_zero(Config) when is_list(Config) -> + %% All the econnreset tests will prove that {linger, {true, 0}} aborts + %% a connection when the driver queue is empty. We will test here + %% that it also works when the driver queue is not empty. + {OS, _} = os:type(), + {ok, L} = gen_tcp:listen(0, [{active, false}, + {recbuf, 4096}, + {show_econnreset, true}]), + {ok, Port} = inet:port(L), + {ok, Client} = gen_tcp:connect(localhost, Port, + [{active, false}, + {sndbuf, 4096}]), + {ok, S} = gen_tcp:accept(L), + ok = gen_tcp:close(L), + PayloadSize = 1024 * 1024, + Payload = lists:duplicate(PayloadSize, $.), + ok = gen_tcp:send(Client, Payload), + case erlang:port_info(Client, queue_size) of + {queue_size, N} when N > 0 -> ok; + {queue_size, 0} when OS =:= win32 -> ok; + {queue_size, 0} = T -> ?t:fail(T) + end, + ok = inet:setopts(Client, [{linger, {true, 0}}]), + ok = gen_tcp:close(Client), + ok = ?t:sleep(1), + undefined = erlang:port_info(Client, connected), + {error, econnreset} = gen_tcp:recv(S, PayloadSize). + %% Thanks to Luke Gorrie. Tests for a very specific problem with %% corrupt data. The testcase will be killed by the timetrap timeout @@ -1720,8 +2039,8 @@ multi_accept_close_listen(Config) when is_list(Config) -> spawn(F), spawn(F), gen_tcp:close(LS), - ?EXPECT_ACCEPTS([{_,{error,closed}},{_,{error,closed}}, - {_,{error,closed}},{_,{error,closed}}],4,500). + ok = ?EXPECT_ACCEPTS([{_,{error,closed}},{_,{error,closed}}, + {_,{error,closed}},{_,{error,closed}}],4,500). accept_timeout(suite) -> []; @@ -1732,7 +2051,7 @@ accept_timeout(Config) when is_list(Config) -> Parent = self(), F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS,1000)} end, P = spawn(F), - ?EXPECT_ACCEPTS([{P,{error,timeout}}],1,2000). + ok = ?EXPECT_ACCEPTS([{P,{error,timeout}}],1,2000). accept_timeouts_in_order(suite) -> []; @@ -1745,8 +2064,8 @@ accept_timeouts_in_order(Config) when is_list(Config) -> P2 = spawn(mktmofun(1200,Parent,LS)), P3 = spawn(mktmofun(1300,Parent,LS)), P4 = spawn(mktmofun(1400,Parent,LS)), - ?EXPECT_ACCEPTS([{P1,{error,timeout}},{P2,{error,timeout}}, - {P3,{error,timeout}},{P4,{error,timeout}}],infinity,2000). + ok = ?EXPECT_ACCEPTS([{P1,{error,timeout}},{P2,{error,timeout}}, + {P3,{error,timeout}},{P4,{error,timeout}}],infinity,2000). accept_timeouts_in_order2(suite) -> []; @@ -1759,8 +2078,8 @@ accept_timeouts_in_order2(Config) when is_list(Config) -> P2 = spawn(mktmofun(1300,Parent,LS)), P3 = spawn(mktmofun(1200,Parent,LS)), P4 = spawn(mktmofun(1000,Parent,LS)), - ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P3,{error,timeout}}, - {P2,{error,timeout}},{P1,{error,timeout}}],infinity,2000). + ok = ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P3,{error,timeout}}, + {P2,{error,timeout}},{P1,{error,timeout}}],infinity,2000). accept_timeouts_in_order3(suite) -> []; @@ -1773,8 +2092,74 @@ accept_timeouts_in_order3(Config) when is_list(Config) -> P2 = spawn(mktmofun(1400,Parent,LS)), P3 = spawn(mktmofun(1300,Parent,LS)), P4 = spawn(mktmofun(1000,Parent,LS)), - ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P1,{error,timeout}}, - {P3,{error,timeout}},{P2,{error,timeout}}],infinity,2000). + ok = ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P1,{error,timeout}}, + {P3,{error,timeout}},{P2,{error,timeout}}],infinity,2000). + +accept_timeouts_in_order4(suite) -> + []; +accept_timeouts_in_order4(doc) -> + ["Check that multi-accept timeouts happen in the correct order after " + "mixing millsec and sec timeouts"]; +accept_timeouts_in_order4(Config) when is_list(Config) -> + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(200,Parent,LS)), + P2 = spawn(mktmofun(400,Parent,LS)), + P3 = spawn(mktmofun(1000,Parent,LS)), + P4 = spawn(mktmofun(600,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P1,{error,timeout}},{P2,{error,timeout}}, + {P4,{error,timeout}},{P3,{error,timeout}}],infinity,2000). + +accept_timeouts_in_order5(suite) -> + []; +accept_timeouts_in_order5(doc) -> + ["Check that multi-accept timeouts happen in the correct order after " + "mixing millsec and sec timeouts (more)"]; +accept_timeouts_in_order5(Config) when is_list(Config) -> + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(400,Parent,LS)), + P2 = spawn(mktmofun(1000,Parent,LS)), + P3 = spawn(mktmofun(600,Parent,LS)), + P4 = spawn(mktmofun(200,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P1,{error,timeout}}, + {P3,{error,timeout}},{P2,{error,timeout}}],infinity,2000). + +accept_timeouts_in_order6(suite) -> + []; +accept_timeouts_in_order6(doc) -> + ["Check that multi-accept timeouts happen in the correct order after " + "mixing millsec and sec timeouts (even more)"]; +accept_timeouts_in_order6(Config) when is_list(Config) -> + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(1000,Parent,LS)), + P2 = spawn(mktmofun(400,Parent,LS)), + P3 = spawn(mktmofun(600,Parent,LS)), + P4 = spawn(mktmofun(200,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P2,{error,timeout}}, + {P3,{error,timeout}},{P1,{error,timeout}}],infinity,2000). + +accept_timeouts_in_order7(suite) -> + []; +accept_timeouts_in_order7(doc) -> + ["Check that multi-accept timeouts happen in the correct order after " + "mixing millsec and sec timeouts (even more++)"]; +accept_timeouts_in_order7(Config) when is_list(Config) -> + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(1000,Parent,LS)), + P2 = spawn(mktmofun(200,Parent,LS)), + P3 = spawn(mktmofun(1200,Parent,LS)), + P4 = spawn(mktmofun(600,Parent,LS)), + P5 = spawn(mktmofun(400,Parent,LS)), + P6 = spawn(mktmofun(800,Parent,LS)), + P7 = spawn(mktmofun(1600,Parent,LS)), + P8 = spawn(mktmofun(1400,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P2,{error,timeout}},{P5,{error,timeout}}, + {P4,{error,timeout}},{P6,{error,timeout}}, + {P1,{error,timeout}},{P3,{error,timeout}}, + {P8,{error,timeout}},{P7,{error,timeout}}],infinity,2000). accept_timeouts_mixed(suite) -> []; @@ -1797,7 +2182,7 @@ accept_timeouts_mixed(Config) when is_list(Config) -> ok = ?EXPECT_ACCEPTS([{P2,{ok,Port0}}] when is_port(Port0),infinity,100), ok = ?EXPECT_ACCEPTS([{P3,{error,timeout}}],infinity,2000), gen_tcp:connect("localhost",PortNo,[]), - ?EXPECT_ACCEPTS([{P4,{ok,Port1}}] when is_port(Port1),infinity,100). + ok = ?EXPECT_ACCEPTS([{P4,{ok,Port1}}] when is_port(Port1),infinity,100). killing_acceptor(suite) -> []; diff --git a/lib/mnesia/doc/src/Makefile b/lib/mnesia/doc/src/Makefile index 6a72b98ebc..f41442f739 100644 --- a/lib/mnesia/doc/src/Makefile +++ b/lib/mnesia/doc/src/Makefile @@ -58,7 +58,6 @@ XML_CHAPTER_FILES = \ Mnesia_App_A.xml \ Mnesia_App_B.xml \ Mnesia_App_C.xml \ - Mnesia_App_D.xml \ notes.xml BOOK_FILES = book.xml diff --git a/lib/mnesia/doc/src/Mnesia_App_A.xml b/lib/mnesia/doc/src/Mnesia_App_A.xml deleted file mode 100644 index 62dbffa14a..0000000000 --- a/lib/mnesia/doc/src/Mnesia_App_A.xml +++ /dev/null @@ -1,87 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>1997</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - The contents of this file are subject to the Erlang Public License, - Version 1.1, (the "License"); you may not use this file except in - compliance with the License. You should have received a copy of the - Erlang Public License along with this software. If not, it can be - retrieved online at http://www.erlang.org/. - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - the License for the specific language governing rights and limitations - under the License. - - </legalnotice> - - <title>Appendix A: Mnesia Error Messages</title> - <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> - <responsible>Bjarne Däcker</responsible> - <docno></docno> - <approved>Bjarne Däcker</approved> - <checked>Bjarne Däcker</checked> - <date>96-11-20</date> - <rev>B</rev> - <file>Mnesia_App_A.xml</file> - </header> - <p>Whenever an operation returns an error in Mnesia, a description - of the error is available. For example, the functions - <c>mnesia:transaction(Fun)</c>, or <c>mnesia:create_table(N,L)</c> - may return the tuple <c>{aborted, Reason}</c>, where <c>Reason</c> - is a term describing the error. The following function is used to - retrieve more detailed information about the error: - </p> - <list type="bulleted"> - <item><c>mnesia:error_description(Error)</c></item> - </list> - - <section> - <title>Errors in Mnesia</title> - <p>The following is a list of valid errors in Mnesia.</p> - <list type="bulleted"> - <item><c>badarg</c>. Bad or invalid argument, possibly bad type. - </item> - <item><c>no_transaction</c>. Operation not allowed outside transactions. - </item> - <item><c>combine_error</c>. Table options were illegally combined. - </item> - <item><c>bad_index</c>. Index already exists, or was out of bounds. - </item> - <item><c>already_exists</c>. Schema option to be activated is already on. - </item> - <item><c>index_exists</c>. Some operations cannot be performed on tables with an index. - </item> - <item><c>no_exists</c>.; Tried to perform operation on non-existing (non-alive) item. - </item> - <item><c>system_limit</c>.; A system limit was exhausted. - </item> - <item><c>mnesia_down</c>. A transaction involves records on a - remote node which became unavailable before the transaction - was completed. Record(s) are no longer available elsewhere in - the network.</item> - <item><c>not_a_db_node</c>. A node was mentioned which does not exist in the schema.</item> - <item><c>bad_type</c>.; Bad type specified in argument.</item> - <item><c>node_not_running</c>. Node is not running.</item> - <item><c>truncated_binary_file</c>. Truncated binary in file.</item> - <item><c>active</c>. Some delete operations require that all active records are removed.</item> - <item><c>illegal</c>. Operation not supported on this record.</item> - </list> - <p>The following example illustrates a function which returns an error, and the method to retrieve more detailed error information. - </p> - <p>The function <c>mnesia:create_table(bar, [{attributes, 3.14}])</c> will return the tuple <c>{aborted,Reason}</c>, where <c>Reason</c> is the tuple - <c>{bad_type,bar,3.14000}</c>. - </p> - <p>The function <c>mnesia:error_description(Reason)</c>, returns the term - <c>{"Bad type on some provided arguments",bar,3.14000}</c> which is an error - description suitable - for display.</p> - </section> -</chapter> - diff --git a/lib/mnesia/doc/src/Mnesia_App_D.xmlsrc b/lib/mnesia/doc/src/Mnesia_App_A.xmlsrc index b7a4c270ad..21366766d8 100644 --- a/lib/mnesia/doc/src/Mnesia_App_D.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_App_A.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2002</year><year>2013</year> + <year>1997</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,31 +13,28 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> - <title>Appendix D: The Fragmented Table Hashing Call Back Interface</title> - <prepared>Håkan Mattsson</prepared> - <responsible></responsible> + <title>Appendix A: Backup Callback Interface</title> + <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> + <responsible>Bjarne Däcker</responsible> <docno></docno> - <approved></approved> - <checked></checked> - <date></date> - <rev></rev> - <file>Mnesia_App_D.xml</file> + <approved>Bjarne Däcker</approved> + <checked>Bjarne Däcker</checked> + <date>97-05-27</date> + <rev>C</rev> + <file>Mnesia_App_A.xml</file> </header> <section> - <title>mnesia_frag_hash callback behavior</title> + <title>mnesia_backup Callback Behavior</title> <p></p> - <codeinclude file="../../src/mnesia_frag_hash.erl" tag="%header_doc_include" type="erl"></codeinclude> - <p></p> - <codeinclude file="../../src/mnesia_frag_hash.erl" tag="%impl_doc_include" type="erl"></codeinclude> + <codeinclude file="../../src/mnesia_backup.erl" tag="%0" type="erl"></codeinclude> </section> </chapter> - diff --git a/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc b/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc index f02e424ca4..9f65888190 100644 --- a/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1998</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -21,21 +21,23 @@ </legalnotice> - <title>Appendix B: The Backup Call Back Interface</title> - <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> - <responsible>Bjarne Däcker</responsible> + <title>Appendix B: Activity Access Callback Interface</title> + <prepared>Håkan Mattsson</prepared> + <responsible></responsible> <docno></docno> - <approved>Bjarne Däcker</approved> - <checked>Bjarne Däcker</checked> - <date>97-05-27</date> - <rev>C</rev> + <approved></approved> + <checked></checked> + <date></date> + <rev></rev> <file>Mnesia_App_B.xml</file> </header> <section> - <title>mnesia_backup callback behavior</title> + <title>mnesia_access Callback Behavior</title> <p></p> - <codeinclude file="../../src/mnesia_backup.erl" tag="%0" type="erl"></codeinclude> + <codeinclude file="../../src/mnesia_frag.erl" tag="%header_doc_include" type="erl"></codeinclude> + <p></p> + <codeinclude file="../../src/mnesia_frag.erl" tag="%impl_doc_include" type="erl"></codeinclude> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc b/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc index f7fefa36c4..eb7cc0431f 100644 --- a/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1998</year><year>2013</year> + <year>2002</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -21,7 +21,7 @@ </legalnotice> - <title>Appendix C: The Activity Access Call Back Interface</title> + <title>Appendix C: Fragmented Table Hashing Callback Interface</title> <prepared>Håkan Mattsson</prepared> <responsible></responsible> <docno></docno> @@ -33,11 +33,11 @@ </header> <section> - <title>mnesia_access callback behavior</title> + <title>mnesia_frag_hash Callback Behavior</title> <p></p> - <codeinclude file="../../src/mnesia_frag.erl" tag="%header_doc_include" type="erl"></codeinclude> + <codeinclude file="../../src/mnesia_frag_hash.erl" tag="%header_doc_include" type="erl"></codeinclude> <p></p> - <codeinclude file="../../src/mnesia_frag.erl" tag="%impl_doc_include" type="erl"></codeinclude> + <codeinclude file="../../src/mnesia_frag_hash.erl" tag="%impl_doc_include" type="erl"></codeinclude> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap1.xml b/lib/mnesia/doc/src/Mnesia_chap1.xml index 540008cdc5..0e9b769557 100644 --- a/lib/mnesia/doc/src/Mnesia_chap1.xml +++ b/lib/mnesia/doc/src/Mnesia_chap1.xml @@ -31,235 +31,82 @@ <rev>C</rev> <file>Mnesia_chap1.xml</file> </header> - <p>This book describes the Mnesia DataBase Management - System (DBMS). <em>Mnesia</em> is a distributed Database Management - System, appropriate for telecommunications applications and other - Erlang applications which require continuous operation and soft - real-time properties. It is one section of the Open Telecom Platform - (OTP), which is a control system platform for building - telecommunications applications.</p> - - <section> - <title>About Mnesia</title> - <p>The management of data in telecommunications system has many - aspects whereof some, but not all, are addressed by traditional - commercial DBMSs (Data Base Management Systems). In particular the - very high level of fault tolerance which is required in many nonstop - systems, combined with requirements on the DBMS to run in the same - address space as the application, have led us to implement a brand new - DBMS. called Mnesia. Mnesia is implemented in, and very tightly - connected to, the programming language Erlang and it provides the - functionality that is necessary for the implementation of fault - tolerant telecommunications systems. Mnesia is a multiuser Distributed - DBMS specially made for industrial telecommunications applications - written in the symbolic programming language Erlang, which is also - the intended target language. Mnesia tries to address all of the data - management issues required for typical telecommunications systems and - it has a number of features that are not normally found in traditional - databases. <br></br> - - In telecommunications applications there are different needs - from the features provided by traditional DBMSs. The applications now - implemented in the Erlang language need a mixture of a broad range - of features, which generally are not satisfied by traditional DBMSs. - Mnesia is designed with requirements like the following in mind:</p> - <list type="ordered"> - <item>Fast real-time key/value lookup</item> - <item>Complicated non real-time queries mainly for - operation and maintenance</item> - <item>Distributed data due to distributed - applications</item> - <item>High fault tolerance</item> - <item>Dynamic re-configuration</item> - <item>Complex objects</item> - </list> - <p>What - sets Mnesia apart from most other DBMSs is that it is designed with - the typical data management problems of telecommunications applications - in mind. Hence Mnesia combines many concepts found in traditional - databases, such as transactions and queries with concepts found in data - management systems for telecommunications applications, such as very - fast real-time operations, configurable degree of fault tolerance (by - means of replication) and the ability to reconfigure the system without - stopping or suspending it. Mnesia is also interesting due to its tight - coupling to the programming language Erlang, thus almost turning Erlang - into a database programming language. This has many benefits, the - foremost is that - the impedance mismatch between data format used by the - DBMS and data format used by the programming language, which is used - to manipulate the data, completely disappears. <br></br> -</p> - </section> - - <section> - <title>The Mnesia DataBase Management System (DBMS)</title> - <p></p> + <p>The <c>Mnesia</c> application provides a heavy duty real-time + distributed database.</p> <section> - <title>Features</title> - <p>Mnesia contains the following features which combine to produce a fault-tolerant, - distributed database management system written in Erlang: - </p> + <title>Scope</title> + <p>This User's Guide describes how to + build <c>Mnesia</c> database applications, and how to integrate + and use the <c>Mnesia</c> database management system with + OTP. Programming constructs are described, and numerous + programming examples are included to illustrate the use of + <c>Mnesia</c>.</p> + <p>This User's Guide is organized as follows:</p> <list type="bulleted"> - <item>Database schema can be dynamically reconfigured at runtime. + <item><seealso marker="Mnesia_overview">Mnesia</seealso> + provides an introduction to + <c>Mnesia</c>. + </item> + <item><seealso marker="Mnesia_chap2">Getting Started</seealso> + introduces <c>Mnesia</c> with an example database. Examples + are included how to start an Erlang session, specify a + <c>Mnesia</c> database directory, initialize a database + schema, start <c>Mnesia</c>, and create tables. Initial + prototyping of record definitions is also discussed. + </item> + <item><seealso marker="Mnesia_chap3">Build a Mnesia + Database</seealso> more formally describes the steps + introduced in the previous section, namely the <c>Mnesia</c> + functions that define a database schema, start <c>Mnesia</c>, + and create the required tables. + </item> + <item><seealso marker="Mnesia_chap4">Transactions and Other Access Contexts</seealso> + describes the transactions properties that make <c>Mnesia</c> into + a fault tolerant, real-time distributed database management + system. This section also describes the concept of locking + to ensure consistency in tables, and "dirty + operations", or short cuts, which bypass the transaction system + to improve speed and reduce overheads. </item> - <item>Tables can be declared to have properties such as location, - replication, and persistence. + <item><seealso marker="Mnesia_chap5">Miscellaneous Mnesia + Features</seealso> describes features that enable the + construction of more complex database applications. These + features include indexing, checkpoints, distribution and fault + tolerance, disc-less nodes, replication manipulation, local + content tables, concurrency, and object-based programming in + <c>Mnesia</c>. </item> - <item>Tables can be moved or replicated to several nodes to improve - fault tolerance. The rest of the system can still access the tables - to read, write, and delete records. + <item><seealso marker="Mnesia_chap7">Mnesia System + Information</seealso> describes the files contained in the + <c>Mnesia</c> database directory, database configuration data, + core and table dumps, as well as the important subject of + backup, fall-back, and disaster recovery principles. </item> - <item>Table locations are transparent to the programmer. - Programs address table names and the system itself keeps track of - table locations. + <item><seealso marker="Mnesia_chap8">Combine Mnesia with + SNMP</seealso> is a short section that outlines <c>Mnesia</c> + integrated with SNMP. </item> - <item>Database transactions can be distributed, and a large number of - functions can be called within one transaction. + <item><seealso marker="Mnesia_App_A">Appendix A: Backup + Callback Interface</seealso> is a program listing of the + default implementation of this facility. </item> - <item>Several transactions can run concurrently, and their execution is - fully synchronized by the database management system. - Mnesia ensures that no two processes manipulate data simultaneously. + <item><seealso marker="Mnesia_App_B">Appendix B: Activity + Access Callback Interface</seealso> is a program outlining + one possible implementation of this facility. </item> - <item>Transactions can be assigned the property of being executed on - all nodes in the system, or on none. Transactions can also be bypassed - in favor of running so called "dirty operations", which reduce - overheads and run very fast. + <item><seealso marker="Mnesia_App_C">Appendix C: Fragmented + Table Hashing Callback Interface</seealso> is a program + outlining one possible implementation of this facility. </item> </list> - <p>Details of these features are described in the following sections.</p> - </section> - <p></p> - - <section> - <title>Add-on Applications</title> - <p>QLC and Mnesia Session can be used in conjunction with Mnesia to produce - specialized functions which enhance the operational ability of Mnesia. - Both Mnesia Session and QLC have their own documentation as part - of the OTP documentation set. Below are the main features of Mnesia Session - and QLC when used in conjunction with Mnesia:</p> - <list type="bulleted"> - <item><em>QLC</em> has the ability to optimize the query - compiler for the Mnesia Database Management System, essentially making - the DBMS more efficient.</item> - <item><em>QLC</em>, can be used as a database programming - language for Mnesia. It includes a notation called "list - comprehensions" and can be used to make complex database - queries over a set of tables.</item> - <item><em>Mnesia Session</em> is an interface for the Mnesia Database - Management System</item> - <item><em>Mnesia Session</em> enables access to the - Mnesia DBMS from foreign programming languages (i.e. other - languages than Erlang).</item> - </list> - <p></p> - - <section> - <title>When to Use Mnesia</title> - <p>Use Mnesia with the following types of applications: - </p> - <list type="bulleted"> - <item>Applications that need to replicate data. - </item> - <item>Applications that perform complicated searches on data. - </item> - <item>Applications that need to use atomic transactions to - update several records simultaneously. - </item> - <item>Applications that use soft real-time characteristics. - </item> - </list> - <p>On the other hand, Mnesia may not be appropriate with the - following types of applications: - </p> - <list type="bulleted"> - <item>Programs that process plain text or binary data files - </item> - <item>Applications that merely need a look-up dictionary - which can be stored to disc can utilize the standard - library module <c>dets</c>, which is a disc based version - of the module <c>ets</c>. - </item> - <item>Applications which need disc logging facilities can - utilize the module <c>disc_log</c> by preference. - </item> - <item>Not suitable for hard real time systems. - </item> - </list> - </section> - </section> - - <section> - <title>Scope and Purpose</title> - <p>This manual is included in the OTP document set. It describes - how to build Mnesia database applications, and how to integrate - and utilize the Mnesia database management system with - OTP. Programming constructs are described, and numerous - programming examples are included to illustrate the use of - Mnesia. - </p> </section> <section> <title>Prerequisites</title> - <p>Readers of this manual are assumed to be familiar with system - development principles and database management systems. Readers - are also assumed to be familiar with the Erlang programming - language.</p> - </section> - - <section> - <title>About This Book</title> - <p>This book contains the following chapters: - </p> - <list type="bulleted"> - <item>Chapter 2, "Getting Started with Mnesia", introduces - Mnesia with an example database. Examples are shown of how to - start an Erlang session, specify a Mnesia database directory, - initialize a database schema, start Mnesia, and create - tables. Initial prototyping of record definitions is also - discussed. - </item> - <item>Chapter 3, "Building a Mnesia Database", more formally - describes the steps introduced in Chapter 2, namely the Mnesia - functions which define a database schema, start Mnesia, and - create the required tables. - </item> - <item>Chapter 4, "Transactions and other access contexts", - describes the transactions properties which make Mnesia into a - fault tolerant, real-time distributed database management - system. This chapter also describes the concept of locking in - order to ensure consistency in tables, and so called "dirty - operations", or short cuts which bypass the transaction system - to improve speed and reduce overheads. - </item> - <item>Chapter 5, "Miscellaneous Mnesia Features", describes - features which enable the construction of more complex - database applications. These features includes indexing, - checkpoints, distribution and fault tolerance, disc-less - nodes, replication manipulation, local content tables, concurrency, - and object based programming in Mnesia. - </item> - <item>Chapter 6, "Mnesia System Information", describes the - files contained in the Mnesia database directory, database - configuration data, core and table dumps, as well as the - important subject of backup, fall-back, and disaster recovery - principles. - </item> - <item>Chapter 7, "Combining Mnesia with SNMP", is a short - chapter which outlines Mnesia integrated with SNMP. - </item> - <item>Appendix A, "Mnesia Errors Messages", lists Mnesia error - messages and their meanings. - </item> - <item>Appendix B, "The Backup Call Back Interface", is a - program listing of the default implementation of this facility. - </item> - <item>Appendix C, "The Activity Access Call Back Interface", - is a program outlining of one possible implementations of this facility. - </item> - </list> + <p>It is assumed that the reader is familiar with the Erlang + programming language, system development principles, and + database management systems.</p> </section> - </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc index f464135a89..14999228a8 100644 --- a/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc @@ -21,7 +21,7 @@ </legalnotice> - <title>Getting Started with Mnesia</title> + <title>Getting Started</title> <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> <responsible></responsible> <docno></docno> @@ -31,27 +31,31 @@ <rev>C</rev> <file>Mnesia_chap2.xml</file> </header> - <p>This chapter introduces Mnesia. Following a brief discussion - about the first initial setup, a Mnesia database example is - demonstrated. This database example will be referenced in the - following chapters, where this example is modified in order to - illustrate various program constructs. In this chapter, the - following mandatory procedures are illustrated by examples: - </p> + <marker id="getting_started"></marker> + <p>This section introduces <c>Mnesia</c> with an example database. + This example is referenced in the + following sections, where the example is modified to + illustrate various program constructs. This section illustrates + the following mandatory procedures through examples:</p> <list type="bulleted"> - <item>Starting an Erlang session and specifying a directory for the - Mnesia database. + <item>Starting the Erlang session. </item> - <item>Initializing a database schema. + <item>Specifying the <c>Mnesia</c> directory where the database + is to be stored. + </item> + <item>Initializing a new database schema with an attribute that + specifies on which node, or nodes, that database is to operate. + </item> + <item>Starting <c>Mnesia</c>. + </item> + <item>Creating and populating the database tables. </item> - <item>Starting Mnesia and creating the required tables.</item> </list> <section> - <title>Starting Mnesia for the first time</title> - <p>Following is a simplified demonstration of a Mnesia system startup. This is the dialogue from the Erlang - shell: - </p> + <title>Starting Mnesia for the First Time</title> + <p>This section provides a simplified demonstration of a <c>Mnesia</c> + system startup. The dialogue from the Erlang shell is as follows:</p> <pre><![CDATA[ unix> erl -mnesia dir '"/tmp/funky"' Erlang (BEAM) emulator version 4.9 @@ -89,79 +93,62 @@ 0 transactions waits for other nodes: [] ok ]]></pre> - <p>In the example above the following actions were performed: - </p> + <p>In this example, the following actions are performed:</p> <list type="bulleted"> - <item>The Erlang system was started from the UNIX prompt - with a flag <c>-mnesia dir '"/tmp/funky"'</c>. This flag indicates - to Mnesia which directory will store the data. + <item><em>Step 1:</em> The Erlang system is started from the UNIX + prompt with a flag <c>-mnesia dir '"/tmp/funky"'</c>, which indicates + in which directory to store the data. </item> - <item>A new empty schema was initialized on the local node by evaluating - <c>mnesia:create_schema([node()]).</c> The schema contains - information about the database in general. This will be - thoroughly explained later on. + <item><em>Step 2:</em> A new empty schema is initialized on the local + node by evaluating + <seealso marker="mnesia#create_schema/1">mnesia:create_schema([node()])</seealso>. + The schema contains information about the database in general. + This is explained in detail later. </item> - <item>The DBMS was started by evaluating <c>mnesia:start()</c>. + <item><em>Step 3:</em> The DBMS is started by evaluating + <seealso marker="mnesia#start/0">mnesia:start()</seealso>. </item> - <item>A first table was created, called <c>funky</c> by evaluating - the expression <c>mnesia:create_table(funky, [])</c>. The table - was given default properties. + <item><em>Step 4:</em> A first table is created, called <c>funky</c>, + by evaluating the expression <c>mnesia:create_table(funky, [])</c>. + The table is given default properties. </item> - <item><c>mnesia:info()</c> was evaluated and subsequently displayed - information regarding the status of the database on the terminal. + <item><em>Step 5:</em> <seealso marker="mnesia#info/0">mnesia:info()</seealso> + is evaluated to + display information on the terminal about the status of the database. </item> </list> </section> <section> - <title>An Introductory Example</title> - <p>A Mnesia database is organized as a set of tables. + <title>Example</title> + <p>A <c>Mnesia</c> database is organized as a set of tables. Each table is populated with instances (Erlang records). - A table also has a number of properties, such as location and - persistence. - </p> - <p>In this example we shall: - </p> - <list type="bulleted"> - <item>Start an Erlang system, and specify the directory where - the database will be located. - </item> - <item>Initiate a new schema with an attribute that specifies - on which node, or nodes, the database will operate. - </item> - <item>Start Mnesia itself. - </item> - <item>Create and populate the database tables. - </item> - </list> + A table has also a number of properties, such as location and + persistence.</p> <section> - <title>The Example Database</title> + <title>Database</title> </section> - <p>In this database example, we will create the database and - relationships depicted in the following diagram. We will call this - database the <em>Company</em> database. - </p> + <p>This example shows how to create a database called <c>Company</c> + and the relationships shown in the following diagram:</p> <image file="company.gif"> <icaption>Company Entity-Relation Diagram</icaption> </image> - <p>The database model looks as follows: - </p> + <p>The database model is as follows:</p> <list type="bulleted"> - <item>There are three entities: employee, project, and - department. + <item>There are three entities: department, employee, and project. </item> <item> <p>There are three relationships between these entities:</p> <list type="ordered"> - <item>A department is managed by an employee, hence the - <em>manager</em> relationship. + <item>A department is managed by an employee, + hence the <c>manager</c> relationship. </item> <item>An employee works at a department, hence the - <em>at_dep</em> relationship. + <c>at_dep</c> relationship. </item> - <item>Each employee works on a number of projects, hence - the <em>in_proj</em> relationship. + <item>Each employee works on a number of projects, + hence the <c>in_proj</c> relationship. </item> </list> </item> @@ -169,35 +156,32 @@ <section> <title>Defining Structure and Content</title> - <p>We first enter our record definitions into a text file + <p>First the record definitions are entered into a text file named <c>company.hrl</c>. This file defines the following - structure for our sample database: - </p> + structure for the example database: </p> <codeinclude file="company.hrl" tag="%0" type="erl"></codeinclude> - <p>The structure defines six tables in our database. In Mnesia, - the function <c>mnesia:create_table(Name, ArgList)</c> is - used to create tables. <c>Name</c> is the table - name <em>Note:</em> The current version of Mnesia does - not require that the name of the table is the same as the record - name, See Chapter 4: - <seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names Versus Table Names.</seealso></p> - <p>For example, the table - for employees will be created with the function - <c>mnesia:create_table(employee, [{attributes, record_info(fields, employee)}]).</c> The table + <p>The structure defines six tables in the database. In <c>Mnesia</c>, + the function + <seealso marker="mnesia#create_table/2">mnesia:create_table(Name, ArgList)</seealso> + creates tables. <c>Name</c> is the table name.</p> + <note><p>The current version of <c>Mnesia</c> does not require that + the name of the table is the same as the record name, see + <seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names versus Table Names.</seealso>.</p></note> + <p>For example, the table for employees is created with the + function <c>mnesia:create_table(employee, + [{attributes, record_info(fields, employee)}])</c>. The table name <c>employee</c> matches the name for records specified - in <c>ArgList</c>. The expression <c>record_info(fields, RecordName)</c> is processed by the Erlang preprocessor and - evaluates to a list containing the names of the different - fields for a record. - </p> + in <c>ArgList</c>. The expression + <c>record_info(fields, RecordName)</c> is processed by the Erlang + preprocessor and evaluates to a list containing the names of the + different fields for a record.</p> </section> <section> - <title>The Program</title> - <p>The following shell interaction starts Mnesia and - initializes the schema for our <c>company</c> database: - </p> + <title>Program</title> + <p>The following shell interaction starts <c>Mnesia</c> and + initializes the schema for the <c>Company</c> database:</p> <pre> - % <input>erl -mnesia dir '"/ldisc/scratch/Mnesia.Company"'</input> Erlang (BEAM) emulator version 4.9 @@ -205,37 +189,39 @@ 1> mnesia:create_schema([node()]). ok 2> mnesia:start(). - ok - </pre> - <p>The following program module creates and populates previously defined tables: - </p> + ok</pre> + <p>The following program module creates and populates previously + defined tables:</p> <codeinclude file="company.erl" tag="%0" type="erl"></codeinclude> </section> <section> - <title>The Program Explained</title> - <p>The following commands and functions were used to initiate the - Company database: - </p> + <title>Program Explained</title> + <p>The following commands and functions are used to initiate the + <c>Company</c> database:</p> <list type="bulleted"> - <item><c>% erl -mnesia dir '"/ldisc/scratch/Mnesia.Company"'.</c> This is a UNIX - command line entry which starts the Erlang system. The flag - <c>-mnesia dir Dir</c> specifies the location of the + <item><c>% erl -mnesia dir '"/ldisc/scratch/Mnesia.Company"'</c>. + This is a UNIX + command-line entry that starts the Erlang system. The flag + <c>-mnesia dir Dir</c> specifies the location of the database directory. The system responds and waits for - further input with the prompt <em>1></em>. + further input with the prompt <c>1></c>. </item> - <item><c>mnesia:create_schema([node()]).</c> This function - has the format <c>mnesia:create_schema(DiscNodeList)</c> and - initiates a new schema. In this example, we have created a - non-distributed system using only one node. Schemas are fully - explained in Chapter 3:<seealso marker="Mnesia_chap3#def_schema">Defining a Schema</seealso>. + <item> + <seealso marker="mnesia#create_schema/1">mnesia:create_schema([node()])</seealso>. + This function + has the format <c>mnesia:create_schema(DiscNodeList)</c> and + initiates a new schema. In this example, a non-distributed system + using only one node is created. Schemas are fully explained in + <seealso marker="Mnesia_chap3#def_schema">Define a Schema</seealso>. </item> - <item><c>mnesia:start().</c> This function starts - Mnesia. This function is fully explained in Chapter 3: - <seealso marker="Mnesia_chap3#start_mnesia">Starting Mnesia</seealso>. + <item><seealso marker="mnesia#start/0">mnesia:start()</seealso>. + This function starts <c>Mnesia</c> and is fully explained in + <seealso marker="Mnesia_chap3#start_mnesia">Start Mnesia</seealso>. </item> </list> - <p>Continuing the dialogue with the Erlang shell will produce the following:</p> + <p>Continuing the dialogue with the Erlang shell produces the + following:</p> <pre><![CDATA[ 3> company:init(). {atomic,ok} @@ -271,63 +257,58 @@ 0 transactions waits for other nodes: [] ok ]]></pre> - <p>A set of tables is created: - </p> - <list type="bulleted"> - <item><c>mnesia:create_table(Name,ArgList)</c>. This - function is used to create the required database tables. The - options available with <c>ArgList</c> are explained in - Chapter 3: <seealso marker="Mnesia_chap3#create_tables">Creating New Tables</seealso>. </item> - </list> - <p>The <c>company:init/0</c> function creates our tables. Two tables are - of type <c>bag</c>. This is the <c>manager</c> relation as well - the <c>in_proj</c> relation. This shall be interpreted as: An + <p>A set of tables is created. The function + <seealso marker="mnesia#create_table/2">mnesia:create_table(Name, ArgList)</seealso> + creates the required database tables. The + options available with <c>ArgList</c> are explained in + <seealso marker="Mnesia_chap3#create_tables">Create New Tables</seealso>.</p> + <p>The function <c>company:init/0</c> creates the tables. Two tables + are of type <c>bag</c>. This is the <c>manager</c> relation as well + the <c>in_proj</c> relation. This is interpreted as: an employee can be manager over several departments, and an employee can participate in several projects. However, the <c>at_dep</c> - relation is <c>set</c> because an employee can only work in one department. - In this data model we have examples of relations that are one-to-one (<c>set</c>), - as well as one-to-many (<c>bag</c>). - </p> - <p><c>mnesia:info()</c> now indicates that a database - which has seven local tables, of which, six are our - user defined tables and one is the schema. - Six transactions have been committed, as six successful transactions were run when - creating the tables. - </p> - <p>To write a function which inserts an employee record into the database, there must be an - <c>at_dep</c> record and a set of <c>in_proj</c> records inserted. Examine the following - code used to complete this action: - </p> - <codeinclude file="company.erl" tag="%1" type="erl"></codeinclude> + relation is <c>set</c>, as an employee can only work in one department. + In this data model, there are examples of relations that are 1-to-1 + (<c>set</c>) and 1-to-many (<c>bag</c>).</p> + <p><seealso marker="mnesia#info/0">mnesia:info()</seealso> + now indicates that a database has seven + local tables, where six are the user-defined tables and one is + the schema. Six transactions have been committed, as six successful + transactions were run when creating the tables.</p> + <p>To write a function that inserts an employee record into the + database, there must be an <c>at_dep</c> record and a set of + <c>in_proj</c> records inserted. Examine the following + code used to complete this action:</p> + <codeinclude file="company.erl" tag="%1" type="erl"></codeinclude> <list type="bulleted"> <item> - <p><c>insert_emp(Emp, DeptId, ProjNames) -></c>. The - <c>insert_emp/3</c> arguments are:</p> + <p>The <c>insert_emp/3</c> arguments are as follows:</p> <list type="ordered"> <item><c>Emp</c> is an employee record. </item> - <item><c>DeptId</c> is the identity of the department where the employee is working. + <item><c>DeptId</c> is the identity of the department where + the employee works. </item> - <item><c>ProjNames</c> is a list of the names of the projects where the employee are working.</item> + <item><c>ProjNames</c> is a list of the names of the projects + where the employee works.</item> </list> </item> </list> - <p>The <c>insert_emp(Emp, DeptId, ProjNames) -></c> function - creates a <em>functional object</em>. Functional objects - are identified by the term <c>Fun</c>. The Fun is passed + <p>The function <c>insert_emp/3</c> creates a Functional Object (Fun). + <c>Fun</c> is passed as a single argument to the function - <c>mnesia:transaction(Fun)</c>. This means that Fun is - run as a transaction with the following properties: - </p> + <seealso marker="mnesia#transaction/2">mnesia:transaction(Fun)</seealso>. + This means that <c>Fun</c> is + run as a transaction with the following properties:</p> <list type="bulleted"> - <item>Fun either succeeds or fails completely. + <item>A <c>Fun</c> either succeeds or fails. </item> - <item>Code which manipulates the same data records can be + <item>Code that manipulates the same data records can be run concurrently without the different processes interfering with each other. </item> </list> - <p>The function can be used as:</p> + <p>The function can be used as follows:</p> <code type="none"> Emp = #employee{emp_no= 104732, name = klacke, @@ -335,20 +316,17 @@ sex = male, phone = 98108, room_no = {221, 015}}, - insert_emp(Me, 'B/SFR', [Erlang, mnesia, otp]). - </code> - <note> - <p>Functional Objects (Funs) are described in the - Erlang Reference Manual, "Fun Expressions". - </p> + insert_emp(Me, 'B/SFR', [Erlang, mnesia, otp]).</code> + <note><p>For information about Funs, see "Fun Expressions" in + section <c>Erlang Reference Manual</c> in System + Documentation..</p> </note> </section> <section> <title>Initial Database Content</title> - <p>After the insertion of the employee named <c>klacke</c> - we have the following records in the database: - </p> + <p>After the insertion of the employee named <c>klacke</c>, + the databse has the following records:</p> <marker id="table2_1"></marker> <table> <row> @@ -364,14 +342,14 @@ <cell align="left" valign="middle">klacke</cell> <cell align="left" valign="middle">7</cell> <cell align="left" valign="middle">male</cell> - <cell align="left" valign="middle">99586</cell> + <cell align="left" valign="middle">98108</cell> <cell align="left" valign="middle">{221, 015}</cell> </row> - <tcaption> -Employee</tcaption> + <tcaption>employee Database Record</tcaption> </table> - <p>An employee record has the following Erlang record/tuple - representation: <c>{employee, 104732, klacke, 7, male, 98108, {221, 015}}</c></p> + <p>This <c>employee</c> record has the Erlang record/tuple + representation + <c>{employee, 104732, klacke, 7, male, 98108, {221, 015}}</c>.</p> <marker id="table2_2"></marker> <table> <row> @@ -382,12 +360,10 @@ Employee</tcaption> <cell align="left" valign="middle">klacke</cell> <cell align="left" valign="middle">B/SFR</cell> </row> - <tcaption> -At_dep</tcaption> + <tcaption>at_dep Database Record</tcaption> </table> - <p>At_dep has the following Erlang tuple representation: - <c>{at_dep, klacke, 'B/SFR'}</c>. - </p> + <p>This <c>at_dep</c> record has the Erlang tuple representation + <c>{at_dep, klacke, 'B/SFR'}</c>.</p> <marker id="table3_3"></marker> <table> <row> @@ -406,39 +382,36 @@ At_dep</tcaption> <cell align="left" valign="middle">klacke</cell> <cell align="left" valign="middle">mnesia</cell> </row> - <tcaption> -In_proj</tcaption> + <tcaption>in_proj Database Record</tcaption> </table> - <p>In_proj has the following Erlang tuple representation: - <c>{in_proj, klacke, 'Erlang', klacke, 'otp', klacke, 'mnesia'}</c></p> - <p>There is no difference between rows in a table and Mnesia - records. Both concepts are the same and will be used - interchangeably throughout this book. - </p> - <p>A Mnesia table is populated by Mnesia records. For example, - the tuple <c>{boss, klacke, bjarne}</c> is a record. The - second element in this tuple is the key. In order to uniquely - identify a table row both the key and the table name is - needed. The term <em>object identifier</em>, - (oid) is sometimes used for the arity two tuple {Tab, Key}. The oid for - the <c>{boss, klacke, bjarne}</c> record is the arity two + <p>This <c>in_proj</c> record has the Erlang tuple representation + <c>{in_proj, klacke, 'Erlang', klacke, 'otp', klacke, + 'mnesia'}</c>.</p> + <p>There is no difference between rows in a table and <c>Mnesia</c> + records. Both concepts are the same and are used + interchangeably throughout this User's Guide.</p> + <p>A <c>Mnesia</c> table is populated by <c>Mnesia</c> records. For + example, the tuple <c>{boss, klacke, bjarne}</c> is a record. The + second element in this tuple is the key. To identify a table + uniquely, both the key and the table name is needed. + The term Object Identifier (OID) is + sometimes used for the arity two tuple {Tab, Key}. The OID for + the record <c>{boss, klacke, bjarne}</c> is the arity two tuple <c>{boss, klacke}</c>. The first element of the tuple is the type of the record and the second element is the key. An - oid can lead to zero, one, or more records depending on - whether the table type is <c>set</c> or <c>bag</c>. - </p> - <p>We were also able to insert the <c>{boss, klacke, bjarne}</c> record which contains an implicit reference to - another employee which does not yet exist in the - database. Mnesia does not enforce this. - </p> + OID can lead to zero, one, or more records depending on + whether the table type is <c>set</c> or <c>bag</c>.</p> + <p>The record <c>{boss, klacke, bjarne}</c> can also be inserted. + This record contains an implicit reference to + another employee that does not yet exist in the + database. <c>Mnesia</c> does not enforce this.</p> </section> <section> - <title>Adding Records and Relationships to the Database</title> - <p>After adding additional record to the Company database, we - may end up with the following records: - </p> - <p><em>Employees</em></p> + <title>Adding Records and Relationships to Database</title> + <p>After adding more records to the <c>Company</c> database, the + result can be the following records:</p> + <p><c>employees</c>:</p> <code type="none"> {employee, 104465, "Johnson Torbjorn", 1, male, 99184, {242,038}}. {employee, 107912, "Carlsson Tuula", 2, female,94556, {242,056}}. @@ -447,16 +420,13 @@ In_proj</tcaption> {employee, 104659, "Tornkvist Torbjorn", 2, male, 99514, {222,022}}. {employee, 104732, "Wikstrom Claes", 2, male, 99586, {221,015}}. {employee, 117716, "Fedoriw Anna", 1, female,99143, {221,031}}. - {employee, 115018, "Mattsson Hakan", 3, male, 99251, {203,348}}. - </code> - <p><em>Dept</em></p> + {employee, 115018, "Mattsson Hakan", 3, male, 99251, {203,348}}.</code> + <p><c>dept</c>:</p> <code type="none"> - {dept, 'B/SF', "Open Telecom Platform"}. {dept, 'B/SFP', "OTP - Product Development"}. - {dept, 'B/SFR', "Computer Science Laboratory"}. - </code> - <p><em>Projects</em></p> + {dept, 'B/SFR', "Computer Science Laboratory"}.</code> + <p><c>projects</c>:</p> <code type="none"> %% projects {project, erlang, 1}. @@ -465,23 +435,19 @@ In_proj</tcaption> {project, mnesia, 5}. {project, wolf, 6}. {project, documentation, 7}. - {project, www, 8}. - </code> - <p>The above three tables, titled <c>employees</c>, - <c>dept</c>, and <c>projects</c>, are the tables which are + {project, www, 8}.</code> + <p>These three tables, <c>employees</c>, <c>dept</c>, and + <c>projects</c>, are made up of real records. The following database content is - stored in the tables which is built on - relationships. These tables are titled <c>manager</c>, - <c>at_dep</c>, and <c>in_proj</c>. - </p> - <p><em>Manager</em></p> + stored in the tables and is built on + relationships. These tables are <c>manager</c>, + <c>at_dep</c>, and <c>in_proj</c>.</p> + <p><c>manager</c>:</p> <code type="none"> - {manager, 104465, 'B/SF'}. {manager, 104465, 'B/SFP'}. - {manager, 114872, 'B/SFR'}. - </code> - <p><em>At_dep</em></p> + {manager, 114872, 'B/SFR'}.</code> + <p><c>at_dep</c>:</p> <code type="none"> {at_dep, 104465, 'B/SF'}. {at_dep, 107912, 'B/SF'}. @@ -490,9 +456,8 @@ In_proj</tcaption> {at_dep, 104659, 'B/SFR'}. {at_dep, 104732, 'B/SFR'}. {at_dep, 117716, 'B/SFP'}. - {at_dep, 115018, 'B/SFP'}. - </code> - <p><em>In_proj</em></p> + {at_dep, 115018, 'B/SFP'}.</code> + <p><c>in_proj</c>:</p> <code type="none"> {in_proj, 104465, otp}. {in_proj, 107912, otp}. @@ -508,136 +473,118 @@ In_proj</tcaption> {in_proj, 117716, otp}. {in_proj, 117716, documentation}. {in_proj, 115018, otp}. - {in_proj, 115018, mnesia}. - </code> + {in_proj, 115018, mnesia}.</code> <p>The room number is an attribute of the employee - record. This is a structured attribute which consists of a + record. This is a structured attribute that consists of a tuple. The first element of the tuple identifies a corridor, - and the second element identifies the actual room in the - corridor. We could have chosen to represent this as a record + and the second element identifies the room in that + corridor. An alternative is to represent this as a record <c>-record(room, {corr, no}).</c> instead of an anonymous - tuple representation. - </p> - <p>The Company database is now initialized and contains - data. </p> + tuple representation.</p> + <p>The <c>Company</c> database is now initialized and contains + data.</p> </section> <section> <title>Writing Queries</title> - <p>Retrieving data from DBMS should usually be done with <c>mnesia:read/3</c> or - <c>mnesia:read/1</c> functions. The following function raises the salary:</p> + <p>Retrieving data from DBMS is usually to be done with the + functions + <seealso marker="mnesia#read/3">mnesia:read/3</seealso> or + <seealso marker="mnesia#read/2">mnesia:read/1</seealso>. + The following function raises the salary:</p> <codeinclude file="company.erl" tag="%5" type="erl"></codeinclude> - <p>Since we want to update the record using <c>mnesia:write/1</c> after we have - increased the salary we acquire a write lock (third argument to read) when we read the - record from the table. - </p> - <p>It is not always the case that we can directly read the values from the table, - we might need to search the table or several tables to get the data we want, this - is done by writing database queries. Queries are always more expensive operations - than direct lookups done with <c>mnesia:read</c> and should be avoided in performance - critical code.</p> - <p>There are two methods for writing database queries: - </p> + <p>Since it is desired to update the record using the function + <seealso marker="mnesia#write/1">mnesia:write/1</seealso> + after the salary has been increased, a write + lock (third argument to <c>read</c>) is acquired when the record from + the table is read.</p> + <p>To read the values from the table directly is not always possible. + It can be needed to search one or more tables to get the + wanted data, and this is done by writing database queries. Queries + are always more expensive operations than direct lookups done with + <c>mnesia:read</c>. Therefore, avoid queries in + performance-critical code.</p> + <p>Two methods are available for writing database queries:</p> <list type="bulleted"> - <item>Mnesia functions - </item> + <item><c>Mnesia</c> functions</item> <item>QLC</item> </list> <section> - <title>Mnesia functions </title> - <p></p> + <title>Using Mnesia Functions</title> <p>The following function extracts the names of the female employees - stored in the database: - </p> + stored in the database:</p> <pre> -mnesia:select(employee, [{#employee{sex = female, name = '$1', _ = '_'},[], ['$1']}]). - </pre> - <p>Select must always run within an activity such as a - transaction. To be able to call from the shell we might - construct a function as: - </p> +mnesia:select(employee, [{#employee{sex = female, name = '$1', _ = '_'},[], ['$1']}]).</pre> + <p><c>select</c> must always run within an activity, such as a + transaction. The following function can be constructed to call + from the shell:</p> <codeinclude file="company.erl" tag="%20" type="erl"></codeinclude> - <p>The select expression matches all entries in table employee with - the field sex set to female. - </p> - <p>This function can be called from the shell as follows: - </p> + <p>The <c>select</c> expression matches all entries in table + employee with the field <c>sex</c> set to <c>female</c>.</p> + <p>This function can be called from the shell as follows:</p> <pre> (klacke@gin)1> <input>company:all_females().</input> - {atomic, ["Carlsson Tuula", "Fedoriw Anna"]} - </pre> - <p>See also the <seealso marker="Mnesia_chap4#matching">Pattern Matching </seealso> - chapter for a description of select and its syntax. - </p> + {atomic, ["Carlsson Tuula", "Fedoriw Anna"]}</pre> + <p>For a description of <c>select</c> and its syntax, see + <seealso marker="Mnesia_chap4#matching">Pattern Matching</seealso>. + </p> </section> <section> <title>Using QLC </title> - <p>This section contains simple introductory examples - only. Refer to <em>QLC reference manual</em> for a - full description of the QLC query language. Using QLC - might be more expensive than using Mnesia functions directly but - offers a nice syntax. - </p> + <p>This section contains simple introductory examples only. For + a full description of the QLC query language, see the + <seealso marker="stdlib:qlc">qlc</seealso> manual page in + <c>STDLIB</c>.</p> + <p>Using QLC can be more expensive than using <c>Mnesia</c> + functions directly but offers a nice syntax.</p> <p>The following function extracts a list of female employees - from the database: - </p> + from the database:</p> <pre> Q = qlc:q([E#employee.name || E <![CDATA[<-]]> mnesia:table(employee), E#employee.sex == female]), - qlc:e(Q), - </pre> - <p>Accessing mnesia tables from a QLC list comprehension must + qlc:e(Q),</pre> + <p>Accessing <c>Mnesia</c> tables from a QLC list comprehension must always be done within a transaction. Consider the following - function: - </p> + function:</p> <codeinclude file="company.erl" tag="%2" type="erl"></codeinclude> - <p>This function can be called from the shell as follows: - </p> + <p>This function can be called from the shell as follows:</p> <pre> (klacke@gin)1> <input>company:females().</input> - {atomic, ["Carlsson Tuula", "Fedoriw Anna"]} - </pre> - <p>In traditional relational database terminology, the above - operation would be called a selection, followed by a projection. - </p> - <p>The list comprehension expression shown above contains a - number of syntactical elements. - </p> + {atomic, ["Carlsson Tuula", "Fedoriw Anna"]}</pre> + <p>In traditional relational database terminology, this + operation is called a selection, followed by a projection.</p> + <p>The previous list comprehension expression contains a + number of syntactical elements:</p> <list type="bulleted"> - <item>the first <c>[</c> bracket should be read as "build the - list" + <item>The first <c>[</c> bracket is read as "build the + list". </item> - <item>the <c>||</c> "such that" and the arrow <c><![CDATA[<-]]></c> should - be read as "taken from" + <item>The <c>||</c> "such that" and the arrow <c><![CDATA[<-]]></c> + is read as "taken from". </item> </list> - <p>Hence, the above list comprehension demonstrates the - formation of the list <c>E#employee.name</c> such that <c>E</c> is - taken from the table of employees and the <c>sex</c> attribute - of each records is equal with the atom <c>female</c>. - </p> - <p>The whole list comprehension must be given to the - <c>qlc:q/1</c> function. - </p> - <p>It is possible to combine list comprehensions with low - level Mnesia functions in the same transaction. If we want to - raise the salary of all female employees we execute: - </p> + <p>Hence, the previous list comprehension demonstrates the + formation of the list <c>E#employee.name</c> such that <c>E</c> is + taken from the table of employees, and attribute <c>sex</c> + of each record is equal to the atom <c>female</c>.</p> + <p>The whole list comprehension must be given to the function + <c>qlc:q/1</c>.</p> + <p>List comprehensions with low-level <c>Mnesia</c> functions + can be combined in the same transaction. To raise the + salary of all female employees, execute the following:</p> <codeinclude file="company.erl" tag="%4" type="erl"></codeinclude> <p>The function <c>raise_females/1</c> returns the tuple <c>{atomic, Number}</c>, where <c>Number</c> is the number of - female employees who received a salary increase. Should an error - occur, the value <c>{aborted, Reason}</c> is returned. In the - case of an error, Mnesia guarantees that the salary is not - raised for any employees at all. - </p> + female employees who received a salary increase. If an error + occurs, the value <c>{aborted, Reason}</c> is returned, and + <c>Mnesia</c> guarantees that the salary is not + raised for any employee.</p> + <p><em>Example:</em></p> <pre> - 33><input>company:raise_females(33).</input> - {atomic,2} - </pre> + {atomic,2}</pre> </section> </section> </section> diff --git a/lib/mnesia/doc/src/Mnesia_chap3.xml b/lib/mnesia/doc/src/Mnesia_chap3.xml deleted file mode 100644 index ae704b4199..0000000000 --- a/lib/mnesia/doc/src/Mnesia_chap3.xml +++ /dev/null @@ -1,556 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>1997</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - The contents of this file are subject to the Erlang Public License, - Version 1.1, (the "License"); you may not use this file except in - compliance with the License. You should have received a copy of the - Erlang Public License along with this software. If not, it can be - retrieved online at http://www.erlang.org/. - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - the License for the specific language governing rights and limitations - under the License. - - </legalnotice> - - <title>Building A Mnesia Database</title> - <prepared></prepared> - <responsible></responsible> - <docno></docno> - <approved></approved> - <checked></checked> - <date></date> - <rev></rev> - <file>Mnesia_chap3.xml</file> - </header> - <p>This chapter details the basic steps involved when designing - a Mnesia database and the programming constructs which make different - solutions available to the programmer. The chapter includes the following - sections: - </p> - <list type="bulleted"> - <item>defining a schema</item> - <item>the datamodel</item> - <item>starting Mnesia</item> - <item>creating new tables.</item> - </list> - - <section> - <marker id="def_schema"></marker> - <title>Defining a Schema</title> - <p>The configuration of a Mnesia system is described in the - schema. The schema is a special table which contains information - such as the table names and each table's - storage type, (i.e. whether a table should be stored in RAM, - on disc or possibly on both, as well as its location). - </p> - <p>Unlike data tables, information contained in schema tables can only be - accessed and modified by using the schema related functions - described in this section. - </p> - <p>Mnesia has various functions for defining the - database schema. It is possible to move tables, delete tables, - or reconfigure the layout of tables. - </p> - <p>An important aspect of these functions is that the system can access a - table while it is being reconfigured. For example, it is possible to move a - table and simultaneously perform write operations to the same - table. This feature is essential for applications that require - continuous service. - </p> - <p>The following section describes the functions available for schema management, - all of which return a tuple: - </p> - <list type="bulleted"> - <item><c>{atomic, ok}</c>; or, - </item> - <item><c>{aborted, Reason}</c> if unsuccessful.</item> - </list> - - <section> - <title>Schema Functions</title> - <list type="bulleted"> - <item><c>mnesia:create_schema(NodeList)</c>. This function is - used to initialize a new, empty schema. This is a mandatory - requirement before Mnesia can be started. Mnesia is a truly - distributed DBMS and the schema is a system table that is - replicated on all nodes in a Mnesia system. - The function will fail if a schema is already present on any of - the nodes in <c>NodeList</c>. This function requires Mnesia - to be stopped on the all - <c>db_nodes</c> contained in the parameter <c>NodeList</c>. - Applications call this function only once, - since it is usually a one-time activity to initialize a new - database. - </item> - <item><c>mnesia:delete_schema(DiscNodeList)</c>. This function - erases any old schemas on the nodes in - <c>DiscNodeList</c>. It also removes all old tables together - with all data. This function requires Mnesia to be stopped - on all <c>db_nodes</c>. - </item> - <item><c>mnesia:delete_table(Tab)</c>. This function - permanently deletes all replicas of table <c>Tab</c>. - </item> - <item><c>mnesia:clear_table(Tab)</c>. This function - permanently deletes all entries in table <c>Tab</c>. - </item> - <item><c>mnesia:move_table_copy(Tab, From, To)</c>. This - function moves the copy of table <c>Tab</c> from node - <c>From</c> to node <c>To</c>. The table storage type, - <c>{type}</c> is preserved, so if a RAM table is moved from - one node to another node, it remains a RAM table on the new - node. It is still possible for other transactions to perform - read and write operation to the table while it is being - moved. - </item> - <item><c>mnesia:add_table_copy(Tab, Node, Type)</c>. This - function creates a replica of the table <c>Tab</c> at node - <c>Node</c>. The <c>Type</c> argument must be either of the - atoms <c>ram_copies</c>, <c>disc_copies</c>, or - <c>disc_only_copies</c>. If we add a copy of the system - table <c>schema</c> to a node, this means that we want the - Mnesia schema to reside there as well. This action then - extends the set of nodes that comprise this particular - Mnesia system. - </item> - <item><c>mnesia:del_table_copy(Tab, Node)</c>. This function - deletes the replica of table <c>Tab</c> at node <c>Node</c>. - When the last replica of a table is removed, the table is - deleted. - </item> - <item> - <p><c>mnesia:transform_table(Tab, Fun, NewAttributeList, NewRecordName)</c>. This - function changes the format on all records in table - <c>Tab</c>. It applies the argument <c>Fun</c> to all - records in the table. <c>Fun</c> shall be a function which - takes a record of the old type, and returns the record of the new - type. The table key may not be changed.</p> - <code type="none"> --record(old, {key, val}). --record(new, {key, val, extra}). - -Transformer = - fun(X) when record(X, old) -> - #new{key = X#old.key, - val = X#old.val, - extra = 42} - end, -{atomic, ok} = mnesia:transform_table(foo, Transformer, - record_info(fields, new), - new), - </code> - <p>The <c>Fun</c> argument can also be the atom - <c>ignore</c>, it indicates that only the meta data about the table will - be updated. Usage of <c>ignore</c> is not recommended (since it creates - inconsistencies between the meta data and the actual data) but included - as a possibility for the user to do his own (off-line) transform.</p> - </item> - <item><c>change_table_copy_type(Tab, Node, ToType)</c>. This - function changes the storage type of a table. For example, a - RAM table is changed to a disc_table at the node specified - as <c>Node</c>.</item> - </list> - </section> - </section> - - <section> - <title>The Data Model</title> - <p>The data model employed by Mnesia is an extended - relational data model. Data is organized as a set of - tables and relations between different data records can - be modeled as additional tables describing the actual - relationships. - Each table contains instances of Erlang records - and records are represented as Erlang tuples. - </p> - <p>Object identifiers, also known as oid, are made up of a table name and a key. - For example, if we have an employee record represented by the tuple - <c>{employee, 104732, klacke, 7, male, 98108, {221, 015}}</c>. - This record has an object id, (Oid) which is the tuple - <c>{employee, 104732}</c>. - </p> - <p>Thus, each table is made up of records, where the first element - is a record name and the second element of the table is a key - which identifies the particular record in that table. The - combination of the table name and a key, is an arity two tuple - <c>{Tab, Key}</c> called the Oid. See Chapter 4:<seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names Versus Table Names</seealso>, for more information - regarding the relationship between the record name and the table - name. - </p> - <p>What makes the Mnesia data model an extended relational model - is the ability to store arbitrary Erlang terms in the attribute - fields. One attribute value could for example be a whole tree of - oids leading to other terms in other tables. This - type of record is hard to model in traditional relational - DBMSs.</p> - </section> - - <section> - <marker id="start_mnesia"></marker> - <title>Starting Mnesia</title> - <p>Before we can start Mnesia, we must initialize an empty schema - on all the participating nodes. - </p> - <list type="bulleted"> - <item>The Erlang system must be started. - </item> - <item>Nodes with disc database schema must be defined and - implemented with the function <c>create_schema(NodeList).</c></item> - </list> - <p>When running a distributed system, with two or more - participating nodes, then the <c>mnesia:start( ).</c> function - must be executed on each participating node. Typically this would - be part of the boot script in an embedded environment. - In a test environment or an interactive environment, - <c>mnesia:start()</c> can also be used either from the - Erlang shell, or another program. - </p> - - <section> - <title>Initializing a Schema and Starting Mnesia</title> - <p>To use a known example, we illustrate how to run the - Company database described in Chapter 2 on two separate nodes, - which we call <c>a@gin</c> and <c>b@skeppet</c>. Each of these - nodes must have have a Mnesia directory as well as an - initialized schema before Mnesia can be started. There are two - ways to specify the Mnesia directory to be used: - </p> - <list type="bulleted"> - <item> - <p>Specify the Mnesia directory by providing an application - parameter either when starting the Erlang shell or in the - application script. Previously the following example was used - to create the directory for our Company database:</p> - <pre> -%<input>erl -mnesia dir '"/ldisc/scratch/Mnesia.Company"'</input> - </pre> - </item> - <item>If no command line flag is entered, then the Mnesia - directory will be the current working directory on the node - where the Erlang shell is started.</item> - </list> - <p>To start our Company database and get it running on the two - specified nodes, we enter the following commands: - </p> - <list type="ordered"> - <item> - <p>On the node called gin:</p> - <pre> - gin %<input>erl -sname a -mnesia dir '"/ldisc/scratch/Mnesia.company"'</input> - </pre> - </item> - <item> - <p>On the node called skeppet:</p> - <pre> -skeppet %<input>erl -sname b -mnesia dir '"/ldisc/scratch/Mnesia.company"'</input> - </pre> - </item> - <item> - <p>On one of the two nodes:</p> - <pre> -(a@gin1)><input>mnesia:create_schema([a@gin, b@skeppet]).</input> - </pre> - </item> - <item>The function <c>mnesia:start()</c> is called on both - nodes. - </item> - <item> - <p>To initialize the database, execute the following - code on one of the two nodes.</p> - <codeinclude file="company.erl" tag="%12" type="erl"></codeinclude> - </item> - </list> - <p>As illustrated above, the two directories reside on different nodes, because the - <c>/ldisc/scratch</c> (the "local" disc) exists on the two different - nodes. - </p> - <p>By executing these commands we have configured two Erlang - nodes to run the Company database, and therefore, initialize the - database. This is required only once when setting up, the next time the - system is started <c>mnesia:start()</c> is called - on both nodes, to initialize the system from disc. - </p> - <p>In a system of Mnesia nodes, every node is aware of the - current location of all tables. In this example, data is - replicated on both nodes and functions which manipulate the - data in our tables can be executed on either of the two nodes. - Code which manipulate Mnesia data behaves identically - regardless of where the data resides. - </p> - <p>The function <c>mnesia:stop()</c> stops Mnesia on the node - where the function is executed. Both the <c>start/0</c> and - the <c>stop/0</c> functions work on the "local" Mnesia system, - and there are no functions which start or stop a set of nodes. - </p> - </section> - - <section> - <title>The Start-Up Procedure</title> - <p>Mnesia is started by calling the following function: - </p> - <code type="none"> - mnesia:start(). - </code> - <p>This function initiates the DBMS locally. </p> - <p>The choice of configuration will alter the location and load - order of the tables. The alternatives are listed below: <br></br> -</p> - <list type="ordered"> - <item>Tables that are stored locally only, are initialized - from the local Mnesia directory. - </item> - <item>Replicated tables that reside locally - as well as somewhere else are either initiated from disc or - by copying the entire table from the other node depending on - which of the different replicas is the most recent. Mnesia - determines which of the tables is the most recent. - </item> - <item>Tables that reside on remote nodes are available to other nodes as soon - as they are loaded.</item> - </list> - <p>Table initialization is asynchronous, the function - call <c>mnesia:start()</c> returns the atom <c>ok</c> and - then starts to initialize the different tables. Depending on - the size of the database, this may take some time, and the - application programmer must wait for the tables that the - application needs before they can be used. This achieved by using - the function:</p> - <list type="bulleted"> - <item><c>mnesia:wait_for_tables(TabList, Timeout)</c></item> - </list> - <p>This function suspends the caller until all tables - specified in <c>TabList</c> are properly initiated. - </p> - <p>A problem can arise if a replicated table on one node is - initiated, but Mnesia deduces that another (remote) - replica is more recent than the replica existing on - the local node, the initialization procedure will not proceed. - In this situation, a call to to - <c>mnesia:wait_for_tables/2</c> suspends the caller until the - remote node has initiated the table from its local disc and - the node has copied the table over the network to the local node. - </p> - <p>This procedure can be time consuming however, the shortcut function - shown below will load all the tables from disc at a faster rate: - </p> - <list type="bulleted"> - <item><c>mnesia:force_load_table(Tab)</c>. This function forces - tables to be loaded from disc regardless of the network - situation.</item> - </list> - <p>Thus, we can assume that if an application - wishes to use tables <c>a</c> and <c>b</c>, then the - application must perform some action similar to the below code before it can utilize the tables. - </p> - <pre> - case mnesia:wait_for_tables([a, b], 20000) of - {timeout, RemainingTabs} -> - panic(RemainingTabs); - ok -> - synced - end. - </pre> - <warning> - <p>When tables are forcefully loaded from the local disc, - all operations that were performed on the replicated table - while the local node was down, and the remote replica was - alive, are lost. This can cause the database to become - inconsistent.</p> - </warning> - <p>If the start-up procedure fails, the - <c>mnesia:start()</c> function returns the cryptic tuple - <c>{error,{shutdown, {mnesia_sup,start,[normal,[]]}}}</c>. - Use command line arguments -boot start_sasl as argument to - the erl script in order to get more information - about the start failure. - </p> - </section> - </section> - - <section> - <marker id="create_tables"></marker> - <title>Creating New Tables</title> - <p>Mnesia provides one function to create new tables. This - function is: <c>mnesia:create_table(Name, ArgList).</c></p> - <p>When executing this function, it returns one of the following - responses: - </p> - <list type="bulleted"> - <item><c>{atomic, ok}</c> if the function executes - successfully - </item> - <item><c>{aborted, Reason}</c> if the function fails. - </item> - </list> - <p>The function arguments are: - </p> - <list type="bulleted"> - <item><c>Name</c> is the atomic name of the table. It is - usually the same name as the name of the records that - constitute the table. (See <c>record_name</c> for more - details.) - </item> - <item> - <p><c>ArgList</c> is a list of <c>{Key,Value}</c> tuples. - The following arguments are valid: - </p> - <list type="bulleted"> - <item> - <p><c>{type, Type}</c> where <c>Type</c> must be either of the - atoms <c>set</c>, <c>ordered_set</c> or <c>bag</c>. - The default value is - <c>set</c>. Note: currently 'ordered_set' - is not supported for 'disc_only_copies' tables. - A table of type <c>set</c> or <c>ordered_set</c> has either zero or - one record per key. Whereas a table of type <c>bag</c> can - have an arbitrary number of records per key. The key for - each record is always the first attribute of the record.</p> - <p>The following example illustrates the difference between - type <c>set</c> and <c>bag</c>: </p> - <pre> - f() -> F = fun() -> - mnesia:write({foo, 1, 2}), mnesia:write({foo, 1, 3}), - mnesia:read({foo, 1}) end, mnesia:transaction(F). </pre> - <p>This transaction will return the list <c>[{foo,1,3}]</c> if - the <c>foo</c> table is of type <c>set</c>. However, list - <c>[{foo,1,2}, {foo,1,3}]</c> will return if the table is - of type <c>bag</c>. Note the use of <c>bag</c> and - <c>set</c> table types. </p> - <p>Mnesia tables can never contain - duplicates of the same record in the same table. Duplicate - records have attributes with the same contents and key. - </p> - </item> - <item> - <p><c>{disc_copies, NodeList}</c>, where <c>NodeList</c> is a - list of the nodes where this table will reside on disc.</p> - <p>Write operations to a table replica of type - <c>disc_copies</c> will write data to the disc copy as well - as to the RAM copy of the table. </p> - <p>It is possible to have a - replicated table of type <c>disc_copies</c> on one node, and - the same table stored as a different type on another node. - The default value is <c>[]</c>. This arrangement is - desirable if we want the following operational - characteristics are required:</p> - <list type="ordered"> - <item>read operations must be very fast and performed in RAM - </item> - <item>all write operations must be written to persistent - storage.</item> - </list> - <p>A write operation on a <c>disc_copies</c> table - replica will be performed in two steps. First the write - operation is appended to a log file, then the actual - operation is performed in RAM. - </p> - </item> - <item> - <p><c>{ram_copies, NodeList}</c>, where <c>NodeList</c> is a - list of the nodes where this table is stored in RAM. The - default value for <c>NodeList</c> is <c>[node()]</c>. If the - default value is used to create a new table, it will be - located on the local node only. </p> - <p>Table replicas of type - <c>ram_copies</c> can be dumped to disc with the function - <c>mnesia:dump_tables(TabList)</c>. - </p> - </item> - <item><c>{disc_only_copies, NodeList}</c>. These table - replicas are stored on disc only and are therefore slower to - access. However, a disc only replica consumes less memory than - a table replica of the other two storage types. - </item> - <item><c>{index, AttributeNameList}</c>, where - <c>AttributeNameList</c> is a list of atoms specifying the - names of the attributes Mnesia shall build and maintain. An - index table will exist for every element in the list. The - first field of a Mnesia record is the key and thus need no - extra index. - <br></br> -The first field of a record is the second element of the - tuple, which is the representation of the record. - </item> - <item><c>{snmp, SnmpStruct}</c>. <c>SnmpStruct</c> is - described in the SNMP User Guide. Basically, if this attribute - is present in <c>ArgList</c> of <c>mnesia:create_table/2</c>, - the table is immediately accessible by means of the Simple - Network Management Protocol (SNMP). - <br></br> -It is easy to design applications which use SNMP to - manipulate and control the system. Mnesia provides a direct - mapping between the logical tables that make up an SNMP - control application and the physical data which make up a - Mnesia table. <c>[]</c> - is default. - </item> - <item><c>{local_content, true}</c> When an application needs a - table whose contents should be locally unique on each - node, - <c>local_content</c> tables may be used. The name of the - table is known to all Mnesia nodes, but its contents is - unique for each node. Access to this type of table must be - done locally. </item> - <item> - <p><c>{attributes, AtomList}</c> is a list of the attribute - names for the records that are supposed to populate the - table. The default value is the list <c>[key, val]</c>. The - table must at least have one extra attribute besides the - key. When accessing single attributes in a record, it is not - recommended to hard code the attribute names as atoms. Use - the construct <c>record_info(fields,record_name)</c> - instead. The expression - <c>record_info(fields,record_name)</c> is processed by the - Erlang macro pre-processor and returns a list of the - record's field names. With the record definition - <c>-record(foo, {x,y,z}).</c> the expression - <c>record_info(fields,foo)</c> is expanded to the list - <c>[x,y,z]</c>. Accordingly, it is possible to provide the - attribute names yourself, or to use the <c>record_info/2</c> - notation. </p> - <p>It is recommended that - the <c>record_info/2</c> notation be used as it is easier to - maintain the program and it will be more robust with regards - to future record changes. - </p> - </item> - <item> - <p><c>{record_name, Atom}</c> specifies the common name of - all records stored in the table. All records, stored in - the table, must have this name as their first element. - The <c>record_name</c> defaults to the name of the - table. For more information see Chapter 4:<seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names Versus Table Names</seealso>.</p> - </item> - </list> - </item> - </list> - <p>As an example, assume we have the record definition:</p> - <pre> - -record(funky, {x, y}). - </pre> - <p>The below call would create a table which is replicated on two - nodes, has an additional index on the <c>y</c> attribute, and is - of type - <c>bag</c>.</p> - <pre> - mnesia:create_table(funky, [{disc_copies, [N1, N2]}, {index, - [y]}, {type, bag}, {attributes, record_info(fields, funky)}]). - </pre> - <p>Whereas a call to the below default code values: </p> - <pre> -mnesia:create_table(stuff, []) </pre> - <p>would return a table with a RAM copy on the - local node, no additional indexes and the attributes defaulted to - the list <c>[key,val]</c>.</p> - </section> -</chapter> - diff --git a/lib/mnesia/doc/src/Mnesia_chap3.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap3.xmlsrc new file mode 100644 index 0000000000..d965c98ab9 --- /dev/null +++ b/lib/mnesia/doc/src/Mnesia_chap3.xmlsrc @@ -0,0 +1,519 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>1997</year><year>2013</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Build a Mnesia Database</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date></date> + <rev></rev> + <file>Mnesia_chap3.xml</file> + </header> + <p>This section describes the basic steps when designing a + <c>Mnesia</c> database and the programming constructs that make different + solutions available to the programmer. The following topics are + included:</p> + <list type="bulleted"> + <item>Define a schema</item> + <item>Data model</item> + <item>Start <c>Mnesia</c></item> + <item>Create tables</item> + </list> + + <section> + <marker id="def_schema"></marker> + <title>Define a Schema</title> + <p>The configuration of a <c>Mnesia</c> system is described in a + schema. The schema is a special table that includes information + such as the table names and the storage type of each table + (that is, whether a table is to be stored in RAM, + on disc, or on both, as well as its location).</p> + <p>Unlike data tables, information in schema tables can only be + accessed and modified by using the schema-related functions + described in this section.</p> + <p><c>Mnesia</c> has various functions for defining the + database schema. Tables can be moved or deleted, and the + table layout can be reconfigured.</p> + <p>An important aspect of these functions is that the system can access + a table while it is being reconfigured. For example, it is possible + to move a + table and simultaneously perform write operations to the same + table. This feature is essential for applications that require + continuous service.</p> + <p>This section describes the functions available for schema management, + all which return either of the following tuples:</p> + <list type="bulleted"> + <item><c>{atomic, ok}</c> if successful</item> + <item><c>{aborted, Reason}</c> if unsuccessful</item> + </list> + + <section> + <title>Schema Functions</title> + <p>The schema functions are as follows:</p> + <list type="bulleted"> + <item><seealso marker="mnesia#create_schema/1">mnesia:create_schema(NodeList)</seealso> + initializes a new, empty schema. This is a mandatory requirement + before <c>Mnesia</c> can be started. <c>Mnesia</c> is a truly + distributed DBMS and the schema is a system table that is + replicated on all nodes in a <c>Mnesia</c> system. + This function fails if a schema is already present on any of + the nodes in <c>NodeList</c>. The function requires <c>Mnesia</c> + to be stopped on the all + <c>db_nodes</c> contained in parameter <c>NodeList</c>. + Applications call this function only once, as + it is usually a one-time activity to initialize a new database. + </item> + <item><seealso marker="mnesia#delete_schema/1">mnesia:delete_schema(DiscNodeList)</seealso> + erases any old schemas on the nodes in + <c>DiscNodeList</c>. It also removes all old tables together + with all data. This function requires <c>Mnesia</c> to be stopped + on all <c>db_nodes</c>. + </item> + <item><seealso marker="mnesia#delete_table/1">mnesia:delete_table(Tab)</seealso> + permanently deletes all replicas of table <c>Tab</c>. + </item> + <item><seealso marker="mnesia#clear_table/1">mnesia:clear_table(Tab)</seealso> + permanently deletes all entries in table <c>Tab</c>. + </item> + <item><seealso marker="mnesia#move_table_copy/3">mnesia:move_table_copy(Tab, From, To)</seealso> + moves the copy of table <c>Tab</c> from node + <c>From</c> to node <c>To</c>. The table storage type + <c>{type}</c> is preserved, so if a RAM table is moved from + one node to another, it remains a RAM table on the new + node. Other transactions can still perform read + and write operation to the table while it is being moved. + </item> + <item><seealso marker="mnesia#add_table_copy/3">mnesia:add_table_copy(Tab, Node, Type)</seealso> + creates a replica of table <c>Tab</c> at node + <c>Node</c>. Argument <c>Type</c> must be either of the + atoms <c>ram_copies</c>, <c>disc_copies</c>, or + <c>disc_only_copies</c>. If you add a copy of the system + table <c>schema</c> to a node, you want the <c>Mnesia</c> + schema to reside there as well. This action + extends the set of nodes that comprise this particular + <c>Mnesia</c> system. + </item> + <item><seealso marker="mnesia#del_table_copy/2">mnesia:del_table_copy(Tab, Node)</seealso> + deletes the replica of table <c>Tab</c> at node <c>Node</c>. + When the last replica of a table is removed, the table is + deleted. + </item> + <item> + <p><seealso marker="mnesia#transform_table/4">mnesia:transform_table(Tab, Fun, NewAttributeList, NewRecordName)</seealso> + changes the format on all records in table + <c>Tab</c>. It applies argument <c>Fun</c> to all + records in the table. <c>Fun</c> must be a function that + takes a record of the old type, and returns the record of the + new type. The table key must not be changed.</p> + <p><em>Example:</em></p> + <code type="none"> +-record(old, {key, val}). +-record(new, {key, val, extra}). + +Transformer = + fun(X) when record(X, old) -> + #new{key = X#old.key, + val = X#old.val, + extra = 42} + end, +{atomic, ok} = mnesia:transform_table(foo, Transformer, + record_info(fields, new), + new), +</code> + <p>Argument <c>Fun</c> can also be the atom + <c>ignore</c>, which indicates that only the metadata about + the table is updated. Use of <c>ignore</c> is not recommended + (as it creates inconsistencies between the metadata and the + actual data) but it is included as a possibility for the user + do to an own (offline) transform.</p> + </item> + <item><c>change_table_copy_type(Tab, Node, ToType)</c> + changes the storage type of a table. For example, a + RAM table is changed to a <c>disc_table</c> at the node specified + as <c>Node</c>.</item> + </list> + </section> + </section> + + <section> + <title>Data Model</title> + <p>The data model employed by <c>Mnesia</c> is an extended + relational data model. Data is organized as a set of + tables and relations between different data records can + be modeled as more tables describing the relationships. + Each table contains instances of Erlang records. + The records are represented as Erlang tuples.</p> + <p>Each Object Identifier (OID) is made up of a table name and a key. + For example, if an employee record is represented by the tuple + <c>{employee, 104732, klacke, 7, male, 98108, {221, 015}}</c>, + this record has an OID, which is the tuple + <c>{employee, 104732}</c>.</p> + <p>Thus, each table is made up of records, where the first element + is a record name and the second element of the table is a key, + which identifies the particular record in that table. The + combination of the table name and a key is an arity two tuple + <c>{Tab, Key}</c> called the OID. For more information about + the relationship beween the record name and the table name, see + <seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names versus Table Names</seealso>. + </p> + <p>What makes the <c>Mnesia</c> data model an extended relational model + is the ability to store arbitrary Erlang terms in the attribute + fields. One attribute value can, for example, be a whole tree of + OIDs leading to other terms in other tables. This type + of record is difficult to model in traditional relational DBMSs.</p> + </section> + + <section> + <marker id="start_mnesia"></marker> + <title>Start Mnesia</title> + <p>Before starting <c>Mnesia</c>, the following must be done: + </p> + <list type="bulleted"> + <item>An empty schema must be initialized on all the + participating nodes.</item> + <item>The Erlang system must be started.</item> + <item>Nodes with disc database schema must be defined and + implemented with the function + <seealso marker="mnesia#create_schema/1">mnesia:create_schema(NodeList)</seealso>.</item> + </list> + <p>When running a distributed system with two or more + participating nodes, the function + <seealso marker="mnesia#start/0">mnesia:start()</seealso> + must be executed on each participating node. This would typically + be part of the boot script in an embedded environment. + In a test environment or an interactive environment, + <c>mnesia:start()</c> can also be used either from the + Erlang shell or another program. + </p> + + <section> + <title>Initialize a Schema and Start Mnesia</title> + <p>Let us use the example database <c>Company</c>, described in + <seealso marker="Mnesia_chap2#getting_started">Getting Started</seealso> to + illustrate how to run a database on two separate nodes, + called <c>a@gin</c> and <c>b@skeppet</c>. Each of these + nodes must have a <c>Mnesia</c> directory and an + initialized schema before <c>Mnesia</c> can be started. There are + two ways to specify the <c>Mnesia</c> directory to be used:</p> + <list type="bulleted"> + <item> + <p>Specify the <c>Mnesia</c> directory by providing an application + parameter either when starting the Erlang shell or in the + application script. Previously, the following example was used + to create the directory for the <c>Company</c> database:</p> + <pre> +%<input>erl -mnesia dir '"/ldisc/scratch/Mnesia.Company"'</input> + </pre> + </item> + <item>If no command-line flag is entered, the <c>Mnesia</c> + directory becomes the current working directory on the node + where the Erlang shell is started.</item> + </list> + <p>To start the <c>Company</c> database and get it running on the two + specified nodes, enter the following commands:</p> + <list type="ordered"> + <item> + <p>On the node <c>a@gin</c>:</p> + <pre> + gin %<input>erl -sname a -mnesia dir '"/ldisc/scratch/Mnesia.company"'</input></pre> + </item> + <item><p>On the node <c>b@skeppet</c>:</p> + <pre> +skeppet %<input>erl -sname b -mnesia dir '"/ldisc/scratch/Mnesia.company"'</input></pre> + </item> + <item> + <p>On one of the two nodes:</p> + <pre> +(a@gin)1><input>mnesia:create_schema([a@gin, b@skeppet]).</input></pre> + </item> + <item>The function + <seealso marker="mnesia#start/0">mnesia:start()</seealso> + is called on both nodes. + </item> + <item><p>To initialize the database, execute the following + code on one of the two nodes:</p> + <codeinclude file="company.erl" tag="%12" type="erl"></codeinclude> + </item> + </list> + <p>As illustrated, the two directories reside on different nodes, + because <c>/ldisc/scratch</c> (the "local" disc) exists on + the two different nodes.</p> + <p>By executing these commands, two Erlang nodes are configured to + run the <c>Company</c> database, and therefore, initialize the + database. This is required only once when setting up. The next time + the system is started, + <seealso marker="mnesia#start/0">mnesia:start()</seealso> + is called + on both nodes, to initialize the system from disc.</p> + <p>In a system of <c>Mnesia</c> nodes, every node is aware of the + current location of all tables. In this example, data is + replicated on both nodes and functions that manipulate the + data in the tables can be executed on either of the two nodes. + Code that manipulate <c>Mnesia</c> data behaves identically + regardless of where the data resides.</p> + <p>The function <seealso marker="mnesia#stop/0">mnesia:stop()</seealso> + stops <c>Mnesia</c> on the node + where the function is executed. The functions <c>mnesia:start/0</c> + and <c>mnesia:stop/0</c> work on the "local" <c>Mnesia</c> system. + No functions start or stop a set of nodes.</p> + </section> + + <section> + <title>Startup Procedure</title> + <p>Start <c>Mnesia</c> by calling the following function:</p> + <code type="none"> + mnesia:start().</code> + <p>This function initiates the DBMS locally.</p> + <p>The choice of configuration alters the location and load + order of the tables. The alternatives are as follows:</p> + <list type="ordered"> + <item>Tables that are only stored locally are initialized + from the local <c>Mnesia</c> directory. + </item> + <item>Replicated tables that reside locally + as well as somewhere else are either initiated from disc or + by copying the entire table from the other node, depending on + which of the different replicas are the most recent. <c>Mnesia</c> + determines which of the tables are the most recent. + </item> + <item>Tables that reside on remote nodes are available to other + nodes as soon as they are loaded.</item> + </list> + <p>Table initialization is asynchronous. The function + call <seealso marker="mnesia#start/0">mnesia:start()</seealso> + returns the atom <c>ok</c> and + then starts to initialize the different tables. Depending on + the size of the database, this can take some time, and the + application programmer must wait for the tables that the + application needs before they can be used. This is achieved by + using the function + <seealso marker="mnesia#wait_for_tables/2">mnesia:wait_for_tables(TabList, Timeout)</seealso>, + which suspends the caller until all tables + specified in <c>TabList</c> are properly initiated.</p> + <p>A problem can arise if a replicated table on one node is + initiated, but <c>Mnesia</c> deduces that another (remote) + replica is more recent than the replica existing on the + local node, and the initialization procedure does not proceed. + In this situation, a call to + <seealso marker="mnesia#wait_for_tables/2">mnesia:wait_for_tables/2</seealso>, + suspends the caller until the + remote node has initialized the table from its local disc and + the node has copied the table over the network to the local node.</p> + <p>However, this procedure can be time-consuming, the shortcut function + <seealso marker="mnesia#force_load_table/1">mnesia:force_load_table(Tab)</seealso> + loads all the tables from disc at a faster rate. The function forces + tables to be loaded from disc regardless of the network + situation.</p> + <p>Thus, it can be assumed that if an application wants to use + tables <c>a</c> and <c>b</c>, the application must perform + some action similar to following before it can use the tables:</p> + <pre> + case mnesia:wait_for_tables([a, b], 20000) of + {timeout, RemainingTabs} -> + panic(RemainingTabs); + ok -> + synced + end.</pre> + <warning> + <p>When tables are forcefully loaded from the local disc, + all operations that were performed on the replicated table + while the local node was down, and the remote replica was + alive, are lost. This can cause the database to become + inconsistent.</p> + </warning> + <p>If the startup procedure fails, the function + <seealso marker="mnesia#start/0">mnesia:start()</seealso> + returns the cryptic tuple + <c>{error,{shutdown, {mnesia_sup,start,[normal,[]]}}}</c>. + To get more information about the start failure, use + command-line arguments <c>-boot start_sasl</c> as argument to + the <c>erl</c> script.</p> + </section> + </section> + + <section> + <marker id="create_tables"></marker> + <title>Create Tables</title> + <p>The function + <seealso marker="mnesia#create_table/2">mnesia:create_table(Name, ArgList)</seealso> + creates tables. When executing this function, it returns one of + the following responses:</p> + <list type="bulleted"> + <item><c>{atomic, ok}</c> if the function executes successfully + </item> + <item><c>{aborted, Reason}</c> if the function fails + </item> + </list> + <p>The function arguments are as follows:</p> + <list type="bulleted"> + <item><c>Name</c> is the name of the table. It is + usually the same name as the name of the records that + constitute the table. For details, see <c>record_name</c>. + </item> + <item> + <p><c>ArgList</c> is a list of <c>{Key,Value}</c> tuples. + The following arguments are valid:</p> + <list type="bulleted"> + <item> + <p><c>{type, Type}</c>, where <c>Type</c> must be either of + the atoms <c>set</c>, <c>ordered_set</c>, or <c>bag</c>. + Default is <c>set</c>.</p> + <p>Notice that currently <c>ordered_set</c> is not + supported for <c>disc_only_copies</c> tables.</p> + <p>A table of type + <c>set</c> or <c>ordered_set</c> has either zero or + one record per key, whereas a table of type <c>bag</c> can + have an arbitrary number of records per key. The key for + each record is always the first attribute of the record.</p> + <p>The following example illustrates the difference between + type <c>set</c> and <c>bag</c>:</p> + <pre> + f() -> + F = fun() -> + mnesia:write({foo, 1, 2}), + mnesia:write({foo, 1, 3}), + mnesia:read({foo, 1}) + end, + mnesia:transaction(F).</pre> + <p>This transaction returns the list <c>[{foo,1,3}]</c> if + table <c>foo</c> is of type <c>set</c>. However, the list + <c>[{foo,1,2}, {foo,1,3}]</c> is returned if the table is + of type <c>bag</c>.</p> + <p><c>Mnesia</c> tables can never contain + duplicates of the same record in the same table. Duplicate + records have attributes with the same contents and key.</p> + </item> + <item> + <p><c>{disc_copies, NodeList}</c>, where <c>NodeList</c> is a + list of the nodes where this table is to reside on disc.</p> + <p>Write operations to a table replica of type + <c>disc_copies</c> write data to the disc copy and + to the RAM copy of the table.</p> + <p>It is possible to have a + replicated table of type <c>disc_copies</c> on one node, and + the same table stored as a different type on another node. + Default is <c>[]</c>. This arrangement is + desirable if the following operational + characteristics are required:</p> + <list type="ordered"> + <item>Read operations must be fast and performed in RAM.</item> + <item>All write operations must be written to persistent + storage.</item> + </list> + <p>A write operation on a <c>disc_copies</c> table + replica is performed in two steps. First the write + operation is appended to a log file, then the actual + operation is performed in RAM.</p> + </item> + <item> + <p><c>{ram_copies, NodeList}</c>, where <c>NodeList</c> is a + list of the nodes where this table is stored in RAM. + Default is <c>[node()]</c>. If the default value is used + to create a table, it is located on the local node only.</p> + <p>Table replicas of type + <c>ram_copies</c> can be dumped to disc with the function + <seealso marker="mnesia#dump_tables/1">mnesia:dump_tables(TabList)</seealso>.</p> + </item> + <item><c>{disc_only_copies, NodeList}</c>. These table + replicas are stored on disc only and are therefore slower to + access. However, a disc-only replica consumes less memory than + a table replica of the other two storage types. + </item> + <item><p><c>{index, AttributeNameList}</c>, where + <c>AttributeNameList</c> is a list of atoms specifying the + names of the attributes <c>Mnesia</c> is to build and maintain. + An index table exists for every element in the list. The first + field of a <c>Mnesia</c> record is the key and thus need no + extra index.</p> + <p>The first field of a record is the second element of the + tuple, which is the representation of the record.</p> + </item> + <item><p><c>{snmp, SnmpStruct}</c>. <c>SnmpStruct</c> is + described in the + <seealso marker="snmp:index">SNMP</seealso> User's Guide. + Basically, if this attribute is present in <c>ArgList</c> of + <seealso marker="mnesia#create_table/2">mnesia:create_table/2</seealso>, + the table is immediately accessible the SNMP.</p> + <p>It is easy to design applications that use SNMP to + manipulate and control the system. <c>Mnesia</c> provides a + direct mapping between the logical tables that make up an SNMP + control application and the physical data that makes up a + <c>Mnesia</c> table. The default value is <c>[]</c>.</p> + </item> + <item><c>{local_content, true}</c>. When an application needs a + table whose contents is to be locally unique on each node, + <c>local_content</c> tables can be used. The name of the + table is known to all <c>Mnesia</c> nodes, but its contents is + unique for each node. Access to this type of table must be + done locally.</item> + <item> + <p><c>{attributes, AtomList}</c> is a list of the attribute + names for the records that are supposed to populate the + table. Default is the list <c>[key, val]</c>. The + table must at least have one extra attribute besides the + key. When accessing single attributes in a record, it is not + recommended to hard code the attribute names as atoms. Use + the construct <c>record_info(fields, record_name)</c> + instead.</p> + <p>The expression + <c>record_info(fields, record_name)</c> is processed by the + Erlang preprocessor and returns a list of the + record field names. With the record definition + <c>-record(foo, {x,y,z}).</c>, the expression + <c>record_info(fields,foo)</c> is expanded to the list + <c>[x,y,z]</c>. It is therefore possible for you to provide + the attribute names or to use the <c>record_info/2</c> + notation.</p> + <p>It is recommended to use the <c>record_info/2</c> notation, + as it becomes easier to maintain the program and the program + becomes more robust with regards to future record changes.</p> + </item> + <item> + <p><c>{record_name, Atom}</c> specifies the common name of + all records stored in the table. All records stored in + the table must have this name as their first element. + <c>record_name</c> defaults to the name of the table. + For more information, see + <seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names versus Table Names</seealso>.</p> + </item> + </list> + </item> + </list> + <p>As an example, consider the following record definition:</p> + <pre> + -record(funky, {x, y}).</pre> + <p>The following call would create a table that is replicated on two + nodes, has an extra index on attribute <c>y</c>, and is of type + <c>bag</c>.</p> + <pre> + mnesia:create_table(funky, [{disc_copies, [N1, N2]}, {index, + [y]}, {type, bag}, {attributes, record_info(fields, funky)}]).</pre> + <p>Whereas a call to the following default code values would return + a table with a RAM copy on the local node, no extra indexes, and the + attributes defaulted to the list <c>[key,val]</c>.</p> + <pre> +mnesia:create_table(stuff, [])</pre> + </section> +</chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc index a18f853662..1671faa9b6 100644 --- a/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc @@ -31,163 +31,146 @@ <rev></rev> <file>Mnesia_chap4.xml</file> </header> - <p>This chapter describes the Mnesia transaction system and the - transaction properties which make Mnesia a fault tolerant, - distributed database management system. - </p> - <p>Also covered in this chapter are the locking functions, + <p>This section describes the <c>Mnesia</c> transaction system and + the transaction properties that make <c>Mnesia</c> a fault-tolerant, + distributed Database Management System (DBMS).</p> + <p>This section also describes the locking functions, including table locks and sticky locks, as well as alternative - functions which bypass the transaction system in favor of improved - speed and reduced overheads. These functions are called "dirty - operations". We also describe the usage of nested transactions. - This chapter contains the following sections: - </p> + functions that bypass the transaction system in favor of improved + speed and reduced overhead. These functions are called "dirty + operations". The use of nested transactions is also described. + The following topics are included:</p> <list type="bulleted"> - <item>transaction properties, which include atomicity, - consistency, isolation, and durability - </item> - <item>Locking - </item> - <item>Dirty operations - </item> - <item>Record names vs table names - </item> - <item>Activity concept and various access contexts - </item> - <item>Nested transactions - </item> - <item>Pattern matching - </item> - <item>Iteration - </item> + <item>Transaction properties, which include atomicity, + consistency, isolation, and durability</item> + <item>Locking</item> + <item>Dirty operations</item> + <item>Record names versus table names</item> + <item>Activity concept and various access contexts</item> + <item>Nested transactions</item> + <item>Pattern matching</item> + <item>Iteration</item> </list> <section> <marker id="trans_prop"></marker> <title>Transaction Properties</title> - <p>Transactions are an important tool when designing fault - tolerant, distributed systems. A Mnesia transaction is a mechanism + <p>Transactions are important when designing fault-tolerant, + distributed systems. A <c>Mnesia</c> transaction is a mechanism by which a series of database operations can be executed as one - functional block. The functional block which is run as a + functional block. The functional block that is run as a transaction is called a Functional Object (Fun), and this code can - read, write, or delete Mnesia records. The Fun is evaluated as a - transaction which either commits, or aborts. If a transaction - succeeds in executing Fun it will replicate the action on all nodes - involved, or abort if an error occurs. - </p> - <p>The following example shows a transaction which raises the - salary of certain employee numbers. - </p> + read, write, and delete <c>Mnesia</c> records. The Fun is evaluated + as a transaction that either commits or terminates. If a transaction + succeeds in executing the Fun, it replicates the action on all nodes + involved, or terminates if an error occurs.</p> + <p>The following example shows a transaction that raises the + salary of certain employee numbers:</p> <codeinclude file="company.erl" tag="%5" type="erl"></codeinclude> - <p>The transaction <c>raise(Eno, Raise) - ></c> contains a Fun - made up of four lines of code. This Fun is called by the statement - <c>mnesia:transaction(F)</c> and returns a value. - </p> - <p>The Mnesia transaction system facilitates the construction of + <p>The function <c>raise/2</c> contains a Fun + made up of four code lines. This Fun is called by the statement + <c>mnesia:transaction(F)</c> and returns a value.</p> + <p>The <c>Mnesia</c> transaction system facilitates the construction of reliable, distributed systems by providing the following important - properties: - </p> + properties:</p> <list type="bulleted"> - <item>The transaction handler ensures that a Fun which is placed - inside a transaction does not interfere with operations embedded + <item>The transaction handler ensures that a Fun, which is placed + inside a transaction, does not interfere with operations embedded in other transactions when it executes a series of operations on tables. </item> <item>The transaction handler ensures that either all operations in the transaction are performed successfully on all nodes atomically, or the transaction fails without permanent effect on - any of the nodes. + any node. </item> - <item>The Mnesia transactions have four important properties, - which we call <em>A</em>tomicity, - <em>C</em>onsistency,<em>I</em>solation, and - <em>D</em>urability, or ACID for short. These properties are - described in the following sub-sections.</item> + <item>The <c>Mnesia</c> transactions have four important properties, + called <em>A</em>tomicity, + <em>C</em>onsistency, <em>I</em>solation, and + <em>D</em>urability (ACID). These properties are + described in the following sections.</item> </list> <section> <title>Atomicity</title> - <p><em>Atomicity</em> means that database changes which are + <p>Atomicity means that database changes that are executed by a transaction take effect on all nodes involved, or - on none of the nodes. In other words, the transaction either - succeeds entirely, or it fails entirely. - </p> - <p>Atomicity is particularly important when we want to - atomically write more than one record in the same - transaction. The <c>raise/2</c> function, shown as an example - above, writes one record only. The <c>insert_emp/3</c> function, - shown in the program listing in Chapter 2, writes the record - <c>employee</c> as well as employee relations such as - <c>at_dep</c> and <c>in_proj</c> into the database. If we run - this latter code inside a transaction, then the transaction + on none of the nodes. That is, the transaction either + succeeds entirely, or it fails entirely.</p> + <p>Atomicity is important when it is needed to write + atomically more than one record in the same + transaction. The function <c>raise/2</c>, shown in the previous + example, writes one record only. The function <c>insert_emp/3</c>, + shown in the program listing in + <seealso marker="Mnesia_chap2#getting_started">Getting Started</seealso>, writes the record + <c>employee</c> as well as employee relations, such as + <c>at_dep</c> and <c>in_proj</c>, into the database. If this + latter code is run inside a transaction, the transaction handler ensures that the transaction either succeeds completely, - or not at all. - </p> - <p>Mnesia is a distributed DBMS where data can be replicated on - several nodes. In many such applications, it is important that a + or not at all.</p> + <p><c>Mnesia</c> is a distributed DBMS where data can be replicated + on several nodes. In many applications, it is important that a series of write operations are performed atomically inside a transaction. The atomicity property ensures that a transaction - take effect on all nodes, or none at all. </p> + takes effect on all nodes, or none.</p> </section> <section> <title>Consistency</title> - <p><em>Consistency</em>. This transaction property ensures that + <p>The consistency property ensures that a transaction always leaves the DBMS in a consistent state. For - example, Mnesia ensures that inconsistencies will not occur if - Erlang, Mnesia or the computer crashes while a write operation - is in progress. - </p> + example, <c>Mnesia</c> ensures that no inconsistencies occur if + Erlang, <c>Mnesia</c>, or the computer crashes while a write + operation is in progress.</p> </section> <section> <title>Isolation</title> - <p><em>Isolation</em>. This transaction property ensures that - transactions which execute on different nodes in a network, and - access and manipulate the same data records, will not interfere - with each other. - </p> - <p>The isolation property makes it possible to concurrently execute - the <c>raise/2</c> function. A classical problem in concurrency control - theory is the so called "lost update problem". - </p> - <p>The isolation property is extremely useful if the following - circumstances occurs where an employee (with an employee number - 123) and two processes, (P1 and P2), are concurrently trying to - raise the salary for the employee. The initial value of the - employees salary is, for example, 5. Process P1 then starts to execute, - it reads the employee record and adds 2 to the salary. At this - point in time, process P1 is for some reason preempted and - process P2 has the opportunity to run. P2 reads the record, adds 3 - to the salary, and finally writes a new employee record with - the salary set to 8. Now, process P1 start to run again and + <p>The isolation property ensures that + transactions that execute on different nodes in a network, and + access and manipulate the same data records, do not interfere + with each other. The isolation property makes it possible to + execute the function <c>raise/2</c> concurrently. A classical + problem in concurrency control theory is the "lost update + problem".</p> + <p>The isolation property is in particular useful if the following + circumstances occur where an employee (with employee number + 123) and two processes (P1 and P2) are concurrently trying to + raise the salary for the employee:</p> + <list type="bulleted"> + <item><em>Step 1:</em> The initial value of the employees salary + is, for example, 5. Process P1 starts to execute, reads the + employee record, and adds 2 to the salary.</item> + <item><em>Step 2:</em> Process P1 is for some reason pre-empted + and process P2 has the opportunity to run.</item> + <item><em>Step 3:</em> Process P2 reads the record, adds 3 to + the salary, and finally writes a new employee record with + the salary set to 8.</item> + <item><em>Step 4:</em> Process P1 starts to run again and writes its employee record with salary set to 7, thus effectively overwriting and undoing the work performed by - process P2. The update performed by P2 is lost. - </p> - <p>A transaction system makes it possible to concurrently - execute two or more processes which manipulate the same - record. The programmer does not need to check that the - updates are synchronous, this is overseen by the + process P2. The update performed by P2 is lost.</item> + </list> + <p>A transaction system makes it possible to execute two or more + processes concurrently that manipulate the same record. + The programmer does not need to check that the + updates are synchronous; this is overseen by the transaction handler. All programs accessing the database through - the transaction system may be written as if they had sole access - to the data. - </p> + the transaction system can be written as if they had sole access + to the data.</p> </section> <section> <title>Durability</title> - <p><em>Durability</em>. This transaction property ensures that + <p>The durability property ensures that changes made to the DBMS by a transaction are permanent. Once a - transaction has been committed, all changes made to the database - are durable - i.e. they are written safely to disc and will not - be corrupted or disappear. - </p> + transaction is committed, all changes made to the database are + durable, that is, they are written safely to disc and do not + become corrupted and do not disappear.</p> <note> - <p>The durability feature described does not entirely apply to - situations where Mnesia is configured as a "pure" primary memory - database. - </p> + <p>The described durability feature does not entirely apply to + situations where <c>Mnesia</c> is configured as a "pure" + primary memory database.</p> </note> </section> </section> @@ -195,397 +178,383 @@ <section> <title>Locking</title> <p>Different transaction managers employ different strategies to - satisfy the isolation property. Mnesia uses the standard technique - of two-phase locking. This means that locks are set on records - before they are read or written. Mnesia uses five different kinds - of locks. - </p> + satisfy the isolation property. <c>Mnesia</c> uses the standard + technique of two phase locking. That is, locks are set on records + before they are read or written. <c>Mnesia</c> uses the following + lock types:</p> <list type="bulleted"> <item><em>Read locks</em>. A read lock is set on one replica of a record before it can be read. </item> - <item><em>Write locks</em>. Whenever a transaction writes to an + <item><em>Write locks</em>. Whenever a transaction writes to a record, write locks are first set on all replicas of that - particular record. + particular record. </item> <item><em>Read table locks</em>. If a transaction traverses an - entire table in search for a record which satisfy some + entire table in search for a record that satisfies some particular property, it is most inefficient to set read locks on - the records, one by one. It is also very memory consuming, since - the read locks themselves may take up considerable space if the - table is very large. For this reason, Mnesia can set a read lock - on an entire table. + the records one by one. It is also memory consuming, as + the read locks themselves can take up considerable space if the + table is large. Therefore, <c>Mnesia</c> can set a read lock + on an entire table. </item> - <item><em>Write table locks</em>. If a transaction writes a - large number of records to one table, it is possible to set a - write lock on the entire table. + <item><em>Write table locks</em>. If a transaction writes many + records to one table, a write lock can be set on the entire table. </item> <item><em>Sticky locks</em>. These are write locks that stay in - place at a node after the transaction which initiated the lock - has terminated. </item> + place at a node after the transaction that initiated the lock + has terminated.</item> </list> - <p>Mnesia employs a strategy whereby functions such as - <c>mnesia:read/1</c> acquire the necessary locks dynamically as - the transactions execute. Mnesia automatically sets and releases - the locks and the programmer does not have to code these - operations. - </p> + <p><c>Mnesia</c> employs a strategy whereby functions, such as + <seealso marker="mnesia#read/1">mnesia:read/1</seealso> + acquire the necessary locks dynamically as + the transactions execute. <c>Mnesia</c> automatically sets and + releases the locks and the programmer does not need to code these + operations.</p> <p>Deadlocks can occur when concurrent processes set and release - locks on the same records. Mnesia employs a "wait-die" strategy to - resolve these situations. If Mnesia suspects that a deadlock can + locks on the same records. <c>Mnesia</c> employs a "wait-die" + strategy to resolve + these situations. If <c>Mnesia</c> suspects that a deadlock can occur when a transaction tries to set a lock, the transaction is - forced to release all its locks and sleep for a while. The - Fun in the transaction will be evaluated one more time. - </p> - <p>For this reason, it is important that the code inside the Fun given to - <c>mnesia:transaction/1</c> is pure. Some strange results can + forced to release all its locks and sleep for a while. The Fun + in the transaction is evaluated once more.</p> + <p>It is therefore important that the code inside the Fun given to + <seealso marker="mnesia#transaction/2"><c>mnesia:transaction/1</c></seealso> + is pure. Some strange results can occur if, for example, messages are sent by the transaction - Fun. The following example illustrates this situation: - </p> + Fun. The following example illustrates this situation:</p> <codeinclude file="company.erl" tag="%6" type="erl"></codeinclude> - <p>This transaction could write the text <c>"Trying to write ... "</c> a thousand times to the terminal. Mnesia does guarantee, - however, that each and every transaction will eventually run. As a - result, Mnesia is not only deadlock free, but also livelock - free. - </p> - <p>The Mnesia programmer cannot prioritize one particular - transaction to execute before other transactions which are waiting - to execute. As a result, the Mnesia DBMS transaction system is not - suitable for hard real time applications. However, Mnesia contains - other features that have real time properties. - </p> - <p>Mnesia dynamically sets and releases locks as - transactions execute, therefore, it is very dangerous to execute code with + <p>This transaction can write the text <c>"Trying to write ... "</c> + 1000 times to the terminal. However, <c>Mnesia</c> guarantees + that each transaction will eventually run. As a result, + <c>Mnesia</c> is not only deadlock free, but also livelock free.</p> + <p>The <c>Mnesia</c> programmer cannot prioritize one particular + transaction to execute before other transactions that are waiting + to execute. As a result, the <c>Mnesia</c> DBMS transaction system is + not suitable for hard real-time applications. However, <c>Mnesia</c> + contains other features that have real-time properties.</p> + <p><c>Mnesia</c> dynamically sets and releases locks as transactions + execute. It is therefore dangerous to execute code with transaction side-effects. In particular, a <c>receive</c> statement inside a transaction can lead to a situation where the transaction hangs and never returns, which in turn can cause locks - not to release. This situation could bring the whole system to a - standstill since other transactions which execute in other + not to release. This situation can bring the whole system to a + standstill, as other transactions that execute in other processes, or on other nodes, are forced to wait for the defective - transaction. - </p> - <p>If a transaction terminates abnormally, Mnesia will - automatically release the locks held by the transaction. - </p> - <p>We have shown examples of a number of functions that can be - used inside a transaction. The following list shows the - <em>simplest</em> Mnesia functions that work with transactions. It - is important to realize that these functions must be embedded in a + transaction.</p> + <p>If a transaction terminates abnormally, <c>Mnesia</c> + automatically releases the locks held by the transaction.</p> + <p>Up to now, examples of a number of functions that can be used + inside a transaction have been shown. The following list shows + the <em>simplest</em> <c>Mnesia</c> functions that work with + transactions. Notice that these functions must be embedded in a transaction. If no enclosing transaction (or other enclosing - Mnesia activity) exists, they will all fail. - </p> + <c>Mnesia</c> activity) exists, they all fail.</p> <list type="bulleted"> - <item><c>mnesia:transaction(Fun) -> {aborted, Reason} |{atomic, Value}</c>. This function executes one transaction with the - functional object <c>Fun</c> as the single parameter. + <item><seealso marker="mnesia#transaction/2">mnesia:transaction(Fun) -> {aborted, Reason} |{atomic, Value}</seealso> + executes one transaction with the + functional object <c>Fun</c> as the single parameter. </item> - <item><c>mnesia:read({Tab, Key}) -> transaction abort | RecordList</c>. This function reads all records with <c>Key</c> - as key from table <c>Tab</c>. This function has the same semantics + <item><seealso marker="mnesia#read/1">mnesia:read({Tab, Key}) -> transaction abort | RecordList</seealso> + reads all records with <c>Key</c> + as key from table <c>Tab</c>. This function has the same semantics regardless of the location of <c>Table</c>. If the table is of - type <c>bag</c>, the <c>read({Tab, Key})</c> can return an arbitrarily + type <c>bag</c>, <c>read({Tab, Key})</c> can return an arbitrarily long list. If the table is of type <c>set</c>, the list is - either of length one, or <c>[]</c>. + either of length one or <c>[]</c>. </item> - <item><c>mnesia:wread({Tab, Key}) -> transaction abort | RecordList</c>. This function behaves the same way as the - previously listed <c>read/1</c> function, except that it - acquires a write lock instead of a read lock. If we execute a - transaction which reads a record, modifies the record, and then + <item><seealso marker="mnesia#wread/1">mnesia:wread({Tab, Key}) -> transaction abort | RecordList</seealso> + behaves the same way as the + previously listed function <c>read/1</c>, except that it + acquires a write lock instead of a read lock. To execute a + transaction that reads a record, modifies the record, and then writes the record, it is slightly more efficient to set the - write lock immediately. In cases where we issue a - <c>mnesia:read/1</c>, followed by a <c>mnesia:write/1</c>, the - first read lock must be upgraded to a write lock when the write - operation is executed. + write lock immediately. When a <seealso marker="mnesia#read/1">mnesia:read/1</seealso> + is issued, followed by a + <seealso marker="mnesia#write/1">mnesia:write/1</seealso> + the first read lock must be upgraded to a write lock when the + write operation is executed. </item> - <item><c>mnesia:write(Record) -> transaction abort | ok</c>. This function writes a record into the database. The - <c>Record</c> argument is an instance of a record. The function - returns <c>ok</c>, or aborts the transaction if an error should - occur. + <item><seealso marker="mnesia#write/1">mnesia:write(Record) -> transaction abort | ok</seealso> + writes a record into the database. Argument + <c>Record</c> is an instance of a record. The function returns + <c>ok</c>, or terminates the transaction if an error occurs. </item> - <item><c>mnesia:delete({Tab, Key}) -> transaction abort | ok</c>. This - function deletes all records with the given key. + <item><seealso marker="mnesia#delete/1">mnesia:delete({Tab, Key}) -> transaction abort | ok</seealso> + deletes all records with the given key. </item> - <item><c>mnesia:delete_object(Record) -> transaction abort | ok</c>. This function deletes records with object id - <c>Record</c>. This function is used when we want to delete only - some records in a table of type <c>bag</c>. </item> + <item><seealso marker="mnesia#delete_object/1">mnesia:delete_object(Record) -> transaction abort | ok</seealso> + deletes records with the OID <c>Record</c>. Use this function to + delete only some records in a table of type <c>bag</c>.</item> </list> <section> <title>Sticky Locks</title> - <p>As previously stated, the locking strategy used by Mnesia is - to lock one record when we read a record, and lock all replicas - of a record when we write a record. However, there are - applications which use Mnesia mainly for its fault-tolerant - qualities, and these applications may be configured with one - node doing all the heavy work, and a standby node which is ready - to take over in case the main node fails. Such applications may + <p>As previously stated, the locking strategy used by <c>Mnesia</c> + is to lock one record when reading a record, and lock all replicas + of a record when writing a record. However, some + applications use <c>Mnesia</c> mainly for its fault-tolerant + qualities. These applications can be configured with one + node doing all the heavy work, and a standby node that is ready + to take over if the main node fails. Such applications can benefit from using sticky locks instead of the normal locking - scheme. - </p> - <p>A sticky lock is a lock which stays in place at a node after - the transaction which first acquired the lock has terminated. To - illustrate this, assume that we execute the following - transaction: - </p> + scheme.</p> + <p>A sticky lock is a lock that stays in place at a node, after + the transaction that first acquired the lock has terminated. To + illustrate this, assume that the following transaction is + executed:</p> <code type="none"> F = fun() -> mnesia:write(#foo{a = kalle}) end, - mnesia:transaction(F). - </code> + mnesia:transaction(F).</code> <p>The <c>foo</c> table is replicated on the two nodes <c>N1</c> - and <c>N2</c>. - <br></br> -Normal locking requires: - </p> + and <c>N2</c>.</p> + <p>Normal locking requires the following:</p> <list type="bulleted"> - <item>one network rpc (2 messages) to acquire the write lock + <item>One network RPC (two messages) to acquire the write lock </item> - <item>three network messages to execute the two-phase commit protocol. + <item>Three network messages to execute the two-phase commit + protocol </item> </list> - <p>If we use sticky locks, we must first change the code as follows: - </p> + <p>If sticky locks are used, the code must first be changed as + follows:</p> <code type="none"> - F = fun() -> mnesia:s_write(#foo{a = kalle}) end, - mnesia:transaction(F). - </code> - <p>This code uses the <c>s_write/1</c> function instead of the - <c>write/1</c> function. The <c>s_write/1</c> function sets a + mnesia:transaction(F).</code> + <p>This code uses the function + <seealso marker="mnesia#s_write/1">s_write/1</seealso> + instead of the function + <seealso marker="mnesia#write/1">write/1</seealso> + The function <c>s_write/1</c> sets a sticky lock instead of a normal lock. If the table is not replicated, sticky locks have no special effect. If the table is - replicated, and we set a sticky lock on node <c>N1</c>, this - lock will then stick to node <c>N1</c>. The next time we try to - set a sticky lock on the same record at node <c>N1</c>, Mnesia - will see that the lock is already set and will not do a network - operation in order to acquire the lock. - </p> - <p>It is much more efficient to set a local lock than it is to set - a networked lock, and for this reason sticky locks can benefit - application that use a replicated table and perform most of the - work on only one of the nodes. - </p> - <p>If a record is stuck at node <c>N1</c> and we try to set a + replicated, and a sticky lock is set on node <c>N1</c>, this + lock then sticks to node <c>N1</c>. The next time you try to + set a sticky lock on the same record at node <c>N1</c>, + <c>Mnesia</c> detects that the lock is already set and do no + network operation to acquire the lock.</p> + <p>It is more efficient to set a local lock than it is to set + a networked lock. Sticky locks can therefore benefit an + application that uses a replicated table and perform most of the + work on only one of the nodes.</p> + <p>If a record is stuck at node <c>N1</c> and you try to set a sticky lock for the record on node <c>N2</c>, the record must be - unstuck. This operation is expensive and will reduce performance. The unsticking is - done automatically if we issue <c>s_write/1</c> requests at - <c>N2</c>. - </p> + unstuck. This operation is expensive and reduces performance. + The unsticking is done automatically if you issue <c>s_write/1</c> + requests at <c>N2</c>.</p> </section> <section> <title>Table Locks</title> - <p>Mnesia supports read and write locks on whole tables as a + <p><c>Mnesia</c> supports read and write locks on whole tables as a complement to the normal locks on single records. As previously - stated, Mnesia sets and releases locks automatically, and the - programmer does not have to code these operations. However, - transactions which read and write a large number of records in a - specific table will execute more efficiently if we start the - transaction by setting a table lock on this table. This will - block other concurrent transactions from the table. The - following two function are used to set explicit table locks for - read and write operations: - </p> + stated, <c>Mnesia</c> sets and releases locks automatically, and + the programmer does not need to code these operations. However, + transactions that read and write many records in a + specific table execute more efficiently if the + transaction is started by setting a table lock on this table. This + blocks other concurrent transactions from the table. The + following two functions are used to set explicit table locks for + read and write operations:</p> <list type="bulleted"> - <item><c>mnesia:read_lock_table(Tab)</c> Sets a read lock on - the table <c>Tab</c></item> - <item><c>mnesia:write_lock_table(Tab)</c> Sets a write lock on - the table <c>Tab</c></item> + <item><seealso marker="mnesia#read_lock_table/1">mnesia:read_lock_table(Tab)</seealso> + sets a read lock on table <c>Tab</c>.</item> + <item><seealso marker="mnesia#write_lock_table/1">mnesia:write_lock_table(Tab)</seealso> + sets a write lock on table <c>Tab</c>.</item> </list> - <p>Alternate syntax for acquisition of table locks is as follows: - </p> + <p>Alternative syntax for acquisition of table locks is as + follows:</p> <code type="none"> mnesia:lock({table, Tab}, read) - mnesia:lock({table, Tab}, write) - </code> - <p>The matching operations in Mnesia may either lock the entire - table or just a single record (when the key is bound in the - pattern). - </p> + mnesia:lock({table, Tab}, write)</code> + <p>The matching operations in <c>Mnesia</c> can either lock the + entire table or only a single record (when the key is bound in + the pattern).</p> </section> <section> <title>Global Locks</title> <p>Write locks are normally acquired on all nodes where a replica of the table resides (and is active). Read locks are - acquired on one node (the local one if a local - replica exists). - </p> - <p>The function <c>mnesia:lock/2</c> is intended to support - table locks (as mentioned previously) - but also for situations when locks need to be - acquired regardless of how tables have been replicated: - </p> + acquired on one node (the local one if a local + replica exists).</p> + <p>The function + <seealso marker="mnesia#lock/2">mnesia:lock/2</seealso> + is intended to support table locks (as mentioned previously) + but also for situations when locks need to be + acquired regardless of how tables have been replicated:</p> <code type="none"> mnesia:lock({global, GlobalKey, Nodes}, LockKind) - LockKind ::= read | write | ... - </code> - <p>The lock is acquired on the LockItem on all Nodes in the - nodes list.</p> + LockKind ::= read | write | ...</code> + <p>The lock is acquired on <c>LockItem</c> on all nodes in the + node list.</p> </section> </section> <section> <title>Dirty Operations</title> <p>In many applications, the overhead of processing a transaction - may result in a loss of performance. Dirty operation are short - cuts which bypass much of the processing and increase the speed - of the transaction. - </p> - <p>Dirty operation are useful in many situations, for example in a datagram routing - application where Mnesia stores the routing table, and it is time + can result in a loss of performance. Dirty operation are short + cuts that bypass much of the processing and increase the speed + of the transaction.</p> + <p>Dirty operation are often useful, for example, in a + datagram routing application + where <c>Mnesia</c> stores the routing table, and it is time consuming to start a whole transaction every time a packet is - received. For this reason, Mnesia has functions which manipulate + received. <c>Mnesia</c> has therefore functions that manipulate tables without using transactions. This alternative - to processing is known as a dirty operation. However, it is important to - realize the trade-off in avoiding the overhead of transaction - processing: - </p> + to processing is known as a dirty operation. However, notice the + trade-off in avoiding the overhead of transaction processing:</p> <list type="bulleted"> - <item>The atomicity and the isolation properties of Mnesia are lost. + <item>The atomicity and the isolation properties of <c>Mnesia</c> + are lost. </item> <item>The isolation property is compromised, because other Erlang processes, which use transaction to manipulate the data, - do not get the benefit of isolation if we simultaneously use - dirty operations to read and write records from the same table. + do not get the benefit of isolation if dirty operations + simultaneously are used to read and write records from the same + table. </item> </list> <p>The major advantage of dirty operations is that they execute - much faster than equivalent operations that are processed as - functional objects within a transaction. - </p> + much faster than equivalent operations that are processed as + functional objects within a transaction.</p> <p>Dirty operations are written to disc if they are performed on a table of type - <c>disc_copies</c>, or type <c>disc_only_copies</c>. Mnesia also - ensures that all replicas of a table are updated if a - dirty write operation is performed on a table. - </p> - <p>A dirty operation will ensure a certain level of consistency. - For example, it is not possible for dirty operations to return - garbled records. Hence, each individual read or write operation - is performed in an atomic manner. - </p> - <p>All dirty functions execute a call to <c>exit({aborted, Reason})</c> on failure. Even if the following functions are - executed inside a transaction no locks will be acquired. The - following functions are available: - </p> + <c>disc_copies</c> or type <c>disc_only_copies</c>. <c>Mnesia</c> + also ensures that all replicas of a table are updated if a + dirty write operation is performed on a table.</p> + <p>A dirty operation ensures a certain level of consistency. + For example, dirty operations cannot return + garbled records. Hence, each individual read or write operation + is performed in an atomic manner.</p> + <p>All dirty functions execute a call to <c>exit({aborted, Reason})</c> + on failure. Even if the following functions are + executed inside a transaction no locks are acquired. The + following functions are available:</p> <list type="bulleted"> - <item><c>mnesia:dirty_read({Tab, Key})</c>. This function reads - record(s) from Mnesia. + <item><seealso marker="mnesia#dirty_read/1">mnesia:dirty_read({Tab, Key})</seealso> + reads one or more records from <c>Mnesia</c>. + </item> + <item><seealso marker="mnesia#dirty_write/1">mnesia:dirty_write(Record)</seealso> + writes the record <c>Record</c>. </item> - <item><c>mnesia:dirty_write(Record)</c>. This function writes - the record <c>Record</c></item> - <item><c>mnesia:dirty_delete({Tab, Key})</c>. This function deletes - record(s) with the key <c>Key</c>. + <item><seealso marker="mnesia#dirty_delete/1">mnesia:dirty_delete({Tab, Key})</seealso> + deletes one or more records with key <c>Key</c>. + </item> + <item><seealso marker="mnesia#dirty_delete_object/1">mnesia:dirty_delete_object(Record)</seealso> + is the dirty operation alternative to the function + <seealso marker="mnesia#delete_object/1">delete_object/1</seealso>. </item> - <item><c>mnesia:dirty_delete_object(Record)</c> This function is - the dirty operation alternative to the function - <c>delete_object/1</c></item> <item> - <p><c>mnesia:dirty_first(Tab)</c>. This function returns the - "first" key in the table <c>Tab</c>. </p> - <p>Records in <c>set</c> or <c>bag</c> tables are not sorted. - However, there is - a record order which is not known to the user. - This means that it is possible to traverse a table by means of - this function in conjunction with the <c>dirty_next/2</c> - function. - </p> - <p>If there are no records at all in the table, this function - will return the atom <c>'$end_of_table'</c>. It is not + <p><seealso marker="mnesia#dirty_first/1">mnesia:dirty_first(Tab)</seealso> + returns the "first" key in table <c>Tab</c>.</p> + <p>Records in <c>set</c> or <c>bag</c> tables are not sorted. + However, there is a record order that is unknown to the user. + This means that a table can be traversed by this function + with the function + <seealso marker="mnesia#dirty_next/2">mnesia:dirty_next/2</seealso>.</p> + <p>If there are no records in the table, this function + returns the atom <c>'$end_of_table'</c>. It is not recommended to use this atom as the key for any user - records. - </p> + records.</p> </item> - <item><c>mnesia:dirty_next(Tab, Key)</c>. This function returns - the "next" key in the table <c>Tab</c>. This function makes it + <item><p><seealso marker="mnesia#dirty_next/2">mnesia:dirty_next(Tab, Key)</seealso> + returns the "next" key in table <c>Tab</c>. This function makes it possible to traverse a table and perform some operation on all - records in the table. When the end of the table is reached the + records in the table. When the end of the table is reached, the special key <c>'$end_of_table'</c> is returned. Otherwise, the - function returns a key which can be used to read the actual - record. - <br></br> -The behavior is undefined if any process perform a write - operation on the table while we traverse the table with the - <c>dirty_next/2</c> function. This is because <c>write</c> - operations on a Mnesia table may lead to internal reorganizations - of the table itself. This is an implementation detail, but remember - the dirty functions are low level functions. + function returns a key that can be used to read the actual + record.</p> + <p>The behavior is undefined if any process performs a write + operation on the table while traversing the table with the + function + <seealso marker="mnesia#dirty_next/2">dirty_next/2</seealso> + This is because <c>write</c> + operations on a <c>Mnesia</c> table can lead to internal + reorganizations of the table itself. This is an implementation + detail, but remember that the dirty functions are low-level + functions.</p> </item> - <item><c>mnesia:dirty_last(Tab)</c> This function works exactly like - <c>mnesia:dirty_first/1</c> but returns the last object in - Erlang term order for the <c>ordered_set</c> table type. For + <item><seealso marker="mnesia#dirty_last/1">mnesia:dirty_last(Tab)</seealso> + works exactly like + <seealso marker="mnesia#dirty_first/1">mnesia:dirty_first/1</seealso> + but returns the last object in + Erlang term order for the table type <c>ordered_set</c>. For all other table types, <c>mnesia:dirty_first/1</c> and - <c>mnesia:dirty_last/1</c> are synonyms. + <c>mnesia:dirty_last/1</c> are synonyms. </item> - <item><c>mnesia:dirty_prev(Tab, Key)</c> This function works exactly like - <c>mnesia:dirty_next/2</c> but returns the previous object in - Erlang term order for the ordered_set table type. For - all other table types, <c>mnesia:dirty_next/2</c> and - <c>mnesia:dirty_prev/2</c> are synonyms. + <item><seealso marker="mnesia#dirty_prev/2">mnesia:dirty_prev(Tab, Key)</seealso> + works exactly like + <c>mnesia:dirty_next/2</c> but returns the previous object in + Erlang term order for the table type <c>ordered_set</c>. For + all other table types, <c>mnesia:dirty_next/2</c> and + <c>mnesia:dirty_prev/2</c> are synonyms. </item> <item> - <p><c>mnesia:dirty_slot(Tab, Slot)</c></p> - <p>Returns the list of records that are associated with Slot + <p><seealso marker="mnesia#dirty_slot/2">mnesia:dirty_slot(Tab, Slot)</seealso> + returns the list of records that are associated with <c>Slot</c> in a table. It can be used to traverse a table in a manner - similar to the <c>dirty_next/2</c> function. A table has a + similar to the function <c>dirty_next/2</c>. A table has a number of slots that range from zero to some unknown upper bound. The function <c>dirty_slot/2</c> returns the special atom <c>'$end_of_table'</c> when the end of the table is - reached. - <br></br> -The behavior of this function is undefined if the + reached.</p> + <p>The behavior of this function is undefined if the table is written on while being - traversed. <c>mnesia:read_lock_table(Tab)</c> may be used to - ensure that no transaction protected writes are performed - during the iteration. - </p> + traversed. The function + <seealso marker="mnesia#read_lock_table/1">mnesia:read_lock_table(Tab)</seealso> + can be used to ensure that no transaction-protected writes + are performed during the iteration.</p> </item> - <item> - <p><c>mnesia:dirty_update_counter({Tab,Key}, Val)</c>. </p> - <p>Counters are positive integers with a value greater than or - equal to zero. Updating a counter will add the <c>Val</c> and - the counter where <c>Val</c> is a positive or negative integer. - <br></br> - There exists no special counter records in - Mnesia. However, records on the form of <c>{TabName, Key, Integer}</c> can be used as counters, and can be - persistent. - </p> - <p>It is not possible to have transaction protected updates of - counter records. - </p> + <item><p><seealso marker="mnesia#dirty_update_counter/2">mnesia:dirty_update_counter({Tab,Key}, Val)</seealso>. + Counters are positive integers with a value greater than or + equal to zero. Updating a counter adds <c>Val</c> and the + counter where <c>Val</c> is a positive or negative integer.</p> + <p><c>Mnesia</c> has no special counter records. However, records + of the form <c>{TabName, Key, Integer}</c> can be used as + counters, and can be persistent.</p> + <p>Transaction-protected updates of counter records are not + possible.</p> <p>There are two significant differences when using this function instead of reading the record, performing the - arithmetic, and writing the record: - </p> + arithmetic, and writing the record:</p> <list type="ordered"> - <item>it is much more efficient + <item>It is much more efficient. </item> - <item>the <c>dirty_update_counter/2</c> function is - performed as an atomic operation although it is not protected by - a transaction. Accordingly, no table update is lost if two - processes simultaneously execute the - <c>dirty_update_counter/2</c> function. + <item>The funcion + <seealso marker="mnesia#dirty_update_counter/2">dirty_update_counter/2</seealso> + is performed as an atomic operation although it is not protected + by a transaction. Therfore no table update is lost if two + processes simultaneously execute the function + <c>dirty_update_counter/2</c>. </item> </list> </item> - <item><c>mnesia:dirty_match_object(Pat)</c>. This function is - the dirty equivalent of <c>mnesia:match_object/1</c>. + <item><seealso marker="mnesia#dirty_match_object/2">mnesia:dirty_match_object(Pat)</seealso> + is the dirty equivalent of + <seealso marker="mnesia#match_object/1">mnesia:match_object/1</seealso>. </item> - <item><c>mnesia:dirty_select(Tab, Pat)</c>. This function is - the dirty equivalent of <c>mnesia:select/2</c>. + <item><seealso marker="mnesia#dirty_select/2">mnesia:dirty_select(Tab, Pat)</seealso> + is the dirty equivalent of + <seealso marker="mnesia#select/2"> mnesia:select/2</seealso>. </item> - <item><c>mnesia:dirty_index_match_object(Pat, Pos)</c>. This - function is the dirty equivalent of - <c>mnesia:index_match_object/2</c>. + <item><seealso marker="mnesia#dirty_index_match_object/2">mnesia:dirty_index_match_object(Pat, Pos)</seealso> + is the dirty equivalent of + <seealso marker="mnesia#index_match_object/2">mnesia:index_match_object/2</seealso>. </item> - <item><c>mnesia:dirty_index_read(Tab, SecondaryKey, Pos)</c>. This - function is the dirty equivalent of <c>mnesia:index_read/3</c>. + <item><seealso marker="mnesia#dirty_index_read/3">mnesia:dirty_index_read(Tab, SecondaryKey, Pos)</seealso> + is the dirty equivalent of + <seealso marker="mnesia#index_read/3">mnesia:index_read/3</seealso>. </item> - <item><c>mnesia:dirty_all_keys(Tab)</c>. This function is the - dirty equivalent of <c>mnesia:all_keys/1</c>. + <item><seealso marker="mnesia#dirty_all_keys/1">mnesia:dirty_all_keys(Tab)</seealso> + is the dirty equivalent of <seealso marker="mnesia#all_keys/1"> +mnesia:all_keys/1</seealso>. </item> </list> </section> @@ -593,42 +562,38 @@ The behavior of this function is undefined if the <section> <marker id="recordnames_tablenames"></marker> <title>Record Names versus Table Names</title> - <p>In Mnesia, all records in a table must have the same name. All - the records must be instances of the same - record type. The record name does however not necessarily be - the same as the table name. Even though that it is the case in - the most of the examples in this document. If a table is created - without the <c>record_name</c> property the code below will - ensure all records in the tables have the same name as the table: - </p> + <p>In <c>Mnesia</c>, all records in a table must have the same name. + All the records must be instances of the same + record type. The record name, however, does not necessarily have + to be the same as the table name, although this is the case in + most of the examples in this User's Guide. If a table is created + without property <c>record_name</c>, the following code ensures + that all records in the tables have the same name as the table:</p> <code type="none"> - mnesia:create_table(subscriber, []) - </code> - <p>However, if the table is is created with an explicit record name - as argument, as shown below, it is possible to store subscriber - records in both of the tables regardless of the table names: - </p> + mnesia:create_table(subscriber, [])</code> + <p>However, if the table is created with an explicit record name + as argument, as shown in the following example, subscriber records + can be stored in both of the tables regardless of the table + names:</p> <code type="none"> TabDef = [{record_name, subscriber}], mnesia:create_table(my_subscriber, TabDef), - mnesia:create_table(your_subscriber, TabDef). - </code> - <p>In order to access such - tables it is not possible to use the simplified access functions - as described earlier in the document. For example, - writing a subscriber record into a table requires a - <c>mnesia:write/3</c>function instead of the simplified functions - <c>mnesia:write/1</c> and <c>mnesia:s_write/1</c>: - </p> + mnesia:create_table(your_subscriber, TabDef).</code> + <p>To access such tables, simplified access functions + (as described earlier) cannot be used. For example, + writing a subscriber record into a table requires the function + <seealso marker="mnesia#write/3">mnesia:write/3</seealso> + instead of the simplified functions + <seealso marker="mnesia#write/1">mnesia:write/1</seealso> + and + <seealso marker="mnesia#s_write/1">mnesia:s_write/1</seealso>:</p> <code type="none"> mnesia:write(subscriber, #subscriber{}, write) mnesia:write(my_subscriber, #subscriber{}, sticky_write) - mnesia:write(your_subscriber, #subscriber{}, write) - </code> - <p>The following simplified piece of code illustrates the + mnesia:write(your_subscriber, #subscriber{}, write)</code> + <p>The following simple code illustrates the relationship between the simplified access functions used in - most examples and their more flexible counterparts: - </p> + most of the examples and their more flexible counterparts:</p> <code type="none"> mnesia:dirty_write(Record) -> Tab = element(1, Record), @@ -676,7 +641,7 @@ The behavior of this function is undefined if the mnesia:s_delete_object(Record) -> Tab = element(1, Record), - mnesia:delete_object(Tab, Record. sticky_write). + mnesia:delete_object(Tab, Record, sticky_write). mnesia:read({Tab, Key}) -> mnesia:read(Tab, Key, read). @@ -690,217 +655,222 @@ The behavior of this function is undefined if the mnesia:index_match_object(Pattern, Attr) -> Tab = element(1, Pattern), - mnesia:index_match_object(Tab, Pattern, Attr, read). - </code> + mnesia:index_match_object(Tab, Pattern, Attr, read).</code> </section> <section> <title>Activity Concept and Various Access Contexts</title> - <p>As previously described, a functional object (Fun) performing - table access operations as listed below may be - passed on as arguments to the function - <c>mnesia:transaction/1,2,3</c>: - </p> + <p>As previously described, a Functional Object (Fun) performing + table access operations, as listed here, can be passed + on as arguments to the function + <seealso marker="mnesia#transaction/2">mnesia:transaction/1,2,3</seealso>: + </p> <list type="bulleted"> <item> - <p>mnesia:write/3 (write/1, s_write/1)</p> + <seealso marker="mnesia#write/3">mnesia:write/3 (write/1, s_write/1)</seealso> </item> <item> - <p>mnesia:delete/3 (delete/1, s_delete/1)</p> + <seealso marker="mnesia#delete/3">mnesia:delete/3</seealso> + (<seealso marker="mnesia#delete/1">mnesia:delete/1</seealso>, + <seealso marker="mnesia#s_delete/1">mnesia:s_delete/1</seealso>) </item> <item> - <p>mnesia:delete_object/3 (delete_object/1, s_delete_object/1)</p> + <seealso marker="mnesia#delete_object/3">mnesia:delete_object/3</seealso> + (<seealso marker="mnesia#delete_object/1">mnesia:delete_object/1</seealso>, + <seealso marker="mnesia#s_delete_object/1">mnesia:s_delete_object/1</seealso>) </item> <item> - <p>mnesia:read/3 (read/1, wread/1)</p> + <seealso marker="mnesia#read/3">mnesia:read/3</seealso> + (<seealso marker="mnesia#read/1">mnesia:read/1</seealso>, + <seealso marker="mnesia#wread/1">mnesia:wread/1</seealso>) </item> <item> - <p>mnesia:match_object/2 (match_object/1)</p> + <seealso marker="mnesia#match_object/3">mnesia:match_object/2</seealso> + (<seealso marker="mnesia#match_object/1">mnesia:match_object/1</seealso>) </item> <item> - <p>mnesia:select/3 (select/2)</p> + <seealso marker="mnesia#select/2">mnesia:select/3</seealso> + (<seealso marker="mnesia#select/2">mnesia:select/2</seealso>) </item> <item> - <p>mnesia:foldl/3 (foldl/4, foldr/3, foldr/4)</p> + <seealso marker="mnesia#foldl/3">mnesia:foldl/3</seealso> + (<c>mnesia:foldl/4</c>, + <seealso marker="mnesia#foldr/3">mnesia:foldr/3</seealso>, + <c>mnesia:foldr/4</c>) </item> <item> - <p>mnesia:all_keys/1</p> + <seealso marker="mnesia#all_keys/1">mnesia:all_keys/1</seealso> </item> <item> - <p>mnesia:index_match_object/4 (index_match_object/2)</p> + <seealso marker="mnesia#index_match_object/4">mnesia:index_match_object/4</seealso> + (<seealso marker="mnesia#index_match_object/2">mnesia:index_match_object/2</seealso>) </item> <item> - <p>mnesia:index_read/3</p> + <seealso marker="mnesia#index_read/3">mnesia:index_read/3</seealso> </item> <item> - <p>mnesia:lock/2 (read_lock_table/1, write_lock_table/1)</p> + <seealso marker="mnesia#lock/2">mnesia:lock/2</seealso> + (<seealso marker="mnesia#read_lock_table/1">mnesia:read_lock_table/1</seealso>, + <seealso marker="mnesia#write_lock_table/1">mnesia:write_lock_table/1</seealso>) </item> <item> - <p>mnesia:table_info/2</p> + <seealso marker="mnesia#table_info/2">mnesia:table_info/2</seealso> </item> </list> - <p>These functions will be performed in a - transaction context involving mechanisms like locking, logging, - replication, checkpoints, subscriptions, commit protocols - etc.However, the same function may also be - evaluated in other activity contexts. - <br></br> -The following activity access contexts are currently supported: - </p> + <p>These functions are performed in a + transaction context involving mechanisms, such as locking, logging, + replication, checkpoints, subscriptions, and commit protocols. + However, the same function can also be + evaluated in other activity contexts.</p> + <p>The following activity access contexts are currently supported:</p> <list type="bulleted"> - <item> - <p>transaction </p> - </item> - <item> - <p>sync_transaction</p> - </item> - <item> - <p>async_dirty</p> - </item> - <item> - <p>sync_dirty</p> - </item> - <item> - <p>ets</p> - </item> + <item><c>transaction</c></item> + <item><c>sync_transaction</c></item> + <item><c>async_dirty</c></item> + <item><c>sync_dirty</c></item> + <item><c>ets</c></item> </list> <p>By passing the same "fun" as argument to the function - <c>mnesia:sync_transaction(Fun [, Args])</c> it will be performed - in synced transaction context. Synced transactions waits until all + <seealso marker="mnesia#sync_transaction/3">mnesia:sync_transaction(Fun [, Args])</seealso> + it is performed + in synced transaction context. Synced transactions wait until all active replicas has committed the transaction (to disc) before - returning from the mnesia:sync_transaction call. Using - sync_transaction is useful for applications that are executing on - several nodes and want to be sure that the update is performed on - the remote nodes before a remote process is spawned or a message - is sent to a remote process, and also when combining transaction - writes with dirty_reads. This is also useful in situations where - an application performs frequent or voluminous updates which may - overload Mnesia on other nodes. - </p> + returning from the <c>mnesia:sync_transaction</c> call. Using + <c>sync_transaction</c> is useful in the following cases:</p> + <list type="bulleted"> + <item>When an application executes on several nodes and wants to + be sure that the update is performed on the remote nodes before + a remote process is spawned or a message is sent to a remote + process.</item> + <item>When a combining transaction writes with "dirty_reads", that + is, the functions <c>dirty_match_object</c>, <c>dirty_read</c>, + <c>dirty_index_read</c>, <c>dirty_select</c>, and so on.</item> + <item>When an application performs frequent or voluminous updates + that can overload <c>Mnesia</c> on other nodes.</item> + </list> <p>By passing the same "fun" as argument to the function - <c>mnesia:async_dirty(Fun [, Args])</c> it will be performed in - dirty context. The function calls will be mapped to the - corresponding dirty functions. This will still involve logging, - replication and subscriptions but there will be no locking, - local transaction storage or commit protocols involved. - Checkpoint retainers will be updated but will be updated - "dirty". Thus, they will be updated asynchronously. The - functions will wait for the operation to be performed on one - node but not the others. If the table resides locally no waiting - will occur. - </p> + <seealso marker="mnesia#async_dirty/2">mnesia:async_dirty(Fun [, Args])</seealso>, + it is performed in dirty context. The function calls are mapped to + the corresponding dirty functions. This still involves logging, + replication, and subscriptions but no locking, + local transaction storage, or commit protocols are involved. + Checkpoint retainers are updated but updated + "dirty". Thus, they are updated asynchronously. The + functions wait for the operation to be performed on one + node but not the others. If the table resides locally, no waiting + occurs.</p> <p>By passing the same "fun" as an argument to the function - <c>mnesia:sync_dirty(Fun [, Args])</c> it will be performed in - almost the same context as <c>mnesia:async_dirty/1,2</c>. The - difference is that the operations are performed - synchronously. The caller will wait for the updates to be - performed on all active replicas. Using sync_dirty is useful for - applications that are executing on several nodes and want to be - sure that the update is performed on the remote nodes before a remote - process is spawned or a message is sent to a remote process. This - is also useful in situations where an application performs frequent or - voluminous updates which may overload Mnesia on other - nodes. - </p> - <p>You can check if your code is executed within a transaction with - <c>mnesia:is_transaction/0</c>, it returns <c>true</c> when called - inside a transaction context and false otherwise.</p> - - <p>Mnesia tables with storage type RAM_copies and disc_copies - are implemented internally as "ets-tables" and - it is possible for applications to access the these tables - directly. This is only recommended if all options have been weighed - and the possible outcomes are understood. By passing the earlier - mentioned "fun" to the function - <c>mnesia:ets(Fun [, Args])</c> it will be performed but in a very raw - context. The operations will be performed directly on the - local ets tables assuming that the local storage type are - RAM_copies and that the table is not replicated on other - nodes. Subscriptions will not be triggered nor - checkpoints updated, but this operation is blindingly fast. Disc resident - tables should not be updated with the ets-function since the - disc will not be updated. - </p> - <p>The Fun may also be passed as an argument to the function - <c>mnesia:activity/2,3,4</c> which enables usage of customized + <seealso marker="mnesia#sync_dirty/2">mnesia:sync_dirty(Fun [, Args])</seealso>, + it is performed in almost the same context as the function + <seealso marker="mnesia#async_dirty/2">mnesia:async_dirty/1,2</seealso>. + The difference is that the operations are performed + synchronously. The caller waits for the updates to be + performed on all active replicas. Using <c>mnesia:sync_dirty/1,2</c> + is useful in the following cases:</p> + <list type="bulleted"> + <item>When an application executes on several nodes and wants to + be sure that the update is performed on the remote nodes before + a remote process is spawned or a message is sent to a remote + process.</item> + <item>When an application performs frequent or voluminous updates + that can overload <c>Mnesia</c> on the nodes.</item> + </list> + <p>To check if your code is executed within a transaction, use + the function + <seealso marker="mnesia#is_transaction/0">mnesia:is_transaction/0</seealso>. + It returns <c>true</c> when called + inside a transaction context, otherwise <c>false</c>.</p> + <p><c>Mnesia</c> tables with storage type <c>RAM_copies</c> and + <c>disc_copies</c> are implemented internally as + <c>ets</c> tables. Applications can access the these tables + directly. This is only + recommended if all options have been weighed and the possible + outcomes are understood. By passing the earlier mentioned "fun" + to the function + <seealso marker="mnesia#ets/2">mnesia:ets(Fun [, Args])</seealso>, + it is performed but in a raw + context. The operations are performed directly on the + local <c>ets</c> tables, assuming that the local storage type is + <c>RAM_copies</c> and that the table is not replicated on other + nodes.</p> + <p>Subscriptions are not triggered and no checkpoints are updated, + but this operation is blindingly fast. Disc resident + tables are not to be updated with the <c>ets</c> function, as the + disc is not updated.</p> + <p>The Fun can also be passed as an argument to the function + <seealso marker="mnesia#activity-4">mnesia:activity/2,3,4</seealso>, + which enables use of customized activity access callback modules. It can either be obtained - directly by stating the module name as argument or implicitly - by usage of the <c>access_module</c> configuration parameter. A - customized callback module may be used for several purposes, - such as providing triggers, integrity constraints, run time - statistics, or virtual tables. - <br></br> - The callback module does - not have to access real Mnesia tables, it is free to do whatever - it likes as long as the callback interface is fulfilled. - <br></br> - In Appendix C "The Activity Access Call Back Interface" the source - code for one alternate implementation is provided - (mnesia_frag.erl). The context sensitive function - <c>mnesia:table_info/2</c> may be used to provide virtual - information about a table. One usage of this is to perform + directly by stating the module name as argument, or implicitly + by use of configuration parameter <c>access_module</c>. A + customized callback module can be used for several purposes, + such as providing triggers, integrity constraints, runtime + statistics, or virtual tables.</p> + <p>The callback module does not have + to access real <c>Mnesia</c> tables, it is free to do whatever + it wants as long as the callback interface is fulfilled.</p> + <p><seealso marker="Mnesia_App_B">Appendix B, + Activity Access Callback Interface</seealso> provides the + source code, <c>mnesia_frag.erl</c>, for one alternative + implementation. The context-sensitive function + <seealso marker="mnesia#table_info/2">mnesia:table_info/2</seealso> + can be used to provide virtual + information about a table. One use of this is to perform <c>QLC</c> queries within an activity context with a customized callback module. By providing table information about - table indices and other <c>QLC</c> requirements, - <c>QLC</c> may be used as a generic query language to - access virtual tables. - </p> - <p>QLC queries may be performed in all these activity - contexts (transaction, sync_transaction, async_dirty, sync_dirty - and ets). The ets activity will only work if the table has no - indices. - </p> + table indexes and other <c>QLC</c> requirements, <c>QLC</c> can + be used as a generic query language to access virtual tables.</p> + <p>QLC queries can be performed in all these activity + contexts (<c>transaction</c>, <c>sync_transaction</c>, + <c>async_dirty</c>, <c>sync_dirty</c>, and <c>ets</c>). The + <c>ets</c> activity only works if the table has no indexes.</p> <note> - <p>The mnesia:dirty_* function always executes with - async_dirty semantics regardless of which activity access contexts - are invoked. They may even invoke contexts without any - enclosing activity access context.</p> + <p>The function <c>mnesia:dirty_*</c> always executes with + <c>async_dirty</c> semantics regardless of which activity + access contexts that are started. It can even start contexts + without any enclosing activity access context.</p> </note> </section> <section> - <title>Nested transactions</title> - <p>Transactions may be nested in an arbitrary fashion. A child transaction - must run in the same process as its parent. When a child transaction - aborts, the caller of the child transaction will get the - return value <c>{aborted, Reason}</c> and any work performed - by the child will be erased. If a child transaction commits, the - records written by the child will be propagated to the parent. - </p> + <title>Nested Transactions</title> + <p>Transactions can be nested in an arbitrary fashion. A child + transaction must run in the same process as its parent. When a + child transaction terminates, the caller of the child transaction + gets return value <c>{aborted, Reason}</c> and any work performed + by the child is erased. If a child transaction commits, the + records written by the child are propagated to the parent.</p> <p>No locks are released when child transactions terminate. Locks - created by a sequence of nested transactions are kept until - the topmost transaction terminates. Furthermore, any updates - performed by a nested transaction are only propagated + created by a sequence of nested transactions are kept until + the topmost transaction terminates. Furthermore, any update + performed by a nested transaction is only propagated in such a manner so that the parent of the nested transaction - sees the updates. No final commitment will be done until - the top level transaction is terminated. + sees the updates. No final commitment is done until + the top-level transaction terminates. So, although a nested transaction returns <c>{atomic, Val}</c>, - if the enclosing parent transaction is aborted, the entire - nested operation is aborted. - </p> + if the enclosing parent transaction terminates, the entire + nested operation terminates.</p> <p>The ability to have nested transaction with identical semantics - as top level transaction makes it easier to write - library functions that manipulate mnesia tables. - </p> - <p>Say for example that we have a function that adds a - new subscriber to a telephony system:</p> + as top-level transaction makes it easier to write + library functions that manipulate <c>Mnesia</c> tables.</p> + <p>Consider a function that adds a subscriber to a telephony + system:</p> <pre> add_subscriber(S) -> mnesia:transaction(fun() -> - case mnesia:read( .......... - </pre> + case mnesia:read( ..........</pre> <p>This function needs to be called as a transaction. - Now assume that we wish to write a function that - both calls the <c>add_subscriber/1</c> function and + Assume that you wish to write a function that + both calls the function <c>add_subscriber/1</c> and is in itself protected by the context of a transaction. - By simply calling the <c>add_subscriber/1</c> from within - another transaction, a nested transaction is created. - </p> - <p>It is also possible to mix different activity access contexts while nesting, - but the dirty ones (async_dirty,sync_dirty and ets) will inherit the transaction - semantics if they are called inside a transaction and thus it will grab locks and - use two or three phase commit. - </p> + By calling <c>add_subscriber/1</c> from within + another transaction, a nested transaction is created.</p> + <p>Also, different activity access contexts can be mixed while + nesting. However, the dirty ones (<c>async_dirty</c>, + <c>sync_dirty</c>, and <c>ets</c>) inherit the transaction + semantics if they are called inside a transaction and thus + grab locks and use two or three phase commit.</p> + <p><em>Example:</em></p> <pre> add_subscriber(S) -> mnesia:transaction(fun() -> @@ -915,17 +885,18 @@ The following activity access contexts are currently supported: mnesia:read({some_tab, some_data}), mnesia:transaction(fun() -> %% In a transaction context. - case mnesia:read( ..) ..end), end). - </pre> + case mnesia:read( ..) ..end), end).</pre> </section> <section> <title>Pattern Matching</title> <marker id="matching"></marker> - <p>When it is not possible to use <c>mnesia:read/3</c> Mnesia + <p>When the function + <seealso marker="mnesia#read/3">mnesia:read/3</seealso> + cannot be used, <c>Mnesia</c> provides the programmer with several functions for matching - records against a pattern. The most useful functions of these are: - </p> + records against a pattern. The most useful ones + are the following:</p> <code type="none"> mnesia:select(Tab, MatchSpecification, LockKind) -> transaction abort | [ObjectList] @@ -934,170 +905,178 @@ The following activity access contexts are currently supported: mnesia:select(Cont) -> transaction abort | {[Object],Continuation} | '$end_of_table' mnesia:match_object(Tab, Pattern, LockKind) -> - transaction abort | RecordList - </code> - <p>These functions matches a <c>Pattern</c> against all records in - table <c>Tab</c>. In a <c>mnesia:select</c> call <c>Pattern</c> is - a part of <c>MatchSpecification</c> described below. It is not - necessarily performed as an exhaustive search of the entire - table. By utilizing indices and bound values in the key of the - pattern, the actual work done by the function may be condensed - into a few hash lookups. Using <c>ordered_set</c> tables may reduce the - search space if the keys are partially bound. - </p> + transaction abort | RecordList</code> + <p>These functions match a <c>Pattern</c> against all records in + table <c>Tab</c>. In a + <seealso marker="mnesia#select/2">mnesia:select</seealso> + call, <c>Pattern</c> is + a part of <c>MatchSpecification</c> described in the following. It + is not necessarily performed as an exhaustive search of the entire + table. By using indexes and bound values in the key of the + pattern, the actual work done by the function can be condensed + into a few hash lookups. Using <c>ordered_set</c> tables can reduce + the search space if the keys are partially bound.</p> <p>The pattern provided to the functions must be a valid record, and the first element of the provided tuple must be the <c>record_name</c> of the table. The special element <c>'_'</c> matches any data structure in Erlang (also known as an Erlang - term). The special elements <c><![CDATA['$<number>']]></c> behaves as Erlang - variables i.e. matches anything and binds the first occurrence and - matches the coming occurrences of that variable against the bound value. - </p> - <p>Use the function <c>mnesia:table_info(Tab, wild_pattern)</c> - to obtain a basic pattern which matches all records in a table + term). The special elements <c><![CDATA['$<number>']]></c> + behave as Erlang variables, that is, they match anything, + bind the first occurrence, and match the + coming occurrences of that variable against the bound value.</p> + <p>Use function + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, wild_pattern)</seealso> + to obtain a basic pattern, which matches all records in a table, or use the default value in record creation. - Do not make the pattern hard coded since it will make your code more - vulnerable to future changes of the record definition. - </p> + Do not make the pattern hard-coded, as this makes the code more + vulnerable to future changes of the record definition.</p> + <p><em>Example:</em></p> <code type="none"> Wildpattern = mnesia:table_info(employee, wild_pattern), %% Or use - Wildpattern = #employee{_ = '_'}, - </code> - <p>For the employee table the wild pattern will look like:</p> + Wildpattern = #employee{_ = '_'},</code> + <p>For the employee table, the wild pattern looks as follows:</p> <code type="none"> - {employee, '_', '_', '_', '_', '_',' _'}. - </code> - <p>In order to constrain the match you must replace some + {employee, '_', '_', '_', '_', '_',' _'}.</code> + <p>To constrain the match, it is needed to replace some of the <c>'_'</c> elements. The code for matching out - all female employees, looks like: - </p> + all female employees looks as follows:</p> <code type="none"> Pat = #employee{sex = female, _ = '_'}, F = fun() -> mnesia:match_object(Pat) end, - Females = mnesia:transaction(F). - </code> - <p>It is also possible to use the match function if we want to - check the equality of different attributes. Assume that we want - to find all employees which happens to have a employee number - which is equal to their room number: - </p> + Females = mnesia:transaction(F).</code> + <p>The match function can also be used to check the equality of + different attributes. For example, to find all employees with + an employee number equal to their room number:</p> <code type="none"> Pat = #employee{emp_no = '$1', room_no = '$1', _ = '_'}, F = fun() -> mnesia:match_object(Pat) end, - Odd = mnesia:transaction(F). - </code> - <p>The function <c>mnesia:match_object/3</c> lacks some important - features that <c>mnesia:select/3</c> have. For example + Odd = mnesia:transaction(F).</code> + <p>The function + <seealso marker="mnesia#match_object/3">mnesia:match_object/3</seealso> + lacks some important features that + <seealso marker="mnesia#select/2">mnesia:select/3</seealso> + have. For example, <c>mnesia:match_object/3</c> can only return the matching records, - and it can not express constraints other then equality. - If we want to find the names of the male employees on the second floor - we could write: - </p> + and it cannot express constraints other than equality. To find + the names of the male employees on the second floor:</p> <codeinclude file="company.erl" tag="%21" type="erl"></codeinclude> - <p>Select can be used to add additional constraints and create - output which can not be done with <c>mnesia:match_object/3</c>. </p> - <p>The second argument to select is a <c>MatchSpecification</c>. - A <c>MatchSpecification</c> is list of <c>MatchFunctions</c>, where + <p>The function <c>select</c> can be used to add more constraints + and create output that cannot be done with + <c>mnesia:match_object/3</c>.</p> + <p>The second argument to <c>select</c> is a <c>MatchSpecification</c>. + A <c>MatchSpecification</c> is a list of <c>MatchFunction</c>s, where each <c>MatchFunction</c> consists of a tuple containing - <c>{MatchHead, MatchCondition, MatchBody}</c>. <c>MatchHead</c> - is the same pattern used in <c>mnesia:match_object/3</c> - described above. <c>MatchCondition</c> is a list of additional - constraints applied to each record, and <c>MatchBody</c> is used - to construct the return values. - </p> - <p>A detailed explanation of match specifications can be found in - the <em>Erts users guide: Match specifications in Erlang </em>, - and the ets/dets documentations may provide some additional - information. - </p> - <p>The functions <c>select/4</c> and <c>select/1</c> are used to - get a limited number of results, where the <c>Continuation</c> - are used to get the next chunk of results. Mnesia uses the - <c>NObjects</c> as an recommendation only, thus more or less - results then specified with <c>NObjects</c> may be returned in - the result list, even the empty list may be returned despite there - are more results to collect. - </p> + <c>{MatchHead, MatchCondition, MatchBody}</c>:</p> + <list type="bulleted"> + <item><c>MatchHead</c> is the same pattern as used in + <c>mnesia:match_object/3</c> described earlier.</item> + <item><c>MatchCondition</c> is a list of extra constraints + applied to each record.</item> + <item><c>MatchBody</c> constructs the return values.</item> + </list> + <p>For details about the match specifications, see + "Match Specifications in Erlang" in + <seealso marker="erts:index">ERTS</seealso> User's Guide. + For more information, see the + <seealso marker="stdlib:ets">ets</seealso> and + <seealso marker="stdlib:dets">dets</seealso> + manual pages in <c>STDLIB</c>.</p> + <p>The functions + <seealso marker="mnesia#select/4">select/4</seealso> and + <seealso marker="mnesia#select/2">select/1</seealso> + are used to + get a limited number of results, where <c>Continuation</c> + gets the next chunk of results. <c>Mnesia</c> uses + <c>NObjects</c> as a recommendation only. Thus, more or less + results than specified with <c>NObjects</c> can be returned in + the result list, even the empty list can be returned even + if there are more results to collect.</p> <warning> <p>There is a severe performance penalty in using - <c>mnesia:select/[1|2|3|4]</c> after any modifying operations - are done on that table in the same transaction, i.e. avoid using - <c>mnesia:write/1</c> or <c>mnesia:delete/1</c> before a - <c>mnesia:select</c> in the same transaction.</p> + <c>mnesia:select/[1|2|3|4]</c> after any modifying operation + is done on that table in the same transaction. That is, avoid + using + <seealso marker="mnesia#write/1">mnesia:write/1</seealso> or + <seealso marker="mnesia#delete/1">mnesia:delete/1</seealso> + before <c>mnesia:select</c> in the same transaction.</p> </warning> <p>If the key attribute is bound in a pattern, the match operation - is very efficient. However, if the key attribute in a pattern is - given as <c>'_'</c>, or <c>'$1'</c>, the whole <c>employee</c> + is efficient. However, if the key attribute in a pattern is + given as <c>'_'</c> or <c>'$1'</c>, the whole <c>employee</c> table must be searched for records that match. Hence if the table is - large, this can become a time consuming operation, but it can be - remedied with indices (refer to Chapter 5: <seealso marker="Mnesia_chap5#indexing">Indexing</seealso>) if - <c>mnesia:match_object</c> is used. - </p> - <p>QLC queries can also be used to search Mnesia tables. By - using <c>mnesia:table/[1|2]</c> as the generator inside a QLC - query you let the query operate on a mnesia table. Mnesia - specific options to <c>mnesia:table/2</c> are {lock, Lock}, - {n_objects,Integer} and {traverse, SelMethod}. The <c>lock</c> - option specifies whether mnesia should acquire a read or write - lock on the table, and <c>n_objects</c> specifies how many - results should be returned in each chunk to QLC. The last option is - <c>traverse</c> and it specifies which function mnesia should - use to traverse the table. Default <c>select</c> is used, but by using - <c>{traverse, {select, MatchSpecification}}</c> as an option to - <c>mnesia:table/2</c> the user can specify it's own view of the - table. - </p> - <p>If no options are specified a read lock will acquired and 100 - results will be returned in each chunk, and select will be used - to traverse the table, i.e.: - </p> + large, this can become a time-consuming operation, but it can be + remedied with indexes (see + <seealso marker="Mnesia_chap5#indexing">Indexing</seealso>) + if the function + <seealso marker="mnesia#match_object/1">mnesia:match_object</seealso> + is used.</p> + <p>QLC queries can also be used to search <c>Mnesia</c> tables. By + using the function + <seealso marker="mnesia#table/1">mnesia:table/[1|2]</seealso> + as the generator inside a QLC + query, you let the query operate on a <c>Mnesia</c> table. + <c>Mnesia</c>-specific options to <c>mnesia:table/2</c> are + <c>{lock, Lock}</c>, <c>{n_objects,Integer}</c>, and + <c>{traverse, SelMethod}</c>:</p> + <list type="bulleted"> + <item><c>lock</c> specifies whether <c>Mnesia</c> is to acquire a + read or write lock on the table.</item> + <item><c>n_objects</c> specifies how many results are to be + returned in each chunk to QLC.</item> + <item><c>traverse</c> specifies which function <c>Mnesia</c> is + to use to traverse the table. Default <c>select</c> is used, but + by using <c>{traverse, {select, MatchSpecification}}</c> as an + option to + <seealso marker="mnesia#table/1">mnesia:table/2</seealso> + the user can specify its own view of the table.</item> + </list> + <p>If no options are specified, a read lock is acquired, 100 + results are returned in each chunk, and <c>select</c> is used + to traverse the table, that is:</p> <code type="none"> mnesia:table(Tab) -> - mnesia:table(Tab, [{n_objects,100},{lock, read}, {traverse, select}]). - </code> - <p>The function <c>mnesia:all_keys(Tab)</c> returns all keys in a - table.</p> + mnesia:table(Tab, [{n_objects,100},{lock, read}, {traverse, select}]).</code> + <p>The function + <seealso marker="mnesia#all_keys/1">mnesia:all_keys(Tab)</seealso> + returns all keys in a table.</p> </section> <section> <title>Iteration</title> <marker id="iteration"></marker> - <p>Mnesia provides a couple of functions which iterates over all - the records in a table. - </p> + <p><c>Mnesia</c> provides the following functions that iterate over all + the records in a table:</p> <code type="none"> mnesia:foldl(Fun, Acc0, Tab) -> NewAcc | transaction abort mnesia:foldr(Fun, Acc0, Tab) -> NewAcc | transaction abort mnesia:foldl(Fun, Acc0, Tab, LockType) -> NewAcc | transaction abort - mnesia:foldr(Fun, Acc0, Tab, LockType) -> NewAcc | transaction abort - </code> - <p>These functions iterate over the mnesia table <c>Tab</c> and - apply the function <c>Fun</c> to each record. The <c>Fun</c> - takes two arguments, the first argument is a record from the - table and the second argument is the accumulator. The - <c>Fun</c> return a new accumulator. </p> - <p>The first time the <c>Fun</c> is applied <c>Acc0</c> will - be the second argument. The next time the <c>Fun</c> is called - the return value from the previous call, will be used as the - second argument. The term the last call to the Fun returns - will be the return value of the <c>fold[lr]</c> function. - </p> - <p>The difference between <c>foldl</c> and <c>foldr</c> is the - order the table is accessed for <c>ordered_set</c> tables, - for every other table type the functions are equivalent. - </p> - <p><c>LockType</c> specifies what type of lock that shall be + mnesia:foldr(Fun, Acc0, Tab, LockType) -> NewAcc | transaction abort</code> + <p>These functions iterate over the <c>Mnesia</c> table <c>Tab</c> + and apply the function <c>Fun</c> to each record. <c>Fun</c> + takes two arguments, the first is a record from the + table, and the second is the accumulator. + <c>Fun</c> returns a new accumulator.</p> + <p>The first time <c>Fun</c> is applied, <c>Acc0</c> is + the second argument. The next time <c>Fun</c> is called, + the return value from the previous call is used as the + second argument. The term the last call to <c>Fun</c> returns + is the return value of the function + <seealso marker="mnesia#foldl/3">mnesia:foldl/3</seealso> or + <seealso marker="mnesia#foldr/3">mnesia:foldr/3</seealso>.</p> + <p>The difference between these functions is the + order the table is accessed for <c>ordered_set</c> tables. + For other table types the functions are equivalent.</p> + <p><c>LockType</c> specifies what type of lock that is to be acquired for the iteration, default is <c>read</c>. If - records are written or deleted during the iteration a write - lock should be acquired. </p> - <p>These functions might be used to find records in a table - when it is impossible to write constraints for - <c>mnesia:match_object/3</c>, or when you want to perform - some action on certain records. - </p> - <p>For example finding all the employees who has a salary - below 10 could look like:</p> + records are written or deleted during the iteration, a write + lock is to be acquired.</p> + <p>These functions can be used to find records in a table + when it is impossible to write constraints for the function + <seealso marker="mnesia#match_object/3">mnesia:match_object/3</seealso>, + or when you want to perform some action on certain records.</p> + <p>For example, finding all the employees who have a salary + less than 10 can look as follows:</p> <code type="none"><![CDATA[ find_low_salaries() -> Constraint = @@ -1109,7 +1088,7 @@ The following activity access contexts are currently supported: Find = fun() -> mnesia:foldl(Constraint, [], employee) end, mnesia:transaction(Find). ]]></code> - <p>Raising the salary to 10 for everyone with a salary below 10 + <p>To raise the salary to 10 for everyone with a salary less than 10 and return the sum of all raises:</p> <code type="none"><![CDATA[ increase_low_salaries() -> @@ -1124,48 +1103,54 @@ The following activity access contexts are currently supported: IncLow = fun() -> mnesia:foldl(Increase, 0, employee, write) end, mnesia:transaction(IncLow). ]]></code> - <p>A lot of nice things can be done with the iterator functions - but some caution should be taken about performance and memory - utilization for large tables. </p> - <p>Call these iteration functions on nodes that contain a replica of the - table. Each call to the function <c>Fun</c> access the table and if the table - resides on another node it will generate a lot of unnecessary - network traffic. </p> - <p>Mnesia also provides some functions that make it possible for - the user to iterate over the table. The order of the - iteration is unspecified if the table is not of the <c>ordered_set</c> - type. </p> + <p>Many nice things can be done with the iterator functions but take + some caution about performance and memory use for large tables.</p> + <p>Call these iteration functions on nodes that contain a replica of + the table. Each call to the function <c>Fun</c> access the table + and if the table resides on another node it generates much + unnecessary network traffic.</p> + <p><c>Mnesia</c> also provides some functions that make it possible + for the user to iterate over the table. The order of the iteration + is unspecified if the table is not of type <c>ordered_set</c>:</p> <code type="none"> mnesia:first(Tab) -> Key | transaction abort mnesia:last(Tab) -> Key | transaction abort mnesia:next(Tab,Key) -> Key | transaction abort mnesia:prev(Tab,Key) -> Key | transaction abort - mnesia:snmp_get_next_index(Tab,Index) -> {ok, NextIndex} | endOfTable - </code> - <p>The order of first/last and next/prev are only valid for - <c>ordered_set</c> tables, for all other tables, they are synonyms. - When the end of the table is reached the special key + mnesia:snmp_get_next_index(Tab,Index) -> {ok, NextIndex} | endOfTable</code> + <p>The order of <c>first</c>/<c>last</c> and <c>next</c>/<c>prev</c> + is only valid for + <c>ordered_set</c> tables, they are synonyms for other tables. + When the end of the table is reached, the special key <c>'$end_of_table'</c> is returned.</p> <p>If records are written and deleted during the traversal, use - <c>mnesia:fold[lr]/4</c> with a <c>write</c> lock. Or - <c>mnesia:write_lock_table/1</c> when using first and next.</p> + the function + <seealso marker="mnesia#foldl">mnesia:foldl/3</seealso> or + <seealso marker="mnesia#foldr">mnesia:foldr/3</seealso> + with a <c>write</c> lock. Or the function + <seealso marker="mnesia#write_lock_table/1">mnesia:write_lock_table/1</seealso> + when using <c>first</c> and <c>next</c>.</p> <p>Writing or deleting in transaction context creates a local copy - of each modified record, so modifying each record in a large - table uses a lot of memory. Mnesia will compensate for every + of each modified record. Thus, modifying each record in a large + table uses much memory. <c>Mnesia</c> compensates for every written or deleted record during the iteration in a transaction - context, which may reduce the performance. If possible avoid writing + context, which can reduce the performance. If possible, avoid writing or deleting records in the same transaction before iterating over the table.</p> - <p>In dirty context, i.e. <c>sync_dirty</c> or <c>async_dirty</c>, + <p>In dirty context, that is, <c>sync_dirty</c> or <c>async_dirty</c>, the modified records are not stored in a local copy; instead, - each record is updated separately. This generates a lot of + each record is updated separately. This generates much network traffic if the table has a replica on another node and has all the other drawbacks that dirty operations - have. Especially for the <c>mnesia:first/1</c> and - <c>mnesia:next/2</c> commands, the same drawbacks as described - above for <c>dirty_first</c> and <c>dirty_next</c> applies, i.e. - no writes to the table should be done during iteration.</p> - <p></p> + have. Especially for commands + <seealso marker="mnesia#first/1">mnesia:first/1</seealso> and + <seealso marker="mnesia#next/2">mnesia:next/2</seealso>, + the same drawbacks as described previously for + <seealso marker="mnesia#dirty_first/1">mnesia:dirty_first/1</seealso> + and + <seealso marker="mnesia#dirty_next/2">mnesia:dirty_next/2</seealso> + applies, that + is, no writing to the table is to be done during iteration.</p> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc index 127c23e0f7..813731e0b8 100644 --- a/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc @@ -31,222 +31,205 @@ <rev></rev> <file>Mnesia_chap5.xml</file> </header> - <p>The earlier chapters of this User Guide described how to get - started with Mnesia, and how to build a Mnesia database. In this - chapter, we will describe the more advanced features available - when building a distributed, fault tolerant Mnesia database. This - chapter contains the following sections: - </p> + + <p>The previous sections describe how to get started + with <c>Mnesia</c> and how to build a <c>Mnesia</c> database. This + section describes the more advanced features available + when building a distributed, fault-tolerant <c>Mnesia</c> database. + The following topics are included:</p> <list type="bulleted"> - <item>Indexing - </item> - <item>Distribution and Fault Tolerance - </item> - <item>Table fragmentation. - </item> - <item>Local content tables. - </item> - <item>Disc-less nodes. - </item> - <item>More about schema management - </item> - <item>Debugging a Mnesia application - </item> - <item>Concurrent Processes in Mnesia - </item> - <item>Prototyping - </item> - <item>Object Based Programming with Mnesia. - </item> + <item>Indexing</item> + <item>Distribution and fault tolerance</item> + <item>Table fragmentation</item> + <item>Local content tables</item> + <item>Disc-less nodes</item> + <item>More about schema management</item> + <item><c>Mnesia</c> event handling</item> + <item>Debugging <c>Mnesia</c> applications</item> + <item>Concurrent processes in <c>Mnesia</c></item> + <item>Prototyping</item> + <item>Object-based programming with <c>Mnesia</c></item> </list> <section> <marker id="indexing"></marker> <title>Indexing</title> - <p>Data retrieval and matching can be performed very efficiently - if we know the key for the record. Conversely, if the key is not - known, all records in a table must be searched. The larger the - table the more time consuming it will become. To remedy this - problem Mnesia's indexing capabilities are used to improve data - retrieval and matching of records. - </p> - <p>The following two functions manipulate indexes on existing tables: - </p> + <p>Data retrieval and matching can be performed efficiently + if the key for the record is known. Conversely, if the key is + unknown, all records in a table must be searched. The larger the + table, the more time consuming it becomes. To remedy this + problem, <c>Mnesia</c> indexing capabilities are used to improve + data retrieval and matching of records.</p> + <p>The following two functions manipulate indexes on existing + tables:</p> <list type="bulleted"> - <item><c>mnesia:add_table_index(Tab, AttributeName) -> {aborted, R} |{atomic, ok}</c></item> - <item><c>mnesia:del_table_index(Tab, AttributeName) -> {aborted, R} |{atomic, ok}</c></item> - </list> - <p>These functions create or delete a table index on field - defined by <c>AttributeName</c>. To illustrate this, add an - index to the table definition <c>(employee, {emp_no, name, salary, sex, phone, room_no}</c>, which is the example table - from the Company database. The function - which adds an index on the element <c>salary</c> can be expressed in - the following way: - </p> - <list type="ordered"> - <item><c>mnesia:add_table_index(employee, salary)</c></item> + <item><seealso marker="mnesia#add_table_index/2">mnesia:add_table_index(Tab, AttributeName) + -> {aborted, R} |{atomic, ok}</seealso></item> + <item><seealso marker="mnesia#del_table_index/2">mnesia:del_table_index(Tab, AttributeName) + -> {aborted, R} |{atomic, ok}</seealso></item> </list> - <p>The indexing capabilities of Mnesia are utilized with the - following three functions, which retrieve and match records on the - basis of index entries in the database. - </p> + <p>These functions create or delete a table index on a field + defined by <c>AttributeName</c>. To illustrate this, add an + index to the table definition <c>(employee, {emp_no, name, + salary, sex, phone, room_no})</c>, which is the example table + from the <c>Company</c> database. The function that + adds an index on element <c>salary</c> can be expressed + as <c>mnesia:add_table_index(employee, salary)</c>.</p> + <p>The indexing capabilities of <c>Mnesia</c> are used with the + following three functions, which retrieve and match records + based on index entries in the database:</p> <list type="bulleted"> - <item><c>mnesia:index_read(Tab, SecondaryKey, AttributeName) -> transaction abort | RecordList</c>. - Avoids an exhaustive search of the entire table, by looking up - the <c>SecondaryKey</c> in the index to find the primary keys. + <item> + <seealso marker="mnesia#index_read/3">mnesia:index_read(Tab, SecondaryKey, AttributeName) + -> transaction abort | RecordList</seealso> + avoids an exhaustive search of the entire table, by looking up + <c>SecondaryKey</c> in the index to find the primary keys. </item> - <item><c>mnesia:index_match_object(Pattern, AttributeName) -> transaction abort | RecordList</c> - Avoids an exhaustive search of the entire table, by looking up + <item> + <seealso marker="mnesia#index_match_object/2">mnesia:index_match_object(Pattern, AttributeName) + -> transaction abort | RecordList</seealso> + avoids an exhaustive search of the entire table, by looking up the secondary key in the index to find the primary keys. - The secondary key is found in the <c>AttributeName</c> field of - the <c>Pattern</c>. The secondary key must be bound. + The secondary key is found in field <c>AttributeName</c> of + <c>Pattern</c>. The secondary key must be bound. </item> - <item><c>mnesia:match_object(Pattern) -> transaction abort | RecordList</c> - Uses indices to avoid exhaustive search of the entire table. - Unlike the other functions above, this function may utilize + <item> + <seealso marker="mnesia#match_object/1">mnesia:match_object(Pattern) + -> transaction abort | RecordList</seealso> + uses indexes to avoid exhaustive search of the entire table. + Unlike the previous functions, this function can use any index as long as the secondary key is bound.</item> </list> <p>These functions are further described and exemplified in - Chapter 4: <seealso marker="Mnesia_chap4#matching">Pattern matching</seealso>. - </p> + <seealso marker="Mnesia_chap4#matching">Pattern Matching</seealso>. + </p> </section> <section> <title>Distribution and Fault Tolerance</title> - <p>Mnesia is a distributed, fault tolerant DBMS. It is possible - to replicate tables on different Erlang nodes in a variety of - ways. The Mnesia programmer does not have to state + <p><c>Mnesia</c> is a distributed, fault-tolerant DBMS. Tables + can be replicated on different Erlang nodes in various + ways. The <c>Mnesia</c> programmer does not need to state where the different tables reside, only the names of the - different tables are specified in the program code. This is - known as "location transparency" and it is an important - concept. In particular: - </p> + different tables need to be specified in the program code. This + is known as "location transparency" and is an important + concept. In particular:</p> <list type="bulleted"> - <item>A program will work regardless of the - location of the data. It makes no difference whether the data - resides on the local node, or on a remote node. <em>Note:</em> The program - will run slower if the data is located on a remote node. + <item><p>A program works regardless of the data + location. It makes no difference whether the data + resides on the local node or on a remote node.</p> + <p>Notice that the program runs slower if the data + is located on a remote node.</p> </item> <item>The database can be reconfigured, and tables can be - moved between nodes. These operations do not effect the user + moved between nodes. These operations do not affect the user programs. </item> </list> - <p>We have previously seen that each table has a number of - system attributes, such as <c>index</c> and - <c>type</c>. - </p> + <p>It has previously been shown that each table has a number of + system attributes, such as <c>index</c> and <c>type</c>.</p> <p>Table attributes are specified when the table is created. For - example, the following function will create a new table with two - RAM replicas: - </p> + example, the following function creates a table with two + RAM replicas:</p> <pre> mnesia:create_table(foo, [{ram_copies, [N1, N2]}, - {attributes, record_info(fields, foo)}]). - </pre> + {attributes, record_info(fields, foo)}]).</pre> <p>Tables can also have the following properties, - where each attribute has a list of Erlang nodes as its value. - </p> + where each attribute has a list of Erlang nodes as its value:</p> <list type="bulleted"> <item> - <p><c>ram_copies</c>. The value of the node list is a list of - Erlang nodes, and a RAM replica of the table will reside on - each node in the list. This is a RAM replica, and it is - important to realize that no disc operations are performed when - a program executes write operations to these replicas. However, - should permanent RAM replicas be a requirement, then the + <p><c>ram_copies</c>. The value of the node list is a list + of Erlang nodes, and a RAM replica of the table resides on + each node in the list.</p> + <p>Notice that no disc operations are performed when + a program executes write operations to these replicas. + However, if permanent RAM replicas are required, the following alternatives are available:</p> <list type="ordered"> - <item>The <c>mnesia:dump_tables/1</c> function can be used - to dump RAM table replicas to disc. + <item>The function + <seealso marker="mnesia#dump_tables/1">mnesia:dump_tables/1</seealso> + can be used to dump RAM table replicas to disc. </item> - <item>The table replicas can be backed up; either from - RAM, or from disc if dumped there with the above - function. + <item>The table replicas can be backed up, either from + RAM, or from disc if dumped there with this function. </item> </list> </item> <item><c>disc_copies</c>. The value of the attribute is a list - of Erlang nodes, and a replica of the table will reside both + of Erlang nodes, and a replica of the table resides both in RAM and on disc on each node in the list. Write operations - addressed to the table will address both the RAM and the disc + addressed to the table address both the RAM and the disc copy of the table. </item> <item><c>disc_only_copies</c>. The value of the attribute is a - list of Erlang nodes, and a replica of the table will reside + list of Erlang nodes, and a replica of the table resides only as a disc copy on each node in the list. The major disadvantage of this type of table replica is the access speed. The major advantage is that the table does not occupy space in memory. </item> </list> - <p>It is also possible to set and change table properties on - existing tables. Refer to Chapter 3: <seealso marker="Mnesia_chap3#def_schema">Defining the Schema</seealso> for full - details. - </p> + <p>In addition, table properties can be set and changed. + For details, see + <seealso marker="Mnesia_chap3#def_schema">Define a Schema</seealso>. + </p> <p>There are basically two reasons for using more than one table - replica: fault tolerance, or speed. It is worthwhile to note + replica: fault tolerance and speed. Notice that table replication provides a solution to both of these - system requirements. - </p> - <p>If we have two active table replicas, all information is - still available if one of the replicas fail. This can be a very + system requirements.</p> + <p>If there are two active table replicas, all information is + still available if one replica fails. This can be an important property in many applications. Furthermore, if a table - replica exists at two specific nodes, applications which execute + replica exists at two specific nodes, applications that execute at either of these nodes can read data from the table without - accessing the network. Network operations are considerably - slower and consume more resources than local operations. - </p> + accessing the network. Network operations are considerably + slower and consume more resources than local operations.</p> <p>It can be advantageous to create table replicas for a - distributed application which reads data often, but writes data - seldom, in order to achieve fast read operations on the local + distributed application that reads data often, but writes data + seldom, to achieve fast read operations on the local node. The major disadvantage with replication is the increased time to write data. If a table has two replicas, every write operation must access both table replicas. Since one of these write operations must be a network operation, it is considerably more expensive to perform a write operation to a replicated - table than to a non-replicated table. - </p> + table than to a non-replicated table.</p> </section> <section> <title>Table Fragmentation</title> <section> - <title>The Concept</title> - <p>A concept of table fragmentation has been introduced in - order to cope with very large tables. The idea is to split a - table into several more manageable fragments. Each fragment - is implemented as a first class Mnesia table and may be - replicated, have indices etc. as any other table. But the - tables may neither have <c>local_content</c> nor have the - <c>snmp</c> connection activated. - </p> - <p>In order to be able to access a record in a fragmented - table, Mnesia must determine to which fragment the - actual record belongs. This is done by the - <c>mnesia_frag</c> module, which implements the - <c>mnesia_access</c> callback behaviour. Please, read the - documentation about <c>mnesia:activity/4</c> to see how - <c>mnesia_frag</c> can be used as a <c>mnesia_access</c> - callback module. - </p> - <p>At each record access <c>mnesia_frag</c> first computes - a hash value from the record key. Secondly the name of the - table fragment is determined from the hash value. And - finally the actual table access is performed by the same + <title>Concept</title> + <p>A concept of table fragmentation has been introduced + to cope with large tables. The idea is to split a + table into several manageable fragments. Each fragment is + implemented as a first class <c>Mnesia</c> table and can be + replicated, have indexes, and so on, as any other table. But + the tables cannot have <c>local_content</c> or have the + <c>snmp</c> connection activated.</p> + <p>To be able to access a record in a fragmented + table, <c>Mnesia</c> must determine to which fragment the + actual record belongs. This is done by module + <c>mnesia_frag</c>, which implements the <c>mnesia_access</c> + callback behavior. It is recommended to read the + documentation about the function + <seealso marker="mnesia#activity/4">mnesia:activity/4</seealso> + to see how <c>mnesia_frag</c> + can be used as a <c>mnesia_access</c> callback module.</p> + <p>At each record access, <c>mnesia_frag</c> first computes + a hash value from the record key. Second, the name of the + table fragment is determined from the hash value. + Finally the actual table access is performed by the same functions as for non-fragmented tables. When the key is not known beforehand, all fragments are searched for - matching records. Note: In <c>ordered_set</c> tables - the records will be ordered per fragment, and the - the order is undefined in results returned by select and - match_object. - </p> - <p>The following piece of code illustrates - how an existing Mnesia table is converted to be a - fragmented table and how more fragments are added later on. - </p> + matching records.</p> + <p>Notice that in <c>ordered_set</c> tables, the records + are ordered per fragment, and the the order is undefined in + results returned by <c>select</c> and <c>match_object</c>.</p> + <p>The following code illustrates how a <c>Mnesia</c> table is + converted to be a fragmented table and how more fragments + are added later:</p> <code type="none"><![CDATA[ Eshell V4.7.3.3 (abort with ^G) (a@sam)1> mnesia:start(). @@ -299,102 +282,96 @@ ok <section> <title>Fragmentation Properties</title> - <p>There is a table property called - <c>frag_properties</c> and may be read with - <c>mnesia:table_info(Tab, frag_properties)</c>. The - fragmentation properties is a list of tagged tuples with - the arity 2. By default the list is empty, but when it is - non-empty it triggers Mnesia to regard the table as - fragmented. The fragmentation properties are: - </p> + <p>The table property <c>frag_properties</c> can be read with + the function + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, frag_properties)</seealso>. + The fragmentation properties are a list of tagged tuples with + arity 2. By default the list is empty, but when it is + non-empty it triggers <c>Mnesia</c> to regard the table as + fragmented. The fragmentation properties are as follows:</p> <taglist> <tag><c>{n_fragments, Int}</c></tag> <item> <p><c>n_fragments</c> regulates how many fragments - that the table currently has. This property may explicitly + that the table currently has. This property can explicitly be set at table creation and later be changed with <c>{add_frag, NodesOrDist}</c> or - <c>del_frag</c>. <c>n_fragment</c>s defaults to <c>1</c>. - </p> + <c>del_frag</c>. <c>n_fragments</c> defaults to <c>1</c>.</p> </item> <tag><c>{node_pool, List}</c></tag> <item> - <p>The node pool contains a list of nodes and may + <p>The node pool contains a list of nodes and can explicitly be set at table creation and later be changed - with <c>{add_node, Node}</c> or <c>{del_node, Node}</c>. At table creation Mnesia tries to distribute + with <c>{add_node, Node}</c> or <c>{del_node, Node}</c>. + At table creation <c>Mnesia</c> tries to distribute the replicas of each fragment evenly over all the nodes in - the node pool. Hopefully all nodes will end up with the + the node pool. Hopefully all nodes end up with the same number of replicas. <c>node_pool</c> defaults to the - return value from <c>mnesia:system_info(db_nodes)</c>. - </p> + return value from the function + <seealso marker="mnesia#system_info/1">mnesia:system_info(db_nodes)</seealso>.</p> </item> <tag><c>{n_ram_copies, Int}</c></tag> <item> <p>Regulates how many <c>ram_copies</c> replicas - that each fragment should have. This property may - explicitly be set at table creation. The default is + that each fragment is to have. This property can + explicitly be set at table creation. Defaults is <c>0</c>, but if <c>n_disc_copies</c> and <c>n_disc_only_copies</c> also are <c>0</c>, - <c>n_ram_copies</c> will default be set to <c>1</c>. - </p> + <c>n_ram_copies</c> defaults to <c>1</c>.</p> </item> <tag><c>{n_disc_copies, Int}</c></tag> <item> - <p>Regulates how many <c>disc_copies</c> replicas - that each fragment should have. This property may - explicitly be set at table creation. The default is <c>0</c>. - </p> + <p>Regulates how many <c>disc_copies</c> replicas that + each fragment is to have. This property can explicitly + be set at table creation. Default is <c>0</c>.</p> </item> <tag><c>{n_disc_only_copies, Int}</c></tag> <item> <p>Regulates how many <c>disc_only_copies</c> replicas - that each fragment should have. This property may - explicitly be set at table creation. The default is <c>0</c>. - </p> + that each fragment is to have. This property can + explicitly be set at table creation. Defaults is + <c>0</c>.</p> </item> <tag><c>{foreign_key, ForeignKey}</c></tag> <item> - <p><c>ForeignKey</c> may either be the atom + <p><c>ForeignKey</c> can either be the atom <c>undefined</c> or the tuple <c>{ForeignTab, Attr}</c>, - where <c>Attr</c> denotes an attribute which should be + where <c>Attr</c> denotes an attribute that is to be interpreted as a key in another fragmented table named - <c>ForeignTab</c>. Mnesia will ensure that the number of + <c>ForeignTab</c>. <c>Mnesia</c> ensures that the number of fragments in this table and in the foreign table are - always the same. When fragments are added or deleted - Mnesia will automatically propagate the operation to all - fragmented tables that has a foreign key referring to this + always the same.</p> + <p>When fragments are added or deleted, <c>Mnesia</c> + automatically propagates the operation to all + fragmented tables that have a foreign key referring to this table. Instead of using the record key to determine which - fragment to access, the value of the <c>Attr</c> field is - used. This feature makes it possible to automatically - co-locate records in different tables to the same - node. <c>foreign_key</c> defaults to - <c>undefined</c>. However if the foreign key is set to - something else it will cause the default values of the + fragment to access, the value of field <c>Attr</c> is + used. This feature makes it possible to colocate records + automatically in different tables to the same node. + <c>foreign_key</c> defaults to + <c>undefined</c>. However, if the foreign key is set to + something else, it causes the default values of the other fragmentation properties to be the same values as - the actual fragmentation properties of the foreign table. - </p> + the actual fragmentation properties of the foreign table.</p> </item> <tag><c>{hash_module, Atom}</c></tag> <item> - <p>Enables definition of an alternate hashing scheme. - The module must implement the <c>mnesia_frag_hash</c> - callback behaviour (see the reference manual). This - property may explicitly be set at table creation. - The default is <c>mnesia_frag_hash</c>.</p> - <p>Older tables that was created before the concept of - user defined hash modules was introduced, uses - the <c>mnesia_frag_old_hash</c> module in order to - be backwards compatible. The <c>mnesia_frag_old_hash</c> - is still using the poor deprecated <c>erlang:hash/1</c> - function. - </p> + <p>Enables definition of an alternative hashing scheme. + The module must implement the + <seealso marker="mnesia_frag_hash">mnesia_frag_hash</seealso> + callback behavior. This property can explicitly be set at + table creation. Default is <c>mnesia_frag_hash</c>.</p> + <p>Older tables, that were created before the concept of + user-defined hash modules was introduced, use module + <c>mnesia_frag_old_hash</c> to be backwards compatible. + <c>mnesia_frag_old_hash</c> still uses the poor + deprecated function <c>erlang:hash/1</c>.</p> </item> <tag><c>{hash_state, Term}</c></tag> <item> - <p>Enables a table specific parameterization - of a generic hash module. This property may explicitly - be set at table creation. - The default is <c>undefined</c>.</p> + <p>Enables a table-specific parameterization of a + generic hash module. This property can explicitly be set + at table creation. Default is <c>undefined</c>.</p> <code type="none"><![CDATA[ Eshell V4.7.3.3 (abort with ^G) (a@sam)1> mnesia:start(). @@ -463,177 +440,159 @@ ok <title>Management of Fragmented Tables</title> <p>The function <c>mnesia:change_table_frag(Tab, Change)</c> is intended to be used for reconfiguration of fragmented - tables. The <c>Change</c> argument should have one of the - following values: - </p> + tables. Argument <c>Change</c> is to have one of the + following values:</p> <taglist> <tag><c>{activate, FragProps}</c></tag> <item> <p>Activates the fragmentation properties of an - existing table. <c>FragProps</c> should either contain - <c>{node_pool, Nodes}</c> or be empty. - </p> + existing table. <c>FragProps</c> is either to contain + <c>{node_pool, Nodes}</c> or be empty.</p> </item> <tag><c>deactivate</c></tag> <item> <p>Deactivates the fragmentation properties of a - table. The number of fragments must be <c>1</c>. No other - tables may refer to this table in its foreign key. - </p> + table. The number of fragments must be <c>1</c>. No other + table can refer to this table in its foreign key.</p> </item> <tag><c>{add_frag, NodesOrDist}</c></tag> <item> - <p>Adds one new fragment to a fragmented table. All - records in one of the old fragments will be rehashed and - about half of them will be moved to the new (last) - fragment. All other fragmented tables, which refers to this - table in their foreign key, will automatically get a new - fragment, and their records will also be dynamically - rehashed in the same manner as for the main table. - </p> - <p>The <c>NodesOrDist</c> argument may either be a list - of nodes or the result from <c>mnesia:table_info(Tab, frag_dist)</c>. The <c>NodesOrDist</c> argument is + <p>Adds a fragment to a fragmented table. All + records in one of the old fragments are rehashed and + about half of them are moved to the new (last) + fragment. All other fragmented tables, which refer to this + table in their foreign key, automatically get a new + fragment. Also, their records are dynamically + rehashed in the same manner as for the main table.</p> + <p>Argument <c>NodesOrDist</c> can either be a list of + nodes or the result from the function + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, frag_dist)</seealso>. + Argument <c>NodesOrDist</c> is assumed to be a sorted list with the best nodes to host new replicas first in the list. The new fragment - will get the same number of replicas as the first - fragment (see <c>n_ram_copies</c>, <c>n_disc_copies</c> + gets the same number of replicas as the first + fragment (see <c>n_ram_copies</c>, <c>n_disc_copies</c>, and <c>n_disc_only_copies</c>). The <c>NodesOrDist</c> list must at least contain one element for each - replica that needs to be allocated. - </p> + replica that needs to be allocated.</p> </item> <tag><c>del_frag</c></tag> <item> - <p>Deletes one fragment from a fragmented table. All - records in the last fragment will be moved to one of the other - fragments. All other fragmented tables which refers to - this table in their foreign key, will automatically lose - their last fragment and their records will also be + <p>Deletes a fragment from a fragmented table. All + records in the last fragment are moved to one of the other + fragments. All other fragmented tables, which refer to + this table in their foreign key, automatically lose + their last fragment. Also, their records are dynamically rehashed in the same manner as for the main - table. - </p> + table.</p> </item> <tag><c>{add_node, Node}</c></tag> <item> - <p>Adds a new node to the <c>node_pool</c>. The new - node pool will affect the list returned from - <c>mnesia:table_info(Tab, frag_dist)</c>. - </p> + <p>Adds a node to <c>node_pool</c>. The new + node pool affects the list returned from the function + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, frag_dist)</seealso>. + </p> </item> <tag><c>{del_node, Node}</c></tag> <item> - <p>Deletes a new node from the <c>node_pool</c>. The - new node pool will affect the list returned from - <c>mnesia:table_info(Tab, frag_dist)</c>.</p> + <p>Deletes a node from <c>node_pool</c>. The new + node pool affects the list returned from the function + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, frag_dist)</seealso>. + </p> </item> </taglist> </section> <section> <title>Extensions of Existing Functions</title> - <p>The function <c>mnesia:create_table/2</c> is used to - create a brand new fragmented table, by setting the table - property <c>frag_properties</c> to some proper values. - </p> - <p>The function <c>mnesia:delete_table/1</c> is used to - delete a fragmented table including all its - fragments. There must however not exist any other - fragmented tables which refers to this table in their foreign key. - </p> - <p>The function <c>mnesia:table_info/2</c> now understands - the <c>frag_properties</c> item. - </p> - <p>If the function <c>mnesia:table_info/2</c> is invoked in - the activity context of the <c>mnesia_frag</c> module, - information of several new items may be obtained: - </p> + <p>The function + <seealso marker="mnesia#create_table/2">mnesia:create_table/2</seealso> + creates a brand new fragmented table, by setting table + property <c>frag_properties</c> to some proper values.</p> + <p>The function + <seealso marker="mnesia#delete_table/1">mnesia:delete_table/1</seealso> + deletes a fragmented table including all its + fragments. There must however not exist any other fragmented + tables that refer to this table in their foreign key.</p> + <p>The function + <seealso marker="mnesia#table_info/2">mnesia:table_info/2</seealso> + now understands item <c>frag_properties</c>.</p> + <p>If the function <c>mnesia:table_info/2</c> is started in + the activity context of module <c>mnesia_frag</c>, + information of several new items can be obtained:</p> <taglist> <tag><c>base_table</c></tag> - <item> - <p>the name of the fragmented table - </p> - </item> + <item>The name of the fragmented table</item> <tag><c>n_fragments</c></tag> - <item> - <p>the actual number of fragments - </p> - </item> + <item>The actual number of fragments</item> <tag><c>node_pool</c></tag> - <item> - <p>the pool of nodes - </p> - </item> + <item>The pool of nodes</item> <tag><c>n_ram_copies</c></tag> <item></item> <tag><c>n_disc_copies</c></tag> <item></item> <tag><c>n_disc_only_copies</c></tag> <item> - <p>the number of replicas with storage type - <c>ram_copies</c>, <c>disc_copies</c> and <c>disc_only_copies</c> + <p>The number of replicas with storage type <c>ram_copies</c>, + <c>disc_copies</c>, and <c>disc_only_copies</c>, respectively. The actual values are dynamically derived from the first fragment. The first fragment serves as a - pro-type and when the actual values needs to be computed - (e.g. when adding new fragments) they are simply - determined by counting the number of each replicas for - each storage type. This means, when the functions - <c>mnesia:add_table_copy/3</c>, - <c>mnesia:del_table_copy/2</c> and<c>mnesia:change_table_copy_type/2</c> are applied on the - first fragment, it will affect the settings on + protype. When the actual values need to be computed + (for example, when adding new fragments) they are + determined by counting the number of each replica for + each storage type. This means that when the functions + <seealso marker="mnesia#add_table_copy/3">mnesia:add_table_copy/3</seealso>, + + <seealso marker="mnesia#del_table_copy/2">mnesia:del_table_copy/2</seealso>, + and + <seealso marker="mnesia#change_table_copy_type/3">mnesia:change_table_copy_type/2</seealso> are applied on the + first fragment, it affects the settings on <c>n_ram_copies</c>, <c>n_disc_copies</c>, and - <c>n_disc_only_copies</c>. - </p> + <c>n_disc_only_copies</c>.</p> </item> <tag><c>foreign_key</c></tag> <item> - <p>the foreign key. - </p> + <p>The foreign key</p> </item> <tag><c>foreigners</c></tag> <item> - <p>all other tables that refers to this table in - their foreign key. - </p> + <p>All other tables that refer to this table in + their foreign key</p> </item> <tag><c>frag_names</c></tag> <item> - <p>the names of all fragments. - </p> + <p>The names of all fragments</p> </item> <tag><c>frag_dist</c></tag> <item> - <p>a sorted list of <c>{Node, Count}</c> tuples - which is sorted in increasing <c>Count</c> order. The + <p>A sorted list of <c>{Node, Count}</c> tuples + that are sorted in increasing <c>Count</c> order. <c>Count</c> is the total number of replicas that this fragmented table hosts on each <c>Node</c>. The list - always contains at least all nodes in the - <c>node_pool</c>. The nodes which not belongs to the - <c>node_pool</c> will be put last in the list even if - their <c>Count</c> is lower. - </p> + always contains at least all nodes in + <c>node_pool</c>. Nodes that do not belong to + <c>node_pool</c> are put last in the list even if + their <c>Count</c> is lower.</p> </item> <tag><c>frag_size</c></tag> <item> - <p>a list of <c>{Name, Size}</c> tuples where - <c>Name</c> is a fragment <c>Name</c> and <c>Size</c> is - how many records it contains. - </p> + <p>A list of <c>{Name, Size}</c> tuples, where + <c>Name</c> is a fragment <c>Name</c>, and <c>Size</c> is + how many records it contains</p> </item> <tag><c>frag_memory</c></tag> <item> - <p>a list of <c>{Name, Memory}</c> tuples where - <c>Name</c> is a fragment <c>Name</c> and <c>Memory</c> is - how much memory it occupies. - </p> + <p>A list of <c>{Name, Memory}</c> tuples, where + <c>Name</c> is a fragment <c>Name</c>, and <c>Memory</c> is + how much memory it occupies</p> </item> <tag><c>size</c></tag> <item> - <p>total size of all fragments - </p> + <p>Total size of all fragments</p> </item> <tag><c>memory</c></tag> <item> - <p>the total memory of all fragments</p> + <p>Total memory of all fragments</p> </item> </taglist> </section> @@ -642,42 +601,45 @@ ok <title>Load Balancing</title> <p>There are several algorithms for distributing records in a fragmented table evenly over a - pool of nodes. No one is best, it simply depends of the - application needs. Here follows some examples of - situations which may need some attention: - </p> - <p><c>permanent change of nodes</c> when a new permanent - <c>db_node</c> is introduced or dropped, it may be time to - change the pool of nodes and re-distribute the replicas - evenly over the new pool of nodes. It may also be time to - add or delete a fragment before the replicas are re-distributed. - </p> - <p><c>size/memory threshold</c> when the total size or + pool of nodes. No one is best, it depends on the + application needs. The following examples of + situations need some attention:</p> + <list type="bulleted"> + <item><c>permanent change of nodes</c>. When a new permanent + <c>db_node</c> is introduced or dropped, it can be time to + change the pool of nodes and redistribute the replicas + evenly over the new pool of nodes. It can also be time to + add or delete a fragment before the replicas are redistributed. + </item> + <item><c>size/memory threshold</c>. When the total size or total memory of a fragmented table (or a single - fragment) exceeds some application specific threshold, it - may be time to dynamically add a new fragment in order - obtain a better distribution of records. - </p> - <p><c>temporary node down</c> when a node temporarily goes - down it may be time to compensate some fragments with new - replicas in order to keep the desired level of - redundancy. When the node comes up again it may be time to - remove the superfluous replica. - </p> - <p><c>overload threshold</c> when the load on some node is - exceeds some application specific threshold, it may be time to - either add or move some fragment replicas to nodes with lesser - load. Extra care should be taken if the table has a foreign - key relation to some other table. In order to avoid severe - performance penalties, the same re-distribution must be - performed for all of the related tables. - </p> - <p>Use <c>mnesia:change_table_frag/2</c> to add new fragments + fragment) exceeds some application-specific threshold, it + can be time to add a new fragment dynamically to + obtain a better distribution of records. + </item> + <item><c>temporary node down</c>. When a node temporarily goes + down, it can be time to compensate some fragments with new + replicas to keep the desired level of + redundancy. When the node comes up again, it can be time to + remove the superfluous replica. + </item> + <item><c>overload threshold</c>. When the load on some node + exceeds some application-specific threshold, it can be time to + either add or move some fragment replicas to nodes with lower + load. Take extra care if the table has a foreign + key relation to some other table. To avoid severe + performance penalties, the same redistribution must be + performed for all the related tables. + </item> + </list> + <p>Use the function + <c>mnesia:change_table_frag/2</c> to add new fragments and apply the usual schema manipulation functions (such as - <c>mnesia:add_table_copy/3</c>, <c>mnesia:del_table_copy/2</c> - and <c>mnesia:change_table_copy_type/2</c>) on each fragment - to perform the actual re-distribution. - </p> + <seealso marker="mnesia#add_table_copy/3">mnesia:add_table_copy/3</seealso>, + <seealso marker="mnesia#del_table_copy/2">mnesia:del_table_copy/2</seealso>, + and + <seealso marker="mnesia#change_table_copy_type/3">mnesia:change_table_copy_type/2</seealso>) + on each fragment to perform the actual redistribution.</p> </section> </section> @@ -685,356 +647,369 @@ ok <title>Local Content Tables</title> <p>Replicated tables have the same content on all nodes where they are replicated. However, it is sometimes advantageous to - have tables but different content on different nodes. - </p> - <p>If we specify the attribute <c>{local_content, true}</c> when - we create the table, the table will reside on the nodes where - we specify that the table shall exist, but the write operations on the - table will only be performed on the local copy. - </p> - <p>Furthermore, when the table is initialized at start-up, the - table will only be initialized locally, and the table - content will not be copied from another node. - </p> + have tables, but different content on different nodes.</p> + <p>If attribute <c>{local_content, true}</c> is specified when + you create the table, the table resides on the nodes where you + specify the table to exist, but the write operations on the + table are only performed on the local copy.</p> + <p>Furthermore, when the table is initialized at startup, the + table is only initialized locally, and the table + content is not copied from another node.</p> </section> <section> - <title>Disc-less Nodes</title> - <p>It is possible to run Mnesia on nodes that do not have a - disc. It is of course not possible to have replicas - of neither <c>disc_copies</c>, nor <c>disc_only_copies</c> - on such nodes. This especially troublesome for the - <c>schema</c> table since Mnesia need the schema in order - to initialize itself. - </p> - <p>The schema table may, as other tables, reside on one or - more nodes. The storage type of the schema table may either - be <c>disc_copies</c> or <c>ram_copies</c> - (not <c>disc_only_copies</c>). At - start-up Mnesia uses its schema to determine with which - nodes it should try to establish contact. If any - of the other nodes are already started, the starting node + <title>Disc-Less Nodes</title> + <p><c>Mnesia</c> can be run on nodes that do not have a disc. + Replicas of <c>disc_copies</c> or <c>disc_only_copies</c> are + not possible on such nodes. This is especially troublesome for + the <c>schema</c> table, as <c>Mnesia</c> needs the schema + to initialize itself.</p> + <p>The schema table can, as other tables, reside on one or + more nodes. The storage type of the schema table can either + be <c>disc_copies</c> or <c>ram_copies</c> + (but not <c>disc_only_copies</c>). At + startup, <c>Mnesia</c> uses its schema to determine with which + nodes it is to try to establish contact. If any + other node is started already, the starting node merges its table definitions with the table definitions brought from the other nodes. This also applies to the - definition of the schema table itself. The application - parameter <c>extra_db_nodes</c> contains a list of nodes which - Mnesia also should establish contact with besides the ones - found in the schema. The default value is the empty list - <c>[]</c>. - </p> + definition of the schema table itself. Application + parameter <c>extra_db_nodes</c> contains a list of nodes that + <c>Mnesia</c> also is to establish contact with besides those + found in the schema. Default is <c>[]</c> (empty list).</p> <p>Hence, when a disc-less node needs to find the schema - definitions from a remote node on the network, we need to supply - this information through the application parameter <c>-mnesia extra_db_nodes NodeList</c>. Without this - configuration parameter set, Mnesia will start as a single node - system. It is also possible to use <c>mnesia:change_config/2</c> - to assign a value to 'extra_db_nodes' and force a connection - after mnesia have been started, i.e. - mnesia:change_config(extra_db_nodes, NodeList). - </p> - <p>The application parameter schema_location controls where - Mnesia will search for its schema. The parameter may be one of - the following atoms: - </p> + definitions from a remote node on the network, this + information must be supplied through application parameter + <c>-mnesia extra_db_nodes NodeList</c>. Without this + configuration parameter set, <c>Mnesia</c> starts as a single + node system. Also, the function + <seealso marker="mnesia#change_config/2">mnesia:change_config/2</seealso> + can be used to assign a value to <c>extra_db_nodes</c> and force + a connection after <c>Mnesia</c> has been started, that is, + <c>mnesia:change_config(extra_db_nodes, NodeList)</c>.</p> + <p>Application parameter <c>schema_location</c> controls where + <c>Mnesia</c> searches for its schema. The parameter can be one + of the following atoms:</p> <taglist> <tag><c>disc</c></tag> <item> <p>Mandatory disc. The schema is assumed to be located - on the Mnesia directory. And if the schema cannot be found, - Mnesia refuses to start. - </p> + in the <c>Mnesia</c> directory. If the schema cannot be found, + <c>Mnesia</c> refuses to start.</p> </item> <tag><c>ram</c></tag> <item> - <p>Mandatory ram. The schema resides in ram - only. At start-up a tiny new schema is generated. This - default schema contains just the definition of the schema - table and only resides on the local node. Since no other - nodes are found in the default schema, the configuration - parameter <c>extra_db_nodes</c> must be used in order to let the - node share its table definitions with other nodes. (The - <c>extra_db_nodes</c> parameter may also be used on disc-full nodes.) - </p> + <p>Mandatory RAM. The schema resides in RAM + only. At startup, a tiny new schema is generated. This + default schema contains only the definition of the schema + table and resides on the local node only. Since no other + nodes are found in the default schema, configuration + parameter <c>extra_db_nodes</c> must be used to let the + node share its table definitions with other nodes. (Parameter + <c>extra_db_nodes</c> can also be used on disc-full nodes.)</p> </item> <tag><c>opt_disc</c></tag> <item> - <p>Optional disc. The schema may reside on either disc - or ram. If the schema is found on disc, Mnesia starts as a - disc-full node (the storage type of the schema table is - disc_copies). If no schema is found on disc, Mnesia starts - as a disc-less node (the storage type of the schema table is - ram_copies). The default value for the application parameter - is - <c>opt_disc</c>. </p> + <p>Optional disc. The schema can reside on either disc or + RAM. If the schema is found on disc, <c>Mnesia</c> starts as + a disc-full node (the storage type of the schema table is + disc_copies). If no schema is found on disc, <c>Mnesia</c> + starts as a disc-less node (the storage type of the schema + table is <c>ram_copies</c>). The default for the + application parameter is <c>opt_disc</c>.</p> </item> </taglist> - <p>When the <c>schema_location</c> is set to opt_disc the - function <c>mnesia:change_table_copy_type/3</c> may be used to - change the storage type of the schema. - This is illustrated below: - </p> + <p>When <c>schema_location</c> is set to <c>opt_disc</c>, the + function + <seealso marker="mnesia#change_table_copy_type/3">mnesia:change_table_copy_type/3</seealso> + can be used to change the storage type of the schema. + This is illustrated as follows:</p> <pre> 1> mnesia:start(). ok 2> mnesia:change_table_copy_type(schema, node(), disc_copies). - {atomic, ok} - </pre> - <p>Assuming that the call to <c>mnesia:start</c> did not - find any schema to read on the disc, then Mnesia has started - as a disc-less node, and then changed it to a node that - utilizes the disc to locally store the schema. - </p> + {atomic, ok}</pre> + <p>Assuming that the call to + <seealso marker="mnesia#start/0">mnesia:start/0</seealso> does not + find any schema to read on the disc, <c>Mnesia</c> starts + as a disc-less node, and then change it to a node that + use the disc to store the schema locally.</p> </section> <section> - <title>More Schema Management</title> - <p>It is possible to add and remove nodes from a Mnesia system. - This can be done by adding a copy of the schema to those nodes. - </p> - <p>The functions <c>mnesia:add_table_copy/3</c> and - <c>mnesia:del_table_copy/2</c> may be used to add and delete - replicas of the schema table. Adding a node to the list - of nodes where the schema is replicated will affect two - things. First it allows other tables to be replicated to - this node. Secondly it will cause Mnesia to try to contact - the node at start-up of disc-full nodes. - </p> - <p>The function call <c>mnesia:del_table_copy(schema, mynode@host)</c> deletes the node 'mynode@host' from the - Mnesia system. The call fails if mnesia is running on - 'mynode@host'. The other mnesia nodes will never try to connect - to that node again. Note, if there is a disc - resident schema on the node 'mynode@host', the entire mnesia - directory should be deleted. This can be done with - <c>mnesia:delete_schema/1</c>. If - mnesia is started again on the the node 'mynode@host' and the - directory has not been cleared, mnesia's behaviour is undefined. - </p> - <p>If the storage type of the schema is ram_copies, i.e, we - have disc-less node, Mnesia - will not use the disc on that particular node. The disc - usage is enabled by changing the storage type of the table - <c>schema</c> to disc_copies. - </p> - <p>New schemas are - created explicitly with <c>mnesia:create_schema/1</c> or implicitly - by starting Mnesia without a disc resident schema. Whenever - a table (including the schema table) is created it is - assigned its own unique cookie. The schema table is not created with - <c>mnesia:create_table/2</c> as normal tables. - </p> - <p>At start-up Mnesia connects different nodes to each other, - then they exchange table definitions with each other and the - table definitions are merged. During the merge procedure Mnesia + <title>More about Schema Management</title> + <p>Nodes can be added to and removed from a <c>Mnesia</c> system. + This can be done by adding a copy of the schema to those nodes.</p> + <p>The functions + <seealso marker="mnesia#add_table_copy/3">mnesia:add_table_copy/3</seealso> + and + <seealso marker="mnesia#del_table_copy/2">mnesia:del_table_copy/2</seealso> + can be used to add and delete + replicas of the schema table. Adding a node to the list of + nodes where the schema is replicated affects the following:</p> + <list type="bulleted"> + <item>It allows other tables to be replicated to this node. + </item> + <item>It causes <c>Mnesia</c> to try to contact the node at + startup of disc-full nodes. + </item> + </list> + <p>The function call <c>mnesia:del_table_copy(schema, + mynode@host)</c> deletes node <c>mynode@host</c> from the + <c>Mnesia</c> system. The call fails if <c>Mnesia</c> is running + on <c>mynode@host</c>. The other <c>Mnesia</c> nodes never try to + connect to that node again. Notice that if there is a disc resident + schema on node <c>mynode@host</c>, the entire <c>Mnesia</c> + directory is to be deleted. This is done with the function + <seealso marker="mnesia#delete_schema/1">mnesia:delete_schema/1</seealso>. + If <c>Mnesia</c> is started again + on node <c>mynode@host</c> and the directory has not been + cleared, the behavior of <c>Mnesia</c> is undefined.</p> + <p>If the storage type of the schema is <c>ram_copies</c>, + that is, a disc-less node, <c>Mnesia</c> + does not use the disc on that particular node. The disc + use is enabled by changing the storage type of table + <c>schema</c> to <c>disc_copies</c>.</p> + <p>New schemas are created explicitly with the function + <seealso marker="mnesia#create_schema/1">mnesia:create_schema/1</seealso> + or implicitly by starting + <c>Mnesia</c> without a disc resident schema. Whenever + a table (including the schema table) is created, it is + assigned its own unique cookie. The schema table is not created + with the function + <seealso marker="mnesia#create_table/2">mnesia:create_table/2</seealso> + as normal tables.</p> + <p>At startup, <c>Mnesia</c> connects different nodes to each other, + then they exchange table definitions with each other, and the table + definitions are merged. During the merge procedure, <c>Mnesia</c> performs a sanity test to ensure that the table definitions are - compatible with each other. If a table exists on several nodes - the cookie must be the same, otherwise Mnesia will shutdown one - of the nodes. This unfortunate situation will occur if a table + compatible with each other. If a table exists on several nodes, + the cookie must be the same, otherwise <c>Mnesia</c> shut down one + of the nodes. This unfortunate situation occurs if a table has been created on two nodes independently of each other while - they were disconnected. To solve the problem, one of the tables - must be deleted (as the cookies differ we regard it to be two - different tables even if they happen to have the same name). - </p> - <p>Merging different versions of the schema table, does not + they were disconnected. To solve this, one of the tables + must be deleted (as the cookies differ, it is regarded to be two + different tables even if they have the same name).</p> + <p>Merging different versions of the schema table does not always require the cookies to be the same. If the storage - type of the schema table is disc_copies, the cookie is - immutable, and all other db_nodes must have the same - cookie. When the schema is stored as type ram_copies, + type of the schema table is <c>disc_copies</c>, the cookie is + immutable, and all other <c>db_nodes</c> must have the same + cookie. When the schema is stored as type <c>ram_copies</c>, its cookie can be replaced with a cookie from another node - (ram_copies or disc_copies). The cookie replacement (during - merge of the schema table definition) is performed each time - a RAM node connects to another node. - </p> - <p><c>mnesia:system_info(schema_location)</c> and - <c>mnesia:system_info(extra_db_nodes)</c> may be used to determine - the actual values of schema_location and extra_db_nodes - respectively. <c>mnesia:system_info(use_dir)</c> may be used to - determine whether Mnesia is actually using the Mnesia - directory. <c>use_dir</c> may be determined even before - Mnesia is started. The function <c>mnesia:info/0</c> may now be - used to printout some system information even before Mnesia - is started. When Mnesia is started the function prints out - more information. - </p> - <p>Transactions which update the definition of a table, - requires that Mnesia is started on all nodes where the - storage type of the schema is disc_copies. All replicas of + (<c>ram_copies</c> or <c>disc_copies</c>). The cookie replacement + (during merge of the schema table definition) is performed each + time a RAM node connects to another node.</p> + <p>Further, the following applies:</p> + <list type ="bulleted"> + <item><seealso marker="mnesia#system_info/1">mnesia:system_info(schema_location)</seealso> + and + <seealso marker="mnesia#system_info/1">mnesia:system_info(extra_db_nodes)</seealso> + can be used to determine the actual values of <c>schema_location</c> + and <c>extra_db_nodes</c>, respectively. + </item> + <item><seealso marker="mnesia#system_info/1">mnesia:system_info(use_dir)</seealso> + can be used to determine whether <c>Mnesia</c> is actually + using the <c>Mnesia</c> directory. + </item> + <item><c>use_dir</c> can be determined even before + <c>Mnesia</c> is started. + </item> + </list> + <p>The function <seealso marker="mnesia#info/0">mnesia:info/0</seealso> + can now be used to print + some system information even before <c>Mnesia</c> is started. + When <c>Mnesia</c> is started, the function prints more + information.</p> + <p>Transactions that update the definition of a table + requires that <c>Mnesia</c> is started on all nodes where the + storage type of the schema is <c>disc_copies</c>. All replicas of the table on these nodes must also be loaded. There are a - few exceptions to these availability rules. Tables may be - created and new replicas may be added without starting all - of the disc-full nodes. New replicas may be added before all - other replicas of the table have been loaded, it will suffice - when one other replica is active. - </p> + few exceptions to these availability rules:</p> + <list type="bulleted"> + <item>Tables can be created and new replicas can be added + without starting all the disc-full nodes. + </item> + <item>New replicas can be added before all other replicas of + the table have been loaded, provided that at least one other + replica is active. + </item> + </list> </section> <section> <marker id="event_handling"></marker> <title>Mnesia Event Handling</title> - <p>System events and table events are the two categories of events - that Mnesia will generate in various situations. - </p> - <p>It is possible for user process to subscribe on the - events generated by Mnesia. - We have the following two functions:</p> + <p>System events and table events are the two event categories + that <c>Mnesia</c> generates in various situations.</p> + <p>A user process can subscribe on the events generated by + <c>Mnesia</c>. The following two functions are provided:</p> <taglist> - <tag><c>mnesia:subscribe(Event-Category)</c></tag> - <item> - <p>Ensures that a copy of all events of type - <c>Event-Category</c> are sent to the calling process. - </p> - </item> - <tag><c>mnesia:unsubscribe(Event-Category)</c></tag> + <tag><seealso marker="mnesia#subscribe/1">mnesia:subscribe(Event-Category)</seealso> + </tag> + <item>Ensures that a copy of all events of type + <c>Event-Category</c> are sent to the calling process</item> + <tag><seealso marker="mnesia#unsubscribe/1">mnesia:unsubscribe(Event-Category)</seealso> + </tag> <item>Removes the subscription on events of type - <c>Event-Category</c></item> + <c>Event-Category</c> + </item> </taglist> - <p><c>Event-Category</c> may either be the atom <c>system</c>, the atom <c>activity</c>, or - one of the tuples <c>{table, Tab, simple}</c>, <c>{table, Tab, detailed}</c>. The old event-category <c>{table, Tab}</c> is the same - event-category as <c>{table, Tab, simple}</c>. - The subscribe functions activate a subscription + <p><c>Event-Category</c> can be either of the following:</p> + <list type="bulleted"> + <item>The atom <c>system</c> + </item> + <item>The atom <c>activity</c> + </item> + <item>The tuple <c>{table, Tab, simple}</c> + </item> + <item>The tuple <c>{table, Tab, detailed}</c> + </item> + </list> + <p>The old event category <c>{table, Tab}</c> is the same + event category as <c>{table, Tab, simple}</c>.</p> + <p>The subscribe functions activate a subscription of events. The events are delivered as messages to the process - evaluating the <c>mnesia:subscribe/1</c> function. The syntax of - system events is <c>{mnesia_system_event, Event}</c>, - <c>{mnesia_activity_event, Event}</c> for activity events, and - <c>{mnesia_table_event, Event}</c> for table events. What the various - event types mean is described below.</p> - <p>All system events are subscribed by Mnesia's - gen_event handler. The default gen_event handler is - <c>mnesia_event</c>. But it may be changed by using the application - parameter <c>event_module</c>. The value of this parameter must be - the name of a module implementing a complete handler - as specified by the <c>gen_event</c> module in - STDLIB. <c>mnesia:system_info(subscribers)</c> and - <c>mnesia:table_info(Tab, subscribers)</c> may be used to determine - which processes are subscribed to various - events. - </p> + evaluating the function + <seealso marker="mnesia#subscribe/1">mnesia:subscribe/1</seealso> + The syntax is as follows:</p> + <list type="bulleted"> + <item><c>{mnesia_system_event, Event}</c> for system events + </item> + <item><c>{mnesia_activity_event, Event}</c> for activity events + </item> + <item><c>{mnesia_table_event, Event}</c> for table events + </item> + </list> + <p>The event types are described in the next sections.</p> + <p>All system events are subscribed by the <c>Mnesia</c> + <c>gen_event</c> handler. The default <c>gen_event</c> handler + is <c>mnesia_event</c>, but it can be changed by using + application parameter <c>event_module</c>. The value of this + parameter must be the name of a module implementing a complete + handler, as specified by the + <seealso marker="stdlib:gen_event">gen_event</seealso> module + in <c>STDLIB</c>.</p> + <p><seealso marker="mnesia#system_info/1">mnesia:system_info(subscribers)</seealso> + and + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, subscribers)</seealso> + can be used to determine which processes are subscribed to + various events.</p> <section> <title>System Events</title> - <p>The system events are detailed below:</p> + <p>The system events are as follows:</p> <taglist> <tag><c>{mnesia_up, Node}</c></tag> - <item> - <p>Mnesia has been started on a node. - Node is the name of the node. By default this event is ignored. - </p> + <item>Mnesia is started on a node. <c>Node</c> is the node + name. By default this event is ignored. </item> <tag><c>{mnesia_down, Node}</c></tag> - <item> - <p>Mnesia has been stopped on a node. - Node is the name of the node. By default this event is - ignored. - </p> + <item>Mnesia is stopped on a node. <c>Node</c> is the node + name. By default this event is ignored. </item> <tag><c>{mnesia_checkpoint_activated, Checkpoint}</c></tag> - <item> - <p>a checkpoint with the name - <c>Checkpoint</c> has been activated and that the current node is - involved in the checkpoint. Checkpoints may be activated - explicitly with <c>mnesia:activate_checkpoint/1</c> or implicitly - at backup, adding table replicas, internal transfer of data - between nodes etc. By default this event is ignored. - </p> + <item>A checkpoint with the name <c>Checkpoint</c> is + activated and the current node is involved in the + checkpoint. Checkpoints can be activated explicitly with + the function + <seealso marker="mnesia#activate_checkpoint/1">mnesia:activate_checkpoint/1</seealso> + or implicitly at + backup, when adding table replicas, at internal transfer of + data between nodes, and so on. By default this event is + ignored. </item> <tag><c>{mnesia_checkpoint_deactivated, Checkpoint}</c></tag> - <item> - <p>A checkpoint with the name - <c>Checkpoint</c> has been deactivated and that the current node was - involved in the checkpoint. Checkpoints may explicitly be - deactivated with <c>mnesia:deactivate/1</c> or implicitly when the - last replica of a table (involved in the checkpoint) - becomes unavailable, e.g. at node down. By default this - event is ignored. - </p> + <item>A checkpoint with the name <c>Checkpoint</c> is + deactivated and the current node is involved in the + checkpoint. Checkpoints can be deactivated explicitly with + the function + <seealso marker="mnesia#deactivate_checkpoint/1">mnesia:deactivate/1</seealso> + or implicitly when the last + replica of a table (involved in the checkpoint) becomes + unavailable, for example, at node-down. By default this + event is ignored. </item> <tag><c>{mnesia_overload, Details}</c></tag> - <item> - <p>Mnesia on the current node is - overloaded and the subscriber should take action. - </p> + <item><p><c>Mnesia</c> on the current node is + overloaded and the subscriber is to take action.</p> <p>A typical overload situation occurs when the - applications are performing more updates on disc - resident tables than Mnesia is able to handle. Ignoring - this kind of overload may lead into a situation where + applications perform more updates on disc resident + tables than <c>Mnesia</c> can handle. Ignoring + this kind of overload can lead to a situation where the disc space is exhausted (regardless of the size of - the tables stored on disc). - <br></br> - Each update is appended to - the transaction log and occasionally(depending of how it + the tables stored on disc).</p> + <p>Each update is appended to the transaction log and + occasionally (depending on how it is configured) dumped to the tables files. The table file storage is more compact than the transaction log storage, especially if the same record is updated - over and over again. If the thresholds for dumping the - transaction log have been reached before the previous - dump was finished an overload event is triggered. - </p> + repeatedly. If the thresholds for dumping the + transaction log are reached before the previous + dump is finished, an overload event is triggered.</p> <p>Another typical overload situation is when the transaction manager cannot commit transactions at the - same pace as the applications are performing updates of - disc resident tables. When this happens the message - queue of the transaction manager will continue to grow + same pace as the applications perform updates of + disc resident tables. When this occurs, the message + queue of the transaction manager continues to grow until the memory is exhausted or the load - decreases. - </p> - <p>The same problem may occur for dirty updates. The overload - is detected locally on the current node, but its cause may - be on another node. Application processes may cause heavy - loads if any table are residing on other nodes (replicated or not). By default this event - is reported to the error_logger. - </p> + decreases.</p> + <p>The same problem can occur for dirty updates. The overload + is detected locally on the current node, but its cause can + be on another node. Application processes can cause high + load if any table resides on another node (replicated + or not). By default this event + is reported to <c>error_logger.</c></p> </item> <tag><c>{inconsistent_database, Context, Node}</c></tag> - <item> - <p>Mnesia regards the database as - potential inconsistent and gives its applications a chance - to recover from the inconsistency, e.g. by installing a - consistent backup as fallback and then restart the system - or pick a <c>MasterNode</c> from <c>mnesia:system_info(db_nodes)</c>) - and invoke <c>mnesia:set_master_node([MasterNode])</c>. By default an - error is reported to the error logger. - </p> + <item><c>Mnesia</c> regards the database as potential + inconsistent and gives its applications a chance to + recover from the inconsistency. For example, by installing a + consistent backup as fallback and then restart the system. + An alternative is to pick a <c>MasterNode</c> from + <seealso marker="mnesia#system_info/1">mnesia:system_info(db_nodes)</seealso> + and invoke + <seealso marker="mnesia#set_master_nodes/1">mnesia:set_master_node([MasterNode])</seealso>. + By default an error is reported to <c>error_logger</c>. </item> <tag><c>{mnesia_fatal, Format, Args, BinaryCore}</c></tag> <item> - <p>Mnesia has encountered a fatal error - and will (in a short period of time) be terminated. The reason for - the fatal error is explained in Format and Args which may - be given as input to <c>io:format/2</c> or sent to the - error_logger. By default it will be sent to the - error_logger. <c>BinaryCore</c> is a binary containing a summary of - Mnesia's internal state at the time the when the fatal error was - encountered. By default the binary is written to a - unique file name on current directory. On RAM nodes the - core is ignored. - </p> + <p><c>Mnesia</c> detected a fatal error and + terminates soon. The fault reason is explained in + <c>Format</c> and <c>Args</c>, which can be given as input + to <c>io:format/2</c> or sent to <c>error_logger</c>. By + default it is sent to <c>error_logger</c>.</p> + <p><c>BinaryCore</c> is a binary containing a summary of the + <c>Mnesia</c> internal state at the time when the fatal + error was detected. By default the binary is written to a + unique filename on the current directory. On RAM nodes, the + core is ignored.</p> </item> <tag><c>{mnesia_info, Format, Args}</c></tag> - <item> - <p>Mnesia has detected something that - may be of interest when debugging the system. This is explained - in <c>Format</c> and <c>Args</c> which may appear - as input to <c>io:format/2</c> or sent to the error_logger. By - default this event is printed with <c>io:format/2</c>. - </p> + <item><c>Mnesia</c> detected something that can be of + interest when debugging the system. This is explained in + <c>Format</c> and <c>Args</c>, which can appear as input + to <c>io:format/2</c> or sent to <c>error_logger</c>. By + default this event is printed with <c>io:format/2</c>. </item> <tag><c>{mnesia_error, Format, Args}</c></tag> - <item> - <p>Mnesia has encountered an error. The - reason for the error is explained i <c>Format</c> and <c>Args</c> - which may be given as input to <c>io:format/2</c> or sent to the - error_logger. By default this event is reported to the error_logger. - </p> + <item><c>Mnesia</c> has detected an error. The fault reason is + explained in <c>Format</c> and <c>Args</c>, which can be + given as input to <c>io:format/2</c> or sent to + <c>error_logger</c>. By default this event is reported to + <c>error_logger</c>. </item> <tag><c>{mnesia_user, Event}</c></tag> - <item> - <p>An application has invoked the - function <c>mnesia:report_event(Event)</c>. <c>Event</c> may be any Erlang - data structure. When tracing a system of Mnesia applications - it is useful to be able to interleave Mnesia's own events with - application related events that give information about the - application context. Whenever the application starts with - a new and demanding Mnesia activity or enters a - new and interesting phase in its execution it may be a good idea - to use <c>mnesia:report_event/1</c>. </p> + <item>An application started the function + <seealso marker="mnesia#report_event/1">mnesia:report_event(Event)</seealso>. + <c>Event</c> can be + any Erlang data structure. When tracing a system of + <c>Mnesia</c> applications, it is useful to be able to + interleave own events of <c>Mnesia</c> with application-related + events that give information about the application context. + Whenever the application starts with a new and demanding + <c>Mnesia</c> activity, or enters a new and interesting + phase in its execution, it can be a good idea to use + <c>mnesia:report_event/1</c>. </item> </taglist> </section> @@ -1045,80 +1020,86 @@ ok <taglist> <tag><c>{complete, ActivityID}</c></tag> <item> - <p>This event occurs when a transaction that caused a modification to the database - has completed. It is useful for determining when a set of table events - (see below) caused by a given activity have all been sent. Once the this event - has been received, it is guaranteed that no further table events with the same - ActivityID will be received. Note that this event may still be received even - if no table events with a corresponding ActivityID were received, depending on + <p>This event occurs when a transaction that caused a modification + to the database is completed. It is useful for determining when + a set of table events (see the next section), caused by a given + activity, have been sent. Once this event is received, it is + guaranteed that no further table events with the same + <c>ActivityID</c> will be received. Notice that this event can + still be received even if no table events with a corresponding + <c>ActivityID</c> were received, depending on the tables to which the receiving process is subscribed.</p> - <p>Dirty operations always only contain one update and thus no activity event is sent.</p> + <p>Dirty operations always contain only one update and thus no + activity event is sent.</p> </item> </taglist> </section> <section> <title>Table Events</title> - <p>The final category of events are table events, which are - events related to table updates. There are two types of table - events simple and detailed. - </p> - <p>The simple table events are tuples looking like this: - <c>{Oper, Record, ActivityId}</c>. Where <c>Oper</c> is the - operation performed. <c>Record</c> is the record involved in the - operation and <c>ActivityId</c> is the identity of the - transaction performing the operation. Note that the name of the - record is the table name even when the <c>record_name</c> has - another setting. The various table related events that may - occur are: - </p> + <p>Table events are events related to table updates. There are + two types of table events, simple and detailed.</p> + <p>The <em>simple table events</em> are tuples like + <c>{Oper, Record, ActivityId}</c>, where:</p> + <list type="bulleted"> + <item><c>Oper</c> is the operation performed. + </item> + <item><c>Record</c> is the record involved in the operation. + </item> + <item><c>ActivityId</c> is the identity of the transaction + performing the operation. + </item> + </list> + <p>Notice that the record name is the table name even when + <c>record_name</c> has another setting.</p> + <p>The table-related events that can occur are as follows:</p> <taglist> <tag><c>{write, NewRecord, ActivityId}</c></tag> - <item> - <p>a new record has been written. - NewRecord contains the new value of the record. - </p> + <item>A new record has been written. <c>NewRecord</c> contains + the new record value. </item> <tag><c>{delete_object, OldRecord, ActivityId}</c></tag> - <item> - <p>a record has possibly been deleted - with <c>mnesia:delete_object/1</c>. <c>OldRecord</c> - contains the value of the old record as stated as argument - by the application. Note that, other records with the same - key may be remaining in the table if it is a bag. - </p> + <item>A record has possibly been deleted with + <seealso marker="mnesia#delete_object/1">mnesia:delete_object/1</seealso>. + <c>OldRecord</c> + contains the value of the old record, as stated as argument + by the application. Notice that other records with the same + key can remain in the table if it is of type <c>bag</c>. </item> <tag><c>{delete, {Tab, Key}, ActivityId}</c></tag> - <item> - <p>one or more records possibly has - been deleted. All records with the key Key in the table - <c>Tab</c> have been deleted. </p> + <item>One or more records have possibly been deleted. + All records with the key <c>Key</c> in the table + <c>Tab</c> have been deleted. </item> </taglist> - <p>The detailed table events are tuples looking like - this: <c>{Oper, Table, Data, [OldRecs], ActivityId}</c>. - Where <c>Oper</c> is the operation - performed. <c>Table</c> is the table involved in the operation, - <c>Data</c> is the record/oid written/deleted. - <c>OldRecs</c> is the contents before the operation. - and <c>ActivityId</c> is the identity of the transaction - performing the operation. - The various table related events that may occur are: - </p> + <p>The <em>detailed table events</em> are tuples like + <c>{Oper, Table, Data, [OldRecs], ActivityId}</c>, where:</p> + <list type="bulleted"> + <item><c>Oper</c> is the operation performed. + </item> + <item><c>Table</c> is the table involved in the operation. + </item> + <item><c>Data</c> is the record/OID written/deleted. + </item> + <item><c>OldRecs</c> is the contents before the operation. + </item> + <item><c>ActivityId</c> is the identity of the transaction + performing the operation. + </item> + </list> + <p>The table-related events that can occur are as follows:</p> <taglist> <tag><c>{write, Table, NewRecord, [OldRecords], ActivityId}</c></tag> - <item> - <p>a new record has been written. - NewRecord contains the new value of the record and OldRecords - contains the records before the operation is performed. - Note that the new content is dependent on the type of the table.</p> + <item>A new record has been written. <c>NewRecord</c> contains + the new record value and <c>OldRecords</c> contains the + records before the operation is performed. Notice that the + new content depends on the table type. </item> <tag><c>{delete, Table, What, [OldRecords], ActivityId}</c></tag> - <item> - <p>records has possibly been deleted - <c>What</c> is either {Table, Key} or a record {RecordName, Key, ...} - that was deleted. - Note that the new content is dependent on the type of the table.</p> + <item>Records have possibly been deleted. <c>What</c> is + either <c>{Table, Key}</c> or a record + <c>{RecordName, Key, ...}</c> that was deleted. Notice + that the new content depends on the table type. </item> </taglist> </section> @@ -1126,69 +1107,55 @@ ok <section> <title>Debugging Mnesia Applications</title> - <p>Debugging a Mnesia application can be difficult due to a number of reasons, primarily related + <p>Debugging a <c>Mnesia</c> application can be difficult + for various reasons, primarily related to difficulties in understanding how the transaction - and table load mechanisms work. An other source of - confusion may be the semantics of nested transactions. - </p> - <p>We may set the debug level of Mnesia by calling: - </p> - <list type="bulleted"> - <item><c>mnesia:set_debug_level(Level)</c></item> - </list> - <p>Where the parameter <c>Level</c> is: - </p> + and table load mechanisms work. Another source of + confusion can be the semantics of nested transactions.</p> + <p>The debug level of <c>Mnesia</c> is set by calling the function + <seealso marker="mnesia#set_debug_level/1">mnesia:set_debug_level(Level)</seealso>, + where <c>Level</c>is one of the following:</p> <taglist> <tag><c>none</c></tag> - <item> - <p>no trace outputs at all. This is the default. - </p> + <item>No trace outputs. This is the default. </item> <tag><c>verbose</c></tag> - <item> - <p>activates tracing of important debug events. These - debug events will generate <c>{mnesia_info, Format, Args}</c> - system events. Processes may subscribe to these events with - <c>mnesia:subscribe/1</c>. The events are always sent to Mnesia's - event handler. - </p> + <item>Activates tracing of important debug events. These + events generate <c>{mnesia_info, Format, Args}</c> + system events. Processes can subscribe to these events with + the function + <seealso marker="mnesia#subscribe/1">mnesia:subscribe/1</seealso>. + The events are always sent to the <c>Mnesia</c> event handler. </item> <tag><c>debug</c></tag> - <item> - <p>activates all events at the verbose level plus - traces of all debug events. These debug events will generate - <c>{mnesia_info, Format, Args}</c> system events. Processes may - subscribe to these events with <c>mnesia:subscribe/1</c>. The - events are always sent to Mnesia's event handler. On this - debug level Mnesia's event handler starts subscribing - updates in the schema table. - </p> + <item>Activates all events at the verbose level plus + traces of all debug events. These debug events generate + <c>{mnesia_info, Format, Args}</c> system events. Processes + can subscribe to these events with <c>mnesia:subscribe/1</c>. + The events are always sent to the <c>Mnesia</c> event handler. + On this debug level, the <c> Mnesia</c> event handler starts + subscribing to updates in the schema table. </item> <tag><c>trace</c></tag> - <item> - <p>activates all events at the debug level. On this - debug level Mnesia's event handler starts subscribing - updates on all Mnesia tables. This level is only intended - for debugging small toy systems, since many large - events may be generated.</p> + <item>Activates all events at the debug level. On this + level, the <c>Mnesia</c> event handler starts subscribing to + updates on all <c>Mnesia</c> tables. This level is intended + only for debugging small toy systems, as many large + events can be generated. </item> <tag><c>false</c></tag> - <item> - <p>is an alias for none.</p> + <item>An alias for none. </item> <tag><c>true</c></tag> - <item> - <p>is an alias for debug.</p> + <item>An alias for debug. </item> </taglist> - <p>The debug level of Mnesia itself, is also an application - parameter, thereby making it possible to start an Erlang system - in order to turn on Mnesia debug in the initial - start-up phase by using the following code: - </p> + <p>The debug level of <c>Mnesia</c> itself is also an application + parameter, making it possible to start an Erlang system + to turn on <c>Mnesia</c> debug in the initial + startup phase by using the following code:</p> <pre> - % erl -mnesia debug verbose - </pre> + % erl -mnesia debug verbose</pre> </section> <section> @@ -1196,85 +1163,81 @@ ok <p>Programming concurrent Erlang systems is the subject of a separate book. However, it is worthwhile to draw attention to the following features, which permit concurrent processes to - exist in a Mnesia system. - </p> - <p>A group of functions or processes can be called within a - transaction. A transaction may include statements that read, - write or delete data from the DBMS. A large number of such + exist in a <c>Mnesia</c> system:</p> + <list type="bulleted"> + <item><p>A group of functions or processes can be called within a + transaction. A transaction can include statements that read, + write, or delete data from the DBMS. Many such transactions can run concurrently, and the programmer does not - have to explicitly synchronize the processes which manipulate - the data. All programs accessing the database through the - transaction system may be written as if they had sole access to - the data. This is a very desirable property since all + need to explicitly synchronize the processes that manipulate + the data.</p> + <p>All programs accessing the database through the + transaction system can be written as if they had sole access to + the data. This is a desirable property, as all synchronization is taken care of by the transaction handler. If a program reads or writes data, the system ensures that no other - program tries to manipulate the same data at the same time. - </p> - <p>It is possible to move tables, delete tables or reconfigure - the layout of a table in various ways. An important aspect of - the actual implementation of these functions is that it is - possible for user programs to continue to use a table while it - is being reconfigured. For example, it is possible to - simultaneously move a table and perform write operations to the - table . This is important for many applications that - require continuously available services. Refer to Chapter 4: - <seealso marker="Mnesia_chap4#trans_prop">Transactions and other access contexts</seealso> for more information. - </p> + program tries to manipulate the same data at the same time.</p> + </item> + <item>Tables can be moved or deleted, and the layout of a table + can be reconfigured in various ways. An important aspect of + the implementation of these functions is that user programs + can continue to use a table while it + is being reconfigured. For example, it is possible to move a + table and perform write operations to the table at the same + time. This is important for many applications that require + continuously available services. For more information, see + <seealso marker="Mnesia_chap4#trans_prop">Transactions and Other Access Contexts</seealso>. + </item> + </list> </section> <section> <title>Prototyping</title> - <p>If and when we decide that we would like to start and manipulate - Mnesia, it is often easier to write the definitions and + <p>If and when you would like to start and manipulate + <c>Mnesia</c>, it is often easier to write the definitions and data into an ordinary text file. Initially, no tables and no data exist, or which - tables are required. At the initial stages of prototyping it - is prudent write all data into one file, process - that file and have the data in the file inserted into the database. - It is possible to initialize Mnesia with data read from a text file. - We have the following two functions to work with text files. - </p> + tables are required. At the initial stages of prototyping, it + is prudent to write all data into one file, process that + file, and have the data in the file inserted into the database. + <c>Mnesia</c> can be initialized with data read from a text file. + The following two functions can be used to work with text + files.</p> <list type="bulleted"> <item> - <p><c>mnesia:load_textfile(Filename)</c> Which loads a - series of local table definitions and data found in the file - into Mnesia. This function also starts Mnesia and possibly - creates a new schema. The function only operates on the - local node. - </p> + <seealso marker="mnesia#load_textfile/1">mnesia:load_textfile(Filename)</seealso> + loads a series of local table definitions and data found in the + file into <c>Mnesia</c>. This function also starts <c>Mnesia</c> + and possibly creates a new schema. The function operates + on the local node only. </item> <item> - <p><c>mnesia:dump_to_textfile(Filename)</c> Dumps - all local tables of a mnesia system into a text file which can - then be edited (by means of a normal text editor) and then - later reloaded.</p> + <seealso marker="mnesia#dump_to_textfile/1">mnesia:dump_to_textfile(Filename)</seealso> + dumps all local + tables of a <c>Mnesia</c> system into a text file, which + can be edited (with a normal text editor) and later reloaded. </item> </list> - <p>These functions are of course much slower than the ordinary - store and load functions of Mnesia. However, this is mainly intended for minor experiments - and initial prototyping. The major advantages of these functions is that they are very easy - to use. - </p> - <p>The format of the text file is: - </p> + <p>These functions are much slower than the ordinary store and + load functions of <c>Mnesia</c>. However, this is mainly intended + for minor experiments and initial prototyping. The major + advantage of these functions is that they are easy to use.</p> + <p>The format of the text file is as follows:</p> <pre> {tables, [{Typename, [Options]}, {Typename2 ......}]}. - {Typename, Attribute1, Atrribute2 ....}. - {Typename, Attribute1, Atrribute2 ....}. - </pre> + {Typename, Attribute1, Attribute2 ....}. + {Typename, Attribute1, Attribute2 ....}.</pre> <p><c>Options</c> is a list of <c>{Key,Value}</c> tuples conforming - to the options we could give to <c>mnesia:create_table/2</c>. - </p> - <p>For example, if we want to start playing with a small - database for healthy foods, we enter then following data into - the file <c>FRUITS</c>. - </p> + to the options that you can give to + <seealso marker="mnesia#create_table/2">mnesia:create_table/2</seealso>. + </p> + <p>For example, to start playing with a small database for healthy + foods, enter the following data into file <c>FRUITS</c>:</p> <codeinclude file="FRUITS" tag="%0" type="erl"></codeinclude> - <p>The following session with the Erlang shell then shows how - to load the fruits database. - </p> + <p>The following session with the Erlang shell shows how + to load the <c>FRUITS</c> database:</p> <pre><![CDATA[ % erl Erlang (BEAM) emulator version 4.9 @@ -1311,54 +1274,49 @@ ok ok 3> ]]></pre> - <p>Where we can see that the DBMS was initiated from a - regular text file. - </p> + <p>It can be seen that the DBMS was initiated from a + regular text file.</p> </section> <section> - <title>Object Based Programming with Mnesia</title> - <p>The Company database introduced in Chapter 2 has three tables - which store records (employee, dept, project), and three tables - which store relationships (manager, at_dep, in_proj). This is a - normalized data model, which has some advantages over a - non-normalized data model. - </p> - <p>It is more efficient to do a + <title>Object-Based Programming with Mnesia</title> + <p>The <c>Company</c> database, introduced in + <seealso marker="Mnesia_chap2#getting_started">Getting Started</seealso>, + has three tables that store records (<c>employee</c>, + <c>dept</c>, <c>project</c>), and three tables that store + relationships (<c>manager</c>, <c>at_dep</c>, <c>in_proj</c>). + This is a normalized data model, which has some advantages over + a non-normalized data model.</p> + <p>It is more efficient to do a generalized search in a normalized database. Some operations are also easier to perform on a normalized data model. For example, - we can easily remove one project, as the following example - illustrates: - </p> + one project can easily be removed, as the following example + illustrates:</p> <codeinclude file="company.erl" tag="%13" type="erl"></codeinclude> <p>In reality, data models are seldom fully normalized. A realistic alternative to a normalized database model would be - a data model which is not even in first normal form. Mnesia - is very suitable for applications such as telecommunications, - because it is easy to organize data in a very flexible manner. A - Mnesia database is always organized as a set of tables. Each - table is filled with rows/objects/records. What sets Mnesia - apart is that individual fields in a record can contain any type - of compound data structures. An individual field in a record can - contain lists, tuples, functions, and even record code. - </p> + a data model that is not even in first normal form. <c>Mnesia</c> + is suitable for applications such as telecommunications, + because it is easy to organize data in a flexible manner. A + <c>Mnesia</c> database is always organized as a set of tables. + Each table is filled with rows, objects, and records. + What sets <c>Mnesia</c> apart is that individual fields in + a record can contain any type of + compound data structures. An individual field in a record can + contain lists, tuples, functions, and even record code.</p> <p>Many telecommunications applications have unique requirements - on lookup times for certain types of records. If our Company - database had been a part of a telecommunications system, then it - could be that the lookup time of an employee <em>together</em> - with a list of the projects the employee is working on, should - be minimized. If this was the case, we might choose a - drastically different data model which has no direct - relationships. We would only have the records themselves, and - different records could contain either direct references to - other records, or they could contain other records which are not - part of the Mnesia schema. - </p> - <p>We could create the following record definitions: - </p> + on lookup times for certain types of records. If the <c>Company</c> + database had been a part of a telecommunications system, it + could be to minimize the lookup time of an employee + <em>together</em> with a list of the projects the employee is + working on. If this is the case, a drastically different data model + without direct relationships can be chosen. You would then have + only the records themselves, and different records could contain + either direct references to other records, or contain other + records that are not part of the <c>Mnesia</c> schema.</p> + <p>The following record definitions can be created:</p> <codeinclude file="company_o.hrl" tag="%0" type="erl"></codeinclude> - <p>An record which describes an employee might look like this: - </p> + <p>A record that describes an employee can look as follows:</p> <pre> Me = #employee{emp_no= 104732, name = klacke, @@ -1368,50 +1326,43 @@ ok room_no = {221, 015}, dept = 'B/SFR', projects = [erlang, mnesia, otp], - manager = 114872}, - </pre> - <p>This model only has three different tables, and the employee - records contain references to other records. We have the following - references in the record. - </p> + manager = 114872},</pre> + <p>This model has only three different tables, and the employee + records contain references to other records. The record has the + following references:</p> <list type="bulleted"> - <item><c>'B/SFR'</c> refers to a <c>dept</c> record. + <item><c>'B/SFR'</c> refers to a <c>dept</c> record. </item> - <item><c>[erlang, mnesia, otp]</c>. This is a list of three - direct references to three different <c>projects</c> records. + <item><c>[erlang, mnesia, otp]</c> is a list of three + direct references to three different <c>projects</c> records. </item> - <item><c>114872</c>. This refers to another employee record. + <item><c>114872</c> refers to another employee record. </item> </list> - <p>We could also use the Mnesia record identifiers (<c>{Tab, Key}</c>) - as references. In this case, the <c>dept</c> attribute would be - set to the value <c>{dept, 'B/SFR'}</c> instead of - <c>'B/SFR'</c>. - </p> + <p>The <c>Mnesia</c> record identifiers (<c>{Tab, Key}</c>) can + also be used as references. In this case, attribute <c>dept</c> + would be set to value <c>{dept, 'B/SFR'}</c> instead of + <c>'B/SFR'</c>.</p> <p>With this data model, some operations execute considerably - faster than they do with the normalized data model in our - Company database. On the other hand, some other operations + faster than they do with the normalized data model in the + <c>Company</c> database. However, some other operations become much more complicated. In particular, it becomes more difficult to ensure that records do not contain dangling - pointers to other non-existent, or deleted, records. - </p> + pointers to other non-existent, or deleted, records.</p> <p>The following code exemplifies a search with a non-normalized - data model. To find all employees at department - <c>Dep</c> with a salary higher than <c>Salary</c>, use the following code: - </p> + data model. To find all employees at department <c>Dep</c> with + a salary higher than <c>Salary</c>, use the following code:</p> <codeinclude file="company_o.erl" tag="%9" type="erl"></codeinclude> - <p>This code is not only easier to write and to understand, but it - also executes much faster. - </p> - <p>It is easy to show examples of code which executes faster if - we use a non-normalized data model, instead of a normalized - model. The main reason for this is that fewer tables are - required. For this reason, we can more easily combine data from - different tables in join operations. In the above example, the - <c>get_emps/2</c> function was transformed from a join operation - into a simple query which consists of a selection and a projection - on one single table. - </p> + <p>This code is easier to write and to understand, and it + also executes much faster.</p> + <p>It is easy to show examples of code that executes faster if + a non-normalized data model is used, instead of a normalized + model. The main reason is that fewer tables are required. + Therefore, data from different tables can more easily be + combined in join operations. In the previous example, the + function <c>get_emps/2</c> is transformed from a join operation + into a simple query, which consists of a selection and a + projection on one single table.</p> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc index 4458cd3919..573ca79106 100644 --- a/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc @@ -32,47 +32,61 @@ <file>Mnesia_chap7.xml</file> </header> + <p>The following topics are included:</p> + <list type="bulleted"> + <item>Database configuration data</item> + <item>Core dumps</item> + <item>Dumping tables</item> + <item>Checkpoints</item> + <item>Startup files, log file, and data files</item> + <item>Loading tables at startup</item> + <item>Recovery from communication failure</item> + <item>Recovery of transactions</item> + <item>Backup, restore, fallback, and disaster recovery</item> + </list> + <section> <title>Database Configuration Data</title> <p>The following two functions can be used to retrieve system - information. They are described in detail in the reference manual. - </p> + information. For details, see the Reference Manual.</p> <list type="bulleted"> - <item><c>mnesia:table_info(Tab, Key) -></c><c>Info | exit({aborted, Reason})</c>. - Returns information about one table. Such as the - current size of the table, on which nodes it resides etc. + <item><seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, Key) + -> Info | exit({aborted,Reason})</seealso> + returns information about one table, for example, + the current size of the table and on which nodes it resides. </item> - <item><c>mnesia:system_info(Key) -> </c><c>Info | exit({aborted, Reason})</c>. - Returns information about the Mnesia system. For example, transaction - statistics, db_nodes, configuration parameters etc. + <item><seealso marker="mnesia#system_info/1">mnesia:system_info(Key) + -> Info | exit({aborted, Reason})</seealso> + returns information about the <c>Mnesia</c> system, + for example, transaction statistics, <c>db_nodes</c>, and + configuration parameters. </item> </list> </section> <section> <title>Core Dumps</title> - <p>If Mnesia malfunctions, system information is dumped to a file - named <c>MnesiaCore.Node.When</c>. The type of system + <p>If <c>Mnesia</c> malfunctions, system information is dumped to + file <c>MnesiaCore.Node.When</c>. The type of system information contained in this file can also be generated with - the function <c>mnesia_lib:coredump()</c>. If a Mnesia system - behaves strangely, it is recommended that a Mnesia core dump - file be included in the bug report.</p> + the function <c>mnesia_lib:coredump()</c>. If a <c>Mnesia</c> + system behaves strangely, it is recommended that a <c>Mnesia</c> + core dump file is included in the bug report.</p> </section> <section> <title>Dumping Tables</title> <p>Tables of type <c>ram_copies</c> are by definition stored in - memory only. It is possible, however, to dump these tables to - disc, either at regular intervals, or before the system is - shutdown. The function <c>mnesia:dump_tables(TabList)</c> dumps - all replicas of a set of RAM tables to disc. The tables can be - accessed while being dumped to disc. To dump the tables to - disc all replicas must have the storage type <c>ram_copies</c>. - </p> - <p>The table content is placed in a .DCD file on the - disc. When the Mnesia system is started, the RAM table will - initially be loaded with data from its .DCD file. - </p> + memory only. However, these tables can be dumped to + disc, either at regular intervals or before the system is + shut down. The function + <seealso marker="mnesia#dump_tables/1">mnesia:dump_tables(TabList)</seealso> + dumps all replicas of a set of RAM tables to disc. The tables can be + accessed while being dumped to disc. To dump the tables to disc, + all replicas must have the storage type <c>ram_copies</c>.</p> + <p>The table content is placed in a <c>.DCD</c> file on the + disc. When the <c>Mnesia</c> system is started, the RAM table + is initially loaded with data from its <c>.DCD</c> file.</p> </section> <section> @@ -80,137 +94,128 @@ <title>Checkpoints</title> <p>A checkpoint is a transaction consistent state that spans over one or more tables. When a checkpoint is activated, the system - will remember the current content of the set of tables. The + remembers the current content of the set of tables. The checkpoint retains a transaction consistent state of the tables, allowing the tables to be read and updated while the checkpoint - is active. A checkpoint is typically used to + is active. A checkpoint is typically used to back up tables to external media, but they are also used - internally in Mnesia for other purposes. Each checkpoint is - independent and a table may be involved in several checkpoints - simultaneously. - </p> - <p>Each table retains its old contents in a checkpoint retainer - and for performance critical applications, it may be important + internally in <c>Mnesia</c> for other purposes. Each checkpoint + is independent and a table can be involved in several checkpoints + simultaneously.</p> + <p>Each table retains its old contents in a checkpoint retainer. + For performance critical applications, it can be important to realize the processing overhead associated with checkpoints. - In a worst case scenario, the checkpoint retainer will consume - even more memory than the table itself. Each update will also be + In a worst case scenario, the checkpoint retainer consumes + more memory than the table itself. Also, each update becomes slightly slower on those nodes where checkpoint - retainers are attached to the tables. - </p> - <p>For each table it is possible to choose if there should be one + retainers are attached to the tables.</p> + <p>For each table, it is possible to choose if there is to be one checkpoint retainer attached to all replicas of the table, or if it is enough to have only one checkpoint retainer attached to a single replica. With a single checkpoint retainer per table, the - checkpoint will consume less memory, but it will be vulnerable - to node crashes. With several redundant checkpoint retainers the - checkpoint will survive as long as there is at least one active - checkpoint retainer attached to each table. - </p> - <p>Checkpoints may be explicitly deactivated with the function - <c>mnesia:deactivate_checkpoint(Name)</c>, where <c>Name</c> is + checkpoint consumes less memory, but it is vulnerable + to node crashes. With several redundant checkpoint retainers, the + checkpoint survives as long as there is at least one active + checkpoint retainer attached to each table.</p> + <p>Checkpoints can be explicitly deactivated with the function + <seealso marker="mnesia#deactivate_checkpoint/1">mnesia:deactivate_checkpoint(Name)</seealso>, + where <c>Name</c> is the name of an active checkpoint. This function returns - <c>ok</c> if successful, or <c>{error, Reason}</c> in the case - of an error. All tables in a checkpoint must be attached to at + <c>ok</c> if successful or <c>{error, Reason}</c> if there is + an error. All tables in a checkpoint must be attached to at least one checkpoint retainer. The checkpoint is automatically - de-activated by Mnesia, when any table lacks a checkpoint - retainer. This may happen when a node goes down or when a - replica is deleted. Use the <c>min</c> and - <c>max</c> arguments described below, to control the degree of - checkpoint retainer redundancy. - </p> - <p>Checkpoints are activated with the function <marker id="mnesia:chkpt(Args)"></marker> -<c>mnesia:activate_checkpoint(Args)</c>, - where <c>Args</c> is a list of the following tuples: - </p> + deactivated by <c>Mnesia</c>, when any table lacks a checkpoint + retainer. This can occur when a node goes down or when a + replica is deleted. Use arguments <c>min</c> and + <c>max</c> (described in the following list) to control the + degree of checkpoint retainer redundancy.</p> + <marker id="mnesia:chkpt(Args)"></marker> + <p>Checkpoints are activated with the function + <seealso marker="mnesia#activate_checkpoint/1">mnesia:activate_checkpoint(Args)</seealso>, + where <c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> - <item><c>{name,Name}</c>. <c>Name</c> specifies a temporary name - of the checkpoint. The name may be re-used when the checkpoint - has been de-activated. If no name is specified, a name is + <item><c>{name,Name}</c>, where <c>Name</c> specifies a temporary + name of the checkpoint. The name can be reused when the checkpoint + has been deactivated. If no name is specified, a name is generated automatically. </item> - <item><c>{max,MaxTabs}</c>. <c>MaxTabs</c> is a list of tables - which will be included in the checkpoint. The default is - <c>[]</c> (an empty list). For these tables, the redundancy - will be maximized. The old contents of the table will be + <item><c>{max,MaxTabs}</c>, where <c>MaxTabs</c> is a list of + tables that are to be included in the checkpoint. Default is + <c>[]</c> (empty list). For these tables, the redundancy + is maximized. The old content of the table is retained in the checkpoint retainer when the main table is - updated by the applications. The checkpoint becomes more fault + updated by the applications. The checkpoint is more fault tolerant if the tables have several replicas. When new - replicas are added by means of the schema manipulation - function <c>mnesia:add_table_copy/3</c>, it will also - attach a local checkpoint retainer. + replicas are added by the schema manipulation function + <seealso marker="mnesia#add_table_copy/3">mnesia:add_table_copy/3</seealso> + it also attaches a local checkpoint retainer. </item> - <item><c>{min,MinTabs}</c>. <c>MinTabs</c> is a list of tables - that should be included in the checkpoint. The default is - <c>[]</c>. For these tables, the redundancy will be minimized, - and there will be a single checkpoint retainer per table, + <item><c>{min,MinTabs}</c>, where <c>MinTabs</c> is a list of + tables that are to be included in the checkpoint. Default + is <c>[]</c>. For these tables, the redundancy is minimized, + and there is to be single checkpoint retainer per table, preferably at the local node. </item> - <item><c>{allow_remote,Bool}</c>. <c>false</c> means that all - checkpoint retainers must be local. If a table does not reside - locally, the checkpoint cannot be activated. <c>true</c> - allows checkpoint retainers to be allocated on any node. The - defaults is <c>true</c>. + <item><c>{allow_remote,Bool}</c>, where <c>false</c> means that + all checkpoint retainers must be local. If a table does not + reside locally, the checkpoint cannot be activated. <c>true</c> + allows checkpoint retainers to be allocated on any node. + Default is <c>true</c>. </item> <item><c>{ram_overrides_dump,Bool}</c>. This argument only applies to tables of type <c>ram_copies</c>. <c>Bool</c> - specifies if the table state in RAM should override the table + specifies if the table state in RAM is to override the table state on disc. <c>true</c> means that the latest committed records in RAM are included in the checkpoint retainer. These are the records that the application accesses. <c>false</c> - means that the records on the disc .DAT file are - included in the checkpoint retainer. These are the records - that will be loaded on start-up. Default is <c>false</c>.</item> + means that the records on the disc <c>.DAT</c> file are + included in the checkpoint retainer. These records are + loaded on startup. Default is <c>false</c>.</item> </list> - <p>The <c>mnesia:activate_checkpoint(Args)</c> returns one of the - following values: - </p> + <p>The function + <seealso marker="mnesia#activate_checkpoint/1">mnesia:activate_checkpoint(Args)</seealso> + returns one of the following values:</p> <list type="bulleted"> <item><c>{ok, Name, Nodes}</c></item> - <item><c>{error, Reason}</c>.</item> + <item><c>{error, Reason}</c></item> </list> - <p><c>Name</c> is the name of the checkpoint, and <c>Nodes</c> are - the nodes where the checkpoint is known. - </p> + <p><c>Name</c> is the checkpoint name. <c>Nodes</c> are + the nodes where the checkpoint is known.</p> <p>A list of active checkpoints can be obtained with the following - functions: - </p> + functions:</p> <list type="bulleted"> - <item><c>mnesia:system_info(checkpoints)</c>. This function + <item><seealso marker="mnesia#system_info/1">mnesia:system_info(checkpoints)</seealso> returns all active checkpoints on the current node.</item> - <item><c>mnesia:table_info(Tab,checkpoints)</c>. This function + <item><seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, checkpoints)</seealso> returns active checkpoints on a specific table.</item> </list> </section> <section> - <title>Files</title> - <p>This section describes the internal files which are created and maintained by the Mnesia system, - in particular, the workings of the Mnesia log is described. - </p> + <title>Startup Files, Log File, and Data Files</title> + <p>This section describes the internal files that are created + and maintained by the <c>Mnesia</c> system. In particular, + the workings of the <c>Mnesia</c> log are described.</p> <section> - <title>Start-Up Files</title> - </section> - <p>In Chapter 3 we detailed the following pre-requisites for - starting Mnesia (refer Chapter 3: <seealso marker="Mnesia_chap3#start_mnesia">Starting Mnesia</seealso>: - </p> + <title>Startup Files</title> + <p><seealso marker="Mnesia_chap3#start_mnesia">Start Mnesia</seealso> + states the following prerequisites + for starting <c>Mnesia</c>:</p> <list type="bulleted"> - <item>We must start an Erlang session and specify a Mnesia - directory for our database. + <item>An Erlang session must be started and a <c>Mnesia</c> + directory must be specified for the database. </item> - <item>We must initiate a database schema, using the function - <c>mnesia:create_schema/1</c>. + <item>A database schema must be initiated, using the function + <seealso marker="mnesia#create_schema/1">mnesia:create_schema/1</seealso>. </item> </list> - <p>The following example shows how these tasks are performed: - </p> - <list type="ordered"> - <item> - <pre> -% <input>erl -sname klacke -mnesia dir '"/ldisc/scratch/klacke"'</input> </pre> - </item> - <item> - <pre> + <p>The following example shows how these tasks are performed:</p> + <p><em>Step 1:</em> Start an Erlang session and specify a + <c>Mnesia</c> directory for the database:</p> + <pre> +% <input>erl -sname klacke -mnesia dir '"/ldisc/scratch/klacke"'</input></pre> + <pre> Erlang (BEAM) emulator version 4.9 Eshell V4.9 (abort with ^G) @@ -218,679 +223,700 @@ Eshell V4.9 (abort with ^G) ok (klacke@gin)2> <input>^Z</input> -Suspended </pre> - <p>We can inspect the Mnesia directory to see what files have been created. Enter the following command: - </p> - <pre> +Suspended</pre> + <p><em>Step 2:</em> You can inspect the <c>Mnesia</c> directory + to see what files have been created:</p> + <pre> % <input>ls -l /ldisc/scratch/klacke</input> --rw-rw-r-- 1 klacke staff 247 Aug 12 15:06 FALLBACK.BUP </pre> - <p>The response shows that the file FALLBACK.BUP has been created. This is called a backup file, and it contains an initial schema. If we had specified more than one node in the <c>mnesia:create_schema/1</c> function, identical backup files would have been created on all nodes. - </p> - </item> - <item> - <p>Continue by starting Mnesia:</p> - <pre> +-rw-rw-r-- 1 klacke staff 247 Aug 12 15:06 FALLBACK.BUP</pre> + <p>The response shows that the file <c>FALLBACK.BUP</c> has + been created. This is called a backup file, and it contains + an initial schema. If more than one node in the function + <seealso marker="mnesia#create_schema/1">mnesia:create_schema/1</seealso> + had been specified, identical + backup files would have been created on all nodes.</p> + <p><em>Step 3:</em> Start <c>Mnesia</c>:</p> + <pre> (klacke@gin)3><input>mnesia:start( ).</input> -ok </pre> - <p>We can now see the following listing in the Mnesia directory: - </p> - <pre> +ok</pre> + <p><em>Step 4:</em> You can see the following listing in + the <c>Mnesia</c> directory:</p> + <pre> -rw-rw-r-- 1 klacke staff 86 May 26 19:03 LATEST.LOG --rw-rw-r-- 1 klacke staff 34507 May 26 19:03 schema.DAT </pre> - <p>The schema in the backup file FALLBACK.BUP has been used to generate the file <c>schema.DAT.</c> Since we have no other disc resident tables than the schema, no other data files were created. The file FALLBACK.BUP was removed after the successful "restoration". We also see a number of files that are for internal use by Mnesia. - </p> - </item> - <item> - <p>Enter the following command to create a table:</p> - <pre> +-rw-rw-r-- 1 klacke staff 34507 May 26 19:03 schema.DAT</pre> + <p>The schema in the backup file <c>FALLBACK.BUP</c> has been + used to generate the file <c>schema.DAT</c>. Since there are + no other disc resident tables than the schema, no other data + files were created. The file <c>FALLBACK.BUP</c> was removed + after the successful "restoration". You also see some files + that are for internal use by <c>Mnesia</c>.</p> + <p><em>Step 5:</em> Create a table:</p> + <pre> (klacke@gin)4> <input>mnesia:create_table(foo,[{disc_copies, [node()]}]).</input> -{atomic,ok} </pre> - <p>We can now see the following listing in the Mnesia directory: - </p> - <pre> +{atomic,ok}</pre> + <p><em>Step 6:</em> You can see the following listing in + the <c>Mnesia</c> directory:</p> + <pre> % <input>ls -l /ldisc/scratch/klacke</input> -rw-rw-r-- 1 klacke staff 86 May 26 19:07 LATEST.LOG -rw-rw-r-- 1 klacke staff 94 May 26 19:07 foo.DCD --rw-rw-r-- 1 klacke staff 6679 May 26 19:07 schema.DAT </pre> - <p>Where a file <c>foo.DCD</c> has been created. This file will eventually store - all data that is written into the <c>foo</c> table.</p> - </item> - </list> +-rw-rw-r-- 1 klacke staff 6679 May 26 19:07 schema.DAT</pre> + <p>The file <c>foo.DCD</c> has been created. This file will + eventually store all data that is written into the + <c>foo</c> table.</p> + </section> <section> - <title>The Log File</title> - <p>When starting Mnesia, a .LOG file called <c>LATEST.LOG</c> - was created and placed in the database directory. This file is - used by Mnesia to log disc based transactions. This includes all - transactions that write at least one record in a table which is - of storage type <c>disc_copies</c>, or - <c>disc_only_copies</c>. It also includes all operations which - manipulate the schema itself, such as creating new tables. The - format of the log can vary with different implementations of - Mnesia. The Mnesia log is currently implemented with the - standard library module <c>disc_log</c>. - </p> - <p>The log file will grow continuously and must be dumped at - regular intervals. "Dumping the log file" means that Mnesia will - perform all the operations listed in the log and place the - records in the corresponding .DAT, .DCD and .DCL data files. For - example, if the operation "write record <c>{foo, 4, elvis, 6}</c>" - is listed in the log, Mnesia inserts the operation into the - file <c>foo.DCL</c>, later when Mnesia thinks the .DCL has become to large - the data is moved to the .DCD file. - The dumping operation can be time consuming - if the log is very large. However, it is important to realize - that the Mnesia system continues to operate during log dumps. - </p> - <p>By default Mnesia either dumps the log whenever 100 records have - been written in the log or when 3 minutes have passed. + <title>Log File</title> + <p>When starting <c>Mnesia</c>, a <c>.LOG</c> file called + <c>LATEST.LOG</c> is created + and placed in the database directory. This file is used by + <c>Mnesia</c> to log disc-based transactions. This includes all + transactions that write at least one record in a table that is + of storage type <c>disc_copies</c> or <c>disc_only_copies</c>. + The file also includes all operations that + manipulate the schema itself, such as creating new tables. + The log format can vary with different implementations of + <c>Mnesia</c>. The <c>Mnesia</c> log is currently implemented + in the standard library module + <seealso marker="kernel:disk_log">disk_log</seealso> in + <c>Kernel</c>.</p> + <p>The log file grows continuously and must be dumped at + regular intervals. "Dumping the log file" means that <c>Mnesia</c> + performs all the operations listed in the log and place the + records in the corresponding <c>.DAT</c>, <c>.DCD</c>, and + <c>.DCL</c> data files. For example, if the operation "write + record <c>{foo, 4, elvis, 6}</c>" is listed in the log, + <c>Mnesia</c> inserts the operation into the file + <c>foo.DCL</c>. Later, when <c>Mnesia</c> thinks that the + <c>.DCL</c> file is too large, the data is moved to the + <c>.DCD</c> file. The dumping operation can be time consuming + if the log is large. Notice that the <c>Mnesia</c> system + continues to operate during log dumps.</p> + <p>By default <c>Mnesia</c> either dumps the log whenever + 100 records have + been written in the log or when three minutes have passed. This is controlled by the two application parameters <c>-mnesia dump_log_write_threshold WriteOperations</c> and - <c>-mnesia dump_log_time_threshold MilliSecs</c>. - </p> + <c>-mnesia dump_log_time_threshold MilliSecs</c>.</p> <p>Before the log is dumped, the file <c>LATEST.LOG</c> is renamed to <c>PREVIOUS.LOG</c>, and a new <c>LATEST.LOG</c> file is created. Once the log has been successfully dumped, the file - <c>PREVIOUS.LOG</c> is deleted. - </p> - <p>The log is also dumped at start-up and whenever a schema - operation is performed. - </p> + <c>PREVIOUS.LOG</c> is deleted.</p> + <p>The log is also dumped at startup and whenever a schema + operation is performed.</p> </section> <section> - <title>The Data Files</title> - <p>The directory listing also contains one .DAT file. This contain - the schema itself, contained in the <c>schema.DAT</c> - file. The DAT files are indexed files, and it is efficient to - insert and search for records in these files with a specific - key. The .DAT files are used for the schema and for <c>disc_only_copies</c> - tables. The Mnesia data files are currently implemented with the - standard library module <c>dets</c>, and all operations which - can be performed on <c>dets</c> files can also be performed on - the Mnesia data files. For example, <c>dets</c> contains a - function <c>dets:traverse/2</c> which can be used to view the - contents of a Mnesia DAT file. However, this can only be done - when Mnesia is not running. So, to view a our schema file, we - can: </p> + <title>Data Files</title> + <p>The directory listing also contains one <c>.DAT</c> file, + which contains the schema itself, contained in the + <c>schema.DAT</c> file. The <c>DAT</c> files are indexed + files, and it is efficient to insert and search for records + in these files with a specific key. The <c>.DAT</c> files + are used for the schema and for <c>disc_only_copies</c> + tables. The <c>Mnesia</c> data files are currently implemented + in the standard library module + <seealso marker="stdlib:dets">dets</seealso> in + <c>STDLIB</c>.</p> + <p>All operations that can be performed on <c>dets</c> files + can also be performed on the <c>Mnesia</c> data files. For + example, <c>dets</c> contains the function + <c>dets:traverse/2</c>, which can be used to view the + contents of a <c>Mnesia</c> <c>DAT</c> file. However, this + can only be done when <c>Mnesia</c> is not running. So, to + view the schema file, do as follows;</p> <pre> {ok, N} = dets:open_file(schema, [{file, "./schema.DAT"},{repair,false}, {keypos, 2}]), F = fun(X) -> io:format("~p~n", [X]), continue end, dets:traverse(N, F), -dets:close(N). </pre> - <note> - <p>Refer to the Reference Manual, <c>std_lib</c> for information about <c>dets</c>.</p> - </note> +dets:close(N).</pre> <warning> - <p>The DAT files must always be opened with the <c>{repair, false}</c> - option. This ensures that these files are not - automatically repaired. Without this option, the database may - become inconsistent, because Mnesia may - believe that the files were properly closed. Refer to the reference - manual for information about the configuration parameter - <c>auto_repair</c>.</p> + <p>The <c>DAT</c> files must always be opened with option + <c>{repair, false}</c>. This ensures that these files are not + automatically repaired. Without this option, the database can + become inconsistent, because <c>Mnesia</c> can believe that + the files were properly closed. For information about + configuration parameter <c>auto_repair</c>, see the + Reference Manual.</p> </warning> <warning> - <p>It is recommended that Data files are not tampered with while Mnesia is - running. While not prohibited, the behavior of Mnesia is unpredictable. </p> + <p>It is recommended that the data files are not tampered + with while <c>Mnesia</c> is running. While not prohibited, + the behavior of <c>Mnesia</c> is unpredictable.</p> </warning> - <p>The <c>disc_copies</c> tables are stored on disk with .DCL and .DCD files, - which are standard disk_log files. - </p> + <p>The <c>disc_copies</c> tables are stored on disk with + <c>.DCL</c> and <c>.DCD</c> files, which are standard + <c>disk_log</c> files.</p> </section> </section> <section> - <title>Loading of Tables at Start-up</title> - <p>At start-up Mnesia loads tables in order to make them accessible - for its applications. Sometimes Mnesia decides to load all tables - that reside locally, and sometimes the tables may not be - accessible until Mnesia brings a copy of the table - from another node. - </p> - <p>To understand the behavior of Mnesia at start-up it is - essential to understand how Mnesia reacts when it loses contact - with Mnesia on another node. At this stage, Mnesia cannot distinguish - between a communication failure and a "normal" node down. <br></br> - - When this happens, Mnesia will assume that the other node is no longer running. - Whereas, in reality, the communication between the nodes has merely failed. - </p> - <p>To overcome this situation, simply try to restart the ongoing transactions that are - accessing tables on the failing node, and write a <c>mnesia_down</c> entry to a log file. - </p> - <p>At start-up, it must be noted that all tables residing on nodes - without a <c>mnesia_down</c> entry, may have fresher replicas. - Their replicas may have been updated after the termination - of Mnesia on the current node. In order to catch up with the latest + <title>Loading Tables at Startup</title> + <p>At startup, <c>Mnesia</c> loads tables to make them accessible + for its applications. Sometimes <c>Mnesia</c> decides to load + all tables that reside locally, and sometimes the tables are + not accessible until <c>Mnesia</c> brings a copy of the table + from another node.</p> + <p>To understand the behavior of <c>Mnesia</c> at startup, it is + essential to understand how <c>Mnesia</c> reacts when it loses + contact with <c>Mnesia</c> on another node. At this stage, + <c>Mnesia</c> cannot distinguish between a communication + failure and a "normal" node-down. When this occurs, + <c>Mnesia</c> assumes that the other node is no longer running, + whereas, in reality, the communication between the nodes has + failed.</p> + <p>To overcome this situation, try to restart the ongoing + transactions that are accessing tables on the failing node, + and write a <c>mnesia_down</c> entry to a log file.</p> + <p>At startup, notice that all tables residing on nodes + without a <c>mnesia_down</c> entry can have fresher replicas. + Their replicas can have been updated after the termination of + <c>Mnesia</c> on the current node. To catch up with the latest updates, transfer a copy of the table from one of these other - "fresh" nodes. If you are unlucky, other nodes may be down - and you must wait for the table to be - loaded on one of these nodes before receiving a fresh copy of - the table. - </p> + "fresh" nodes. If you are unlucky, other nodes can be down + and you must wait for the table to be loaded on one of these + nodes before receiving a fresh copy of the table.</p> <p>Before an application makes its first access to a table, - <c>mnesia:wait_for_tables(TabList, Timeout)</c> ought to be executed + <seealso marker="mnesia#wait_for_tables/2">mnesia:wait_for_tables(TabList, Timeout)</seealso> + is to be executed to ensure that the table is accessible from the local node. If - the function times out the application may choose to force a + the function times out, the application can choose to force a load of the local replica with - <c>mnesia:force_load_table(Tab)</c> and deliberately lose all - updates that may have been performed on the other nodes while - the local node was down. If - Mnesia already has loaded the table on another node or intends - to do so, we will copy the table from that node in order to - avoid unnecessary inconsistency. - </p> + <seealso marker="mnesia#force_load_table/1">mnesia:force_load_table(Tab)</seealso> + and deliberately lose all + updates that can have been performed on the other nodes while + the local node was down. If <c>Mnesia</c> + has loaded the table on another node already, or intends + to do so, copy the table from that node to + avoid unnecessary inconsistency.</p> <warning> - <p>Keep in mind that it is only - one table that is loaded by <c>mnesia:force_load_table(Tab)</c> - and since committed transactions may have caused updates in - several tables, the tables may now become inconsistent due to - the forced load.</p> + <p>Only one table is loaded by + <seealso marker="mnesia#force_load_table/1">mnesia:force_load_table(Tab)</seealso>. + Since committed + transactions can have caused updates in several tables, the + tables can become inconsistent because of the forced load.</p> </warning> - <p>The allowed <c>AccessMode</c> of a table may be defined to - either be <c>read_only</c> or <c>read_write</c>. And it may be - toggled with the function <c>mnesia:change_table_access_mode(Tab, AccessMode)</c> in runtime. <c>read_only</c> tables and - <c>local_content</c> tables will always be loaded locally, since - there are no need for copying the table from other nodes. Other - tables will primary be loaded remotely from active replicas on - other nodes if the table already has been loaded there, or if - the running Mnesia already has decided to load the table there. - </p> - <p>At start up, Mnesia will assume that its local replica is the - most recent version and load the table from disc if either - situation is detected: - </p> + <p>The allowed <c>AccessMode</c> of a table can be defined to be + <c>read_only</c> or <c>read_write</c>. It can be toggled with + the function + <seealso marker="mnesia#change_table_access_mode/2"> + mnesia:change_table_access_mode(Tab, AccessMode)</seealso> + in runtime. <c>read_only</c> tables and + <c>local_content</c> tables are always loaded locally, as + there is no need for copying the table from other nodes. Other + tables are primarily loaded remotely from active replicas on + other nodes if the table has been loaded there already, or if + the running <c>Mnesia</c> has decided to load the table there + already.</p> + <p>At startup, <c>Mnesia</c> assumes that its local replica is the + most recent version and loads the table from disc if either of + the following situations is detected:</p> <list type="bulleted"> - <item><c>mnesia_down</c> is returned from all other nodes that holds a disc - resident replica of the table; or,</item> - <item>if all replicas are <c>ram_copies</c></item> + <item><c>mnesia_down</c> is returned from all other nodes that + hold a disc resident replica of the table.</item> + <item>All replicas are <c>ram_copies</c>.</item> </list> - <p>This is normally a wise decision, but it may turn out to - be disastrous if the nodes have been disconnected due to a - communication failure, since Mnesia's normal table load - mechanism does not cope with communication failures. - </p> - <p>When Mnesia is loading many tables the default load - order. However, it is possible to - affect the load order by explicitly changing the - <c>load_order</c> property for the tables, with the function - <c>mnesia:change_table_load_order(Tab, LoadOrder)</c>. The - <c>LoadOrder</c> is by default <c>0</c> for all tables, but it - can be set to any integer. The table with the highest - <c>load_order</c> will be loaded first. Changing load order is + <p>This is normally a wise decision, but it can be disastrous + if the nodes have been disconnected because of a communication + failure, as the <c>Mnesia</c> normal table load + mechanism does not cope with communication failures.</p> + <p>When <c>Mnesia</c> loads many tables, the default load order + is used. However, the load order + can be affected, by explicitly changing property + <c>load_order</c> for the tables, with the function + <seealso marker="mnesia#change_table_load_order/2"> + mnesia:change_table_load_order(Tab, LoadOrder)</seealso>. + <c>LoadOrder</c> is by default <c>0</c> for all tables, but + it can be set to any integer. The table with the highest + <c>load_order</c> is loaded first. Changing the load order is especially useful for applications that need to ensure early - availability of fundamental tables. Large peripheral - tables should have a low load order value, perhaps set - below 0. - </p> + availability of fundamental tables. Large peripheral tables + are to have a low load order value, perhaps less than <c>0</c></p> </section> <section> <title>Recovery from Communication Failure</title> - <p>There are several occasions when Mnesia may detect that the - network has been partitioned due to a communication failure. - </p> - <p>One is when Mnesia already is up and running and the Erlang - nodes gain contact again. Then Mnesia will try to contact Mnesia - on the other node to see if it also thinks that the network has - been partitioned for a while. If Mnesia on both nodes has logged - <c>mnesia_down</c> entries from each other, Mnesia generates a - system event, called <c>{inconsistent_database, running_partitioned_network, Node}</c> which is sent to Mnesia's - event handler and other possible subscribers. The default event - handler reports an error to the error logger. - </p> - <p>Another occasion when Mnesia may detect that the network has - been partitioned due to a communication failure, is at start-up. - If Mnesia detects that both the local node and another node received - <c>mnesia_down</c> from each other it generates a - <c>{inconsistent_database, starting_partitioned_network, Node}</c> system event and acts as described above. - </p> + <p>There are several occasions when <c>Mnesia</c> can detect + that the network has been partitioned because of a + communication failure, for example:</p> + <list type="bulleted"> + <item><c>Mnesia</c> is operational already and the Erlang nodes + gain contact again. Then <c>Mnesia</c> tries to contact + <c>Mnesia</c> on the other node to see if it also thinks that + the network has been partitioned for a while. If <c>Mnesia</c> + on both nodes has logged <c>mnesia_down</c> entries from each + other, <c>Mnesia</c> generates a system event, called + <c>{inconsistent_database, running_partitioned_network, Node}</c>, + which is sent to the <c>Mnesia</c> event handler and other + possible subscribers. The default event + handler reports an error to the error logger. + </item> + <item>If <c>Mnesia</c> detects at startup that both the local + node and another node received <c>mnesia_down</c> from each + other, <c>Mnesia</c> generates an + <c>{inconsistent_database, starting_partitioned_network, Node}</c> + system event and acts as described in the previous item. + </item> + </list> <p>If the application detects that there has been a communication - failure which may have caused an inconsistent database, it may - use the function <c>mnesia:set_master_nodes(Tab, Nodes)</c> to - pinpoint from which nodes each table may be loaded.</p> - <p>At start-up Mnesia's normal table load algorithm will be - bypassed and the table will be loaded from one of the master + failure that can have caused an inconsistent database, it can + use the function + <seealso marker="mnesia#set_master_nodes/2">mnesia:set_master_nodes(Tab, Nodes)</seealso> + to pinpoint from which nodes each table can be loaded.</p> + <p>At startup, the <c>Mnesia</c> normal table load algorithm is + bypassed and the table is loaded from one of the master nodes defined for the table, regardless of potential - <c>mnesia_down</c> entries in the log. The <c>Nodes</c> may only - contain nodes where the table has a replica and if it is empty, - the master node recovery mechanism for the particular table will - be reset and the normal load mechanism will be used when next - restarting. - </p> - <p>The function <c>mnesia:set_master_nodes(Nodes)</c> sets master - nodes for all tables. For each table it will determine its - replica nodes and invoke <c>mnesia:set_master_nodes(Tab, TabNodes)</c> with those replica nodes that are included in the - <c>Nodes</c> list (i.e. <c>TabNodes</c> is the intersection of + <c>mnesia_down</c> entries in the log. <c>Nodes</c> can only + contain nodes where the table has a replica. If <c>Nodes</c> + is empty, the master node recovery mechanism for the particular + table is reset and the normal load mechanism is used at the + next restart.</p> + <p>The function + <seealso marker="mnesia#set_master_nodes/1">mnesia:set_master_nodes(Nodes)</seealso> + sets master + nodes for all tables. For each table it determines its replica + nodes and starts + <seealso marker="mnesia#set_master_nodes/2">mnesia:set_master_nodes(Tab, TabNodes)</seealso> + with those replica nodes that are included in the <c>Nodes</c> + list (that is, <c>TabNodes</c> is the intersection of <c>Nodes</c> and the replica nodes of the table). If the - intersection is empty the master node recovery mechanism for the - particular table will be reset and the normal load mechanism - will be used at next restart. - </p> - <p>The functions <c>mnesia:system_info(master_node_tables)</c> and - <c>mnesia:table_info(Tab, master_nodes)</c> may be used to - obtain information about the potential master nodes. - </p> - <p>Determining which data to keep after communication failure is outside - the scope of Mnesia. One approach would be to determine which "island" - contains a majority of the nodes. Using the <c>{majority,true}</c> option - for critical tables can be a way of ensuring that nodes that are not part - of a "majority island" are not able to update those tables. Note that this - constitutes a reduction in service on the minority nodes. This would be - a tradeoff in favour of higher consistency guarantees.</p> - <p>The function <c>mnesia:force_load_table(Tab)</c> may be used to - force load the table regardless of which table load mechanism - is activated. - </p> + intersection is empty, the master node recovery mechanism for + the particular table is reset and the normal load mechanism + is used at the next restart.</p> + <p>The functions + <seealso marker="mnesia#system_info/1">mnesia:system_info(master_node_tables)</seealso> + and + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, master_nodes)</seealso> + can be used to + obtain information about the potential master nodes.</p> + <p>Determining what data to keep after a communication failure + is outside the scope of <c>Mnesia</c>. One approach is to + determine which "island" contains most of the nodes. Using + option <c>{majority,true}</c> for critical tables can be a way + to ensure that nodes that are not part of a "majority island" + cannot update those tables. Notice that this constitutes a + reduction in service on the minority nodes. This would be a + tradeoff in favor of higher consistency guarantees.</p> + <p>The function + <seealso marker="mnesia#force_load_table/1">mnesia:force_load_table(Tab)</seealso> + can be used to force load the table regardless of which table + load mechanism that is activated.</p> </section> <section> <title>Recovery of Transactions</title> - <p>A Mnesia table may reside on one or more nodes. When a table is - updated, Mnesia will ensure that the updates will be replicated - to all nodes where the table resides. If a replica happens to be - inaccessible for some reason (e.g. due to a temporary node down), - Mnesia will then perform the replication later. - </p> - <p>On the node where the application is started, there will be a + <p>A <c>Mnesia</c> table can reside on one or more nodes. When a + table is updated, <c>Mnesia</c> ensures that the updates are + replicated to all nodes where the table resides. If a replica is + inaccessible (for example, because of a temporary node-down), + <c>Mnesia</c> performs the replication later.</p> + <p>On the node where the application is started, there is a transaction coordinator process. If the transaction is - distributed, there will also be a transaction participant process on - all the other nodes where commit work needs to be performed. - </p> - <p>Internally Mnesia uses several commit protocols. The selected - protocol depends on which table that has been updated in - the transaction. If all the involved tables are symmetrically - replicated, (i.e. they all have the same <c>ram_nodes</c>, - <c>disc_nodes</c> and <c>disc_only_nodes</c> currently + distributed, there is also a transaction participant process on + all the other nodes where commit-work needs to be performed.</p> + <p>Internally <c>Mnesia</c> uses several commit protocols. The + selected protocol depends on which table that has been updated + in the transaction. If all the involved tables are symmetrically + replicated (that is, they all have the same <c>ram_nodes</c>, + <c>disc_nodes</c>, and <c>disc_only_nodes</c> currently accessible from the coordinator node), a lightweight transaction - commit protocol is used. - </p> + commit protocol is used.</p> <p>The number of messages that the - transaction coordinator and its participants needs to exchange - is few, since Mnesia's table load mechanism takes care of the - transaction recovery if the commit protocol gets + transaction coordinator and its participants need to exchange + is few, as the <c>Mnesia</c> table load mechanism takes care of + the transaction recovery if the commit protocol gets interrupted. Since all involved tables are replicated - symmetrically the transaction will automatically be recovered by - loading the involved tables from the same node at start-up of a - failing node. We do not really care if the transaction was - aborted or committed as long as we can ensure the ACID - properties. The lightweight commit protocol is non-blocking, - i.e. the surviving participants and their coordinator will - finish the transaction, regardless of some node crashes in the - middle of the commit protocol or not. - </p> - <p>If a node goes down in the middle of a dirty operation the - table load mechanism will ensure that the update will be - performed on all replicas or none. Both asynchronous dirty + symmetrically, the transaction is automatically recovered by + loading the involved tables from the same node at startup of a + failing node. It does not matter if the transaction was + committed or terminated as long as the ACID properties can be + ensured. The lightweight commit protocol is non-blocking, + that is, the surviving participants and their coordinator + finish the transaction, even if any node crashes in the + middle of the commit protocol.</p> + <p>If a node goes down in the middle of a dirty operation, the + table load mechanism ensures that the update is + performed on all replicas, or none. Both asynchronous dirty updates and synchronous dirty updates use the same recovery - principle as lightweight transactions. - </p> + principle as lightweight transactions.</p> <p>If a transaction involves updates of asymmetrically replicated tables or updates of the schema table, a heavyweight commit - protocol will be used. The heavyweight commit protocol is able - to finish the transaction regardless of how the tables are - replicated. The typical usage of a heavyweight transaction is - when we want to move a replica from one node to another. Then we - must ensure that the replica either is entirely moved or left as - it was. We must never end up in a situation with replicas on both - nodes or no node at all. Even if a node crashes in the middle of - the commit protocol, the transaction must be guaranteed to be + protocol is used. This protocol can + finish the transaction regardless of how the tables are + replicated. The typical use of a heavyweight transaction is + when a replica is to be moved from one node to another. Then + ensure that the replica either is entirely moved or left as + it was. Do never end up in a situation with replicas on both + nodes, or on no node at all. Even if a node crashes in the middle + of the commit protocol, the transaction must be guaranteed to be atomic. The heavyweight commit protocol involves more messages between the transaction coordinator and its participants than - a lightweight protocol and it will perform recovery work at - start-up in order to finish the abort or commit work. - </p> + a lightweight protocol, and it performs recovery work at + startup to finish the terminating or commit work.</p> <p>The heavyweight commit protocol is also non-blocking, which allows the surviving participants and their coordinator to finish the transaction regardless (even if a node crashes in the - middle of the commit protocol). When a node fails at start-up, - Mnesia will determine the outcome of the transaction and - recover it. Lightweight protocols, heavyweight protocols and dirty updates, are - dependent on other nodes to be up and running in order to make the - correct heavyweight transaction recovery decision. - </p> - <p>If Mnesia has not started on some of the nodes that are involved in the - transaction AND neither the local node or any of the already - running nodes know the outcome of the transaction, Mnesia will - by default wait for one. In the worst case scenario all other - involved nodes must start before Mnesia can make the correct decision - about the transaction and finish its start-up. - </p> - <p>This means that Mnesia (on one node)may hang if a double fault occurs, i.e. when two nodes crash simultaneously - and one attempts to start when the other refuses to - start e.g. due to a hardware error. - </p> - <p>It is possible to specify the maximum time that Mnesia - will wait for other nodes to respond with a transaction - recovery decision. The configuration parameter - <c>max_wait_for_decision</c> defaults to infinity (which may - cause the indefinite hanging as mentioned above) but if it is - set to a definite time period (eg.three minutes), Mnesia will then enforce a - transaction recovery decision if needed, in order to allow - Mnesia to continue with its start-up procedure. </p> - <p>The downside of an enforced transaction recovery decision, is that the decision may be - incorrect, due to insufficient information regarding the other nodes' - recovery decisions. This may result in an - inconsistent database where Mnesia has committed the transaction - on some nodes but aborted it on others. </p> - <p>In fortunate cases the inconsistency will only appear in tables belonging to a specific - application, but if a schema transaction has been inconsistently - recovered due to the enforced transaction recovery decision, the - effects of the inconsistency can be fatal. + middle of the commit protocol). When a node fails at startup, + <c>Mnesia</c> determines the outcome of the transaction and + recovers it. Lightweight protocols, heavyweight protocols, and + dirty updates, are dependent on other nodes to be operational + to make the correct heavyweight transaction recovery decision.</p> + <p>If <c>Mnesia</c> has not started on some of the nodes that + are involved in the transaction <em>and</em> neither the + local node nor any of the already running nodes know the + outcome of the transaction, <c>Mnesia</c> waits for one, + by default. In the worst case scenario, all other involved + nodes must start before <c>Mnesia</c> can make the correct + decision about the transaction and finish its startup.</p> + <p>Thus, <c>Mnesia</c> (on one node) can hang if a double fault + occurs, that is, when two nodes crash simultaneously + and one attempts to start when the other refuses to + start, for example, because of a hardware error.</p> + <p>The maximum time that <c>Mnesia</c> waits for other nodes to + respond with a transaction recovery decision can be specified. + The configuration parameter <c>max_wait_for_decision</c> + defaults to <c>infinity</c>, which can cause the indefinite + hanging as mentioned earlier. However, if the parameter is + set to a definite time period (for example, three minutes), + <c>Mnesia</c> then enforces a transaction recovery decision, + if needed, to allow <c>Mnesia</c> to continue with its startup + procedure.</p> + <p>The downside of an enforced transaction recovery decision is + that the decision can be incorrect, because of insufficient + information about the recovery decisions from the other nodes. + This can result in an inconsistent database where <c>Mnesia</c> + has committed the transaction on some nodes but terminated it + on others.</p> + <p>In fortunate cases, the inconsistency is only visible in + tables belonging to a specific application. However, if a + schema transaction is inconsistently recovered because of + the enforced transaction recovery decision, the + effects of the inconsistency can be fatal. However, if the higher priority is availability rather than - consistency, then it may be worth the risk. </p> - <p>If Mnesia - encounters a inconsistent transaction decision a - <c>{inconsistent_database, bad_decision, Node}</c> system event - will be generated in order to give the application a chance to - install a fallback or other appropriate measures to resolve the inconsistency. The default - behavior of the Mnesia event handler is the same as if the - database became inconsistent as a result of partitioned network (see - above). - </p> + consistency, it can be worth the risk.</p> + <p>If <c>Mnesia</c> detects an inconsistent transaction decision, + an <c>{inconsistent_database, bad_decision, Node}</c> system event + is generated to give the application a chance to install a + fallback or other appropriate measures to resolve the + inconsistency. The default behavior of the <c>Mnesia</c> + event handler is the same as if the database became + inconsistent as a result of partitioned network (as + described earlier).</p> </section> <section> - <title>Backup, Fallback, and Disaster Recovery</title> - <p>The following functions are used to backup data, to install a - backup as fallback, and for disaster recovery. - </p> + <title>Backup, Restore, Fallback, and Disaster Recovery</title> + <p>The following functions are used to back up data, to install + a backup as fallback, and for disaster recovery:</p> <list type="bulleted"> - <item><c>mnesia:backup_checkpoint(Name, Opaque, [Mod])</c>. This - function performs a backup of the tables included in the - checkpoint. + <item> + <seealso marker="mnesia#backup_checkpoint/2">mnesia:backup_checkpoint(Name, Opaque, [Mod])</seealso> + performs a backup of the tables included in the checkpoint. </item> - <item><c>mnesia:backup(Opaque, [Mod])</c>. This function - activates a new checkpoint which covers all Mnesia tables and + <item> + <seealso marker="mnesia#backup/1">mnesia:backup(Opaque, [Mod])</seealso> + activates a new + checkpoint that covers all <c>Mnesia</c> tables and performs a backup. It is performed with maximum degree of - redundancy (also refer to the function <seealso marker="#checkpoints">mnesia:activate_checkpoint(Args)</seealso>, - <c>{max, MaxTabs} and {min, MinTabs}).</c></item> - <item><c>mnesia:traverse_backup(Source,[SourceMod,]</c><c>Target,[TargetMod,]Fun,Ac)</c>. This function can be used - to read an existing backup, create a new backup from an - existing one, or to copy a backup from one type media to - another. + redundancy (see also the function + <seealso marker="#checkpoints">mnesia:activate_checkpoint(Args)</seealso>, + <c>{max, MaxTabs} and {min, MinTabs})</c>. </item> - <item><c>mnesia:uninstall_fallback()</c>. This function removes - previously installed fallback files. + <item> + <seealso marker="mnesia#traverse_backup/4">mnesia:traverse_backup(Source, [SourceMod,] Target, [TargetMod,] Fun, Acc)</seealso> + can be used to read an existing backup, create a backup from an + existing one, or to copy a backup from one type media to another. + </item> + <item> + <seealso marker="mnesia#uninstall_fallback/0">mnesia:uninstall_fallback()</seealso> + removes previously installed fallback files. </item> - <item><c>mnesia:restore(Opaque, Args)</c>. This function + <item> + <seealso marker="mnesia#restore/2">mnesia:restore(Opaque, Args)</seealso> restores a set of tables from a previous backup. </item> - <item><c>mnesia:install_fallback(Opaque, [Mod])</c>. This - function can be configured to restart the Mnesia and reload data - tables, and possibly schema tables, from an existing + <item> + <seealso marker="mnesia#install_fallback/1">mnesia:install_fallback(Opaque, [Mod])</seealso> + can be configured to restart <c>Mnesia</c> and the reload data + tables, and possibly the schema tables, from an existing backup. This function is typically used for disaster recovery - purposes, when data or schema tables are corrupted.</item> + purposes, when data or schema tables are corrupted. + </item> </list> - <p>These functions are explained in the following - sub-sections. Also refer to the the section <seealso marker="#checkpoints">Checkpoints</seealso> in this chapter, which - describes the two functions used to activate and de-activate - checkpoints. - </p> + <p>These functions are explained in the following sections. + See also <seealso marker="#checkpoints">Checkpoints</seealso>, + which describes the two functions used + to activate and deactivate checkpoints.</p> <section> <title>Backup</title> - <p>Backup operation are performed with the following functions: - </p> + <p>Backup operation are performed with the following functions:</p> <list type="bulleted"> - <item><c>mnesia:backup_checkpoint(Name, Opaque, [Mod])</c></item> - <item><c>mnesia:backup(Opaque, [Mod])</c></item> - <item><c>mnesia:traverse_backup(Source, [SourceMod,],</c><c>Target,[TargetMod,]Fun,Acc)</c>.</item> + <item> + <seealso marker="mnesia#backup_checkpoint/2">mnesia:backup_checkpoint(Name, Opaque, [Mod])</seealso> + </item> + <item> + <seealso marker="mnesia#backup/1">mnesia:backup(Opaque, [Mod])</seealso> + </item> + <item> + <seealso marker="mnesia#traverse_backup/4">mnesia:traverse_backup(Source, [SourceMod,] Target, [TargetMod,] Fun, Acc)</seealso> + </item> </list> <p>By default, the actual access to the backup media is - performed via the <c>mnesia_backup</c> module for both read + performed through module <c>mnesia_backup</c> for both read and write. Currently <c>mnesia_backup</c> is implemented with - the standard library module <c>disc_log</c>, but it is possible to write - your own module with the same interface as - <c>mnesia_backup</c> and configure Mnesia so the alternate - module performs the actual accesses to the backup media. This - means that the user may put the backup on medias that Mnesia + the standard library module <c>disc_log</c>. However, you + can write your own module with the same interface as + <c>mnesia_backup</c> and configure <c>Mnesia</c> so that + the alternative module performs the actual accesses to + the backup media. The user can + therefore put the backup on a media that <c>Mnesia</c> does not know about, possibly on hosts where Erlang is not - running. Use the configuration parameter <c><![CDATA[-mnesia backup_module <module>]]></c> for this purpose. </p> - <p>The source - for a backup is an activated checkpoint. The backup function - most commonly used is <c>mnesia:backup_checkpoint(Name, Opaque,[Mod])</c>. This function returns either <c>ok</c>, or - <c>{error,Reason}</c>. It has the following arguments: - </p> + running. Use configuration parameter + <c><![CDATA[-mnesia backup_module <module>]]></c> + for this purpose.</p> + <p>The source for a backup is an activated checkpoint. + The backup function + <seealso marker="mnesia#backup_checkpoint/2">mnesia:backup_checkpoint(Name, Opaque,[Mod])</seealso> + is most commonly used and returns <c>ok</c> or + <c>{error,Reason}</c>. It has the following arguments:</p> <list type="bulleted"> - <item><c>Name</c> is the name of an activated - checkpoint. Refer to the section <seealso marker="#checkpoints">Checkpoints</seealso> in this chapter, the - function <c>mnesia:activate_checkpoint(ArgList)</c> for - details on how to include table names in checkpoints. + <item><c>Name</c> is the name of an activated checkpoint. + For details on how to include table names in checkpoints, + see the function <c>mnesia:activate_checkpoint(ArgList)</c> + in <seealso marker="#checkpoints">Checkpoints</seealso>. </item> - <item><c>Opaque</c>. Mnesia does not interpret this argument, - but it is forwarded to the backup module. The Mnesia default - backup module, <c>mnesia_backup</c> interprets this argument - as a local file name. + <item><c>Opaque</c>. <c>Mnesia</c> does not interpret this + argument, but it is forwarded to the backup module. The + <c>Mnesia</c> default backup module <c>mnesia_backup</c> + interprets this argument as a local filename. </item> - <item><c>Mod</c>. The name of an alternate backup module. + <item><c>Mod</c> is the name of an alternative backup module. </item> </list> - <p>The function <c>mnesia:backup(Opaque[, Mod])</c> activates a - new checkpoint which covers all Mnesia tables with maximum - degree of redundancy and performs a backup. Maximum + <p>The function + <seealso marker="mnesia#backup/1">mnesia:backup(Opaque [,Mod])</seealso> + activates a + new checkpoint that covers all <c>Mnesia</c> tables with + maximum degree of redundancy and performs a backup. Maximum redundancy means that each table replica has a checkpoint - retainer. Tables with the <c>local_contents</c> property are - backed up as they - look on the current node. - </p> - <p>It is possible to iterate over a backup, either for the - purpose of transforming it into a new backup, or just reading - it. The function <c>mnesia:traverse_backup(Source, [SourceMod,]</c><c>Target, [TargeMod,] Fun, Acc)</c> which normally returns <c>{ok, LastAcc}</c>, is used for both of these purposes. - </p> + retainer. Tables with property <c>local_contents</c> are + backed up as they look on the current node.</p> + <p>You can iterate over a backup, either to transform it + into a new backup, or only read it. The function + <seealso marker="mnesia#traverse_backup/4">mnesia:traverse_backup(Source, [SourceMod,] Target, [TargetMod,] Fun, Acc)</seealso>, + which normally returns <c>{ok, LastAcc}</c>, + is used for both of these purposes.</p> <p>Before the traversal starts, the source backup media is opened with <c>SourceMod:open_read(Source)</c>, and the target backup media is opened with - <c>TargetMod:open_write(Target)</c>. The arguments are: - </p> + <c>TargetMod:open_write(Target)</c>. The arguments are as + follows:</p> <list type="bulleted"> <item><c>SourceMod</c> and <c>TargetMod</c> are module names. </item> <item><c>Source</c> and <c>Target</c> are opaque data used exclusively by the modules <c>SourceMod</c> and - <c>TargetMod</c> for the purpose of initializing the backup - medias. + <c>TargetMod</c> for initializing the backup medias. </item> <item><c>Acc</c> is an initial accumulator value. </item> <item><c>Fun(BackupItems, Acc)</c> is applied to each item in - the backup. The Fun must return a tuple <c>{ValGoodBackupItems, NewAcc}</c>, where <c>ValidBackupItems</c> is a list of valid - backup items, and <c>NewAcc</c> is a new accumulator value. + the backup. The Fun must return a tuple + <c>{ValGoodBackupItems, NewAcc}</c>, + where <c>ValidBackupItems</c> is a list of valid + backup items. <c>NewAcc</c> is a new accumulator value. The <c>ValidBackupItems</c> are written to the target backup with the function <c>TargetMod:write/2</c>. </item> - <item><c>LastAcc</c> is the last accumulator value. I.e. + <item><c>LastAcc</c> is the last accumulator value, that is, the last <c>NewAcc</c> value that was returned by <c>Fun</c>. </item> </list> - <p>It is also possible to perform a read-only traversal of the - source backup without updating a target backup. If - <c>TargetMod==read_only</c>, then no target backup is accessed - at all. - </p> + <p>Also, a read-only traversal of the source backup can be + performed without updating a target backup. If + <c>TargetMod==read_only</c>, no target backup is accessed.</p> <p>By setting <c>SourceMod</c> and <c>TargetMod</c> to different - modules it is possible to copy a backup from one kind of backup - media to another. - </p> - <p>Valid <c>BackupItems</c> are the following tuples: - </p> + modules, a backup can be copied from one backup + media to another.</p> + <p>Valid <c>BackupItems</c> are the following tuples:</p> <list type="bulleted"> <item><c>{schema, Tab}</c> specifies a table to be deleted. </item> <item><c>{schema, Tab, CreateList}</c> specifies a table to be - created. See <c>mnesia_create_table/2</c> for more - information about <c>CreateList</c>. + created. For more information about <c>CreateList</c>, see + <seealso marker="mnesia#create_table/2">mnesia:create_table/2</seealso>. </item> <item><c>{Tab, Key}</c> specifies the full identity of a record - to be deleted. + to be deleted. </item> <item><c>{Record}</c> specifies a record to be inserted. It - can be a tuple with <c>Tab</c> as first field. Note that the + can be a tuple with <c>Tab</c> as first field. Notice that the record name is set to the table name regardless of what - <c>record_name</c> is set to. + <c>record_name</c> is set to. </item> </list> <p>The backup data is divided into two sections. The first - section contains information related to the schema. All schema - related items are tuples where the first field equals the atom - schema. The second section is the record section. It is not - possible to mix schema records with other records and all schema - records must be located first in the backup. - </p> - <p>The schema itself is a table and will possibly be included in - the backup. All nodes where the schema table resides are - regarded as a <c>db_node</c>. - </p> - <p>The following example illustrates how - <c>mnesia:traverse_backup</c> can be used to rename a db_node in - a backup file: - </p> + section contains information related to the schema. All + schema-related items are tuples where the first field equals + the atom schema. The second section is the record section. + Schema records cannot be mixed with other records and all + schema records must be located first in the backup.</p> + <p>The schema itself is a table and is possibly included in + the backup. Each node where the schema table resides is + regarded as a <c>db_node</c>.</p> + <p>The following example shows how + <seealso marker="mnesia#traverse_backup/4">mnesia:traverse_backup</seealso> + can be used to rename a <c>db_node</c> in a backup file:</p> <codeinclude file="bup.erl" tag="%0" type="erl"></codeinclude> </section> <section> <title>Restore</title> - <p>Tables can be restored on-line from a backup without - restarting Mnesia. A restore is performed with the function - <c>mnesia:restore(Opaque,Args)</c>, where <c>Args</c> can - contain the following tuples: - </p> + <p>Tables can be restored online from a backup without + restarting <c>Mnesia</c>. A restore is performed with the + function + <seealso marker="mnesia#restore/2">mnesia:restore(Opaque, Args)</seealso>, + where <c>Args</c> can contain the following tuples:</p> <list type="bulleted"> <item><c>{module,Mod}</c>. The backup module <c>Mod</c> is used to access the backup media. If omitted, the default - backup module will be used.</item> - <item><c>{skip_tables, TableList}</c> Where <c>TableList</c> - is a list of tables which should not be read from the backup.</item> - <item><c>{clear_tables, TableList}</c> Where <c>TableList</c> - is a list of tables which should be cleared, before the - records from the backup are inserted, i.e. all records in + backup module is used. + </item> + <item><c>{skip_tables, TableList}</c>, where <c>TableList</c> + is a list of tables, which is not to be read from the backup. + </item> + <item><c>{clear_tables, TableList}</c>, where <c>TableList</c> + is a list of tables, which is to be cleared before the + records from the backup are inserted. That is, all records in the tables are deleted before the tables are restored. Schema information about the tables is not cleared or read - from backup.</item> - <item><c>{keep_tables, TableList}</c> Where <c>TableList</c> - is a list of tables which should be not be cleared, before - the records from the backup are inserted, i.e. the records - in the backup will be added to the records in the table. + from the backup. + </item> + <item><c>{keep_tables, TableList}</c>, where <c>TableList</c> + is a list of tables, which is not to be cleared before + the records from the backup are inserted. That is, the records + in the backup are added to the records in the table. Schema information about the tables is not cleared or read - from backup.</item> - <item><c>{recreate_tables, TableList}</c> Where <c>TableList</c> - is a list of tables which should be re-created, before the - records from the backup are inserted. The tables are first - deleted and then created with the schema information from the - backup. All the nodes in the backup needs to be up and running.</item> - <item><c>{default_op, Operation}</c> Where <c>Operation</c> is - one of the following operations <c>skip_tables</c>, - <c>clear_tables</c>, <c>keep_tables</c> or - <c>recreate_tables</c>. The default operation specifies - which operation should be used on tables from the backup - which are not specified in any of the lists above. - If omitted, the operation <c>clear_tables</c> will be used. </item> + from the backup. + </item> + <item><c>{recreate_tables, TableList}</c>, where <c>TableList</c> + is a list of tables, which is to be recreated before the + records from the backup are inserted. The tables are first + deleted and then created with the schema information from the + backup. All the nodes in the backup need to be operational. + </item> + <item><c>{default_op, Operation}</c>, where <c>Operation</c> is + one of the operations <c>skip_tables</c>, + <c>clear_tables</c>, <c>keep_tables</c>, or + <c>recreate_tables</c>. The default operation specifies + which operation is to be used on tables from the backup + that are not specified in any of the previous lists. + If omitted, the operation <c>clear_tables</c> is used. + </item> </list> <p>The argument <c>Opaque</c> is forwarded to the backup module. It returns <c>{atomic, TabList}</c> if successful, or the - tuple <c>{aborted, Reason}</c> in the case of an error. - <c>TabList</c> is a list of the restored tables. Tables which - are restored are write locked for the duration of the restore - operation. However, regardless of any lock conflict caused by + tuple <c>{aborted, Reason}</c> if there is an error. + <c>TabList</c> is a list of the restored tables. Tables that + are restored are write-locked during the restore + operation. However, regardless of any lock conflict caused by this, applications can continue to do their work during the - restore operation. - </p> + restore operation.</p> <p>The restoration is performed as a single transaction. If the - database is very large, it may not be possible to restore it - online. In such a case the old database must be restored by - installing a fallback, and then restart. - </p> + database is large, it cannot always be restored + online. The old database must then be restored by + installing a fallback, followed by a restart.</p> </section> <section> - <title>Fallbacks</title> - <p>The function <c>mnesia:install_fallback(Opaque, [Mod])</c> is - used to install a backup as fallback. It uses the backup module + <title>Fallback</title> + <p>The function + <seealso marker="mnesia#install_fallback/2">mnesia:install_fallback(Opaque, [Mod])</seealso> + installs a backup as fallback. It uses the backup module <c>Mod</c>, or the default backup module, to access the backup - media. This function returns <c>ok</c> if successful, or - <c>{error, Reason}</c> in the case of an error. - </p> - <p>Installing a fallback is a distributed operation that is + media. The function returns <c>ok</c> if successful, or + <c>{error, Reason}</c> if there is an error.</p> + <p>Installing a fallback is a distributed operation, which is <em>only</em> performed on all <c>db_nodes</c>. The fallback - is used to restore the database the next time the system is - started. If a Mnesia node with a fallback installed detects that - Mnesia on another node has died for some reason, it will - unconditionally terminate itself. - </p> + restores the database the next time the system is started. + If a <c>Mnesia</c> node with a fallback installed detects that + <c>Mnesia</c> on another node has died, it + unconditionally terminates itself.</p> <p>A fallback is typically used when a system upgrade is performed. A system typically involves the installation of new - software versions, and Mnesia tables are often transformed into - new layouts. If the system crashes during an upgrade, it is - highly probable re-installation of the old - applications will be required and restoration of the database - to its previous state. This can be done if a backup is performed and - installed as a fallback before the system upgrade begins. - </p> - <p>If the system upgrade fails, Mnesia must be restarted on all - <c>db_nodes</c> in order to restore the old database. The - fallback will be automatically de-installed after a successful - start-up. The function <c>mnesia:uninstall_fallback()</c> may - also be used to de-install the fallback after a + software versions, and <c>Mnesia</c> tables are often transformed + into new layouts. If the system crashes during an upgrade, it is + highly probable that reinstallation of the old applications is + required, and restoration of the database to its previous state. + This can be done if a backup is performed and + installed as a fallback before the system upgrade begins.</p> + <p>If the system upgrade fails, <c>Mnesia</c> must be restarted + on all <c>db_nodes</c> to restore the old database. The + fallback is automatically deinstalled after a successful + startup. The function + <seealso marker="mnesia#uninstall_fallback/0">mnesia:uninstall_fallback()</seealso> + can also be used to deinstall the fallback after a successful system upgrade. Again, this is a distributed - operation that is either performed on all <c>db_nodes</c>, or - none. Both the installation and de-installation of fallbacks - require Erlang to be up and running on all <c>db_nodes</c>, but - it does not matter if Mnesia is running or not. - </p> + operation that is either performed on all <c>db_nodes</c> or + none. Both the installation and deinstallation of fallbacks + require Erlang to be operational on all <c>db_nodes</c>, but + it does not matter if <c>Mnesia</c> is running or not.</p> </section> <section> <title>Disaster Recovery</title> - <p>The system may become inconsistent as a result of a power - failure. The UNIX <c>fsck</c> feature can possibly repair the - file system, but there is no guarantee that the file contents - will be consistent. - </p> - <p>If Mnesia detects that a file has not been properly closed, - possibly as a result of a power failure, it will attempt to - repair the bad file in a similar manner. Data may be lost, but - Mnesia can be restarted even if the data is inconsistent. The - configuration parameter <c><![CDATA[-mnesia auto_repair <bool>]]></c> can be - used to control the behavior of Mnesia at start-up. If - <c><![CDATA[<bool>]]></c> has the value <c>true</c>, Mnesia will attempt to - repair the file; if <c><![CDATA[<bool>]]></c> has the value <c>false</c>, - Mnesia will not restart if it detects a suspect file. This - configuration parameter affects the repair behavior of log - files, DAT files, and the default backup media. - </p> - <p>The configuration parameter <c><![CDATA[-mnesia dump_log_update_in_place <bool>]]></c> controls the safety level of - the <c>mnesia:dump_log()</c> function. By default, Mnesia will - dump the transaction log directly into the DAT files. If a power - failure happens during the dump, this may cause the randomly - accessed DAT files to become corrupt. If the parameter is set to - <c>false</c>, Mnesia will copy the DAT files and target the dump + <p>The system can become inconsistent as a result of a power + failure. The UNIX feature <c>fsck</c> can possibly repair the + file system, but there is no guarantee that the file content + is consistent.</p> + <p>If <c>Mnesia</c> detects that a file has not been properly + closed, possibly as a result of a power failure, it tries to + repair the bad file in a similar manner. Data can be lost, but + <c>Mnesia</c> can be restarted even if the data is inconsistent. + Configuration parameter + <c><![CDATA[-mnesia auto_repair <bool>]]></c> can be used + to control the behavior of <c>Mnesia</c> at startup. If + <c><![CDATA[<bool>]]></c> has the value <c>true</c>, + <c>Mnesia</c> tries to repair the file. If + <c><![CDATA[<bool>]]></c> has the value <c>false</c>, + <c>Mnesia</c> does not restart if it detects a suspect file. + This configuration parameter affects the repair behavior of log + files, <c>DAT</c> files, and the default backup media.</p> + <p>Configuration parameter + <c><![CDATA[-mnesia dump_log_update_in_place <bool>]]></c> + controls the safety level of the function + <seealso marker="mnesia#dump_log/0">mnesia:dump_log()</seealso> + By default, <c>Mnesia</c> dumps the + transaction log directly into the <c>DAT</c> files. If a power + failure occurs during the dump, this can cause the randomly + accessed <c>DAT</c> files to become corrupt. If the parameter + is set to <c>false</c>, <c>Mnesia</c> copies the <c>DAT</c> + files and target the dump to the new temporary files. If the dump is successful, the - temporary files will be renamed to their normal DAT + temporary files are renamed to their normal <c>DAT</c> suffixes. The possibility for unrecoverable inconsistencies in - the data files will be much smaller with this strategy. On the - other hand, the actual dumping of the transaction log will be + the data files becomes much smaller with this strategy. + However, the actual dumping of the transaction log becomes considerably slower. The system designer must decide whether - speed or safety is the higher priority. - </p> - <p>Replicas of type <c>disc_only_copies</c> will only be + speed or safety is the higher priority.</p> + <p>Replicas of type <c>disc_only_copies</c> are only affected by this parameter during the initial dump of the log - file at start-up. When designing applications which have - <em>very</em> high requirements, it may be appropriate not to + file at startup. When designing applications with + <em>very</em> high requirements, it can be appropriate not to use <c>disc_only_copies</c> tables at all. The reason for this is the random access nature of normal operating system files. If - a node goes down for reason for a reason such as a power - failure, these files may be corrupted because they are not - properly closed. The DAT files for <c>disc_only_copies</c> are - updated on a per transaction basis. - </p> - <p>If a disaster occurs and the Mnesia database has been - corrupted, it can be reconstructed from a backup. This should be - regarded as a last resort, since the backup contains old data. The - data is hopefully consistent, but data will definitely be lost - when an old backup is used to restore the database. - </p> + a node goes down for a reason such as a power + failure, these files can be corrupted because they are not + properly closed. The <c>DAT</c> files for <c>disc_only_copies</c> + are updated on a per transaction basis.</p> + <p>If a disaster occurs and the <c>Mnesia</c> database is + corrupted, it can be reconstructed from a backup. Regard + this as a last resort, as the backup contains old data. The + data is hopefully consistent, but data is definitely lost + when an old backup is used to restore the database.</p> </section> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap8.xml b/lib/mnesia/doc/src/Mnesia_chap8.xml index d35dd0c539..55309177b1 100644 --- a/lib/mnesia/doc/src/Mnesia_chap8.xml +++ b/lib/mnesia/doc/src/Mnesia_chap8.xml @@ -21,7 +21,7 @@ </legalnotice> - <title>Combining Mnesia with SNMP</title> + <title>Combine Mnesia with SNMP</title> <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> <responsible></responsible> <docno></docno> @@ -33,32 +33,29 @@ </header> <section> - <title>Combining Mnesia and SNMP </title> + <title>Combine Mnesia and SNMP</title> <p>Many telecommunications applications must be controlled and reconfigured remotely. It is sometimes an advantage to perform this remote control with an open protocol such as the Simple Network Management Protocol (SNMP). The alternatives to this would - be: - </p> + be the following:</p> <list type="bulleted"> - <item>Not being able to control the application remotely at all. + <item>Not being able to control the application remotely </item> - <item>Using a proprietary control protocol. + <item>Using a proprietary control protocol </item> - <item>Using a bridge which maps control messages in a + <item>Using a bridge that maps control messages in a proprietary protocol to a standardized management protocol and - vice versa. + conversely </item> </list> - <p>All of these approaches have different advantages and - disadvantages. Mnesia applications can easily be opened to the - SNMP protocol. It is possible to establish a direct one-to-one - mapping between Mnesia tables and SNMP tables. This - means that a Mnesia table can be configured to be <em>both</em> - a Mnesia table and an SNMP table. A number of functions to - control this behavior are described in the Mnesia reference - manual. - </p> + <p>All these approaches have different advantages and + disadvantages. <c>Mnesia</c> applications can easily be opened to + the SNMP protocol. A direct 1-to-1 mapping can be established + between <c>Mnesia</c> tables and SNMP tables. This means + that a <c>Mnesia</c> table can be configured to be <em>both</em> + a <c>Mnesia</c> table and an SNMP table. A number of functions to + control this behavior are described in the Reference Manual.</p> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_overview.xml b/lib/mnesia/doc/src/Mnesia_overview.xml new file mode 100644 index 0000000000..4758612053 --- /dev/null +++ b/lib/mnesia/doc/src/Mnesia_overview.xml @@ -0,0 +1,192 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>1997</year><year>2013</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Mnesia</title> + <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> + <responsible>Bjarne Däcker</responsible> + <docno></docno> + <approved>Bjarne Däcker</approved> + <checked>Bjarne Däcker</checked> + <date></date> + <rev>C</rev> + <file>Mnesia_overview.xml</file> + </header> + + <p>The management of data in telecommunications system has many + aspects, thereof some, but not all, are addressed by traditional + commercial Database Management Systems (DBMSs). In particular the + high level of fault tolerance that is required in many nonstop + systems, combined with requirements on the DBMS to run in the same + address space as the application, have led us to implement a new + DBMS, called <c>Mnesia</c>.</p> + <p><c>Mnesia</c> is implemented in, and tightly connected to Erlang. + It provides the functionality that is necessary for the + implementation of fault tolerant telecommunications systems.</p> + <p><c>Mnesia</c> is a multiuser distributed DBMS specially made for + industrial telecommunications applications written in Erlang, + which is also the intended target language. + <c>Mnesia</c> tries to address all the data + management issues required for typical telecommunications systems. + It has a number of features that are not normally found in traditional + databases.</p> + <p>In telecommunications applications, there are different needs + from the features provided by traditional DBMSs. The applications now + implemented in Erlang need a mixture of a broad range + of features, which generally are not satisfied by traditional DBMSs. + <c>Mnesia</c> is designed with requirements like the following in + mind:</p> + <list type="ordered"> + <item>Fast real-time key/value lookup + </item> + <item>Complicated non-real-time queries mainly for + operation and maintenance + </item> + <item>Distributed data because of distributed applications + </item> + <item>High fault tolerance + </item> + <item>Dynamic reconfiguration + </item> + <item>Complex objects + </item> + </list> + <p><c>Mnesia</c> is designed with the typical data management problems + of telecommunications applications in mind. This sets <c>Mnesia</c> + apart from most other DBMS. Hence <c>Mnesia</c> + combines many concepts found in traditional databases such as + transactions and queries with concepts found in data management + systems for telecommunications applications, for example:</p> + <list type="bulleted"> + <item>Fast real-time operations + </item> + <item>Configurable degree of fault tolerance (by replication) + </item> + <item>The ability to reconfigure the system without stopping or + suspending it. + </item> + </list> + <p><c>Mnesia</c> is also interesting because of its tight coupling to + Erlang, thus almost turning Erlang into a database programming + language. This has many benefits, the foremost is that + the impedance mismatch between the data format used by the DBMS + and the data format used by the programming language, which is used + to manipulate the data, completely disappears.</p> + + <section> + <title>Mnesia Database Management System (DBMS)</title> + <section> + <title>Features</title> + <p><c>Mnesia</c> contains the following features that combine to + produce a fault-tolerant, distributed DBMS written in Erlang: + </p> + <list type="bulleted"> + <item>Database schema can be dynamically reconfigured at runtime. + </item> + <item>Tables can be declared to have properties such as location, + replication, and persistence. + </item> + <item>Tables can be moved or replicated to several nodes to improve + fault tolerance. The rest of the system can still access the tables + to read, write, and delete records. + </item> + <item>Table locations are transparent to the programmer. + Programs address table names and the system itself keeps track of + table locations. + </item> + <item>Database transactions can be distributed, and many + functions can be called within one transaction. + </item> + <item>Several transactions can run concurrently, and their execution + is fully synchronized by the DBMS. <c>Mnesia</c> ensures that no + two processes manipulate data simultaneously. + </item> + <item>Transactions can be assigned the property of being executed on + all nodes in the system, or on none. Transactions can also be + bypassed in favor of running "dirty operations", which reduce + overheads and run fast. + </item> + </list> + <p>Details of these features are described in the following sections.</p> + </section> + + <section> + <title>Add-On Application</title> + <p>Query List Comprehension (QLC) can be used with <c>Mnesia</c> + to produce specialized functions that enhance the operational + ability of <c>Mnesia</c>. QLC has its own documentation as part + of the OTP documentation set. The main features of QLC + when used with <c>Mnesia</c> are as follows:</p> + <list type="bulleted"> + <item>QLC can optimize the query compiler for the <c>Mnesia</c> + DBMS, essentially making the DBMS more efficient. + </item> + <item>QLC can be used as a database programming + language for <c>Mnesia</c>. It includes a notation called "list + comprehensions" and can be used to make complex database + queries over a set of tables. + </item> + </list> + <p>For information about QLC, see the + <seealso marker="stdlib:qlc">qlc</seealso> manual page + in <c>STDLIB</c>.</p> + </section> + + <section> + <title>When to Use Mnesia</title> + <p>Use <c>Mnesia</c> with the following types of applications:</p> + <list type="bulleted"> + <item>Applications that need to replicate data. + </item> + <item>Applications that perform complicated searches on data. + </item> + <item>Applications that need to use atomic transactions to + update several records simultaneously. + </item> + <item>Applications that use soft real-time characteristics. + </item> + </list> + <p><c>Mnesia</c> is not as appropriate with the + following types of applications:</p> + <list type="bulleted"> + <item>Programs that process plain text or binary data files. + </item> + <item>Applications that merely need a look-up dictionary that + can be stored to disc. Those applications use the standard + library module <c>dets</c>, which is a disc-based version + of the module <c>ets</c>. For information about <c>dets</c>, + see the <seealso marker="stdlib:dets">dets</seealso> + manual page in <c>STDLIB</c>. + </item> + <item>Applications that need disc logging facilities. + Those applications can + use the module <c>disk_log</c> by preference. For + information about <c>disk_log</c>, see the + <seealso marker="kernel:disk_log">disk_log</seealso> + manual page in <c>Kernel</c>. + </item> + <item>Hard real-time systems. + </item> + </list> + </section> + </section> +</chapter> diff --git a/lib/mnesia/doc/src/mnesia.xml b/lib/mnesia/doc/src/mnesia.xml index 856a7594a7..1f5c62a5e6 100644 --- a/lib/mnesia/doc/src/mnesia.xml +++ b/lib/mnesia/doc/src/mnesia.xml @@ -32,291 +32,254 @@ <file></file> </header> <module>mnesia</module> - <modulesummary>A Distributed Telecommunications DBMS </modulesummary> + <modulesummary>A distributed telecommunications DBMS</modulesummary> <description> - <p><c>Mnesia</c> is a distributed DataBase Management System (DBMS), - appropriate for telecommunications applications and other Erlang - applications which require continuous operation and exhibit soft - real-time properties. - </p> - <p>Listed below are some of the most important and attractive capabilities, Mnesia provides: - </p> + + <p>The following are some of the most important and attractive + capabilities provided by <c>Mnesia</c>:</p> <list type="bulleted"> - <item> - <p>A relational/object hybrid data model which is - suitable for telecommunications applications. - </p> + <item>A relational/object hybrid data model that is suitable + for telecommunications applications. </item> - <item> - <p>A specifically designed DBMS query language, QLC (as an add-on library). - </p> + <item>A DBMS query language, Query List Comprehension (QLC) as + an add-on library. </item> - <item> - <p>Persistence. Tables may be coherently kept on disc as - well as in main memory. - </p> + <item>Persistence. Tables can be coherently kept on disc and + in the main memory. </item> - <item> - <p>Replication. Tables may be replicated at several nodes. - </p> + <item>Replication. Tables can be replicated at several nodes. </item> - <item> - <p>Atomic transactions. A series of table manipulation - operations can be grouped into a single atomic - transaction. - </p> + <item>Atomic transactions. A series of table manipulation + operations can be grouped into a single atomic transaction. </item> - <item> - <p>Location transparency. Programs can be written without - knowledge of the actual location of data. - </p> + <item>Location transparency. Programs can be written without + knowledge of the actual data location. </item> - <item> - <p>Extremely fast real time data searches. - </p> + <item>Extremely fast real-time data searches. </item> - <item> - <p>Schema manipulation routines. It is possible to - reconfigure the DBMS at runtime without stopping the - system. - </p> + <item>Schema manipulation routines. The DBMS can be + reconfigured at runtime without stopping the system. </item> </list> - <p>This Reference Manual describes the Mnesia API. This includes - functions used to define and manipulate Mnesia tables. - </p> - <p>All functions documented in these pages can be used in any - combination with queries using the list comprehension notation. The - query notation is described in the QLC's man page. - </p> - <p>Data in Mnesia is organized as a set of tables. Each table - has a name which must be an atom. Each table is made up of - Erlang records. The user is responsible for the record - definitions. Each table also has a set of properties. Below - are some of the properties that are associated with each - table: - </p> + <p>This Reference Manual describes the <c>Mnesia</c> API. This + includes functions that define and manipulate <c>Mnesia</c> + tables.</p> + <p>All functions in this Reference Manual can be used in any + combination with queries using the list comprehension notation. + For information about the query notation, see the + <seealso marker="stdlib:qlc">qlc</seealso> + manual page in <c>STDLIB</c>.</p> + <p>Data in <c>Mnesia</c> is organized as a set of tables. Each table + has a name that must be an atom. Each table is made up of + Erlang records. The user is responsible for the record + definitions. Each table also has a set of properties. The + following are some of the properties that are associated with each + table:</p> <list type="bulleted"> <item> - <p><c>type</c>. Each table can either have 'set', - 'ordered_set' or 'bag' semantics. Note: currently 'ordered_set' - is not supported for 'disc_only_copies'. If a table is of type - 'set' it means that each key leads to either one or zero - records. <br></br> -If a new item is inserted with the same key as - an existing record, the old record is overwritten. On the - other hand, if a table is of type 'bag', each key can map to - several records. However, all records in type bag tables are - unique, only the keys may be duplicated. - </p> + <p><c>type</c>. Each table can have <c>set</c>, + <c>ordered_set</c>, or <c>bag</c> semantics. Notice that + currently <c>ordered_set</c> is not supported for + <c>disc_only_copies</c>.</p> + <p>If a table is of type <c>set</c>, each key leads to + either one or zero records.</p> + <p>If a new item is inserted with the same key as an + existing record, the old record is overwritten. However, + if a table is of type <c>bag</c>, each key can map to + several records. All records in type <c>bag</c> tables are + unique, only the keys can be duplicated.</p> </item> <item> <p><c>record_name</c>. All records stored in a table must - have the same name. You may say that the records must be - instances of the same record type. - </p> + have the same name. The records must be instances of the + same record type.</p> </item> <item> - <p><c>ram_copies</c> A table can be replicated on a number - of Erlang nodes. The <c>ram_copies</c> property specifies a - list of Erlang nodes where RAM copies are kept. These - copies can be dumped to disc at regular intervals. However, + <p><c>ram_copies</c>. A table can be replicated on a number + of Erlang nodes. Property <c>ram_copies</c> specifies a + list of Erlang nodes where RAM copies are kept. These + copies can be dumped to disc at regular intervals. However, updates to these copies are not written to disc on a - transaction basis. - </p> + transaction basis.</p> </item> <item> - <p><c>disc_copies</c> The <c>disc_copies</c> property + <p><c>disc_copies</c>. This property specifies a list of Erlang nodes where the table is kept in - RAM as well as on disc. All updates of the table are - performed on the actual table and are also logged to disc. + RAM and on disc. All updates of the table are + performed in the actual table and are also logged to disc. If a table is of type <c>disc_copies</c> at a certain node, - it means that the entire table is resident in RAM memory as - well as on disc. Each transaction performed on the table is - appended to a LOG file as well as written into the RAM - table. - </p> + the entire table is resident in RAM memory and on disc. + Each transaction performed on the table is appended to a + <c>LOG</c> file and written into the RAM table.</p> </item> <item> - <p><c>disc_only_copies</c> Some, or all, table replicas + <p><c>disc_only_copies</c>. Some, or all, table replicas can be kept on disc only. These replicas are considerably - slower than the RAM based replicas. - </p> + slower than the RAM-based replicas.</p> </item> <item> - <p><c>index</c> This is a list of attribute names, or + <p><c>index</c>. This is a list of attribute names, or integers, which specify the tuple positions on which - Mnesia shall build and maintain an extra index table. - </p> + <c>Mnesia</c> is to build and maintain an extra index + table.</p> </item> <item> - <p><c>local_content</c> When an application requires + <p><c>local_content</c>. When an application requires tables whose contents are local to each node, - <c>local_content</c> tables may be used. The name of the - table is known to all Mnesia nodes, but its contents are + <c>local_content</c> tables can be used. The table name + is known to all <c>Mnesia</c> nodes, but its content is unique on each node. This means that access to such a table - must be done locally. Set the <c>local_content</c> field to - <c>true</c> if you want to enable the <c>local_content</c> - behavior. The default is <c>false</c>. - </p> + must be done locally. Set field <c>local_content</c> to + <c>true</c> to enable the <c>local_content</c> + behavior. Default is <c>false</c>.</p> </item> <item> - <p><c>majority</c> This attribute can be either <c>true</c> or - <c>false</c> (default is <c>false</c>). When <c>true</c>, a majority - of the table replicas must be available for an update to succeed. - Majority checking can be enabled on tables with mission-critical data, - where it is vital to avoid inconsistencies due to network splits. - </p> + <p><c>majority</c>. This attribute is <c>true</c> or + <c>false</c>; default is <c>false</c>. When <c>true</c>, + a majority of the table replicas must be available for an + update to succeed. Majority checking can be enabled on + tables with mission-critical data, where it is vital to + avoid inconsistencies because of network splits.</p> </item> <item> - <p><c>snmp</c> Each (set based) Mnesia table can be - automatically turned into an SNMP ordered table as well. - This property specifies the types of the SNMP keys. - </p> + <p><c>snmp</c>. Each (set-based) <c>Mnesia</c> table can be + automatically turned into a Simple Network Management + Protocol (SNMP) ordered table as well. + This property specifies the types of the SNMP keys.</p> </item> <item> <p><c>attributes</c>. The names of the attributes for the - records that are inserted in the table. - </p> + records that are inserted in the table.</p> </item> </list> - <p>See <c>mnesia:create_table/2</c> about the complete set of - table properties and their details. - </p> - <p>This document uses a table of persons to illustrate various - examples. The following record definition is assumed: - </p> + <p>For information about the complete set of table properties + and their details, see <c>mnesia:create_table/2</c>.</p> + <p>This Reference Manual uses a table of persons to illustrate + various examples. The following record definition is assumed:</p> <code type="none"> -record(person, {name, age = 0, address = unknown, salary = 0, - children = []}), - </code> - <p>The first attribute of the record is the primary key, or key - for short. - </p> - <p>The function descriptions are sorted in alphabetic order. <em>Hint:</em> - start to read about <c>mnesia:create_table/2</c>, - <c>mnesia:lock/2</c> and <c>mnesia:activity/4</c> before you continue on - and learn about the rest. - </p> - <p>Writing or deleting in transaction context creates a local copy - of each modified record during the transaction. During iteration, - i.e. <c>mnesia:fold[lr]/4</c> <c>mnesia:next/2</c> <c>mnesia:prev/2</c> - <c>mnesia:snmp_get_next_index/2</c>, mnesia will compensate for - every written or deleted record, which may reduce the - performance. If possible avoid writing or deleting records in - the same transaction before iterating over the table. - </p> + children = []}),</code> + <p>The first record attribute is the primary key, or key + for short.</p> + <p>The function descriptions are sorted in alphabetical order. + It is recommended to start to read about + <c>mnesia:create_table/2</c>, <c>mnesia:lock/2</c>, and + <c>mnesia:activity/4</c> before you continue and learn + about the rest.</p> + <p>Writing or deleting in transaction-context creates a local + copy of each modified record during the transaction. During + iteration, that is, <c>mnesia:fold[lr]/4</c>, + <c>mnesia:next/2</c>, <c>mnesia:prev/2</c>, and + <c>mnesia:snmp_get_next_index/2</c>, <c>Mnesia</c> + compensates for every written or deleted record, which can + reduce the performance.</p> + <p>If possible, avoid writing or deleting records in the same + transaction before iterating over the table.</p> </description> + <funcs> <func> - <name>abort(Reason) -> transaction abort </name> - <fsummary>Abort the current transaction.</fsummary> + <name>abort(Reason) -> transaction abort</name> + <fsummary>Terminates the current transaction.</fsummary> <desc> - <p>Makes the transaction silently + <p>Makes the transaction silently return the tuple <c>{aborted, Reason}</c>. - The abortion of a Mnesia transaction means that - an exception will be thrown to an enclosing <c>catch</c>. + Termination of a <c>Mnesia</c> transaction means that + an exception is thrown to an enclosing <c>catch</c>. Thus, the expression <c>catch mnesia:abort(x)</c> does - not abort the transaction. </p> + not terminate the transaction.</p> </desc> </func> <func> <name>activate_checkpoint(Args) -> {ok,Name,Nodes} | {error,Reason}</name> - <fsummary>Activate a checkpoint.</fsummary> + <fsummary>Activates a checkpoint.</fsummary> <desc> - <p>A checkpoint is a consistent view of the system. + <marker id="activate_checkpoint"></marker> + <p>A checkpoint is a consistent view of the system. A checkpoint can be activated on a set of tables. - This checkpoint can then be traversed and will - present a view of the system as it existed at the time when - the checkpoint was activated, even if the tables are being or have been - manipulated. - </p> - <p><c>Args</c> is a list of the following tuples: - </p> + This checkpoint can then be traversed and + presents a view of the system as it existed at the time when + the checkpoint was activated, even if the tables are + being or have been manipulated.</p> + <p><c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> <item> - <p><c>{name,Name}</c>. <c>Name</c> of checkpoint. Each - checkpoint must have a name which is unique to the + <p><c>{name,Name}</c>. <c>Name</c> is the checkpoint name. + Each checkpoint must have a name that is unique to the associated nodes. The name can be reused only once the checkpoint has been deactivated. By default, a name - which is probably unique is generated. - </p> + that is probably unique is generated.</p> </item> <item> <p><c>{max,MaxTabs}</c>. <c>MaxTabs</c> is a list of - tables that should be included in the checkpoint. The - default is []. For these tables, the redundancy will be - maximized and checkpoint information will be retained together + tables that are to be included in the checkpoint. + Default is <c>[]</c>. For these tables, the redundancy is + maximized and checkpoint information is retained together with all replicas. The checkpoint becomes more fault tolerant if the tables have several replicas. When a new - replica is added by means of the schema manipulation - function <c>mnesia:add_table_copy/3</c>, a retainer will - also be attached automatically. - </p> + replica is added by the schema manipulation + function <c>mnesia:add_table_copy/3</c>, a retainer is + also attached automatically.</p> </item> <item> <p><c>{min,MinTabs}</c>. <c>MinTabs</c> is a list of - tables that should be included in the checkpoint. The - default is []. For these tables, the redundancy will be - minimized and the checkpoint information will only be retained - with one replica, preferably on the local node. - </p> + tables that are to be included in the checkpoint. + Default is []. For these tables, the redundancy is + minimized and the checkpoint information is only retained + with one replica, preferably on the local node.</p> </item> <item> <p><c>{allow_remote,Bool}</c>. <c>false</c> means that all retainers must be local. The checkpoint cannot be activated if a table does not reside locally. <c>true</c> allows retainers to be allocated on any - node. Default is set to <c>true</c>. - </p> + node. Default is <c>true</c>.</p> </item> <item> <p><c>{ram_overrides_dump,Bool}</c>. Only applicable - for <c>ram_copies</c>. <c>Bool</c> allows you to choose - to backup the table state as it is in RAM, or as it is on - disc. <c>true</c> means that the latest committed - records in RAM should be included in the checkpoint. - These are the records that the application accesses. - <c>false</c> means that the records dumped to DAT files - should be included in the checkpoint. These are the - records that will be loaded at startup. Default is - <c>false</c>. - </p> + for <c>ram_copies</c>. <c>Bool</c> allows you to choose + to back up the table state as it is in RAM, or as it is + on disc. <c>true</c> means that the latest committed + records in RAM are to be included in the checkpoint. + These are the records that the application accesses. + <c>false</c> means that the records dumped to <c>DAT</c> + files are to be included in the checkpoint. These + records are loaded at startup. Default is <c>false</c>.</p> </item> </list> <p>Returns <c>{ok,Name,Nodes}</c> or <c>{error,Reason}</c>. - <c>Name</c> is the (possibly generated) name of the - checkpoint. <c>Nodes</c> are the nodes that + <c>Name</c> is the (possibly generated) checkpoint name. + <c>Nodes</c> are the nodes that are involved in the checkpoint. Only nodes that keep a - checkpoint retainer know about the checkpoint. - </p> + checkpoint retainer know about the checkpoint.</p> </desc> </func> <func> <name>activity(AccessContext, Fun [, Args]) -> ResultOfFun | exit(Reason)</name> - <fsummary>Execute <c>Fun</c>in <c>AccessContext</c>.</fsummary> + <fsummary>Executes <c>Fun</c> in <c>AccessContext</c>.</fsummary> <desc> - <p>Invokes <c>mnesia:activity(AccessContext, Fun, Args, AccessMod)</c> where <c>AccessMod</c> is the default + <marker id="activity_2_3"></marker> + <p>Calls <c>mnesia:activity(AccessContext, Fun, Args, + AccessMod)</c>, where <c>AccessMod</c> is the default access callback module obtained by <c>mnesia:system_info(access_module)</c>. <c>Args</c> - defaults to the empty list <c>[]</c>.</p> + defaults to <c>[]</c> (empty list).</p> </desc> </func> <func> <name>activity(AccessContext, Fun, Args, AccessMod) -> ResultOfFun | exit(Reason)</name> - <fsummary>Execute <c>Fun</c>in <c>AccessContext</c>.</fsummary> - <desc> - <p>This function executes the functional object <c>Fun</c> - with the arguments <c>Args</c>. - </p> - <p>The code which executes inside the activity can - consist of a series of table manipulation functions, which is - performed in a <c>AccessContext</c>. Currently, the following - access contexts are supported: - </p> + <fsummary>Executes <c>Fun</c> in <c>AccessContext</c>.</fsummary> + <desc> + <marker id="activity_4"></marker> + <p>Executes the functional object <c>Fun</c> + with argument <c>Args</c>.</p> + <p>The code that executes inside the activity can + consist of a series of table manipulation functions, which are + performed in an <c>AccessContext</c>. Currently, the following + access contexts are supported:</p> <taglist> <tag><c>transaction</c></tag> <item> @@ -324,10 +287,10 @@ If a new item is inserted with the same key as </item> <tag><c>{transaction, Retries}</c></tag> <item> - <p>Invokes <c>mnesia:transaction(Fun, Args, Retries)</c>. Note that the result from the <c>Fun</c> is - returned if the transaction was successful (atomic), - otherwise the function exits with an abort reason. - </p> + <p>Calls <c>mnesia:transaction(Fun, Args, Retries)</c>. + Notice that the result from <c>Fun</c> is + returned if the transaction is successful (atomic), + otherwise the function exits with an abort reason.</p> </item> <tag><c>sync_transaction</c></tag> <item> @@ -335,540 +298,499 @@ If a new item is inserted with the same key as </item> <tag><c>{sync_transaction, Retries}</c></tag> <item> - <p>Invokes <c>mnesia:sync_transaction(Fun, Args, Retries)</c>. Note that the result from the <c>Fun</c> is - returned if the transaction was successful (atomic), - otherwise the function exits with an abort reason. - </p> + <p>Calls <c>mnesia:sync_transaction(Fun, Args, Retries)</c>. + Notice that the result from <c>Fun</c> is + returned if the transaction is successful (atomic), + otherwise the function exits with an abort reason.</p> </item> <tag><c>async_dirty</c></tag> <item> - <p>Invokes <c>mnesia:async_dirty(Fun, Args)</c>. - </p> + <p>Calls <c>mnesia:async_dirty(Fun, Args)</c>.</p> </item> <tag><c>sync_dirty</c></tag> <item> - <p>Invokes <c>mnesia:sync_dirty(Fun, Args)</c>. - </p> + <p>Calls <c>mnesia:sync_dirty(Fun, Args)</c>.</p> </item> <tag><c>ets</c></tag> <item> - <p>Invokes <c>mnesia:ets(Fun, Args)</c>. - </p> + <p>Calls <c>mnesia:ets(Fun, Args)</c>.</p> </item> </taglist> <p>This function (<c>mnesia:activity/4</c>) differs in an - important aspect from the <c>mnesia:transaction</c>, + important way from the functions <c>mnesia:transaction</c>, <c>mnesia:sync_transaction</c>, - <c>mnesia:async_dirty</c>, <c>mnesia:sync_dirty</c> and - <c>mnesia:ets</c> functions. The <c>AccessMod</c> argument - is the name of a callback module which implements the - <c>mnesia_access</c> behavior. - </p> - <p>Mnesia will forward calls to the following functions: - </p> + <c>mnesia:async_dirty</c>, <c>mnesia:sync_dirty</c>, and + <c>mnesia:ets</c>. Argument <c>AccessMod</c> + is the name of a callback module, which implements the + <c>mnesia_access</c> behavior.</p> + <p><c>Mnesia</c> forwards calls to the following functions:</p> <list type="bulleted"> - <item> - <p>mnesia:lock/2 (read_lock_table/1, write_lock_table/1)</p> + <item>mnesia:lock/2 (read_lock_table/1, write_lock_table/1) </item> - <item> - <p>mnesia:write/3 (write/1, s_write/1)</p> + <item>mnesia:write/3 (write/1, s_write/1) </item> - <item> - <p>mnesia:delete/3 (delete/1, s_delete/1)</p> + <item>mnesia:delete/3 (delete/1, s_delete/1) </item> - <item> - <p>mnesia:delete_object/3 (delete_object/1, s_delete_object/1)</p> + <item>mnesia:delete_object/3 (delete_object/1, s_delete_object/1) </item> - <item> - <p>mnesia:read/3 (read/1, wread/1)</p> + <item>mnesia:read/3 (read/1, wread/1) </item> - <item> - <p>mnesia:match_object/3 (match_object/1)</p> + <item>mnesia:match_object/3 (match_object/1) </item> - <item> - <p>mnesia:all_keys/1</p> + <item>mnesia:all_keys/1 </item> - <item> - <p>mnesia:first/1</p> + <item>mnesia:first/1 </item> - <item> - <p>mnesia:last/1</p> + <item>mnesia:last/1 </item> - <item> - <p>mnesia:prev/2</p> + <item>mnesia:prev/2 </item> - <item> - <p>mnesia:next/2</p> + <item>mnesia:next/2 </item> - <item> - <p>mnesia:index_match_object/4 (index_match_object/2)</p> + <item>mnesia:index_match_object/4 (index_match_object/2) </item> - <item> - <p>mnesia:index_read/3</p> + <item>mnesia:index_read/3 </item> - <item> - <p>mnesia:table_info/2</p> + <item>mnesia:table_info/2 </item> </list> - <p>to the corresponding: - </p> + <p>to the corresponding:</p> <list type="bulleted"> - <item> - <p>AccessMod:lock(ActivityId, Opaque, LockItem, LockKind)</p> + <item>AccessMod:lock(ActivityId, Opaque, LockItem, LockKind) </item> - <item> - <p>AccessMod:write(ActivityId, Opaque, Tab, Rec, LockKind)</p> + <item>AccessMod:write(ActivityId, Opaque, Tab, Rec, LockKind) </item> - <item> - <p>AccessMod:delete(ActivityId, Opaque, Tab, Key, LockKind)</p> + <item>AccessMod:delete(ActivityId, Opaque, Tab, Key, LockKind) </item> - <item> - <p>AccessMod:delete_object(ActivityId, Opaque, Tab, RecXS, LockKind)</p> + <item>AccessMod:delete_object(ActivityId, Opaque, Tab, RecXS, + LockKind) </item> - <item> - <p>AccessMod:read(ActivityId, Opaque, Tab, Key, LockKind)</p> + <item>AccessMod:read(ActivityId, Opaque, Tab, Key, LockKind) </item> - <item> - <p>AccessMod:match_object(ActivityId, Opaque, Tab, Pattern, LockKind)</p> + <item>AccessMod:match_object(ActivityId, Opaque, Tab, Pattern, + LockKind) </item> - <item> - <p>AccessMod:all_keys(ActivityId, Opaque, Tab, LockKind)</p> + <item>AccessMod:all_keys(ActivityId, Opaque, Tab, LockKind) </item> - <item> - <p>AccessMod:first(ActivityId, Opaque, Tab)</p> + <item>AccessMod:first(ActivityId, Opaque, Tab) </item> - <item> - <p>AccessMod:last(ActivityId, Opaque, Tab)</p> + <item>AccessMod:last(ActivityId, Opaque, Tab) </item> - <item> - <p>AccessMod:prev(ActivityId, Opaque, Tab, Key)</p> + <item>AccessMod:prev(ActivityId, Opaque, Tab, Key) </item> - <item> - <p>AccessMod:next(ActivityId, Opaque, Tab, Key)</p> + <item>AccessMod:next(ActivityId, Opaque, Tab, Key) </item> - <item> - <p>AccessMod:index_match_object(ActivityId, Opaque, Tab, Pattern, Attr, LockKind)</p> + <item>AccessMod:index_match_object(ActivityId, Opaque, Tab, + Pattern, Attr, LockKind) </item> - <item> - <p>AccessMod:index_read(ActivityId, Opaque, Tab, SecondaryKey, Attr, LockKind)</p> + <item>AccessMod:index_read(ActivityId, Opaque, Tab, + SecondaryKey, Attr, LockKind) </item> - <item> - <p>AccessMod:table_info(ActivityId, Opaque, Tab, InfoItem)</p> + <item>AccessMod:table_info(ActivityId, Opaque, Tab, InfoItem) </item> </list> - <p>where <c>ActivityId</c> is a record which represents the - identity of the enclosing Mnesia activity. The first field - (obtained with <c>element(1, ActivityId)</c> contains an - atom which may be interpreted as the type of the activity: - <c>'ets'</c>, <c>'async_dirty'</c>, <c>'sync_dirty'</c> or - <c>'tid'</c>. <c>'tid'</c> means that the activity is a + <p><c>ActivityId</c> is a record that represents the identity + of the enclosing <c>Mnesia</c> activity. The first field + (obtained with <c>element(1, ActivityId)</c>) contains an + atom, which can be interpreted as the activity type: + <c>ets</c>, <c>async_dirty</c>, <c>sync_dirty</c>, or + <c>tid</c>. <c>tid</c> means that the activity is a transaction. The structure of the rest of the identity - record is internal to Mnesia. - </p> - <p><c>Opaque</c> is an opaque data structure which is internal - to Mnesia.</p> + record is internal to <c>Mnesia</c>.</p> + <p><c>Opaque</c> is an opaque data structure that is internal + to <c>Mnesia</c>.</p> </desc> </func> <func> <name>add_table_copy(Tab, Node, Type) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Copy a table to a remote node.</fsummary> + <fsummary>Copies a table to a remote node.</fsummary> <desc> - <p>This function makes another copy of a table at the - node <c>Node</c>. The <c>Type</c> argument must be - either of the atoms <c>ram_copies</c>, <c>disc_copies</c>, - or + <marker id="add_table_copy"></marker> + <p>Makes another copy of a table at the node <c>Node</c>. + Argument <c>Type</c> must be either of the atoms + <c>ram_copies</c>, <c>disc_copies</c>, or <c>disc_only_copies</c>. For example, the following call - ensures that a disc replica of the <c>person</c> table also - exists at node <c>Node</c>.</p> + ensures that a disc replica of the <c>person</c> table also + exists at node <c>Node</c>:</p> <code type="none"> -mnesia:add_table_copy(person, Node, disc_copies) - </code> +mnesia:add_table_copy(person, Node, disc_copies)</code> <p>This function can also be used to add a replica of the table named <c>schema</c>.</p> </desc> </func> <func> <name>add_table_index(Tab, AttrName) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Create an index for a table. </fsummary> - <desc> - <p>Table indices can and should be used whenever the user - wants to frequently use some other field than the key field - to look up records. If this other field has an index - associated with it, these lookups can occur in constant time - and space. For example, if our application wishes to use - the age field of persons to efficiently find all person with - a specific age, it might be a good idea to have an index on - the age field. This can be accomplished with the following + <fsummary>Creates an index for a table.</fsummary> + <desc> + <marker id="add_table_index"></marker> + <p>Table indexes can be used whenever the user + wants to use frequently some other field than the key field + to look up records. If this other field has an associated + index, these lookups can occur in constant time + and space. For example, if your application wishes to use + field <c>age</c> to find efficiently all persons with + a specific age, it can be a good idea to have an index on + field <c>age</c>. This can be done with the following call:</p> <code type="none"> -mnesia:add_table_index(person, age) - </code> - <p>Indices do not come free, they occupy space which is - proportional to the size of the table. They also cause insertions - into the table to execute slightly slower. </p> +mnesia:add_table_index(person, age)</code> + <p>Indexes do not come for free. They occupy space that is + proportional to the table size, and they cause insertions + into the table to execute slightly slower.</p> </desc> </func> <func> <name>all_keys(Tab) -> KeyList | transaction abort</name> - <fsummary>Return all keys in a table.</fsummary> + <fsummary>Returns all keys in a table.</fsummary> <desc> - <p>This function returns a list of all keys in the table - named <c>Tab</c>. The semantics of this function is context - sensitive. See <c>mnesia:activity/4</c> for more information. In - transaction context it acquires a read lock on the entire + <marker id="all_keys"></marker> + <p>Returns a list of all keys in the table named <c>Tab</c>. + The semantics of this function is context-sensitive. + For more information, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a read lock on the entire table.</p> </desc> </func> <func> - <name>async_dirty(Fun, [, Args]) -> ResultOfFun | exit(Reason)</name> - <fsummary>Call the Fun in a context which is not protected by a transaction.</fsummary> + <name>async_dirty(Fun, [, Args]) -> ResultOfFun | exit(Reason)</name> + <fsummary>Calls the <c>Fun</c> in a context that is not protected by a transaction.</fsummary> <desc> - <p>Call the <c>Fun</c> in a context which is not protected - by a transaction. The Mnesia function calls performed in the - <c>Fun</c> are mapped to the corresponding dirty - functions. This still involves logging, replication and + <marker id="async_dirty"></marker> + <p>Calls the <c>Fun</c> in a context that is not protected by + a transaction. The <c>Mnesia</c> function calls performed in + the <c>Fun</c> are mapped to the corresponding dirty + functions. This still involves logging, replication, and subscriptions, but there is no locking, local transaction storage, or commit protocols involved. Checkpoint retainers - and indices are updated, but they will be updated dirty. As - for normal mnesia:dirty_* operations, the operations are - performed semi-asynchronously. See - <c>mnesia:activity/4</c> and the Mnesia User's Guide for - more details. - </p> - <p>It is possible to manipulate the Mnesia tables without + and indexes are updated, but they are updated dirty. As + for normal <c>mnesia:dirty_*</c> operations, the operations + are performed semi-asynchronously. For details, see + <c>mnesia:activity/4</c> and the User's Guide.</p> + <p>The <c>Mnesia</c> tables can be manipulated without using transactions. This has some serious disadvantages, but - is considerably faster since the transaction manager is not + is considerably faster, as the transaction manager is not involved and no locks are set. A dirty operation does, - however, guarantee a certain level of consistency and it is - not possible for the dirty operations to return garbled - records. All dirty operations provide location transparency - to the programmer and a program does not have to be aware of - the whereabouts of a certain table in order to function. - </p> - <p><em>Note:</em>It is more than 10 times more efficient to read records dirty - than within a transaction. - </p> - <p>Depending on the application, it may be a good idea to use + however, guarantee a certain level of consistency, and + the dirty operations cannot return garbled records. + All dirty operations provide location transparency + to the programmer, and a program does not have to be aware + of the whereabouts of a certain table to function.</p> + <p>Notice that it is more than ten times more efficient to + read records dirty than within a transaction.</p> + <p>Depending on the application, it can be a good idea to use the dirty functions for certain operations. Almost all - Mnesia functions which can be called within transactions - have a dirty equivalent which is much more - efficient. However, it must be noted that it is possible for - the database to be left in an inconsistent state if dirty - operations are used to update it. Dirty operations should - only be used for performance reasons when it is absolutely - necessary. </p> - <p><em>Note:</em> Calling (nesting) a <c>mnesia:[a]sync_dirty</c> - inside a transaction context will inherit the transaction semantics. - </p> + <c>Mnesia</c> functions that can be called within + transactions have a dirty equivalent, which is much more + efficient.</p> + <p>However, notice that there is a risk that the database can + be left in an inconsistent state if dirty operations are + used to update it. Dirty operations are only to be used + for performance reasons when it is absolutely necessary.</p> + <p>Notice that calling (nesting) <c>mnesia:[a]sync_dirty</c> + inside a transaction-context inherits the transaction + semantics.</p> </desc> </func> <func> <name>backup(Opaque [, BackupMod]) -> ok | {error,Reason}</name> - <fsummary>Back up all tables in the database.</fsummary> + <fsummary>Backs up all tables in the database.</fsummary> <desc> - <p>Activates a new checkpoint covering all Mnesia tables, - including the schema, with maximum degree of redundancy and - performs a backup using <c>backup_checkpoint/2/3</c>. The + <marker id="backup"></marker> + <p>Activates a new checkpoint covering all <c>Mnesia</c> tables, + including the schema, with maximum degree of redundancy, and + performs a backup using <c>backup_checkpoint/2/3</c>. The default value of the backup callback module <c>BackupMod</c> is obtained by <c>mnesia:system_info(backup_module)</c>.</p> </desc> </func> <func> - <name>backup_checkpoint(Name, Opaque [, BackupMod]) -> ok | {error,Reason}</name> - <fsummary>Back up all tables in a checkpoint.</fsummary> + <name>backup_checkpoint(Name, Opaque [, BackupMod]) -> ok | {error,Reason}</name> + <fsummary>Backs up all tables in a checkpoint.</fsummary> <desc> - <p>The tables are backed up to external media using the backup + <marker id="backup_checkpoint"></marker> + <p>The tables are backed up to external media using backup module <c>BackupMod</c>. Tables with the local contents property are backed up as they exist on the current - node. <c>BackupMod</c> is the default backup callback + node. <c>BackupMod</c> is the default backup callback module obtained by - <c>mnesia:system_info(backup_module)</c>. See the User's - Guide about the exact callback interface (the - <c>mnesia_backup behavior</c>).</p> + <c>mnesia:system_info(backup_module)</c>. For information + about the exact callback interface (the + <c>mnesia_backup behavior</c>), see the User's Guide.</p> </desc> </func> <func> <name>change_config(Config, Value) -> {error, Reason} | {ok, ReturnValue}</name> - <fsummary>Change a configuration parameter.</fsummary> + <fsummary>Changes a configuration parameter.</fsummary> <desc> - <p>The <c>Config</c> should be an atom of the following - configuration parameters: </p> + <marker id="change_config"></marker> + <p><c>Config</c> is to be an atom of the following + configuration parameters:</p> <taglist> <tag><c>extra_db_nodes</c></tag> <item> - <p><c>Value</c> is a list of nodes which Mnesia should try to connect to. - The <c>ReturnValue</c> will be those nodes in - <c>Value</c> that Mnesia are connected to. - <br></br> -Note: This function shall only be used to connect to newly started ram nodes - (N.D.R.S.N.) with an empty schema. If for example it is used after the network - have been partitioned it may lead to inconsistent tables. - <br></br> -Note: Mnesia may be connected to other nodes than those - returned in <c>ReturnValue</c>.</p> + <p><c>Value</c> is a list of nodes that <c>Mnesia</c> + is to try to connect to. <c>ReturnValue</c> is those + nodes in <c>Value</c> that <c>Mnesia</c> is connected + to.</p> + <p>Notice that this function must only be used to connect + to newly started RAM nodes (N.D.R.S.N.) with an empty + schema. If, for example, this function is used after + the network has been partitioned, it can lead to + inconsistent tables.</p> + <p>Notice that <c>Mnesia</c> can be connected to other + nodes than those returned in <c>ReturnValue</c>.</p> </item> <tag><c>dc_dump_limit</c></tag> <item> - <p><c>Value</c> is a number. See description in - <c>Configuration Parameters</c> below. - The <c>ReturnValue</c> is the new value. Note this configuration parameter - is not persistent, it will be lost when mnesia stopped.</p> + <p><c>Value</c> is a number. See the description in + <seealso marker="#configuration_parameters">Section + Configuration Parameters</seealso>. <c>ReturnValue</c> + is the new value. Notice that this configuration + parameter is not persistent. It is lost when + <c>Mnesia</c> has stopped.</p> </item> </taglist> </desc> </func> <func> <name>change_table_access_mode(Tab, AccessMode) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Change the access mode for the table.</fsummary> - <desc> - <p>The <c>AcccessMode</c> is by default the atom - <c>read_write</c> but it may also be set to the atom - <c>read_only</c>. If the <c>AccessMode</c> is set to - <c>read_only</c>, it means that it is not possible to perform - updates to the table. At startup Mnesia always loads + <fsummary>Changes the access mode for the table.</fsummary> + <desc> + <marker id="change_table_access_mode"></marker> + <p><c>AcccessMode</c> is by default the atom + <c>read_write</c> but it can also be set to the atom + <c>read_only</c>. If <c>AccessMode</c> is set to + <c>read_only</c>, updates to the table cannot be + performed. At startup, <c>Mnesia</c> always loads <c>read_only</c> tables locally regardless of when and if - Mnesia was terminated on other nodes.</p> + <c>Mnesia</c> is terminated on other nodes.</p> </desc> </func> <func> <name>change_table_copy_type(Tab, Node, To) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Change the storage type of a table.</fsummary> + <fsummary>Changes the storage type of a table.</fsummary> <desc> + <marker id="change_table_copy_type"></marker> <p>For example:</p> <code type="none"> -mnesia:change_table_copy_type(person, node(), disc_copies) - </code> - <p>Transforms our <c>person</c> table from a RAM table into - a disc based table at <c>Node</c>. - </p> - <p>This function can also be used to change the storage type of - the table named <c>schema</c>. The schema table can only - have <c>ram_copies</c> or <c>disc_copies</c> as the storage type. If the - storage type of the schema is <c>ram_copies</c>, no other table - can be disc resident on that node.</p> +mnesia:change_table_copy_type(person, node(), disc_copies)</code> + <p>Transforms the <c>person</c> table from a RAM table into + a disc-based table at <c>Node</c>.</p> + <p>This function can also be used to change the storage type + of the table named <c>schema</c>. The schema table can only + have <c>ram_copies</c> or <c>disc_copies</c> as the storage + type. If the storage type of the schema is <c>ram_copies</c>, + no other table can be disc-resident on that node.</p> </desc> </func> <func> <name>change_table_load_order(Tab, LoadOrder) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Change the load order priority for the table.</fsummary> + <fsummary>Changes the load order priority for the table.</fsummary> <desc> + <marker id="change_table_load_order"></marker> <p>The <c>LoadOrder</c> priority is by default <c>0</c> (zero) - but may be set to any integer. The tables with the highest - <c>LoadOrder</c> priority will be loaded first at startup.</p> + but can be set to any integer. The tables with the highest + <c>LoadOrder</c> priority are loaded first at startup.</p> </desc> </func> <func> <name>change_table_majority(Tab, Majority) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Change the majority check setting for the table.</fsummary> + <fsummary>Changes the majority check setting for the table.</fsummary> <desc> - <p><c>Majority</c> must be a boolean; the default is <c>false</c>. - When <c>true</c>, a majority of the table's replicas must be available - for an update to succeed. When used on fragmented tables, <c>Tab</c> - must be the name base table. Directly changing the majority setting on - individual fragments is not allowed.</p> + <p><c>Majority</c> must be a boolean. Default is <c>false</c>. + When <c>true</c>, a majority of the table replicas must be + available for an update to succeed. When used on fragmented + tables, <c>Tab</c> must be the base table name. Directly + changing the majority setting on individual fragments is + not allowed.</p> </desc> </func> <func> <name>clear_table(Tab) -> {aborted, R} | {atomic, ok}</name> <fsummary>Deletes all entries in a table.</fsummary> <desc> + <marker id="clear_table"></marker> <p>Deletes all entries in the table <c>Tab</c>.</p> </desc> </func> <func> <name>create_schema(DiscNodes) -> ok | {error,Reason}</name> - <fsummary>Create a brand new schema on the specified nodes.</fsummary> + <fsummary>Creates a new schema on the specified nodes.</fsummary> <desc> + <marker id="create_schema"></marker> <p>Creates a new database on disc. Various files are - created in the local Mnesia directory of each node. Note - that the directory must be unique for each node. Two nodes - may never share the same directory. If possible, use a local - disc device in order to improve performance.</p> + created in the local <c>Mnesia</c> directory of each node. + Notice that the directory must be unique for each node. + Two nodes must never share the same directory. If possible, + use a local disc device to improve performance.</p> <p><c>mnesia:create_schema/1</c> fails if any of the Erlang nodes given as <c>DiscNodes</c> are not alive, if - Mnesia is running on anyone of the nodes, or if anyone of - the nodes already has a schema. Use + <c>Mnesia</c> is running on any of the nodes, or if any + of the nodes already have a schema. Use <c>mnesia:delete_schema/1</c> to get rid of old faulty - schemas. - </p> - <p><em>Note:</em> Only nodes with disc should be - included in <c>DiscNodes</c>. Disc-less nodes, that is nodes - where all tables including the schema only resides in RAM, - may not be included.</p> + schemas.</p> + <p>Notice that only nodes with disc are to be included in + <c>DiscNodes</c>. Disc-less nodes, that is, nodes where + all tables including the schema only resides in RAM, + must not be included.</p> </desc> </func> <func> <name>create_table(Name, TabDef) -> {atomic, ok} | {aborted, Reason}</name> - <fsummary>Create a Mnesia table called <c>Name</c>with properties as described by the argument <c>TabDef</c>.</fsummary> + <fsummary>Creates a <c>Mnesia</c> table called <c>Name</c>with properties as described by argument <c>TabDef</c>.</fsummary> <desc> - <p>This function creates a Mnesia table called <c>Name</c> - according to the - argument <c>TabDef</c>. This list must be a list of - <c>{Item, Value}</c> tuples, where the following values are - allowed:</p> + <marker id="create_table"></marker> + <p>Creates a <c>Mnesia</c> table called + <c>Name</c> according to argument <c>TabDef</c>. This + list must be a list of <c>{Item, Value}</c> tuples, + where the following values are allowed:</p> <list type="bulleted"> <item> - <p><c>{access_mode, Atom}</c>. The access mode is by - default the atom <c>read_write</c> but it may also be - set to the atom <c>read_only</c>. If the - <c>AccessMode</c> is set to <c>read_only</c>, it means - that it is not possible to perform updates to the table. - </p> - <p>At startup Mnesia always loads <c>read_only</c> tables - locally regardless of when and if Mnesia was terminated - on other nodes. This argument returns the access mode of - the table. The access mode may either be read_only or - read_write. - </p> - </item> - <item> - <p><c>{attributes, AtomList}</c> a list of the + <p><c>{access_mode, Atom}</c>. The access mode is by + default the atom <c>read_write</c> but it can also be + set to the atom <c>read_only</c>. If <c>AccessMode</c> + is set to <c>read_only</c>, updates to the table + cannot be performed.</p> + <p>At startup, <c>Mnesia</c> always loads <c>read_only</c> + table locally regardless of when and if <c>Mnesia</c> is + terminated on other nodes. This argument returns the + access mode of the table. The access mode can be + <c>read_only</c> or <c>read_write</c>.</p> + </item> + <item> + <p><c>{attributes, AtomList}</c> is a list of the attribute names for the records that are supposed to - populate the table. The default value is <c>[key, val]</c>. The table must have at least one extra - attribute in addition to the key. - </p> - <p>When accessing single attributes in a record, it is not - necessary, or even recommended, to hard code any - attribute names as atoms. Use the construct - <c>record_info(fields, RecordName)</c> instead. It can be - used for records of type <c>RecordName</c></p> + populate the table. Default is <c>[key, val]</c>. + The table must at least have one extra attribute in + addition to the key.</p> + <p>When accessing single attributes in a record, it is + not necessary, or even recommended, to hard code any + attribute names as atoms. Use construct + <c>record_info(fields, RecordName)</c> instead. It can + be used for records of type <c>RecordName</c>.</p> </item> <item> <p><c>{disc_copies, Nodelist}</c>, where <c>Nodelist</c> is a list of the nodes where this table - is supposed to have disc copies. If a table replica is + is supposed to have disc copies. If a table replica is of type <c>disc_copies</c>, all write operations on this - particular replica of the table are written to disc as - well as to the RAM copy of the table. - </p> - <p>It is possible - to have a replicated table of type <c>disc_copies</c> - on one node, and another type on another node. The - default value is <c>[]</c></p> + particular replica of the table are written to disc and + to the RAM copy of the table.</p> + <p>It is possible to have a replicated table of type + <c>disc_copies</c> on one node and another type on + another node. Default is <c>[]</c>.</p> </item> <item> <p><c>{disc_only_copies, Nodelist}</c>, where <c>Nodelist</c> is a list of the nodes where this table - is supposed to have <c>disc_only_copies</c>. A disc only - table replica is kept on disc only and unlike the other - replica types, the contents of the replica will not + is supposed to have <c>disc_only_copies</c>. A disc only + table replica is kept on disc only and unlike the other + replica types, the contents of the replica do not reside in RAM. These replicas are considerably slower - than replicas held in RAM. - </p> + than replicas held in RAM.</p> </item> <item> <p><c>{index, Intlist}</c>, where <c>Intlist</c> is a list of attribute names (atoms) or - record fields for which Mnesia shall build and maintain - an extra index table. The <c>qlc</c> query compiler may - or may not utilize any additional indices while - processing queries on a table. - </p> + record fields for which <c>Mnesia</c> is to build and + maintain an extra index table. The <c>qlc</c> query + compiler <em>may</em> be able to optimize queries + if there are indexes available.</p> </item> <item> <p><c>{load_order, Integer}</c>. The load order - priority is by default <c>0</c> (zero) but may be set to - any integer. The tables with the highest load order - priority will be loaded first at startup. - </p> - </item> - <item> - <p><c>{majority, Flag}</c>, where <c>Flag</c> must be a boolean. - If <c>true</c>, any (non-dirty) update to the table will abort unless - a majority of the table's replicas are available for the commit. - When used on a fragmented table, all fragments will be given - the same majority setting. - </p> - </item> + priority is by default <c>0</c> (zero) but can be set + to any integer. The tables with the highest load order + priority are loaded first at startup.</p> + </item> + <item> + <p><c>{majority, Flag}</c>, where <c>Flag</c> must be a + boolean. If <c>true</c>, any (non-dirty) update to the + table is aborted, unless a majority of the table + replicas are available for the commit. When used on a + fragmented table, all fragments are given the same + the same majority setting.</p> + </item> <item> <p><c>{ram_copies, Nodelist}</c>, where <c>Nodelist</c> is a list of the nodes where this table - is supposed to have RAM copies. A table replica of type - <c>ram_copies</c> is obviously not written to disc on a - per transaction basis. It is possible to dump - <c>ram_copies</c> replicas to disc with the function - <c>mnesia:dump_tables(Tabs)</c>. The default value for - this attribute is <c>[node()]</c>. - </p> + is supposed to have RAM copies. A table replica of type + <c>ram_copies</c> is not written to disc on a per + transaction basis. <c>ram_copies</c> replicas can be + dumped to disc with the function + <c>mnesia:dump_tables(Tabs)</c>. Default value for + this attribute is <c>[node()]</c>.</p> </item> <item> <p><c>{record_name, Name}</c>, where <c>Name</c> must - be an atom. All records, stored in the table, must have + be an atom. All records stored in the table must have this name as the first element. It defaults to the same - name as the name of the table. - </p> - </item> - <item> - <p><c>{snmp, SnmpStruct}</c>. See - <c>mnesia:snmp_open_table/2</c> for a description of - <c>SnmpStruct</c>. If this attribute is present in the - <c>ArgList</c> to <c>mnesia:create_table/2</c>, the - table is immediately accessible by means of the Simple - Network Management Protocol (SNMP). This means that - applications which use SNMP to manipulate and control - the system can be designed easily, since Mnesia provides - a direct mapping between the logical tables that make up - an SNMP control application and the physical data which - makes up a Mnesia table. - </p> - </item> - <item> - <p><c>{storage_properties, [{Backend, Properties}]</c>. - Forwards additional properties to the backend storage. - <c>Backend</c> can currently be <c>ets</c> or <c>dets</c> and - <c>Properties</c> is a list of options sent to the backend storage - during table creation. <c>Properties</c> may not contain properties - already used by mnesia such as <c>type</c> or <c>named_table</c>. - </p> - <p>For example:</p> - <code type="none"> + name as the table name.</p> + </item> + <item> + <p><c>{snmp, SnmpStruct}</c>. For a description of + <c>SnmpStruct</c>, see <c>mnesia:snmp_open_table/2</c>. + If this attribute is present in <c>ArgList</c> to + <c>mnesia:create_table/2</c>, the table is immediately + accessible by SNMP. Therefore applications that use + SNMP to manipulate and control the system can be + designed easily, since <c>Mnesia</c> provides a + direct mapping between the logical tables that make up + an SNMP control application and the physical data that + makes up a <c>Mnesia</c> table.</p> + </item> + <item> + <p><c>{storage_properties, [{Backend, Properties}]</c> + forwards more properties to the back end storage. + <c>Backend</c> can currently be <c>ets</c> or <c>dets</c>. + <c>Properties</c> is a list of options sent to the + back end storage during table creation. <c>Properties</c> + cannot contain properties already used by <c>Mnesia</c>, + such as <c>type</c> or <c>named_table</c>.</p> + <p>For example:</p> + <code type="none"> mnesia:create_table(table, [{ram_copies, [node()]}, {disc_only_copies, nodes()}, - {storage_properties, - [{ets, [compressed]}, {dets, [{auto_save, 5000}]} ]}]) - </code> + {storage_properties, + [{ets, [compressed]}, {dets, [{auto_save, 5000}]} ]}])</code> </item> <item> <p><c>{type, Type}</c>, where <c>Type</c> must be - either of the atoms <c>set</c>, <c>ordered_set</c> or - <c>bag</c>. The default value is <c>set</c>. In a - <c>set</c> all records have unique keys and in a - <c>bag</c> several records may have the same key, but + either of the atoms <c>set</c>, <c>ordered_set</c>, or + <c>bag</c>. Default is <c>set</c>. In a + <c>set</c>, all records have unique keys. In a + <c>bag</c>, several records can have the same key, but the record content is unique. If a non-unique record is - stored the old, conflicting record(s) will simply be - overwritten. Note: currently 'ordered_set' - is not supported for 'disc_only_copies'. - </p> + stored, the old conflicting records are overwritten.</p> + <p>Notice that currently <c>ordered_set</c> is not + supported for <c>disc_only_copies</c>.</p> </item> <item> - <p><c>{local_content, Bool}</c>, where <c>Bool</c> must be - either <c>true</c> or <c>false</c>. The default value is <c>false</c>.</p> + <p><c>{local_content, Bool}</c>, where <c>Bool</c> is + <c>true</c> or <c>false</c>. Default is <c>false</c>.</p> </item> </list> - <p>For example, the following call creates the <c>person</c> table - previously defined and replicates it on 2 nodes: - </p> + <p>For example, the following call creates the <c>person</c> + table (defined earlier) and replicates it on two nodes:</p> <code type="none"> -mnesia:create_table(person, +mnesia:create_table(person, [{ram_copies, [N1, N2]}, - {attributes, record_info(fields,person)}]). - </code> - <p>If it was required that Mnesia build and maintain an extra index - table on the <c>address</c> attribute of all the <c>person</c> - records that are inserted in the table, the following code would be issued: - </p> + {attributes, record_info(fields, person)}]).</code> + <p>If it is required that <c>Mnesia</c> must build and + maintain an extra index table on attribute <c>address</c> + of all the <c>person</c> records that are inserted in the + table, the following code would be issued:</p> <code type="none"> mnesia:create_table(person, [{ram_copies, [N1, N2]}, {index, [address]}, - {attributes, record_info(fields,person)}]). - </code> - <p>The specification of <c>index</c> and <c>attributes</c> may be - hard coded as <c>{index, [4]}</c> and - <c>{attributes, [name, age, address, salary, children]}</c> - respectively. - </p> + {attributes, record_info(fields, person)}]). + </code> + <p>The specification of <c>index</c> and <c>attributes</c> + can be hard-coded as <c>{index, [2]}</c> and + <c>{attributes, [name, age, address, salary, children]}</c>, + respectively.</p> <p><c>mnesia:create_table/2</c> writes records into the - <c>schema</c> table. This function, as well as all other + table <c>schema</c>. This function, and all other schema manipulation functions, are implemented with the normal transaction management system. This guarantees that schema updates are performed on all nodes in an atomic @@ -877,163 +799,169 @@ mnesia:create_table(person, </func> <func> <name>deactivate_checkpoint(Name) -> ok | {error, Reason}</name> - <fsummary>Deactivate a checkpoint.</fsummary> + <fsummary>Deactivates a checkpoint.</fsummary> <desc> + <marker id="deactivate_checkpoint"></marker> <p>The checkpoint is automatically deactivated when some of - the tables involved have no retainer attached to them. This may - happen when nodes go down or when a replica is deleted. - Checkpoints will also be deactivated with this function. + the tables involved have no retainer attached to them. This + can occur when nodes go down or when a replica is deleted. + Checkpoints are also deactivated with this function. <c>Name</c> is the name of an active checkpoint.</p> </desc> </func> <func> <name>del_table_copy(Tab, Node) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Delete the replica of table <c>Tab</c>at node <c>Node</c>.</fsummary> + <fsummary>Deletes the replica of table <c>Tab</c> at node <c>Node</c>.</fsummary> <desc> + <marker id="del_table_copy"></marker> <p>Deletes the replica of table <c>Tab</c> at node <c>Node</c>. When the last replica is deleted with this - function, the table disappears entirely. - </p> - <p>This function may also be used to delete a replica of - the table named <c>schema</c>. Then the mnesia node will be removed. - Note: Mnesia must be stopped on the node first.</p> + function, the table disappears entirely.</p> + <p>This function can also be used to delete a replica of + the table named <c>schema</c>. The <c>Mnesia</c> node is + then removed. Notice that <c>Mnesia</c> must be + stopped on the node first.</p> </desc> </func> <func> <name>del_table_index(Tab, AttrName) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Delete an index in a table. </fsummary> + <fsummary>Deletes an index in a table.</fsummary> <desc> - <p>This function deletes the index on attribute with name + <marker id="del_table_index"></marker> + <p>Deletes the index on attribute with name <c>AttrName</c> in a table.</p> </desc> </func> <func> - <name>delete({Tab, Key}) -> transaction abort | ok </name> - <fsummary>Delete all records in table <c>Tab</c>with the key <c>Key</c>.</fsummary> + <name>delete({Tab, Key}) -> transaction abort | ok</name> + <fsummary>Deletes all records in table <c>Tab</c> with the key <c>Key</c>.</fsummary> <desc> - <p>Invokes <c>mnesia:delete(Tab, Key, write)</c></p> + <marker id="delete_2"></marker> + <p>Calls <c>mnesia:delete(Tab, Key, write)</c>.</p> </desc> </func> <func> - <name>delete(Tab, Key, LockKind) -> transaction abort | ok </name> - <fsummary>Delete all records in table <c>Tab</c>with the key <c>Key</c>.</fsummary> + <name>delete(Tab, Key, LockKind) -> transaction abort | ok</name> + <fsummary>Deletes all records in table <c>Tab</c>with the key <c>Key</c>.</fsummary> <desc> + <marker id="delete_3"></marker> <p>Deletes all records in table <c>Tab</c> with the key - <c>Key</c>. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type <c>LockKind</c> in the - record. Currently the lock types <c>write</c> and + <c>Key</c>.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type + <c>LockKind</c> in the record. + Currently, the lock types <c>write</c> and <c>sticky_write</c> are supported.</p> </desc> </func> <func> - <name>delete_object(Record) -> transaction abort | ok </name> - <fsummary>Delete a record</fsummary> + <name>delete_object(Record) -> transaction abort | ok</name> + <fsummary>Delete a record.</fsummary> <desc> - <p>Invokes <c>mnesia:delete_object(Tab, Record, write)</c> where + <marker id="delete_object_1"></marker> + <p>Calls <c>mnesia:delete_object(Tab, Record, write)</c>, where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> <func> - <name>delete_object(Tab, Record, LockKind) -> transaction abort | ok </name> - <fsummary>Delete a record</fsummary> + <name>delete_object(Tab, Record, LockKind) -> transaction abort | ok</name> + <fsummary>Deletes a record.</fsummary> <desc> - <p>If a table is of type <c>bag</c>, we may sometimes - want to delete only some of the records with a certain - key. This can be done with the <c>delete_object/3</c> - function. A complete record must be supplied to this - function. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type <c>LockKind</c> on the - record. Currently the lock types <c>write</c> and + <marker id="delete_object_3"></marker> + <p>If a table is of type <c>bag</c>, it can sometimes be + needed to delete only some of the records with a certain + key. This can be done with the function <c>delete_object/3</c>. + A complete record must be supplied to this function.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type + <c>LockKind</c> on the record. + Currently, the lock types <c>write</c> and <c>sticky_write</c> are supported.</p> </desc> </func> <func> <name>delete_schema(DiscNodes) -> ok | {error,Reason}</name> - <fsummary>Delete the schema on the given nodes</fsummary> + <fsummary>Deletes the schema on the given nodes.</fsummary> <desc> + <marker id="delete_schema"></marker> <p>Deletes a database created with <c>mnesia:create_schema/1</c>. <c>mnesia:delete_schema/1</c> fails if any of the Erlang - nodes given as <c>DiscNodes</c> is not alive, or if Mnesia - is running on any of the nodes. - </p> - <p>After the database has been deleted, it may still be - possible to start Mnesia as a disc-less node. This depends on - how the configuration parameter <c>schema_location</c> is set. - </p> + nodes given as <c>DiscNodes</c> are not alive, or if + <c>Mnesia</c> is running on any of the nodes.</p> + <p>After the database is deleted, it can still be possible + to start <c>Mnesia</c> as a disc-less node. This depends + on how configuration parameter <c>schema_location</c> is + set.</p> <warning> - <p>This function must be used with extreme - caution since it makes existing persistent data - obsolete. Think twice before using it. </p> + <p>Use this function with extreme caution, as it makes + existing persistent data obsolete. Think twice before + using it.</p> </warning> </desc> </func> <func> - <name>delete_table(Tab) -> {aborted, Reason} | {atomic, ok} </name> - <fsummary>Delete permanently all replicas of table <c>Tab</c>.</fsummary> + <name>delete_table(Tab) -> {aborted, Reason} | {atomic, ok}</name> + <fsummary>Deletes permanently all replicas of table <c>Tab</c>.</fsummary> <desc> + <marker id="delete_table"></marker> <p>Permanently deletes all replicas of table <c>Tab</c>.</p> </desc> </func> <func> - <name>dirty_all_keys(Tab) -> KeyList | exit({aborted, Reason}).</name> + <name>dirty_all_keys(Tab) -> KeyList | exit({aborted, Reason})</name> <fsummary>Dirty search for all record keys in table.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:all_keys/1</c> function.</p> + <marker id="delete_all_keys"></marker> + <p>Dirty equivalent of the function <c>mnesia:all_keys/1</c>.</p> </desc> </func> <func> - <name>dirty_delete({Tab, Key}) -> ok | exit({aborted, Reason}) </name> + <name>dirty_delete({Tab, Key}) -> ok | exit({aborted, Reason})</name> <fsummary>Dirty delete of a record.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_delete(Tab, Key)</c>.</p> + <marker id="dirty_delete"></marker> + <p>Calls <c>mnesia:dirty_delete(Tab, Key)</c>.</p> </desc> </func> <func> - <name>dirty_delete(Tab, Key) -> ok | exit({aborted, Reason}) </name> - <fsummary>Dirty delete of a record. </fsummary> + <name>dirty_delete(Tab, Key) -> ok | exit({aborted, Reason})</name> + <fsummary>Dirty delete of a record.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:delete/3</c> function.</p> + <p>Dirty equivalent of the function <c>mnesia:delete/3</c>.</p> </desc> </func> <func> - <name>dirty_delete_object(Record) </name> + <name>dirty_delete_object(Record)</name> <fsummary>Dirty delete of a record.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_delete_object(Tab, Record)</c> + <marker id="dirty_delete_object_1"></marker> + <p>Calls <c>mnesia:dirty_delete_object(Tab, Record)</c>, where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> <func> - <name>dirty_delete_object(Tab, Record) </name> - <fsummary>Dirty delete of a record. </fsummary> + <name>dirty_delete_object(Tab, Record)</name> + <fsummary>Dirty delete of a record.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:delete_object/3</c> function.</p> + <p>Dirty equivalent of the function <c>mnesia:delete_object/3</c>.</p> </desc> </func> <func> - <name>dirty_first(Tab) -> Key | exit({aborted, Reason}) </name> - <fsummary>Return the key for the first record in a table.</fsummary> + <name>dirty_first(Tab) -> Key | exit({aborted, Reason})</name> + <fsummary>Returns the key for the first record in a table.</fsummary> <desc> + <marker id="dirty_first"></marker> <p>Records in <c>set</c> or <c>bag</c> tables are not ordered. - However, there - is an ordering of the records which is not known - to the user. Accordingly, it is possible to traverse a table by means - of this function in conjunction with the <c>mnesia:dirty_next/2</c> - function. + However, there is an ordering of the records that is unknown + to the user. Therefore, a table can be traversed by this + function with the function <c>mnesia:dirty_next/2</c>. </p> - <p>If there are no records at all in the table, this function - returns the atom <c>'$end_of_table'</c>. For this reason, it - is highly undesirable, but not disallowed, to use this atom + <p>If there are no records in the table, this function + returns the atom <c>'$end_of_table'</c>. It is therefore + highly undesirable, but not disallowed, to use this atom as the key for any user records.</p> </desc> </func> @@ -1041,74 +969,82 @@ mnesia:create_table(person, <name>dirty_index_match_object(Pattern, Pos)</name> <fsummary>Dirty pattern match using index.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_index_match_object(Tab, Pattern, Pos)</c> where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> + <marker id="dirty_index_match_object_2"></marker> + <p>Starts + <c>mnesia:dirty_index_match_object(Tab, Pattern, Pos)</c>, + where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> </desc> </func> <func> <name>dirty_index_match_object(Tab, Pattern, Pos)</name> <fsummary>Dirty pattern match using index.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:index_match_object/4</c> function.</p> + <p>Dirty equivalent of the function + <c>mnesia:index_match_object/4</c>.</p> </desc> </func> <func> <name>dirty_index_read(Tab, SecondaryKey, Pos)</name> <fsummary>Dirty read using index.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:index_read/3</c> function.</p> + <marker id="dirty_index_read"></marker> + <p>Dirty equivalent of the function + <c>mnesia:index_read/3</c>.</p> </desc> </func> <func> - <name>dirty_last(Tab) -> Key | exit({aborted, Reason}) </name> - <fsummary>Return the key for the last record in a table.</fsummary> + <name>dirty_last(Tab) -> Key | exit({aborted, Reason})</name> + <fsummary>Returns the key for the last record in a table.</fsummary> <desc> - <p>This function works exactly like - <c>mnesia:dirty_first/1</c> but returns the last object in - Erlang term order for the <c>ordered_set</c> table type. For - all other table types, <c>mnesia:dirty_first/1</c> and + <marker id="dirty_last"></marker> + <p>Works exactly like <c>mnesia:dirty_first/1</c> but returns + the last object in Erlang term order for the <c>ordered_set</c> + table type. For all other table types, + <c>mnesia:dirty_first/1</c> and <c>mnesia:dirty_last/1</c> are synonyms.</p> </desc> </func> <func> - <name>dirty_match_object(Pattern) -> RecordList | exit({aborted, Reason}).</name> + <name>dirty_match_object(Pattern) -> RecordList | exit({aborted, Reason})</name> <fsummary>Dirty pattern match pattern.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_match_object(Tab, Pattern)</c> + <marker id="dirty_match_object_1"></marker> + <p>Calls <c>mnesia:dirty_match_object(Tab, Pattern)</c>, where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> </desc> </func> <func> - <name>dirty_match_object(Tab, Pattern) -> RecordList | exit({aborted, Reason}).</name> + <name>dirty_match_object(Tab, Pattern) -> RecordList | exit({aborted, Reason})</name> <fsummary>Dirty pattern match pattern.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:match_object/3</c> function.</p> + <p>Dirty equivalent of the function + <c>mnesia:match_object/3</c>.</p> </desc> </func> <func> - <name>dirty_next(Tab, Key) -> Key | exit({aborted, Reason}) </name> - <fsummary>Return the next key in a table. </fsummary> + <name>dirty_next(Tab, Key) -> Key | exit({aborted, Reason})</name> + <fsummary>Return the next key in a table.</fsummary> <desc> - <p>This function makes it possible to traverse a table - and perform operations on all records in the table. When - the end of the table is reached, the special key + <marker id="dirty_next"></marker> + <p>Traverses a table and + performs operations on all records in the table. + When the end of the table is reached, the special key <c>'$end_of_table'</c> is returned. Otherwise, the function - returns a key which can be used to read the actual record.The + returns a key that can be used to read the actual record. The behavior is undefined if another Erlang process performs write operations on the table while it is being traversed with the - <c>mnesia:dirty_next/2</c> function.</p> + function <c>mnesia:dirty_next/2</c>.</p> </desc> </func> <func> - <name>dirty_prev(Tab, Key) -> Key | exit({aborted, Reason}) </name> - <fsummary>Return the previous key in a table. </fsummary> + <name>dirty_prev(Tab, Key) -> Key | exit({aborted, Reason})</name> + <fsummary>Returns the previous key in a table.</fsummary> <desc> - <p>This function works exactly like - <c>mnesia:dirty_next/2</c> but returns the previous object in - Erlang term order for the ordered_set table type. For - all other table types, <c>mnesia:dirty_next/2</c> and + <marker id="dirty_prev"></marker> + <p>Works exactly like <c>mnesia:dirty_next/2</c> but returns + the previous object in Erlang term order for the + <c>ordered_set</c> table type. For all other table types, + <c>mnesia:dirty_next/2</c> and <c>mnesia:dirty_prev/2</c> are synonyms.</p> </desc> </func> @@ -1116,33 +1052,34 @@ mnesia:create_table(person, <name>dirty_read({Tab, Key}) -> ValueList | exit({aborted, Reason}</name> <fsummary>Dirty read of records.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_read(Tab, Key)</c>.</p> + <marker id="dirty_read"></marker> + <p>Calls <c>mnesia:dirty_read(Tab, Key)</c>.</p> </desc> </func> <func> <name>dirty_read(Tab, Key) -> ValueList | exit({aborted, Reason}</name> <fsummary>Dirty read of records.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:read/3</c> function.</p> + <p>Dirty equivalent of the function <c>mnesia:read/3</c>.</p> </desc> </func> <func> <name>dirty_select(Tab, MatchSpec) -> ValueList | exit({aborted, Reason}</name> - <fsummary>Dirty match the objects in <c>Tab</c>against <c>MatchSpec</c>.</fsummary> + <fsummary>Dirty matches the objects in <c>Tab</c> against <c>MatchSpec</c>.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:select/2</c> function.</p> + <marker id="dirty_select"></marker> + <p>Dirty equivalent of the function <c>mnesia:select/2</c>.</p> </desc> </func> <func> <name>dirty_slot(Tab, Slot) -> RecordList | exit({aborted, Reason})</name> - <fsummary>Return the list of records that are associated with Slot in a table.</fsummary> + <fsummary>Returns the list of records that are associated with <c>Slot</c> in a table.</fsummary> <desc> - <p>This function can be used to traverse a table in a - manner similar to the <c>mnesia:dirty_next/2</c> function. - A table has a number of slots which range from 0 (zero) to some - unknown upper bound. The function + <marker id="dirty_slot"></marker> + <p>Traverses a table in a + manner similar to the function <c>mnesia:dirty_next/2</c>. + A table has a number of slots that range from 0 (zero) to + an unknown upper bound. The function <c>mnesia:dirty_slot/2</c> returns the special atom <c>'$end_of_table'</c> when the end of the table is reached. The behavior of this function is undefined if a write @@ -1154,40 +1091,45 @@ mnesia:create_table(person, <name>dirty_update_counter({Tab, Key}, Incr) -> NewVal | exit({aborted, Reason})</name> <fsummary>Dirty update of a counter record.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_update_counter(Tab, Key, Incr)</c>.</p> + <marker id="dirty_update_counter"></marker> + <p>Calls <c>mnesia:dirty_update_counter(Tab, Key, Incr)</c>.</p> </desc> </func> <func> <name>dirty_update_counter(Tab, Key, Incr) -> NewVal | exit({aborted, Reason})</name> <fsummary>Dirty update of a counter record.</fsummary> <desc> - <p>There are no special counter records in Mnesia. However, + <p><c>Mnesia</c> has no special counter records. However, records of the form <c>{Tab, Key, Integer}</c> can be used - as (possibly disc resident) counters, when <c>Tab</c> is a - <c>set</c>. This function updates a counter with a - positive or negative number. However, counters can never become less + as (possibly disc-resident) counters when <c>Tab</c> is a + <c>set</c>. This function updates a counter with a positive + or negative number. However, counters can never become less than zero. There are two significant differences between this function and the action of first reading the record, performing the arithmetics, and then writing the record:</p> <list type="bulleted"> - <item>It is much more efficient</item> - <item><c>mnesia:dirty_update_counter/3</c> is - performed as an atomic operation despite the fact that it is not - protected by a transaction.</item> + <item>It is much more efficient. + </item> + <item><c>mnesia:dirty_update_counter/3</c> is performed + as an atomic operation although it is not protected + by a transaction. + </item> </list> <p>If two processes perform <c>mnesia:dirty_update_counter/3</c> - simultaneously, both updates will take effect without the + simultaneously, both updates take effect without the risk of losing one of the updates. The new value <c>NewVal</c> of the counter is returned.</p> - <p>If <c>Key</c> don't exits, a new record is created with the value - <c>Incr</c> if it is larger than 0, otherwise it is set to 0.</p> + <p>If <c>Key</c> do not exists, a new record is created with + value <c>Incr</c> if it is larger than 0, otherwise it is + set to 0.</p> </desc> </func> <func> <name>dirty_write(Record) -> ok | exit({aborted, Reason})</name> <fsummary>Dirty write of a record.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_write(Tab, Record)</c> + <marker id="dirty_write_1"></marker> + <p>Calls <c>mnesia:dirty_write(Tab, Record)</c>, where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> @@ -1195,623 +1137,607 @@ mnesia:create_table(person, <name>dirty_write(Tab, Record) -> ok | exit({aborted, Reason})</name> <fsummary>Dirty write of a record.</fsummary> <desc> - <p>This is the dirty equivalent of <c>mnesia:write/3</c>.</p> + <p>Dirty equivalent of the function <c>mnesia:write/3</c>.</p> </desc> </func> <func> <name>dump_log() -> dumped</name> - <fsummary>Perform a user initiated dump of the local log file.</fsummary> + <fsummary>Performs a user-initiated dump of the local log file.</fsummary> <desc> - <p>Performs a user initiated dump of the local log file. - This is usually not necessary since Mnesia, by default, - manages this automatically. - See configuration parameters - <seealso marker="#dump_log_time_threshold">dump_log_time_threshold</seealso> and - <seealso marker="#dump_log_write_threshold">dump_log_write_threshold</seealso>. - </p> + <marker id="dump_log"></marker> + <p>Performs a user-initiated dump of the local log file. + This is usually not necessary, as <c>Mnesia</c> by default + manages this automatically. See configuration parameters + <seealso marker="#dump_log_time_threshold">dump_log_time_threshold</seealso> + and + <seealso marker="#dump_log_write_threshold">dump_log_write_threshold</seealso>. + </p> </desc> </func> <func> <name>dump_tables(TabList) -> {atomic, ok} | {aborted, Reason}</name> - <fsummary>Dump all RAM tables to disc.</fsummary> + <fsummary>Dumps all RAM tables to disc.</fsummary> <desc> - <p>This function dumps a set of <c>ram_copies</c> tables + <marker id="dump_tables"></marker> + <p>Dumps a set of <c>ram_copies</c> tables to disc. The next time the system is started, these tables are initiated with the data found in the files that are the - result of this dump. None of the tables may have disc - resident replicas.</p> + result of this dump. None of the tables can have + disc-resident replicas.</p> </desc> </func> <func> - <name>dump_to_textfile(Filename) </name> - <fsummary>Dump local tables into a text file.</fsummary> + <name>dump_to_textfile(Filename)</name> + <fsummary>Dumps local tables into a text file.</fsummary> <desc> - <p>Dumps all local tables of a mnesia system into a text file - which can then be edited (by means of a normal text editor) - and then later be reloaded with + <marker id="dump_to_textfile"></marker> + <p>Dumps all local tables of a <c>Mnesia</c> system into a + text file, which can be edited (by a normal text editor) + and then be reloaded with <c>mnesia:load_textfile/1</c>. Only use this function for educational purposes. Use other functions to deal with real backups.</p> </desc> </func> <func> - <name>error_description(Error) -> String </name> - <fsummary>Return a string describing a particular Mnesia error.</fsummary> + <name>error_description(Error) -> String</name> + <fsummary>Returns a string describing a particular <c>Mnesia</c> error.</fsummary> <desc> - <p>All Mnesia transactions, including all the schema - update functions, either return the value <c>{atomic, Val}</c> or the tuple <c>{aborted, Reason}</c>. The - <c>Reason</c> can be either of the following atoms. The - <c>error_description/1</c> function returns a descriptive - string which describes the error. - </p> + <marker id="error_description"></marker> + <p>All <c>Mnesia</c> transactions, including all the schema + update functions, either return value <c>{atomic, Val}</c> + or the tuple <c>{aborted, Reason}</c>. <c>Reason</c> can + be either of the atoms in the following list. The + function <c>error_description/1</c> returns a descriptive + string that describes the error.</p> <list type="bulleted"> - <item> - <p><c>nested_transaction</c>. Nested transactions are - not allowed in this context. - </p> + <item><c>nested_transaction</c>. Nested transactions are + not allowed in this context. </item> - <item> - <p><c>badarg</c>. Bad or invalid argument, possibly - bad type. - </p> + <item><c>badarg</c>. Bad or invalid argument, possibly + bad type. </item> - <item> - <p><c>no_transaction</c>. Operation not allowed - outside transactions. - </p> + <item><c>no_transaction</c>. Operation not allowed + outside transactions. </item> - <item> - <p><c>combine_error</c>. Table options were illegally - combined. - </p> + <item><c>combine_error</c>. Table options illegally + combined. </item> - <item> - <p><c>bad_index</c>. Index already exists or was out - of bounds. - </p> + <item><c>bad_index</c>. Index already exists, or was out + of bounds. </item> - <item> - <p><c>already_exists</c>. Schema option is already set. - </p> + <item><c>already_exists</c>. Schema option to be activated + is already on. </item> - <item> - <p><c>index_exists</c>. Some operations cannot be performed on - tabs with index. - </p> + <item><c>index_exists</c>. Some operations cannot be + performed on tables with an index. </item> - <item> - <p><c>no_exists</c>. Tried to perform operation on - non-existing, or not alive, item. - </p> + <item><c>no_exists</c>. Tried to perform operation on + non-existing (not-alive) item. </item> - <item> - <p><c>system_limit</c>. Some system_limit was exhausted. - </p> + <item><c>system_limit</c>. A system limit was exhausted. </item> - <item> - <p><c>mnesia_down</c>. A transaction involving - records at some remote node which died while - transaction was executing. Record(s) are no longer - available elsewhere in the network. - </p> + <item><c>mnesia_down</c>. A transaction involves records + on a remote node, which became unavailable before the + transaction was completed. Records are no longer + available elsewhere in the network. </item> - <item> - <p><c>not_a_db_node</c>. A node which does not exist - in the schema was mentioned. - </p> + <item><c>not_a_db_node</c>. A node was mentioned that does + not exist in the schema. </item> - <item> - <p><c>bad_type</c>. Bad type on some arguments. - </p> + <item><c>bad_type</c>. Bad type specified in argument. </item> - <item> - <p><c>node_not_running</c>. Node not running. - </p> + <item><c>node_not_running</c>. Node is not running. </item> - <item> - <p><c>truncated_binary_file</c>. Truncated binary in file. - </p> + <item><c>truncated_binary_file</c>. Truncated binary in file. </item> - <item> - <p><c>active</c>. Some delete operations require that - all active records are removed. - </p> + <item><c>active</c>. Some delete operations require that + all active records are removed. </item> - <item> - <p><c>illegal</c>. Operation not supported on record. - </p> + <item><c>illegal</c>. Operation not supported on this + record. </item> </list> - <p>The <c>Error</c> may be <c>Reason</c>, - <c>{error, Reason}</c>, or <c>{aborted, Reason}</c>. The - <c>Reason</c> may be an atom or a tuple with <c>Reason</c> + <p><c>Error</c> can be <c>Reason</c>, + <c>{error, Reason}</c>, or <c>{aborted, Reason}</c>. + <c>Reason</c> can be an atom or a tuple with <c>Reason</c> as an atom in the first field.</p> + <p>The following examples illustrate a function that returns an error, + and the method to retrieve more detailed error information:</p> + <list type="bulleted"> + <item>The function + <seealso marker="#create_table/2">mnesia:create_table(bar, [{attributes, 3.14}])</seealso> + returns the tuple <c>{aborted,Reason}</c>, where <c>Reason</c> is + the tuple <c>{bad_type,bar,3.14000}</c>.</item> + <item>The function + <seealso marker="#error_description/1">mnesia:error_description(Reason)</seealso> + returns the term <c>{"Bad type on some provided + arguments",bar,3.14000}</c>, which is an error description + suitable for display.</item> + </list> </desc> </func> <func> <name>ets(Fun, [, Args]) -> ResultOfFun | exit(Reason)</name> - <fsummary>Call the Fun in a raw context which is not protected by a transaction.</fsummary> + <fsummary>Calls the <c>Fun</c> in a raw context that is not protected by a transaction.</fsummary> <desc> - <p>Call the <c>Fun</c> in a raw context which is not protected by - a transaction. The Mnesia function call is performed in the - <c>Fun</c> are performed directly on the local <c>ets</c> tables on - the assumption that the local storage type is + <marker id="ets"></marker> + <p>Calls the <c>Fun</c> in a raw context that is not protected by + a transaction. The <c>Mnesia</c> function call is performed in + the <c>Fun</c> and performed directly on the local <c>ets</c> + tables on the assumption that the local storage type is <c>ram_copies</c> and the tables are not replicated to other nodes. Subscriptions are not triggered and checkpoints are - not updated, but it is extremely fast. This function can + not updated, but it is extremely fast. This function can also be applied to <c>disc_copies</c> tables if all - operations are read only. See <c>mnesia:activity/4</c> - and the Mnesia User's Guide for more details.</p> - <p><em>Note:</em> Calling (nesting) a <c>mnesia:ets</c> - inside a transaction context will inherit the transaction semantics.</p> + operations are read only. For details, see + <c>mnesia:activity/4</c> and the User's Guide.</p> + <p>Notice that calling (nesting) a <c>mnesia:ets</c> inside a + transaction-context inherits the transaction semantics.</p> </desc> </func> <func> - <name>first(Tab) -> Key | transaction abort </name> - <fsummary>Return the key for the first record in a table.</fsummary> + <name>first(Tab) -> Key | transaction abort</name> + <fsummary>Returns the key for the first record in a table.</fsummary> <desc> - <p>Records in <c>set</c> or <c>bag</c> tables are not ordered. - However, there - is an ordering of the records which is not known - to the user. Accordingly, it is possible to traverse a table by means - of this function in conjunction with the <c>mnesia:next/2</c> - function. - </p> - <p>If there are no records at all in the table, this function - returns the atom <c>'$end_of_table'</c>. For this reason, it - is highly undesirable, but not disallowed, to use this atom + <marker id="first"></marker> + <p>Records in <c>set</c> or <c>bag</c> tables are not ordered. + However, there is an ordering of the records that is unknown + to the user. A table can therefore be traversed by this + function with the function <c>mnesia:next/2</c>.</p> + <p>If there are no records in the table, this function + returns the atom <c>'$end_of_table'</c>. It is therefore + highly undesirable, but not disallowed, to use this atom as the key for any user records.</p> </desc> </func> <func> - <name>foldl(Function, Acc, Table) -> NewAcc | transaction abort </name> - <fsummary>Call Function for each record in Table </fsummary> + <name>foldl(Function, Acc, Table) -> NewAcc | transaction abort</name> + <fsummary>Calls <c>Function</c> for each record in <c>Table</c>.</fsummary> <desc> - <p>Iterates over the table <c>Table</c> and calls - <c>Function(Record, NewAcc)</c> for each <c>Record</c> in the table. - The term returned from <c>Function</c> will be used as the second - argument in the next call to the <c>Function</c>. - </p> - <p><c>foldl</c> returns the same term as the last call to + <marker id="foldl"></marker> + <p>Iterates over the table <c>Table</c> and calls + <c>Function(Record, NewAcc)</c> for each <c>Record</c> in + the table. The term returned from <c>Function</c> is used + as the second argument in the next call to <c>Function</c>.</p> + <p><c>foldl</c> returns the same term as the last call to <c>Function</c> returned.</p> </desc> </func> <func> - <name>foldr(Function, Acc, Table) -> NewAcc | transaction abort </name> - <fsummary>Call Function for each record in Table </fsummary> + <name>foldr(Function, Acc, Table) -> NewAcc | transaction abort</name> + <fsummary>Calls <c>Function</c> for each record in <c>Table</c>.</fsummary> <desc> - <p>This function works exactly like - <c>foldl/3</c> but iterates the table in the opposite order - for the <c>ordered_set</c> table type. For - all other table types, <c>foldr/3</c> and + <marker id="foldr"></marker> + <p>Works exactly like <c>foldl/3</c> but iterates the table + in the opposite order for the <c>ordered_set</c> table type. + For all other table types, <c>foldr/3</c> and <c>foldl/3</c> are synonyms.</p> </desc> </func> <func> - <name>force_load_table(Tab) -> yes | ErrorDescription </name> - <fsummary>Force a table to be loaded into the system </fsummary> + <name>force_load_table(Tab) -> yes | ErrorDescription</name> + <fsummary>Forces a table to be loaded into the system.</fsummary> <desc> - <p>The Mnesia algorithm for table load might lead to a + <marker id="force_load_table"></marker> + <p>The <c>Mnesia</c> algorithm for table load can lead to a situation where a table cannot be loaded. This situation - occurs when a node is started and Mnesia concludes, or + occurs when a node is started and <c>Mnesia</c> concludes, or suspects, that another copy of the table was active after - this local copy became inactive due to a system crash. - </p> + this local copy became inactive because of a system crash.</p> <p>If this situation is not acceptable, this function can be - used to override the strategy of the Mnesia table load - algorithm. This could lead to a situation where some - transaction effects are lost with a inconsistent database as + used to override the strategy of the <c>Mnesia</c> table + load algorithm. This can lead to a situation where some + transaction effects are lost with an inconsistent database as result, but for some applications high availability is more important than consistent data.</p> </desc> </func> <func> <name>index_match_object(Pattern, Pos) -> transaction abort | ObjList</name> - <fsummary>Match records and utilizes index information.</fsummary> + <fsummary>Matches records and uses index information.</fsummary> <desc> - <p>Invokes <c>mnesia:index_match_object(Tab, Pattern, Pos, read)</c> where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> + <marker id="index_match_object_2"></marker> + <p>Starts + <c>mnesia:index_match_object(Tab, Pattern, Pos, read)</c>, + where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> </desc> </func> <func> <name>index_match_object(Tab, Pattern, Pos, LockKind) -> transaction abort | ObjList</name> - <fsummary>Match records and utilizes index information.</fsummary> - <desc> - <p>In a manner similar to the <c>mnesia:index_read/3</c> - function, we can also utilize any index information when we - try to match records. This function takes a pattern which - obeys the same rules as the <c>mnesia:match_object/3</c> - function with the exception that this function requires the - following conditions: - </p> + <fsummary>Matches records and uses index information.</fsummary> + <desc> + <marker id="index_match_object_4"></marker> + <p>In a manner similar to the function <c>mnesia:index_read/3</c>, + any index information can be used when trying to match records. + This function takes a pattern that obeys the same rules as the + function <c>mnesia:match_object/3</c>, except that this function + requires the following conditions:</p> <list type="bulleted"> <item> <p>The table <c>Tab</c> must have an index on - position <c>Pos</c>. - </p> + position <c>Pos</c>.</p> </item> <item> <p>The element in position <c>Pos</c> in - <c>Pattern</c> must be bound. <c>Pos</c> may either be - an integer (#record.Field), or an attribute name.</p> + <c>Pattern</c> must be bound. <c>Pos</c> is + an integer (<c>#record.Field</c>) or an attribute name.</p> </item> </list> <p>The two index search functions described here are - automatically invoked when searching tables with <c>qlc</c> - list comprehensions and also when using the low level - <c>mnesia:[dirty_]match_object</c> functions. - </p> - <p></p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type <c>LockKind</c> on the - entire table or on a single record. Currently, the lock type - <c>read</c> is supported. - </p> + automatically started when searching tables with <c>qlc</c> + list comprehensions and also when using the low-level + <c>mnesia:[dirty_]match_object</c> functions.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type + <c>LockKind</c> on the entire table or on a single record. + Currently, the lock type <c>read</c> is supported.</p> </desc> </func> <func> - <name>index_read(Tab, SecondaryKey, Pos) -> transaction abort | RecordList </name> - <fsummary>Read records via index table. </fsummary> + <name>index_read(Tab, SecondaryKey, Pos) -> transaction abort | RecordList</name> + <fsummary>Reads records through index table.</fsummary> <desc> - <p>Assume there is an index on position <c>Pos</c> for a + <marker id="index_read"></marker> + <p>Assume that there is an index on position <c>Pos</c> for a certain record type. This function can be used to read the records without knowing the actual key for the record. For - example, with an index in position 1 of the <c>person</c> - table, the call <c>mnesia:index_read(person, 36, #person.age)</c> returns a list of all persons with age - equal to 36. <c>Pos</c> may also be an attribute name - (atom), but if the notation <c>mnesia:index_read(person, 36, age)</c> is used, the field position will be searched for in - runtime, for each call. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a read lock on the entire table.</p> + example, with an index in position 1 of table <c>person</c>, + the call <c>mnesia:index_read(person, 36, #person.age)</c> + returns a list of all persons with age 36. <c>Pos</c> can + also be an attribute name (atom), but if the notation + <c>mnesia:index_read(person, 36, age)</c> is used, the + field position is searched for in runtime, for each call.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a read lock on the entire + table.</p> </desc> </func> <func> - <name>info() -> ok </name> - <fsummary>Print some information about the system on the tty.</fsummary> + <name>info() -> ok</name> + <fsummary>Prints system information on the terminal.</fsummary> <desc> - <p>Prints some information about the system on the tty. - This function may be used even if Mnesia is not started. - However, more information will be displayed if Mnesia is - started.</p> + <marker id="info"></marker> + <p>Prints system information on the terminal. + This function can be used even if <c>Mnesia</c> is not + started. However, more information is displayed if + <c>Mnesia</c> is started.</p> </desc> </func> <func> <name>install_fallback(Opaque) -> ok | {error,Reason}</name> - <fsummary>Install a backup as fallback.</fsummary> + <fsummary>Installs a backup as fallback.</fsummary> <desc> - <p>Invokes <c>mnesia:install_fallback(Opaque, Args)</c> where + <marker id="install_fallback_1"></marker> + <p>Calls <c>mnesia:install_fallback(Opaque, Args)</c>, where <c>Args</c> is <c>[{scope, global}]</c>.</p> </desc> </func> <func> <name>install_fallback(Opaque), BackupMod) -> ok | {error,Reason}</name> - <fsummary>Install a backup as fallback.</fsummary> + <fsummary>Installs a backup as fallback.</fsummary> <desc> - <p>Invokes <c>mnesia:install_fallback(Opaque, Args)</c> where + <p>Calls <c>mnesia:install_fallback(Opaque, Args)</c>, where <c>Args</c> is <c>[{scope, global}, {module, BackupMod}]</c>.</p> </desc> </func> <func> <name>install_fallback(Opaque, Args) -> ok | {error,Reason}</name> - <fsummary>Install a backup as fallback.</fsummary> - <desc> - <p>This function is used to install a backup as fallback. The - fallback will be used to restore the database at the next - start-up. Installation of fallbacks requires Erlang to be up - and running on all the involved nodes, but it does not - matter if Mnesia is running or not. The installation of the - fallback will fail if the local node is not one of the disc - resident nodes in the backup. - </p> - <p><c>Args</c> is a list of the following tuples: - </p> + <fsummary>Installs a backup as fallback.</fsummary> + <desc> + <p>Installs a backup as fallback. The fallback is used to + restore the database at the next startup. Installation of + fallbacks requires Erlang to be operational on all the + involved nodes, but it does not matter if <c>Mnesia</c> + is running or not. The installation of the fallback fails + if the local node is not one of the disc-resident nodes + in the backup.</p> + <p><c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> <item> - <p><c>{module, BackupMod}</c>. - All accesses of the backup media is performed via a - callback module named <c>BackupMod</c>. The - <c>Opaque</c> argument is forwarded to the callback - module which may interpret it as it wish. The default + <p><c>{module, BackupMod}</c>. + All accesses of the backup media are performed through + a callback module named <c>BackupMod</c>. Argument + <c>Opaque</c> is forwarded to the callback module, + which can interpret it as it wishes. The default callback module is called <c>mnesia_backup</c> and it - interprets the <c>Opaque</c> argument as a local + interprets argument <c>Opaque</c> as a local filename. The default for this module is also - configurable via the <c>-mnesia mnesia_backup</c> - configuration parameter. </p> + configurable through configuration parameter + <c>-mnesia mnesia_backup</c>.</p> </item> <item> - <p><c>{scope, Scope}</c> - The <c>Scope</c> of a fallback may either be + <p><c>{scope, Scope}</c>. + The <c>Scope</c> of a fallback is either <c>global</c> for the entire database or <c>local</c> for one node. By default, the installation of a fallback - is a global operation which either is performed all - nodes with disc resident schema or none. Which nodes - that are disc resident or not, is determined from the - schema info in the backup.</p> - <p>If the <c>Scope</c> of the operation is <c>local</c> - the fallback will only be installed on the local node.</p> + is a global operation, which either is performed on all + nodes with a disc-resident schema or none. Which nodes + that are disc-resident is determined from the + schema information in the backup.</p> + <p>If <c>Scope</c> of the operation is <c>local</c>, + the fallback is only installed on the local node.</p> </item> <item> - <p><c>{mnesia_dir, AlternateDir}</c> + <p><c>{mnesia_dir, AlternateDir}</c>. This argument is only valid if the scope of the installation is <c>local</c>. Normally the installation - of a fallback is targeted towards the Mnesia directory - as configured with the <c>-mnesia dir</c> configuration - parameter. But by explicitly supplying an - <c>AlternateDir</c> the fallback will be installed there - regardless of the Mnesia directory configuration + of a fallback is targeted to the <c>Mnesia</c> directory, + as configured with configuration parameter + <c>-mnesia dir</c>. But by explicitly supplying an + <c>AlternateDir</c>, the fallback is installed there + regardless of the <c>Mnesia</c> directory configuration parameter setting. After installation of a fallback on - an alternate Mnesia directory that directory is fully - prepared for usage as an active Mnesia directory. - </p> - <p>This is a somewhat dangerous feature which must be - used with care. By unintentional mixing of directories - you may easily end up with a inconsistent database, if + an alternative <c>Mnesia</c> directory, that directory + is fully prepared for use as an active <c>Mnesia</c> + directory.</p> + <p>This is a dangerous feature that must be + used with care. By unintentional mixing of directories, + you can easily end up with an inconsistent database, if the same backup is installed on more than one directory.</p> </item> </list> </desc> </func> <func> - <name>is_transaction() -> boolean </name> - <fsummary>Check if code is running in a transaction.</fsummary> + <name>is_transaction() -> boolean</name> + <fsummary>Checks if code is running in a transaction.</fsummary> <desc> - <p>When this function is executed inside a transaction context + <marker id="is_transaction"></marker> + <p>When this function is executed inside a transaction-context, it returns <c>true</c>, otherwise <c>false</c>.</p> </desc> </func> <func> - <name>last(Tab) -> Key | transaction abort </name> - <fsummary>Return the key for the last record in a table.</fsummary> + <name>last(Tab) -> Key | transaction abort</name> + <fsummary>Returns the key for the last record in a table.</fsummary> <desc> - <p>This function works exactly like - <c>mnesia:first/1</c> but returns the last object in - Erlang term order for the <c>ordered_set</c> table type. For - all other table types, <c>mnesia:first/1</c> and + <p>Works exactly like + <c>mnesia:first/1</c>, but returns the last object in + Erlang term order for the <c>ordered_set</c> table type. + For all other table types, <c>mnesia:first/1</c> and <c>mnesia:last/1</c> are synonyms.</p> </desc> </func> <func> <name>load_textfile(Filename)</name> - <fsummary>Load tables from a text file.</fsummary> + <fsummary>Loads tables from a text file.</fsummary> <desc> + <marker id="load_textfile"></marker> <p>Loads a series of definitions and data found in the text file (generated with <c>mnesia:dump_to_textfile/1</c>) - into Mnesia. This function also starts Mnesia and possibly - creates a new schema. This function is intended for - educational purposes only and using other functions to deal - with real backups, is recommended.</p> + into <c>Mnesia</c>. This function also starts <c>Mnesia</c> + and possibly creates a new schema. This function is + intended for educational purposes only. It is recommended + to use other functions to deal with real backups.</p> </desc> </func> <func> <name>lock(LockItem, LockKind) -> Nodes | ok | transaction abort</name> <fsummary>Explicit grab lock.</fsummary> <desc> + <marker id="lock"></marker> <p>Write locks are normally acquired on all nodes where a - replica of the table resides (and is active). Read locks are - acquired on one node (the local node if a local - replica exists). Most of the context sensitive access functions - acquire an implicit lock if they are invoked in a - transaction context. The granularity of a lock may either - be a single record or an entire table. - </p> - <p>The normal usage is to call the function without checking - the return value since it exits if it fails and the - transaction is restarted by the transaction manager. It - returns all the locked nodes if a write lock is acquired, and - <c>ok</c> if it was a read lock. - </p> - <p>This function <c>mnesia:lock/2</c> is intended to support - explicit locking on tables but also intended for situations - when locks need to be acquired regardless of how tables are - replicated. Currently, two <c>LockKind</c>'s are supported: - </p> + replica of the table resides (and is active). Read locks + are acquired on one node (the local node if a local + replica exists). Most of the context-sensitive access + functions acquire an implicit lock if they are started in a + transaction-context. The granularity of a lock can either + be a single record or an entire table.</p> + <p>The normal use is to call the function without checking + the return value, as it exits if it fails and the + transaction is restarted by the transaction manager. It + returns all the locked nodes if a write lock is acquired + and <c>ok</c> if it was a read lock.</p> + <p>The function <c>mnesia:lock/2</c> is intended to support + explicit locking on tables, but is also intended for + situations when locks need to be acquired regardless of + how tables are replicated. Currently, two kinds of + <c>LockKind</c> are supported:</p> <taglist> <tag><c>write</c></tag> <item> - <p>Write locks are exclusive, which means that if one + <p>Write locks are exclusive. This means that if one transaction manages to acquire a write lock on an item, - no other transaction may acquire any kind of lock on the - same item. - </p> + no other transaction can acquire any kind of lock on + the same item.</p> </item> <tag><c>read</c></tag> <item> - <p>Read locks may be shared, which means that if one + <p>Read locks can be shared. This means that if one transaction manages to acquire a read lock on an item, - other transactions may also acquire a read lock on the - same item. However, if someone has a read lock no one can - acquire a write lock at the same item. If some one has a - write lock no one can acquire a read lock nor - a write lock at the same item.</p> + other transactions can also acquire a read lock on the + same item. However, if someone has a read lock, no one + can acquire a write lock at the same item. If someone + has a write lock, no one can acquire either a read lock + or a write lock at the same item.</p> </item> </taglist> <p>Conflicting lock requests are automatically queued if there is no risk of a deadlock. Otherwise the transaction must be - aborted and executed again. Mnesia does this automatically - as long as the upper limit of maximum <c>retries</c> is not - reached. See <c>mnesia:transaction/3</c> for the details. - </p> - <p>For the sake of completeness sticky write locks will also - be described here even if a sticky write lock is not - supported by this particular function: - </p> + terminated and executed again. <c>Mnesia</c> does this + automatically as long as the upper limit of the maximum + <c>retries</c> is not reached. For details, see + <c>mnesia:transaction/3</c>.</p> + <p>For the sake of completeness, sticky write locks are also + described here even if a sticky write lock is not + supported by this function:</p> <taglist> <tag><c>sticky_write</c></tag> <item> - <p>Sticky write locks are a mechanism which can be used + <p>Sticky write locks are a mechanism that can be used to optimize write lock acquisition. If your application uses replicated tables mainly for fault tolerance (as opposed to read access optimization purpose), sticky - locks may be the best option available. - </p> - <p>When a sticky write lock is acquired, all nodes will be - informed which node is locked. Subsequently, - sticky lock requests from the same node will be - performed as a local operation without any + locks can be the best option available.</p> + <p>When a sticky write lock is acquired, all nodes are + informed which node is locked. Then, + sticky lock requests from the same node are + performed as a local operation without any communication with other nodes. The sticky lock - lingers on the node even after the transaction has - ended. See the Mnesia User's Guide for more information.</p> + lingers on the node even after the transaction + ends. For details, see the User's Guide.</p> </item> </taglist> - <p>Currently, two kinds of <c>LockItem</c>'s are supported by - this function: - </p> + <p>Currently, this function supports two kinds of + <c>LockItem</c>:</p> <taglist> <tag><c>{table, Tab}</c></tag> <item> <p>This acquires a lock of type <c>LockKind</c> on the - entire table <c>Tab</c>. - </p> + entire table <c>Tab</c>.</p> </item> <tag><c>{global, GlobalKey, Nodes}</c></tag> <item> <p>This acquires a lock of type <c>LockKind</c> on the global resource <c>GlobalKey</c>. The lock is acquired - on all active nodes in the <c>Nodes</c> list. </p> + on all active nodes in the <c>Nodes</c> list.</p> </item> </taglist> - <p>Locks are released when the outermost transaction ends. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires locks otherwise it just ignores the - request.</p> + <p>Locks are released when the outermost transaction ends.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires locks, otherwise it + ignores the request.</p> </desc> </func> <func> - <name>match_object(Pattern) ->transaction abort | RecList </name> - <fsummary>Match <c>Pattern</c>for records. </fsummary> + <name>match_object(Pattern) -> transaction abort | RecList</name> + <fsummary>Matches <c>Pattern</c> for records.</fsummary> <desc> - <p>Invokes <c>mnesia:match_object(Tab, Pattern, read)</c> where + <marker id="match_object_1"></marker> + <p>Calls <c>mnesia:match_object(Tab, Pattern, read)</c>, where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> </desc> </func> <func> - <name>match_object(Tab, Pattern, LockKind) ->transaction abort | RecList </name> - <fsummary>Match <c>Pattern</c>for records. </fsummary> + <name>match_object(Tab, Pattern, LockKind) -> transaction abort | RecList</name> + <fsummary>Matches <c>Pattern</c> for records.</fsummary> <desc> - <p>This function takes a pattern with 'don't care' variables - denoted as a '_' parameter. This function returns a list of - records which matched the pattern. Since the second element + <marker id="match_object_3"></marker> + <p>Takes a pattern with "don't care" variables + denoted as a <c>'_'</c> parameter. This function returns + a list of records that matched the pattern. + Since the second element of a record in a table is considered to be the key for the record, the performance of this function depends on whether - this key is bound or not. - </p> - <p>For example, the call <c>mnesia:match_object(person, {person, '_', 36, '_', '_'}, read)</c> returns a list of all person records with an - age field of thirty-six (36). - </p> + this key is bound or not.</p> + <p>For example, the call <c>mnesia:match_object(person, + {person, '_', 36, '_', '_'}, read)</c> returns a list of + all person records with an <c>age</c> field of 36.</p> <p>The function <c>mnesia:match_object/3</c> - automatically uses indices if these exist. However, no - heuristics are performed in order to select the best - index. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type <c>LockKind</c> on the - entire table or a single record. Currently, the lock type - <c>read</c> is supported.</p> + automatically uses indexes if these exist. However, no + heuristics are performed to select the best index.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type + <c>LockKind</c> on the entire table or a single record. + Currently, the lock type <c>read</c> is supported.</p> </desc> </func> <func> <name>move_table_copy(Tab, From, To) -> {aborted, Reason} | {atomic, ok}</name> - <fsummary>Move the copy of table <c>Tab</c>from node<c>From</c>to node <c>To</c>.</fsummary> + <fsummary>Moves the copy of table <c>Tab</c> from node <c>From</c> to node <c>To</c>.</fsummary> <desc> + <marker id="move_table_copy"></marker> <p>Moves the copy of table <c>Tab</c> from node - <c>From</c> to node <c>To</c>. - </p> + <c>From</c> to node <c>To</c>.</p> <p>The storage type is preserved. For example, a RAM table - moved from one node remains a RAM on the new node. It is - still possible for other transactions to read and write in - the table while it is being moved. - </p> + moved from one node remains a RAM on the new node. Other + transactions can still read and write in + the table while it is being moved.</p> <p>This function cannot be used on <c>local_content</c> tables.</p> </desc> </func> <func> - <name>next(Tab, Key) -> Key | transaction abort </name> - <fsummary>Return the next key in a table. </fsummary> + <name>next(Tab, Key) -> Key | transaction abort</name> + <fsummary>Returns the next key in a table.</fsummary> <desc> - <p>This function makes it possible to traverse a table - and perform operations on all records in the table. When + <marker id="next"></marker> + <p>Traverses a table and + performs operations on all records in the table. When the end of the table is reached, the special key - <c>'$end_of_table'</c> is returned. Otherwise, the function - returns a key which can be used to read the actual record.</p> + <c>'$end_of_table'</c> is returned. Otherwise the function + returns a key that can be used to read the actual record.</p> </desc> </func> <func> - <name>prev(Tab, Key) -> Key | transaction abort </name> - <fsummary>Return the previous key in a table. </fsummary> + <name>prev(Tab, Key) -> Key | transaction abort</name> + <fsummary>Returns the previous key in a table.</fsummary> <desc> - <p>This function works exactly like - <c>mnesia:next/2</c> but returns the previous object in - Erlang term order for the ordered_set table type. For - all other table types, <c>mnesia:next/2</c> and + <p>Works exactly like + <c>mnesia:next/2</c>, but returns the previous object in + Erlang term order for the <c>ordered_set</c> table type. + For all other table types, <c>mnesia:next/2</c> and <c>mnesia:prev/2</c> are synonyms.</p> </desc> </func> <func> - <name>read({Tab, Key}) -> transaction abort | RecordList </name> - <fsummary>Read records(s) with a given key. </fsummary> + <name>read({Tab, Key}) -> transaction abort | RecordList</name> + <fsummary>Reads records(s) with a given key.</fsummary> <desc> - <p>Invokes <c>mnesia:read(Tab, Key, read)</c>.</p> + <marker id="read_2"></marker> + <p>Calls function <c>mnesia:read(Tab, Key, read)</c>.</p> </desc> </func> <func> - <name>read(Tab, Key) -> transaction abort | RecordList </name> - <fsummary>Read records(s) with a given key. </fsummary> + <name>read(Tab, Key) -> transaction abort | RecordList</name> + <fsummary>Reads records(s) with a given key.</fsummary> <desc> - <p>Invokes <c>mnesia:read(Tab, Key, read)</c>.</p> + <p>Calls function <c>mnesia:read(Tab, Key, read)</c>.</p> </desc> </func> <func> - <name>read(Tab, Key, LockKind) -> transaction abort | RecordList </name> - <fsummary>Read records(s) with a given key. </fsummary> + <name>read(Tab, Key, LockKind) -> transaction abort | RecordList</name> + <fsummary>Reads records(s) with a given key.</fsummary> <desc> - <p>This function reads all records from table <c>Tab</c> with + <marker id="read_3"></marker> + <p>Reads all records from table <c>Tab</c> with key <c>Key</c>. This function has the same semantics regardless of the location of <c>Tab</c>. If the table is - of type <c>bag</c>, the <c>mnesia:read(Tab, Key)</c> can + of type <c>bag</c>, the function + <c>mnesia:read(Tab, Key)</c> can return an arbitrarily long list. If the table is of type - <c>set</c>, the list is either of length 1, or <c>[]</c>. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type + <c>set</c>, the list is either of length 1, or <c>[]</c>.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type <c>LockKind</c>. Currently, the lock types <c>read</c>, - <c>write</c> and <c>sticky_write</c> are supported. - </p> - <p>If the user wants to update the record it is more efficient to - use <c>write/sticky_write</c> as the LockKind. If majority checking - is active on the table, it will be checked as soon as a write lock is - attempted. This can be used to quickly abort if the majority condition - isn't met. - </p> + <c>write</c>, and <c>sticky_write</c> are supported.</p> + <p>If the user wants to update the record, it is more + efficient to use <c>write/sticky_write</c> as the + <c>LockKind</c>. If majority checking is active on the + table, it is checked as soon as a write lock is + attempted. This can be used to end quickly if the + majority condition is not met.</p> </desc> </func> <func> <name>read_lock_table(Tab) -> ok | transaction abort</name> - <fsummary>Set a read lock on an entire table.</fsummary> + <fsummary>Sets a read lock on an entire table.</fsummary> <desc> - <p>Invokes <c>mnesia:lock({table, Tab}, read)</c>.</p> + <marker id="read_lock_table"></marker> + <p>Calls the function + <c>mnesia:lock({table, Tab}, read)</c>.</p> </desc> </func> <func> <name>report_event(Event) -> ok</name> - <fsummary>Report a user event to Mnesia's event handler.</fsummary> + <fsummary>Reports a user event to the <c>Mnesia</c> event handler.</fsummary> <desc> - <p>When tracing a system of Mnesia applications it is useful - to be able to interleave Mnesia's own events with - application related events that give information about the - application context. - </p> + <marker id="report_event"></marker> + <p>When tracing a system of <c>Mnesia</c> applications it is + useful to be able to interleave <c>Mnesia</c> own events with + application-related events that give information about the + application context.</p> <p>Whenever the application begins a - new and demanding Mnesia task, or if it is entering a new - interesting phase in its execution, it may be a good idea to - use <c>mnesia:report_event/1</c>. The <c>Event</c> may be + new and demanding <c>Mnesia</c> task, or if it enters a new + interesting phase in its execution, it can be a good idea to + use <c>mnesia:report_event/1</c>. <c>Event</c> can be any term and generates a <c>{mnesia_user, Event}</c> event - for any processes that subscribe to Mnesia system + for any processes that subscribe to <c>Mnesia</c> system events.</p> </desc> </func> @@ -1819,221 +1745,237 @@ mnesia:create_table(person, <name>restore(Opaque, Args) -> {atomic, RestoredTabs} |{aborted, Reason}</name> <fsummary>Online restore of backup.</fsummary> <desc> - <p>With this function, tables may be restored online from a - backup without restarting Mnesia. <c>Opaque</c> is forwarded - to the backup module. <c>Args</c> is a list of the following - tuples: - </p> + <marker id="restore"></marker> + <p>With this function, tables can be restored online from a + backup without restarting <c>Mnesia</c>. + <c>Opaque</c> is forwarded to the backup module. + <c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> <item> - <p><c>{module,BackupMod}</c> The backup module - <c>BackupMod</c> will be used to access the backup - media. If omitted, the default backup module will be - used. - </p> + <c>{module,BackupMod}</c>. The backup module + <c>BackupMod</c> is used to access the backup media. + If omitted, the default backup module is used. </item> - <item><c>{skip_tables, TabList}</c> Where <c>TabList</c> - is a list of tables which should not be read from the + <item><c>{skip_tables, TabList}</c>, where <c>TabList</c> + is a list of tables that is not to be read from the backup. </item> - <item><c>{clear_tables, TabList}</c> Where - <c>TabList</c> is a list of tables which should be - cleared, before the records from the backup are inserted, - ie. all records in the tables are deleted before the - tables are restored. Schema information about the tables - is not cleared or read from backup. - </item> - <item><c>{keep_tables, TabList}</c> Where <c>TabList</c> - is a list of tables which should be not be cleared, before - the records from the backup are inserted, i.e. the records - in the backup will be added to the records in the table. + <item><c>{clear_tables, TabList}</c>, where + <c>TabList</c> is a list of tables that is to be + cleared before the records from the backup are inserted. + That is, all records in the tables are deleted before the + tables are restored. Schema information about the tables + is not cleared or read from the backup. + </item> + <item><c>{keep_tables, TabList}</c>, where <c>TabList</c> + is a list of tables that is not to be cleared before the + records from the backup are inserted. That is, the records + in the backup are added to the records in the table. Schema information about the tables is not cleared or read - from backup. - </item> - <item><c>{recreate_tables, TabList}</c> Where - <c>TabList</c> is a list of tables which should be - re-created, before the records from the backup are - inserted. The tables are first deleted and then created with - the schema information from the backup. All the nodes in the - backup needs to be up and running. - </item> - <item><c>{default_op, Operation}</c> Where <c>Operation</c> is - one of the following operations <c>skip_tables</c>, - <c>clear_tables</c>, <c>keep_tables</c> or - <c>recreate_tables</c>. The default operation specifies - which operation should be used on tables from the backup - which are not specified in any of the lists above. If - omitted, the operation <c>clear_tables</c> will be used. + from the backup. + </item> + <item><c>{recreate_tables, TabList}</c>, where + <c>TabList</c> is a list of tables that is to be + recreated before the records from the backup are inserted. + The tables are first deleted and then created with the + schema information from the backup. All the nodes in the + backup need to be operational. + </item> + <item><c>{default_op, Operation}</c>, where <c>Operation</c> + is either of the operations <c>skip_tables</c>, + <c>clear_tables</c>, <c>keep_tables</c>, or + <c>recreate_tables</c>. The default operation specifies + which operation that is to be used on tables from the backup + that is not specified in any of the mentioned lists. If + omitted, operation <c>clear_tables</c> is used. </item> </list> - <p>The affected tables are write locked during the - restoration, but regardless of the lock conflicts caused by - this, the applications can continue to do their work while + <p>The affected tables are write-locked during the + restoration. However, regardless of the lock conflicts caused + by this, the applications can continue to do their work while the restoration is being performed. The restoration is - performed as one single transaction. - </p> - <p>If the database is - huge, it may not be possible to restore it online. In such - cases, the old database must be restored by installing a + performed as one single transaction.</p> + <p>If the database is huge, + it it not always possible to restore it online. In such + cases, restore the old database by installing a fallback and then restart.</p> </desc> </func> <func> - <name>s_delete({Tab, Key}) -> ok | transaction abort </name> - <fsummary>Set sticky lock and delete records.</fsummary> + <name>s_delete({Tab, Key}) -> ok | transaction abort</name> + <fsummary>Sets sticky lock and delete records.</fsummary> <desc> - <p>Invokes <c>mnesia:delete(Tab, Key, sticky_write)</c></p> + <marker id="s_delete"></marker> + <p>Calls the function + <c>mnesia:delete(Tab, Key, sticky_write)</c></p> </desc> </func> <func> - <name>s_delete_object(Record) -> ok | transaction abort </name> - <fsummary>Set sticky lock and delete record.</fsummary> + <name>s_delete_object(Record) -> ok | transaction abort</name> + <fsummary>Sets sticky lock and delete record.</fsummary> <desc> - <p>Invokes <c>mnesia:delete_object(Tab, Record, sticky_write)</c> where <c>Tab</c> is <c>element(1, Record)</c>.</p> + <marker id="s_delete_object"></marker> + <p>Calls the function + <c>mnesia:delete_object(Tab, Record, sticky_write)</c>, + where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> <func> - <name>s_write(Record) -> ok | transaction abort </name> - <fsummary>Write <c>Record</c>and sets stick lock.</fsummary> + <name>s_write(Record) -> ok | transaction abort</name> + <fsummary>Writes <c>Record</c> and sets sticky lock.</fsummary> <desc> - <p>Invokes <c>mnesia:write(Tab, Record, sticky_write)</c> + <marker id="s_write"></marker> + <p>Calls the function + <c>mnesia:write(Tab, Record, sticky_write)</c>, where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> <func> - <name>schema() -> ok </name> - <fsummary>Print information about all table definitions on the tty. </fsummary> + <name>schema() -> ok</name> + <fsummary>Prints information about all table definitions on the terminal.</fsummary> <desc> - <p>Prints information about all table definitions on the tty.</p> + <p>Prints information about all table definitions on the terminal.</p> </desc> </func> <func> - <name>schema(Tab) -> ok </name> - <fsummary>Print information about one table definition on the tty.</fsummary> + <name>schema(Tab) -> ok</name> + <fsummary>Prints information about one table definition on the terminal.</fsummary> <desc> - <p>Prints information about one table definition on the tty.</p> + <p>Prints information about one table definition on the terminal.</p> </desc> </func> <func> - <name>select(Tab, MatchSpec [, Lock]) -> transaction abort | [Object] </name> - <fsummary>Match the objects in <c>Tab</c>against <c>MatchSpec</c>.</fsummary> + <name>select(Tab, MatchSpec [, Lock]) -> transaction abort | [Object]</name> + <fsummary>Matches the objects in <c>Tab</c> against <c>MatchSpec</c>.</fsummary> <desc> - <p>Matches the objects in the table <c>Tab</c> using a - match_spec as described in the ERTS Users Guide. Optionally a lock + <marker id="select_2_3"></marker> + <p>Matches the objects in table <c>Tab</c> using a + <c>match_spec</c> as described in the + <seealso marker="stdlib:ets#select/3">ets:select/3</seealso>. + Optionally a lock <c>read</c> or <c>write</c> can be given as the third - argument, default is <c>read</c>. The return value depends - on the <c>MatchSpec</c>.</p> - <p><em>Note:</em> for best performance <c>select</c> should - be used before any modifying operations are done on that table - in the same transaction, i.e. don't use <c>write</c> or <c>delete</c> - before a <c>select</c>.</p> - <p>In its simplest forms the match_spec's look like this:</p> + argument. Default is <c>read</c>. The return value depends + on <c>MatchSpec</c>.</p> + <p>Notice that for best performance, <c>select</c> is to be + used before any modifying operations are done on that table + in the same transaction. That is, do not use <c>write</c> + or <c>delete</c> before a <c>select</c>.</p> + <p>In its simplest forms, the <c>match_spec</c> look as + follows:</p> <list type="bulleted"> - <item>MatchSpec = [MatchFunction]</item> - <item>MatchFunction = {MatchHead, [Guard], [Result]}</item> - <item>MatchHead = tuple() | record()</item> - <item>Guard = {"Guardtest name", ...}</item> - <item>Result = "Term construct"</item> + <item><c>MatchSpec = [MatchFunction]</c></item> + <item><c>MatchFunction = {MatchHead, [Guard], [Result]}</c></item> + <item><c>MatchHead = tuple() | record()</c></item> + <item><c>Guard = {"Guardtest name", ...}</c></item> + <item><c>Result = "Term construct"</c></item> </list> - <p>See the ERTS Users Guide and <c>ets</c> documentation for a - complete description of the select.</p> - <p>For example to find the names of all male persons with an age over 30 in table - Tab do:</p> + <p>For a complete description of <c>select</c>, see the + <seealso marker="erts:index">ERTS</seealso> User's Guide and the + <seealso marker="stdlib:ets">ets</seealso> manual page in + <c>STDLIB</c>.</p> + <p>For example, to find the names of all male persons older + than 30 in table <c>Tab</c>:</p> <code type="none"> MatchHead = #person{name='$1', sex=male, age='$2', _='_'}, Guard = {'>', '$2', 30}, Result = '$1', -mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]), - </code> +mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]),</code> </desc> </func> <func> - <name>select(Tab, MatchSpec, NObjects, Lock) -> transaction abort | {[Object],Cont} | '$end_of_table'</name> - <fsummary>Match the objects in <c>Tab</c>against <c>MatchSpec</c>.</fsummary> + <name>select(Tab, MatchSpec, NObjects, Lock) -> transaction abort | {[Object],Cont} | '$end_of_table'</name> + <fsummary>Matches the objects in <c>Tab</c> against <c>MatchSpec</c>.</fsummary> <desc> - <p>Matches the objects in the table <c>Tab</c> using a - match_spec as described in ERTS users guide, and returns - a chunk of terms and a continuation, the wanted number - of returned terms is specified by the <c>NObjects</c> argument. - The lock argument can be <c>read</c> or <c>write</c>. - The continuation should be used as argument to <c>mnesia:select/1</c>, + <marker id="select_4"></marker> + <p>Matches the objects in table <c>Tab</c> using a + <c>match_spec</c> as described in the + <seealso marker="erts:index">ERTS</seealso> User's Guide, + and returns a chunk of terms and a continuation. + The wanted number of returned terms is specified by + argument <c>NObjects</c>. The lock argument can be + <c>read</c> or <c>write</c>. The continuation is to be + used as argument to <c>mnesia:select/1</c>, if more or all answers are needed.</p> - <p><em>Note:</em> for best performance <c>select</c> should - be used before any modifying operations are done on that - table in the same transaction, i.e. don't use + <p>Notice that for best performance, <c>select</c> is to be + used before any modifying operations are done on that table + in the same transaction. That is, do not use <c>mnesia:write</c> or <c>mnesia:delete</c> before a - <c>mnesia:select</c>. For efficiency the <c>NObjects</c> is - a recommendation only and the result may contain anything - from an empty list to all available results. </p> + <c>mnesia:select</c>. For efficiency, <c>NObjects</c> is + a recommendation only and the result can contain anything + from an empty list to all available results.</p> </desc> </func> <func> - <name>select(Cont) -> transaction abort | {[Object],Cont} | '$end_of_table'</name> - <fsummary>Continues selecting objects. </fsummary> + <name>select(Cont) -> transaction abort | {[Object],Cont} | '$end_of_table'</name> + <fsummary>Continues selecting objects.</fsummary> <desc> <p>Selects more objects with the match specification initiated - by <c>mnesia:select/4</c>. - </p> - <p><em>Note:</em> Any modifying operations, i.e. <c>mnesia:write</c> - or <c>mnesia:delete</c>, that are done between the <c>mnesia:select/4</c> - and <c>mnesia:select/1</c> calls will not be visible in the result.</p> + by <c>mnesia:select/4</c>.</p> + <p>Notice that any modifying operations, that is, + <c>mnesia:write</c> or <c>mnesia:delete</c>, that are done + between the <c>mnesia:select/4</c> and <c>mnesia:select/1</c> + calls are not visible in the result.</p> </desc> </func> <func> <name>set_debug_level(Level) -> OldLevel</name> - <fsummary>Change the internal debug level of Mnesia</fsummary> + <fsummary>Changes the internal debug level of <c>Mnesia</c>.</fsummary> <desc> - <p>Changes the internal debug level of Mnesia. See the - chapter about configuration parameters for details.</p> + <marker id="set_debug_level"></marker> + <p>Changes the internal debug level of <c>Mnesia</c>. + For details, see + <seealso marker="#configuration_parameters">Section + Configuration Parameters</seealso>.</p> </desc> </func> <func> - <name>set_master_nodes(MasterNodes) -> ok | {error, Reason} </name> - <fsummary>Set the master nodes for all tables</fsummary> + <name>set_master_nodes(MasterNodes) -> ok | {error, Reason}</name> + <fsummary>Sets the master nodes for all tables.</fsummary> <desc> - <p>For each table Mnesia will determine its replica nodes - (<c>TabNodes</c>) and invoke <c>mnesia:set_master_nodes(Tab, TabMasterNodes)</c> where <c>TabMasterNodes</c> is the - intersection of <c>MasterNodes</c> and <c>TabNodes</c>. See - <c>mnesia:set_master_nodes/2</c> about the semantics.</p> + <marker id="set_master_nodes_1"></marker> + <p>For each table <c>Mnesia</c> determines its replica nodes + (<c>TabNodes</c>) and starts + <c>mnesia:set_master_nodes(Tab, TabMasterNodes)</c>. where + <c>TabMasterNodes</c> is the intersection of + <c>MasterNodes</c> and <c>TabNodes</c>. For semantics, see + <c>mnesia:set_master_nodes/2</c>.</p> </desc> </func> <func> - <name>set_master_nodes(Tab, MasterNodes) -> ok | {error, Reason} </name> - <fsummary>Set the master nodes for a table</fsummary> + <name>set_master_nodes(Tab, MasterNodes) -> ok | {error, Reason}</name> + <fsummary>Sets the master nodes for a table.</fsummary> <desc> - <p>If the application detects that there has been a - communication failure (in a potentially partitioned network) which - may have caused an inconsistent database, it may use the + <marker id="set_master_nodes_2"></marker> + <p>If the application detects a + communication failure (in a potentially partitioned network) + that can have caused an inconsistent database, it can use the function <c>mnesia:set_master_nodes(Tab, MasterNodes)</c> to - define from which nodes each table will be loaded. - At startup Mnesia's normal table load algorithm will be - bypassed and the table will be loaded from one of the master - nodes defined for the table, regardless of when and if Mnesia - was terminated on other nodes. The <c>MasterNodes</c> may only - contain nodes where the table has a replica and if the + define from which nodes each table is to be loaded. + At startup, the <c>Mnesia</c> normal table load algorithm is + bypassed and the table is loaded from one of the master nodes + defined for the table, regardless of when and if <c>Mnesia</c> + terminated on other nodes. <c>MasterNodes</c> can only + contain nodes where the table has a replica. If the <c>MasterNodes</c> list is empty, the master node recovery - mechanism for the particular table will be reset and the - normal load mechanism will be used at next restart. - </p> - <p>The master node setting is always local and it may be - changed regardless of whether Mnesia is started or not. - </p> - <p>The database may also become inconsistent if the - <c>max_wait_for_decision</c> configuration parameter is used + mechanism for the particular table is reset, and the + normal load mechanism is used at the next restart.</p> + <p>The master node setting is always local. It can be + changed regardless if <c>Mnesia</c> is started or not.</p> + <p>The database can also become inconsistent if + configuration parameter <c>max_wait_for_decision</c> is used or if <c>mnesia:force_load_table/1</c> is used.</p> </desc> </func> <func> <name>snmp_close_table(Tab) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Remove the possibility for SNMP to manipulate the table.</fsummary> + <fsummary>Removes the possibility for SNMP to manipulate the table.</fsummary> <desc> - <p>Removes the possibility for SNMP to manipulate the - table.</p> + <p>Removes the possibility for SNMP to manipulate the table.</p> </desc> </func> <func> <name>snmp_get_mnesia_key(Tab, RowIndex) -> {ok, Key} | undefined</name> - <fsummary>Get the corresponding Mnesia key from an SNMP index.</fsummary> + <fsummary>Gets the corresponding <c>Mnesia</c> key from an SNMP index.</fsummary> <type> <v>Tab ::= atom()</v> <v>RowIndex ::= [integer()]</v> @@ -2041,44 +1983,43 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]), <v>key() ::= integer() | string() | [integer()]</v> </type> <desc> - <p>Transforms an SNMP index to the corresponding Mnesia key. - If the SNMP table has multiple keys, the key is a tuple of - the key columns.</p> + <p>Transforms an SNMP index to the corresponding <c>Mnesia</c> + key. If the SNMP table has multiple keys, the key is a tuple + of the key columns.</p> </desc> </func> <func> <name>snmp_get_next_index(Tab, RowIndex) -> {ok, NextIndex} | endOfTable</name> - <fsummary>Get the index of the next lexicographical row.</fsummary> + <fsummary>Gets the index of the next lexicographical row.</fsummary> <type> <v>Tab ::= atom()</v> <v>RowIndex ::= [integer()]</v> <v>NextIndex ::= [integer()]</v> </type> <desc> - <p>The <c>RowIndex</c> may specify a non-existing row. - Specifically, it might be the empty list. Returns the index + <p><c>RowIndex</c> can specify a non-existing row. + Specifically, it can be the empty list. Returns the index of the next lexicographical row. If <c>RowIndex</c> is the - empty list, this function will return the index of the first row + empty list, this function returns the index of the first row in the table.</p> </desc> </func> <func> <name>snmp_get_row(Tab, RowIndex) -> {ok, Row} | undefined</name> - <fsummary>Retrieve a row indexed by an SNMP index.</fsummary> + <fsummary>Retrieves a row indexed by an SNMP index.</fsummary> <type> <v>Tab ::= atom()</v> <v>RowIndex ::= [integer()]</v> <v>Row ::= record(Tab)</v> </type> <desc> - <p>Makes it possible to read a row by its SNMP index. This - index is specified as an SNMP OBJECT IDENTIFIER, a list of - integers.</p> + <p>Reads a row by its SNMP index. This index is specified as + an SNMP Object Identifier, a list of integers.</p> </desc> </func> <func> <name>snmp_open_table(Tab, SnmpStruct) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Organize a Mnesia table as an SNMP table.</fsummary> + <fsummary>Organizes a <c>Mnesia</c> table as an SNMP table.</fsummary> <type> <v>Tab ::= atom()</v> <v>SnmpStruct ::= [{key, type()}]</v> @@ -2086,624 +2027,558 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]), <v>type_spec() ::= fix_string | string | integer</v> </type> <desc> - <p>It is possible to establish a direct one to one mapping - between Mnesia tables and SNMP tables. Many - telecommunication applications are controlled and monitored - by the SNMP protocol. This connection between Mnesia and - SNMP makes it simple and convenient to achieve this. - </p> - <p>The <c>SnmpStruct</c> argument is a list of SNMP + <p>A direct one-to-one mapping can be established between + <c>Mnesia</c> tables and SNMP tables. Many telecommunication + applications are controlled and monitored by the SNMP + protocol. This connection between <c>Mnesia</c> and SNMP + makes it simple and convenient to achieve this mapping.</p> + <p>Argument <c>SnmpStruct</c> is a list of SNMP information. Currently, the only information needed is - information about the key types in the table. It is not - possible to handle multiple keys in Mnesia, but many SNMP + information about the key types in the table. Multiple + keys cannot be handled in <c>Mnesia</c>, but many SNMP tables have multiple keys. Therefore, the following convention is used: if a table has multiple keys, these must - always be stored as a tuple of the keys. Information about + always be stored as a tuple of the keys. Information about the key types is specified as a tuple of atoms describing - the types. The only significant type is - <c>fix_string</c>. This means that a string has fixed - size. For example: - </p> + the types. The only significant type is <c>fix_string</c>. + This means that a string has a fixed size.</p> + <p>For example, the following causes table <c>person</c> + to be ordered as an SNMP table:</p> <code type="none"> -mnesia:snmp_open_table(person, [{key, string}]) - </code> - <p>causes the <c>person</c> table to be ordered as an SNMP - table. - </p> +mnesia:snmp_open_table(person, [{key, string}])</code> <p>Consider the following schema for a table of company employees. Each employee is identified by department number - and name. The other table column stores the telephone number: - </p> + and name. The other table column stores the telephone + number:</p> <code type="none"> mnesia:create_table(employee, [{snmp, [{key, {integer, string}}]}, - {attributes, record_info(fields, employees)}]), - </code> - <p>The corresponding SNMP table would have three columns; - <c>department</c>, <c>name</c> and <c>telno</c>. - </p> - <p>It is possible to have table columns that are not visible + {attributes, record_info(fields, employees)}]),</code> + <p>The corresponding SNMP table would have three columns: + <c>department</c>, <c>name</c>, and <c>telno</c>.</p> + <p>An option is to have table columns that are not visible through the SNMP protocol. These columns must be the last columns of the table. In the previous example, the SNMP table could have columns <c>department</c> and <c>name</c> - only. The application could then use the <c>telno</c> column + only. The application could then use column <c>telno</c> internally, but it would not be visible to the SNMP - managers. - </p> + managers.</p> <p>In a table monitored by SNMP, all elements must be - integers, strings, or lists of integers. - </p> + integers, strings, or lists of integers.</p> <p>When a table is SNMP ordered, modifications are more - expensive than usual, O(logN). And more memory is used. - </p> - <p><em>Note:</em>Only the lexicographical SNMP ordering is - implemented in Mnesia, not the actual SNMP monitoring.</p> + expensive than usual, O(logN). Also, more memory is used.</p> + <p>Notice that only the lexicographical SNMP ordering is + implemented in <c>Mnesia</c>, not the actual SNMP monitoring.</p> </desc> </func> <func> - <name>start() -> ok | {error, Reason} </name> - <fsummary>Start a local Mnesia system.</fsummary> + <name>start() -> ok | {error, Reason}</name> + <fsummary>Starts a local <c>Mnesia</c> system.</fsummary> <desc> - <p>The start-up procedure for a set of Mnesia nodes is a - fairly complicated operation. A Mnesia system consists of a set - of nodes, with Mnesia started locally on all + <marker id="start"></marker> + <p>The startup procedure for a set of <c>Mnesia</c> nodes is a + fairly complicated operation. A <c>Mnesia</c> system consists + of a set of nodes, with <c>Mnesia</c> started locally on all participating nodes. Normally, each node has a directory where - all the Mnesia files are written. This directory will be - referred to as the Mnesia directory. Mnesia may also be - started on disc-less nodes. See <c>mnesia:create_schema/1</c> - and the Mnesia User's Guide for more information about disc-less - nodes. - </p> - <p>The set of nodes which makes up a Mnesia system is kept in - a schema and it is possible to add and remove Mnesia nodes + all the <c>Mnesia</c> files are written. This directory is + referred to as the <c>Mnesia</c> directory. <c>Mnesia</c> can + also be started on disc-less nodes. For more information + about disc-less nodes, see <c>mnesia:create_schema/1</c> + and the User's Guide.</p> + <p>The set of nodes that makes up a <c>Mnesia</c> system is kept + in a schema. <c>Mnesia</c> nodes can be added to or removed from the schema. The initial schema is normally created on disc with the function <c>mnesia:create_schema/1</c>. On disc-less nodes, a tiny default schema is generated each time - Mnesia is started. During the start-up procedure, Mnesia - will exchange schema information between the nodes in order - to verify that the table definitions are compatible. - </p> - <p>Each schema has a unique cookie which may be regarded as a + <c>Mnesia</c> is started. During the startup procedure, + <c>Mnesia</c> exchanges schema information between the nodes + to verify that the table definitions are compatible.</p> + <p>Each schema has a unique cookie, which can be regarded as a unique schema identifier. The cookie must be the same on all - nodes where Mnesia is supposed to run. See the Mnesia - User's Guide for more information about these details. - </p> - <p>The schema file, as well as all other files which Mnesia - needs, are kept in the Mnesia directory. The command line - option <c>-mnesia dir Dir</c> can be used to specify the - location of this directory to the Mnesia system. If no such - command line option is found, the name of the directory - defaults to <c>Mnesia.Node</c>. - </p> - <p><c>application:start(mnesia)</c> may also be used.</p> + nodes where <c>Mnesia</c> is supposed to run. For details, + see the User's Guide.</p> + <p>The schema file and all other files that <c>Mnesia</c> + needs are kept in the <c>Mnesia</c> directory. The + command-line option <c>-mnesia dir Dir</c> can be used to + specify the location of this directory to the <c>Mnesia</c> + system. If no such command-line option is found, the name + of the directory defaults to <c>Mnesia.Node</c>.</p> + <p><c>application:start(mnesia)</c> can also be used.</p> </desc> </func> <func> - <name>stop() -> stopped </name> - <fsummary>Stop Mnesia locally.</fsummary> + <name>stop() -> stopped</name> + <fsummary>Stops <c>Mnesia</c> locally.</fsummary> <desc> - <p>Stops Mnesia locally on the current node. - </p> - <p><c>application:stop(mnesia)</c> may also be used.</p> + <marker id="stop"></marker> + <p>Stops <c>Mnesia</c> locally on the current node.</p> + <p><c>application:stop(mnesia)</c> can also be used.</p> </desc> </func> <func> - <name>subscribe(EventCategory) -> {ok, Node} | {error, Reason} </name> - <fsummary>Subscribe to events of type <c>EventCategory</c>.</fsummary> + <name>subscribe(EventCategory) -> {ok, Node} | {error, Reason}</name> + <fsummary>Subscribes to events of type <c>EventCategory</c>.</fsummary> <desc> + <marker id="subscribe"></marker> <p>Ensures that a copy of all events of type - <c>EventCategory</c> are sent to the caller. The event - types available are described in the Mnesia User's Guide at <seealso marker="Mnesia_chap5#event_handling">Mnesia Event Handling</seealso>.</p> - <p><c>Node</c> is the local node. For table events to be subscribed, mnesia must have a readable local copy of the table on the node.</p> + <c>EventCategory</c> is sent to the caller. The available + event types are described in the <seealso marker="Mnesia_chap5#event_handling">User's Guide</seealso>.</p> </desc> </func> <func> - <name>sync_dirty(Fun, [, Args]) -> ResultOfFun | exit(Reason) </name> - <fsummary>Call the Fun in a context which is not protected by a transaction.</fsummary> + <name>sync_dirty(Fun, [, Args]) -> ResultOfFun | exit(Reason)</name> + <fsummary>Calls the <c>Fun</c> in a context that is not protected by a transaction.</fsummary> <desc> - <p>Call the <c>Fun</c> in a context which is not protected - by a transaction. The Mnesia function calls performed in the - <c>Fun</c> are mapped to the corresponding dirty functions. + <marker id="sync_dirty"></marker> + <p>Calls the <c>Fun</c> in a context that is not protected by + a transaction. The <c>Mnesia</c> function calls performed in + the <c>Fun</c> are mapped to the corresponding dirty functions. It is performed in almost the same context as <c>mnesia:async_dirty/1,2</c>. The difference is that the operations are performed synchronously. The caller waits for the updates to be performed on all active replicas before - the <c>Fun</c> returns. See <c>mnesia:activity/4</c> and the - Mnesia User's Guide for more details.</p> + the <c>Fun</c> returns. For details, see + <c>mnesia:activity/4</c> and the User's Guide.</p> </desc> </func> <func> - <name>sync_log() -> ok | {error, Reason} </name> - <fsummary>Perform a file sync of the local log file.</fsummary> + <name>sync_log() -> ok | {error, Reason}</name> + <fsummary>Performs a file sync of the local log file.</fsummary> <desc> <p>Ensures that the local transaction log file is synced to disk. - On a single node system data written to disk tables, since the last dump, - can be lost in case of a power outage. - See <seealso marker="#dump_log/0">dump_log/0</seealso>. - </p> + On a single node system, data written to disk tables since the + last dump can be lost if there is a power outage. + See <seealso marker="#dump_log/0">dump_log/0</seealso>.</p> </desc> </func> - <func> - <name>sync_transaction(Fun, [[, Args], Retries]) -> {aborted, Reason} | {atomic, ResultOfFun} </name> - <fsummary>Synchronously execute a transaction.</fsummary> + <name>sync_transaction(Fun, [[, Args], Retries]) -> {aborted, Reason} | {atomic, ResultOfFun}</name> + <fsummary>Synchronously executes a transaction.</fsummary> <desc> - <p>This function waits until data have been committed and + <marker id="sync_transaction"></marker> + <p>Waits until data have been committed and logged to disk (if disk is used) on every involved node before - it returns, otherwise it behaves as + it returns, otherwise it behaves as <c>mnesia:transaction/[1,2,3]</c>.</p> - <p>This functionality can be used to avoid that one process may overload - a database on another node.</p> + <p>This functionality can be used to avoid that one process + overloads a database on another node.</p> </desc> </func> <func> <name>system_info(InfoKey) -> Info | exit({aborted, Reason})</name> - <fsummary>Return information about the Mnesia system</fsummary> + <fsummary>Returns information about the <c>Mnesia</c> system.</fsummary> <desc> - <p>Returns information about the Mnesia system, such as - transaction statistics, db_nodes, and configuration parameters. - Valid keys are:</p> + <marker id="system_info"></marker> + <p>Returns information about the <c>Mnesia</c> system, such as + transaction statistics, <c>db_nodes</c>, and configuration + parameters. The valid keys are as follows:</p> <list type="bulleted"> <item> - <p><c>all</c>. This argument returns a list of all - local system information. Each element is a - <c>{InfoKey, InfoVal}</c> tuples.<em>Note:</em> New <c>InfoKey</c>'s may - be added and old undocumented <c>InfoKey</c>'s may be removed without + <p><c>all</c>. Returns a list of all local system + information. Each element is a <c>{InfoKey, InfoVal}</c> + tuple.</p> + <p>New <c>InfoKey</c>s can be added and old + undocumented <c>InfoKey</c>s can be removed without notice.</p> </item> <item> - <p><c>access_module</c>. This argument returns the name of - the module which is configured to be the activity access - callback module. - </p> + <p><c>access_module</c>. Returns the name of module that is + configured to be the activity access callback module.</p> </item> <item> - <p><c>auto_repair</c>. This argument returns - <c>true</c> or <c>false</c> to indicate if Mnesia is - configured to invoke the auto repair facility on corrupted - disc files. - </p> + <p><c>auto_repair</c>. Returns <c>true</c> or <c>false</c> + to indicate if <c>Mnesia</c> is configured to start the + auto-repair facility on corrupted disc files.</p> </item> <item> - <p><c>backup_module</c>. This argument returns the name of - the module which is configured to be the backup - callback module. - </p> + <p><c>backup_module</c>. Returns the name of the module + that is configured to be the backup callback module.</p> </item> <item> - <p><c>checkpoints</c>. This argument - returns a list of the names of the - checkpoints currently active on this node. - </p> + <p><c>checkpoints</c>. Returns a list of the names of the + checkpoints currently active on this node.</p> </item> <item> - <p><c>event_module</c>. This argument returns the name of - the module which is the event handler callback module. - </p> + <p><c>event_module</c>. Returns the name of the module + that is the event handler callback module.</p> </item> <item> - <p><c>db_nodes</c>. This argument returns - the nodes which make up the persistent database. Disc - less nodes will only be included in the list of nodes if - they explicitly has been added to the schema, e.g. with + <p><c>db_nodes</c>. Returns the nodes that make up the + persistent database. Disc-less nodes are only included + in the list of nodes if they explicitly have been added + to the schema, for example, with <c>mnesia:add_table_copy/3</c>. The function can be - invoked even if Mnesia is not yet running. - </p> + started even if <c>Mnesia</c> is not yet running.</p> </item> <item> - <p><c>debug</c>. This argument returns the current - debug level of Mnesia. - </p> + <p><c>debug</c>. Returns the current debug level of + <c>Mnesia</c>.</p> </item> <item> - <p><c>directory</c>. This argument returns the name of - the Mnesia directory. It can be invoked even if Mnesia is - not yet running. - </p> + <p><c>directory</c>. Returns the name of the <c>Mnesia</c> + directory. It can be called even if <c>Mnesia</c> is + not yet running.</p> </item> <item> - <p><c>dump_log_load_regulation</c>. This argument - returns a boolean which tells whether Mnesia is - configured to load regulate the dumper process or not. - This feature is temporary and will disappear in future - releases. - </p> + <p><c>dump_log_load_regulation</c>. Returns a boolean that + tells if <c>Mnesia</c> is configured to regulate the + dumper process load.</p> + <p>This feature is temporary and will be removed in future + releases.</p> </item> <item> - <p><c>dump_log_time_threshold</c>. This argument - returns the time threshold for transaction log dumps in - milliseconds. - </p> + <p><c>dump_log_time_threshold</c>. Returns the time + threshold for transaction log dumps in milliseconds.</p> </item> <item> - <p><c>dump_log_update_in_place</c>. This argument - returns a boolean which tells whether Mnesia is - configured to perform the updates in the dets files - directly or if the updates should be performed in a copy - of the dets files. - </p> + <p><c>dump_log_update_in_place</c>. Returns a boolean that + tells if <c>Mnesia</c> is configured to perform the + updates in the <c>dets</c> files directly, or if the + updates are to be performed in a copy of the <c>dets</c> + files.</p> </item> <item> - <p><c>dump_log_write_threshold</c>. This argument - returns the write threshold for transaction log dumps as - the number of writes to the transaction log. - </p> + <p><c>dump_log_write_threshold</c>. + Returns the write threshold for transaction log dumps as + the number of writes to the transaction log.</p> </item> <item> - <p><c>extra_db_nodes</c>. This argument returns a list - of extra db_nodes to be contacted at start-up. - </p> + <p><c>extra_db_nodes</c>. Returns a list + of extra <c>db_nodes</c> to be contacted at startup.</p> </item> <item> - <p><c>fallback_activated</c>. This argument returns - true if a fallback is activated, otherwise false. - </p> + <p><c>fallback_activated</c>. Returns <c>true</c> + if a fallback is activated, otherwise <c>false</c>.</p> </item> <item> - <p><c>held_locks</c>. This argument returns a list of - all locks held by the local Mnesia lock manager. - </p> + <p><c>held_locks</c>. Returns a list of all + locks held by the local <c>Mnesia</c> lock manager.</p> </item> <item> - <p><c>is_running</c>. This argument returns <c>yes</c> - or <c>no</c> to indicate if Mnesia is running. It may - also return <c>starting</c> or <c>stopping</c>. Can be - invoked even if Mnesia is not yet running. - </p> + <p><c>is_running</c>. Returns <c>yes</c> or <c>no</c> to + indicate if <c>Mnesia</c> is running. It can + also return <c>starting</c> or <c>stopping</c>. Can be + called even if <c>Mnesia</c> is not yet running.</p> </item> <item> - <p><c>local_tables</c>. This argument returns a list - of all tables which are configured to reside locally. - </p> + <p><c>local_tables</c>. Returns a list + of all tables that are configured to reside locally.</p> </item> <item> - <p><c>lock_queue</c>. This argument returns a list of + <p><c>lock_queue</c>. Returns a list of all transactions that are queued for execution by the - local lock manager. - </p> + local lock manager.</p> </item> <item> - <p><c>log_version</c>. This argument returns the - version number of the Mnesia transaction log format. - </p> + <p><c>log_version</c>. Returns the version + number of the <c>Mnesia</c> transaction log format.</p> </item> <item> - <p><c>master_node_tables</c>. This argument returns a - list of all tables with at least one master node. - </p> + <p><c>master_node_tables</c>. Returns a + list of all tables with at least one master node.</p> </item> <item> - <p><c>protocol_version</c>. This argument - returns the version number - of the Mnesia inter-process communication protocol. - </p> + <p><c>protocol_version</c>. Returns the version number of + the <c>Mnesia</c> inter-process communication protocol.</p> </item> <item> - <p><c>running_db_nodes</c>. This argument returns a - list of nodes where Mnesia currently is running. This - function can be invoked even if Mnesia is not yet - running, but it will then have slightly different - semantics. If Mnesia is down on the local node, the - function will return those other <c>db_nodes</c> and - <c>extra_db_nodes</c> that for the moment are up and - running. If Mnesia is started, the function will return - those nodes that Mnesia on the local node is fully - connected to. Only those nodes that Mnesia has exchanged - schema information with are included as + <p><c>running_db_nodes</c>. Returns a list of nodes where + <c>Mnesia</c> currently is running. This function can be + called even if <c>Mnesia</c> is not yet running, but it + then has slightly different semantics.</p> + <p>If <c>Mnesia</c> is down on the local node, the function + returns those other <c>db_nodes</c> and + <c>extra_db_nodes</c> that for the moment are + operational.</p> + <p>If <c>Mnesia</c> is started, the function returns + those nodes that <c>Mnesia</c> on the local node is fully + connected to. Only those nodes that <c>Mnesia</c> has + exchanged schema information with are included as <c>running_db_nodes</c>. After the merge of schemas, the - local Mnesia system is fully operable and applications - may perform access of remote replicas. Before the schema - merge Mnesia will only operate locally. Sometimes there - may be more nodes included in the + local <c>Mnesia</c> system is fully operable and + applications can perform access of remote replicas. + Before the schema merge, <c>Mnesia</c> only operates + locally. Sometimes there are more nodes included in the <c>running_db_nodes</c> list than all <c>db_nodes</c> - and <c>extra_db_nodes</c> together. - </p> + and <c>extra_db_nodes</c> together.</p> </item> <item> - <p><c>schema_location</c>. This argument returns the - initial schema location. - </p> + <p><c>schema_location</c>. Returns the + initial schema location.</p> </item> <item> - <p><c>subscribers</c>. This argument returns a list of - local processes currently subscribing to system events. - </p> + <p><c>subscribers</c>. Returns a list of + local processes currently subscribing to system events.</p> </item> <item> - <p><c>tables</c>. This argument returns a list of all - locally known tables. - </p> + <p><c>tables</c>. Returns a list of all + locally known tables.</p> </item> <item> - <p><c>transactions</c>. This argument returns a list - of all currently active local transactions. - </p> + <p><c>transactions</c>. Returns a list + of all currently active local transactions.</p> </item> <item> - <p><c>transaction_failures</c>. This argument returns - a number which indicates how many transactions have - failed since Mnesia was started. - </p> + <p><c>transaction_failures</c>. Returns a + number that indicates how many transactions have + failed since <c>Mnesia</c> was started.</p> </item> <item> - <p><c>transaction_commits</c>. This argument returns a - number which indicates how many transactions have - terminated successfully since Mnesia was started. - </p> + <p><c>transaction_commits</c>. Returns a + number that indicates how many transactions have + terminated successfully since <c>Mnesia</c> was started.</p> </item> <item> - <p><c>transaction_restarts</c>. This argument returns - a number which indicates how many transactions have been - restarted since Mnesia was started. - </p> + <p><c>transaction_restarts</c>. Returns a + number that indicates how many transactions have been + restarted since <c>Mnesia</c> was started.</p> </item> <item> - <p><c>transaction_log_writes</c>. This argument - returns a number which indicates the number of write - operation that have been performed to the transaction - log since start-up. - </p> + <p><c>transaction_log_writes</c>. + Returns a number that indicates how many write + operations that have been performed to the transaction + log since startup.</p> </item> <item> - <p><c>use_dir</c>. This argument returns a boolean - which indicates whether the Mnesia directory is used or - not. Can be invoked even if Mnesia is not yet running. - </p> + <p><c>use_dir</c>. Returns a boolean that indicates if + the <c>Mnesia</c> directory is used or not. Can be + started even if <c>Mnesia</c> is not yet running.</p> </item> <item> - <p><c>version</c>. This argument returns the current - version number of Mnesia. - </p> + <p><c>version</c>. Returns the current + version number of <c>Mnesia</c>.</p> </item> </list> </desc> </func> <func> - <name>table(Tab [,[Option]]) -> QueryHandle </name> + <name>table(Tab [,[Option]]) -> QueryHandle</name> <fsummary>Return a QLC query handle.</fsummary> <desc> - <p><marker id="qlc_table"></marker> -Returns a QLC (Query List Comprehension) query handle, see - <seealso marker="stdlib:qlc">qlc(3)</seealso>.The module <c>qlc</c> implements a query language, it - can use mnesia tables as sources of data. Calling - <c>mnesia:table/1,2</c> is the means to make the <c>mnesia</c> - table <c>Tab</c> usable to QLC.</p> - <p>The list of Options may contain mnesia options or QLC - options, the following options are recognized by Mnesia: - <c>{traverse, SelectMethod},{lock, Lock},{n_objects,Number}</c>, any other option is forwarded - to QLC. The <c>lock</c> option may be <c>read</c> or - <c>write</c>, default is <c>read</c>. The option - <c>n_objects</c> specifies (roughly) the number of objects - returned from mnesia to QLC. Queries to remote tables may - need a larger chunks to reduce network overhead, default - <c>100</c> objects at a time are returned. The option - <c>traverse</c> determines the method to traverse the whole - table (if needed), the default method is <c>select</c>:</p> + <marker id="qlc_table"></marker> + <marker id="table"></marker> + <p>Returns a Query List Comprehension (QLC) query handle, + see the <seealso marker="stdlib:qlc">qlc(3)</seealso> + manual page in <c>STDLIB</c>. The module <c>qlc</c> + implements a query language that can use <c>Mnesia</c> + tables as sources of data. Calling + <c>mnesia:table/1,2</c> is the means to make the + <c>mnesia</c> table <c>Tab</c> usable to QLC.</p> + <p><c>Option</c> can contain <c>Mnesia</c> + options or QLC options. <c>Mnesia</c> recognizes the + following options (any other option is forwarded to + QLC).</p> + <list type="bulleted"> + <item><c>{lock, Lock}</c>, where <c>lock</c> can be + <c>read</c> or <c>write</c>. Default is <c>read</c>. + </item> + <item><c>{n_objects,Number}</c>, where <c>n_objects</c> + specifies (roughly) the number of objects returned + from <c>Mnesia</c> to QLC. Queries to remote tables + can need a larger chunk to reduce network overhead. + By default, <c>100</c> objects at a time are returned. + </item> + <item><c>{traverse, SelectMethod}</c>, where + <c>traverse</c> determines the method to traverse + the whole table (if needed). The default method is + <c>select</c>. + </item> + </list> + <p>There are two alternatives for <c>select</c>:</p> <list type="bulleted"> <item> <p><c>select</c>. The table is traversed by calling - <c>mnesia:select/4</c> and <c>mnesia:select/1</c>. The - match specification (the second argument of <c>select/3</c>) - is assembled by QLC: simple filters are - translated into equivalent match specifications while - more complicated filters have to be applied to all + <c>mnesia:select/4</c> and <c>mnesia:select/1</c>. + The match specification (the second argument of + <c>select/3</c>) is assembled by QLC: simple filters + are translated into equivalent match specifications. + More complicated filters need to be applied to all objects returned by <c>select/3</c> given a match specification that matches all objects.</p> </item> <item> - <p><c>{select, MatchSpec}</c>. As for <c>select</c> - the table is traversed by calling <c>mnesia:select/3</c> and - <c>mnesia:select/1</c>. The difference is that the match - specification is explicitly given. This is how to state - match specifications that cannot easily be expressed - within the syntax provided by QLC.</p> + <p><c>{select, MatchSpec}</c>. As for <c>select</c>, + the table is traversed by calling <c>mnesia:select/3</c> + and <c>mnesia:select/1</c>. The difference is that the + match specification is explicitly given. This is how to + state match specifications that cannot easily be + expressed within the syntax provided by QLC.</p> </item> </list> </desc> </func> <func> <name>table_info(Tab, InfoKey) -> Info | exit({aborted, Reason})</name> - <fsummary>Return local information about table.</fsummary> + <fsummary>Returns local information about table.</fsummary> <desc> + <marker id="table_info"></marker> <p>The <c>table_info/2</c> function takes two arguments. - The first is the name of a Mnesia table, the second is one of - the following keys: - </p> + The first is the name of a <c>Mnesia</c> table. + The second is one of the following keys:</p> <list type="bulleted"> <item> - <p><c>all</c>. This argument returns a list of all - local table information. Each element is a <c>{InfoKey, ItemVal}</c> tuples. <em>Note:</em> New <c>InfoItem</c>'s may be - added and old undocumented <c>InfoItem</c>'s may be removed without - notice.</p> + <p><c>all</c>. Returns a list of all local table + information. Each element is a + <c>{InfoKey, ItemVal}</c> tuple.</p> + <p>New <c>InfoItem</c>s can be added and old undocumented + <c>InfoItem</c>s can be removed without notice.</p> </item> <item> - <p><c>access_mode</c>. This argument returns the - access mode of the table. The access mode may either be - read_only or read_write. - </p> + <p><c>access_mode</c>. Returns the + access mode of the table. The access mode can be + <c>read_only</c> or <c>read_write</c>.</p> </item> <item> - <p><c>arity</c>. This argument returns the arity of - records in the table as specified in the schema. - </p> + <p><c>arity</c>. Returns the arity of + records in the table as specified in the schema.</p> </item> <item> - <p><c>attributes</c>. This argument returns the table - attribute names which are specified in the schema. - </p> + <p><c>attributes</c>. Returns the table + attribute names that are specified in the schema.</p> </item> <item> - <p><c>checkpoints</c>. This argument returns the names - of the currently active checkpoints which involves this - table on this node. - </p> + <p><c>checkpoints</c>. Returns the names + of the currently active checkpoints, which involve this + table on this node.</p> </item> <item> - <p><c>cookie</c>. This argument returns a table cookie - which is a unique system generated identifier for the + <p><c>cookie</c>. Returns a table cookie, + which is a unique system-generated identifier for the table. The cookie is used internally to ensure that two different table definitions using the same table name cannot accidentally be intermixed. The cookie is - generated when the table is initially created. - </p> + generated when the table is created initially.</p> </item> <item> - <p><c>disc_copies</c>. This argument returns the nodes - where a disc_copy of the table resides according to the - schema. - </p> + <p><c>disc_copies</c>. Returns the nodes where a + <c>disc_copy</c> of the table resides according to the + schema.</p> </item> <item> - <p><c>disc_only_copies </c>. This argument returns the - nodes where a disc_only_copy of the table resides - according to the schema. - </p> + <p><c>disc_only_copies</c>. Returns the nodes where a + <c>disc_only_copy</c> of the table resides + according to the schema.</p> </item> <item> - <p><c>index</c>. This argument returns the list of - index position integers for the table. - </p> + <p><c>index</c>. Returns the list of + index position integers for the table.</p> </item> <item> - <p><c>load_node</c>. This argument returns the name of - the node that Mnesia loaded the table from. The - structure of the returned value is unspecified but may - be useful for debugging purposes. - </p> + <p><c>load_node</c>. Returns the name of + the node that <c>Mnesia</c> loaded the table from. The + structure of the returned value is unspecified, but + can be useful for debugging purposes.</p> </item> <item> - <p><c>load_order</c>. This argument returns the load + <p><c>load_order</c>. Returns the load order priority of the table. It is an integer and - defaults to <c>0</c> (zero). - </p> + defaults to <c>0</c> (zero).</p> </item> <item> - <p><c>load_reason</c>. This argument returns the - reason of why Mnesia decided to load the table. The - structure of the returned value is unspecified but may - be useful for debugging purposes. - </p> + <p><c>load_reason</c>. Returns the + reason of why <c>Mnesia</c> decided to load the table. + The structure of the returned value is unspecified, but + can be useful for debugging purposes.</p> </item> <item> - <p><c>local_content</c>. This argument returns - <c>true</c> or <c>false</c> to indicate whether the - table is configured to have locally unique content on - each node. - </p> + <p><c>local_content</c>. Returns <c>true</c> or + <c>false</c> to indicate if the table is configured to + have locally unique content on each node.</p> </item> <item> - <p><c>master_nodes</c>. This argument returns the - master nodes of a table. - </p> + <p><c>master_nodes</c>. Returns the master nodes of a + table.</p> </item> <item> - <p><c>memory</c>. This argument returns the number of - words allocated to the table on this node. - </p> + <p><c>memory</c>. Returns the number of + words allocated to the table on this node.</p> </item> <item> - <p><c>ram_copies</c>. This argument returns the nodes - where a ram_copy of the table resides according to the - schema. - </p> + <p><c>ram_copies</c>. Returns the nodes where a + <c>ram_copy</c> of the table resides according to the + schema.</p> </item> <item> - <p><c>record_name</c>. This argument returns the - record name, common for all records in the table - </p> + <p><c>record_name</c>. Returns the + record name, common for all records in the table.</p> </item> <item> - <p><c>size</c>. This argument returns the number of - records inserted in the table. - </p> + <p><c>size</c>. Returns the number of + records inserted in the table.</p> </item> <item> - <p><c>snmp</c>. This argument returns the SNMP struct. - <c>[]</c>meaning that the table currently has no SNMP - properties. - </p> + <p><c>snmp</c>. Returns the SNMP struct. <c>[]</c> means + that the table currently has no SNMP properties.</p> </item> <item> - <p><c>storage_type</c>.This argument returns the local + <p><c>storage_type</c>. Returns the local storage type of the table. It can be <c>disc_copies</c>, <c>ram_copies</c>, <c>disc_only_copies</c>, or the atom <c>unknown</c>. <c>unknown</c> is returned for all - tables which only reside remotely. - </p> + tables that only reside remotely.</p> </item> <item> - <p><c>subscribers</c>. This argument returns a list + <p><c>subscribers</c>. Returns a list of local processes currently subscribing to local table - events which involve this table on this node. - </p> + events that involve this table on this node.</p> </item> <item> - <p><c>type</c>. This argument returns the table type, - which is either <c>bag</c>, <c>set</c> or <c>ordered_set</c>.. - </p> + <p><c>type</c>. Returns the table type, which is + <c>bag</c>, <c>set</c>, or <c>ordered_set</c>.</p> </item> <item> - <p><c>user_properties</c>. This argument returns the - user associated table properties of the table. It is a - list of the stored property records. - </p> + <p><c>user_properties</c>. Returns the + user-associated table properties of the table. It is a + list of the stored property records.</p> </item> <item> - <p><c>version</c>. This argument returns the current + <p><c>version</c>. Returns the current version of the table definition. The table version is incremented when the table definition is changed. The - table definition may be incremented directly when the - table definition has been changed in a schema - transaction, or when a committed table definition is - merged with table definitions from other nodes during - start-up. - </p> + table definition can be incremented directly when it + has been changed in a schema transaction, or + when a committed table definition is merged with + table definitions from other nodes during startup.</p> </item> <item> - <p><c>where_to_read</c>.This argument returns the node - where the table can be read. If the value <c>nowhere</c> - is returned, the table is not loaded, or it resides at a - remote node which is not running. - </p> + <p><c>where_to_read</c>. Returns the node + where the table can be read. If value <c>nowhere</c> + is returned, either the table is not loaded or it + resides at a remote node that is not running.</p> </item> <item> - <p><c>where_to_write</c>. This argument returns a list - of the nodes that currently hold an active replica of - the table. - </p> + <p><c>where_to_write</c>. Returns a list of the nodes + that currently hold an active replica of the table.</p> </item> <item> - <p><c>wild_pattern</c>. This argument returns a - structure which can be given to the various match - functions for a certain table. A record tuple is where all - record fields have the value <c>'_'</c>. - </p> + <p><c>wild_pattern</c>. Returns a + structure that can be given to the various match + functions for a certain table. A record tuple is where + all record fields have value <c>'_'</c>.</p> </item> </list> </desc> </func> <func> <name>transaction(Fun [[, Args], Retries]) -> {aborted, Reason} | {atomic, ResultOfFun}</name> - <fsummary>Execute a transaction.</fsummary> + <fsummary>Executes a transaction.</fsummary> <desc> - <p>This function executes the functional object <c>Fun</c> - with arguments <c>Args</c> as a transaction. - </p> - <p>The code which executes inside the transaction + <marker id="transaction"></marker> + <p>Executes the functional object <c>Fun</c> + with arguments <c>Args</c> as a transaction.</p> + <p>The code that executes inside the transaction can consist of a series of table manipulation functions. - If something goes wrong inside the transaction as a result of a - user error or a certain table not being available, the - entire transaction is aborted and the function - <c>transaction/1</c> returns the tuple - <c>{aborted, Reason}</c>. - </p> - <p>If all is well, <c>{atomic, ResultOfFun}</c> is returned where - <c>ResultOfFun</c> is the value of the last expression in - <c>Fun</c>. - </p> - <p>A function which adds a family to the database can be - written as follows if we have a structure <c>{family, Father, Mother, ChildrenList}</c>: - </p> + If something goes wrong inside the transaction as a result + of a user error or a certain table not being available, the + entire transaction is terminated and the function + <c>transaction/1</c> returns the tuple + <c>{aborted, Reason}</c>.</p> + <p>If all is going well, <c>{atomic, ResultOfFun}</c> is + returned, where <c>ResultOfFun</c> is the value of the + last expression in <c>Fun</c>.</p> + <p>A function that adds a family to the database can be + written as follows if there is a structure + <c>{family, Father, Mother, ChildrenList}</c>:</p> <code type="none"> add_family({family, F, M, Children}) -> ChildOids = lists:map(fun oid/1, Children), @@ -2715,23 +2590,20 @@ add_family({family, F, M, Children}) -> end, mnesia:transaction(Trans). -oid(Rec) -> {element(1, Rec), element(2, Rec)}. - </code> - <p>This code adds a set of people to the database. Running this code - within one transaction will ensure that either the whole - family is added to the database, or the whole transaction - aborts. For example, if the last child is badly formatted, - or the executing process terminates due to an +oid(Rec) -> {element(1, Rec), element(2, Rec)}.</code> + <p>This code adds a set of people to the database. Running + this code within one transaction ensures that either the whole + family is added to the database, or the whole transaction + terminates. For example, if the last child is badly formatted, + or the executing process terminates because of an <c>'EXIT'</c> signal while executing the family code, the - transaction aborts. Accordingly, the situation where half a - family is added can never occur. - </p> + transaction terminates. Thus, the situation where half a + family is added can never occur.</p> <p>It is also useful to update the database within a transaction if several processes concurrently update the same records. - For example, the function <c>raise(Name, Amount)</c>, which - adds <c>Amount</c> to the salary field of a person, should - be implemented as follows: - </p> + For example, the function <c>raise(Name, Amount)</c>, which + adds <c>Amount</c> to the salary field of a person, is to + be implemented as follows:</p> <code type="none"> raise(Name, Amount) -> mnesia:transaction(fun() -> @@ -2743,71 +2615,68 @@ raise(Name, Amount) -> _ -> mnesia:abort("No such person") end - end). - </code> - <p>When this function executes within a transaction, + end).</code> + <p>When this function executes within a transaction, several processes running on different nodes can concurrently - execute the <c>raise/2</c> function without interfering - with each other. - </p> - <p>Since Mnesia detects deadlocks, a transaction can be - restarted any number of times. This function will attempt a restart as specified in - <c>Retries</c>. <c>Retries</c> must - be an integer greater than 0 or the atom <c>infinity</c>. Default is - <c>infinity</c>.</p> + execute the function <c>raise/2</c> without interfering + with each other.</p> + <p>Since <c>Mnesia</c> detects deadlocks, a transaction can be + restarted any number of times. This function attempts a + restart as specified in <c>Retries</c>. <c>Retries</c> must + be an integer greater than 0 or the atom <c>infinity</c>. + Default is <c>infinity</c>.</p> </desc> </func> <func> - <name>transform_table(Tab, Fun, NewAttributeList, NewRecordName) -> {aborted, R} | {atomic, ok} </name> - <fsummary>Change format on all records in table. <c>Tab</c></fsummary> + <name>transform_table(Tab, Fun, NewAttributeList, NewRecordName) -> {aborted, R} | {atomic, ok}</name> + <fsummary>Changes format on all records in table <c>Tab</c>.</fsummary> <desc> - <p>This function applies the argument <c>Fun</c> to all - records in the table. <c>Fun</c> is a function which takes a - record of the old type and returns a transformed record of the - new type. The <c>Fun</c> argument can also be the atom - <c>ignore</c>, it indicates that only the meta data about the table will - be updated. Usage of <c>ignore</c> is not recommended but included - as a possibility for the user to do his own transform. - <c>NewAttributeList</c> and <c>NewRecordName</c> - specifies the attributes and the new record type of converted - table. Table name will always remain unchanged, if the - record_name is changed only the mnesia functions which - uses table identifiers will work, e.g. <c>mnesia:write/3</c> - will work but <c>mnesia:write/1</c> will not.</p> + <marker id="transform_table_4"></marker> + <p>Applies argument <c>Fun</c> to all + records in the table. <c>Fun</c> is a function that takes a + record of the old type and returns a transformed record of + the new type. Argument <c>Fun</c> can also be the atom + <c>ignore</c>, which indicates that only the metadata + about the table is updated. Use of + <c>ignore</c> is not recommended, but included + as a possibility for the user do to an own transformation.</p> + <p><c>NewAttributeList</c> and <c>NewRecordName</c> + specify the attributes and the new record type of the + converted table. Table name always remains unchanged. If + <c>record_name</c> is changed, only the <c>Mnesia</c> + functions that use table identifiers work, for example, + <c>mnesia:write/3</c> works, but not <c>mnesia:write/1</c>.</p> </desc> </func> <func> - <name>transform_table(Tab, Fun, NewAttributeList) -> {aborted, R} | {atomic, ok} </name> - <fsummary>Change format on all records in table. <c>Tab</c></fsummary> + <name>transform_table(Tab, Fun, NewAttributeList) -> {aborted, R} | {atomic, ok}</name> + <fsummary>Changes format on all records in table <c>Tab</c>.</fsummary> <desc> - <p>Invokes <c>mnesia:transform_table(Tab, Fun, NewAttributeList, RecName)</c> - where <c>RecName</c> is <c>mnesia:table_info(Tab, record_name)</c>.</p> + <p>Calls <c>mnesia:transform_table(Tab, Fun, + NewAttributeList, RecName)</c>, where <c>RecName</c> is + <c>mnesia:table_info(Tab, record_name)</c>.</p> </desc> </func> <func> <name>traverse_backup(Source, [SourceMod,] Target, [TargetMod,] Fun, Acc) -> {ok, LastAcc} | {error, Reason}</name> <fsummary>Traversal of a backup.</fsummary> <desc> - <p>With this function it is possible to iterate over a backup, - either for the purpose of transforming it into a new backup, - or just reading it. The arguments are explained briefly - below. See the Mnesia User's Guide for additional - details. - </p> + <marker id="traverse_backup"></marker> + <p>Iterates over a backup, either to transform it into a + new backup, or read it. The arguments are explained briefly + here. For details, see the User's Guide.</p> <list type="bulleted"> <item><c>SourceMod</c> and <c>TargetMod</c> are the names of - the modules which actually access the backup - media. + the modules that actually access the backup media. </item> <item><c>Source</c> and <c>Target</c> are opaque data used - exclusively by the modules <c>SourceMod</c> and - <c>TargetMod</c> for the purpose of initializing the - backup media. + exclusively by modules <c>SourceMod</c> and <c>TargetMod</c> + to initialize the backup media. </item> <item><c>Acc</c> is an initial accumulator value. </item> <item><c>Fun(BackupItems, Acc)</c> is applied to each item in - the backup. The Fun must return a tuple + the backup. The <c>Fun</c> must return a tuple <c>{BackupItems,NewAcc}</c>, where <c>BackupItems</c> is a list of valid backup items, and <c>NewAcc</c> is a new accumulator value. The returned backup items are written @@ -2821,202 +2690,190 @@ raise(Name, Amount) -> </func> <func> <name>uninstall_fallback() -> ok | {error,Reason}</name> - <fsummary>Uninstall a fallback.</fsummary> + <fsummary>Uninstalls a fallback.</fsummary> <desc> - <p>Invokes <c>mnesia:uninstall_fallback([{scope, global}])</c>.</p> + <marker id="uninstall_fallback_0"></marker> + <p>Calls the function + <c>mnesia:uninstall_fallback([{scope, global}])</c>.</p> </desc> </func> <func> <name>uninstall_fallback(Args) -> ok | {error,Reason}</name> - <fsummary>Uninstall a fallback.</fsummary> + <fsummary>Uninstalls a fallback.</fsummary> <desc> - <p>This function is used to de-install a fallback before it + <p>Deinstalls a fallback before it has been used to restore the database. This is normally a distributed operation that is either performed on all - nodes with disc resident schema or none. Uninstallation of - fallbacks requires Erlang to be up and running on all - involved nodes, but it does not matter if Mnesia is running - or not. Which nodes that are considered as disc-resident - nodes is determined from the schema info in the local - fallback. - </p> - <p><c>Args</c> is a list of the following tuples: - </p> + nodes with disc resident schema, or none. Uninstallation of + fallbacks requires Erlang to be operational on all + involved nodes, but it does not matter if <c>Mnesia</c> is + running or not. Which nodes that are considered as + disc-resident nodes is determined from the schema + information in the local fallback.</p> + <p><c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> - <item> - <p><c>{module, BackupMod}</c>. - See <c>mnesia:install_fallback/2</c> about the - semantics.</p> + <item><c>{module, BackupMod}</c>. For semantics, + see <c>mnesia:install_fallback/2</c>. </item> - <item> - <p><c>{scope, Scope}</c> - See <c>mnesia:install_fallback/2</c> about the - semantics.</p> + <item><c>{scope, Scope}</c>. For semantics, + see <c>mnesia:install_fallback/2</c>. </item> - <item> - <p><c>{mnesia_dir, AlternateDir}</c> - See <c>mnesia:install_fallback/2</c> about the - semantics.</p> + <item><c>{mnesia_dir, AlternateDir}</c>. For semantics, + see <c>mnesia:install_fallback/2</c>. </item> </list> </desc> </func> <func> - <name>unsubscribe(EventCategory) -> {ok, Node} | {error, Reason} </name> - <fsummary>Subscribe to events of type <c>EventCategory</c>.</fsummary> + <name>unsubscribe(EventCategory) -> {ok, Node} | {error, Reason}</name> + <fsummary>Subscribes to events of type <c>EventCategory</c>.</fsummary> <desc> + <marker id="unsubscribe"></marker> <p>Stops sending events of type <c>EventCategory</c> to the caller.</p> <p><c>Node</c> is the local node.</p> </desc> </func> <func> - <name>wait_for_tables(TabList,Timeout) -> ok | {timeout, BadTabList} | {error, Reason} </name> - <fsummary>Wait for tables to be accessible.</fsummary> + <name>wait_for_tables(TabList, Timeout) -> ok | {timeout, BadTabList} | {error, Reason}</name> + <fsummary>Waits for tables to be accessible.</fsummary> <desc> - <p>Some applications need to wait for certain tables to - be accessible in order to do useful work. - <c>mnesia:wait_for_tables/2</c> hangs until all tables in the - <c>TabList</c> are accessible, or until <c>timeout</c> is - reached.</p> + <marker id="wait_for_tables"></marker> + <p>Some applications need to wait for certain tables to be + accessible to do useful work. <c>mnesia:wait_for_tables/2</c> + either hangs until all tables in <c>TabList</c> are accessible, + or until <c>timeout</c> is reached.</p> </desc> </func> <func> - <name>wread({Tab, Key}) -> transaction abort | RecordList </name> - <fsummary>Read records with given key.</fsummary> + <name>wread({Tab, Key}) -> transaction abort | RecordList</name> + <fsummary>Reads records with given key.</fsummary> <desc> - <p>Invoke <c>mnesia:read(Tab, Key, write)</c>.</p> + <marker id="wread"></marker> + <p>Calls the function <c>mnesia:read(Tab, Key, write)</c>.</p> </desc> </func> <func> - <name>write(Record) -> transaction abort | ok </name> + <name>write(Record) -> transaction abort | ok</name> <fsummary>Writes a record into the database.</fsummary> <desc> - <p>Invoke <c>mnesia:write(Tab, Record, write)</c> where - <c>Tab</c> is <c>element(1, Record)</c>.</p> + <marker id="write_1"></marker> + <p>Calls the function <c>mnesia:write(Tab, Record, write)</c>, + where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> <func> - <name>write(Tab, Record, LockKind) -> transaction abort | ok </name> - <fsummary>Write a record into the database.</fsummary> + <name>write(Tab, Record, LockKind) -> transaction abort | ok</name> + <fsummary>Writes a record into the database.</fsummary> <desc> - <p>Writes the record <c>Record</c> to the table <c>Tab</c>. - </p> - <p>The function returns <c>ok</c>, or aborts if an error - occurs. For example, the transaction aborts if no - <c>person</c> table exists. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type <c>LockKind</c>. The - following lock types are supported: <c>write</c> and - <c>sticky_write</c>.</p> + <marker id="write_3"></marker> + <p>Writes record <c>Record</c> to table <c>Tab</c>.</p> + <p>The function returns <c>ok</c>, or terminates if an error + occurs. For example, the transaction terminates if no + <c>person</c> table exists.</p> + <p>The semantics of this function is context-sensitive. For + details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type + <c>LockKind</c>. The lock types <c>write</c> and + <c>sticky_write</c> are supported.</p> </desc> </func> <func> <name>write_lock_table(Tab) -> ok | transaction abort</name> - <fsummary>Set write lock on an entire table.</fsummary> + <fsummary>Sets write lock on an entire table.</fsummary> <desc> - <p>Invokes <c>mnesia:lock({table, Tab}, write)</c>.</p> + <marker id="write_lock_table"></marker> + <p>Calls the function + <c>mnesia:lock({table, Tab}, write)</c>.</p> </desc> </func> </funcs> <section> <title>Configuration Parameters</title> - <p>Mnesia reads the following application configuration + <marker id="configuration_parameters"></marker> + <p><c>Mnesia</c> reads the following application configuration parameters:</p> <list type="bulleted"> <item> - <p><c>-mnesia access_module Module</c>. The - name of the Mnesia activity access callback module. The default is - <c>mnesia</c>. - </p> + <p><c>-mnesia access_module Module</c>. The name of the + <c>Mnesia</c> activity access callback module. Default is + <c>mnesia</c>.</p> </item> <item> - <p><c>-mnesia auto_repair true | false</c>. This flag controls - whether Mnesia will try to automatically repair - files that have not been properly closed. The default is - <c>true</c>. - </p> + <p><c>-mnesia auto_repair true | false</c>. This flag + controls if <c>Mnesia</c> automatically tries to repair + files that have not been properly closed. Default is + <c>true</c>.</p> </item> <item> - <p><c>-mnesia backup_module Module</c>. The - name of the Mnesia backup callback module. The default is - <c>mnesia_backup</c>. - </p> + <p><c>-mnesia backup_module Module</c>. The name of the + <c>Mnesia</c> backup callback module. Default is + <c>mnesia_backup</c>.</p> </item> <item> - <p><c>-mnesia debug Level</c> - Controls the debug level of Mnesia. - Possible values are:</p> + <p><c>-mnesia debug Level</c>. Controls the debug level + of <c>Mnesia</c>. The possible values are as follows:</p> <taglist> <tag><c>none</c></tag> <item> - <p>No trace outputs at all. This is the default setting. - </p> + <p>No trace outputs. This is the default.</p> </item> <tag><c>verbose</c></tag> <item> <p>Activates tracing of important debug events. These - debug events generate <c>{mnesia_info, Format, Args}</c> - system events. Processes may subscribe to these events with - <c>mnesia:subscribe/1</c>. The events are always sent to Mnesia's - event handler. - </p> + events generate <c>{mnesia_info, Format, Args}</c> + system events. Processes can subscribe to these events with + <c>mnesia:subscribe/1</c>. The events are always sent to + the <c>Mnesia</c> event handler.</p> </item> <tag><c>debug</c></tag> <item> <p>Activates all events at the verbose level plus full trace of all debug events. These debug events generate - <c>{mnesia_info, Format, Args}</c> system events. Processes may - subscribe to these events with <c>mnesia:subscribe/1</c>. The - events are always sent to the Mnesia event handler. On this - debug level, the Mnesia event handler starts subscribing to - updates in the schema table. - </p> + <c>{mnesia_info, Format, Args}</c> system events. + Processes can subscribe to these events with + <c>mnesia:subscribe/1</c>. The events are always sent to + the <c>Mnesia</c> event handler. On this debug level, + the <c>Mnesia</c> event handler starts subscribing to + updates in the schema table.</p> </item> <tag><c>trace</c></tag> <item> - <p>Activates all events at the level debug. On this - debug level, the Mnesia event handler starts subscribing to - updates on all Mnesia tables. This level is only intended - for debugging small toy systems since many large - events may be generated. - </p> + <p>Activates all events at the debug level. On this + level, the <c>Mnesia</c> event handler starts subscribing + to updates on all <c>Mnesia</c> tables. This level is + intended only for debugging small toy systems, as many + large events can be generated.</p> </item> <tag><c>false</c></tag> - <item> - <p>An alias for none. - </p> + <item>An alias for none. </item> <tag><c>true</c></tag> - <item> - <p>An alias for debug. - </p> + <item>An alias for debug. </item> </taglist> </item> <item> <p><c>-mnesia core_dir Directory</c>. The name of the - directory where Mnesia core files is stored or - false. Setting it implies that also ram only nodes, will - generate a core file if a crash occurs. </p> + directory where <c>Mnesia</c> core files is stored, or + false. Setting it implies that also RAM-only nodes + generate a core file if a crash occurs.</p> </item> <item> - <p><c>-mnesia dc_dump_limit Number</c>. - Controls how often <c>disc_copies</c> tables are dumped from memory. + <p><c>-mnesia dc_dump_limit Number</c>. Controls how often + <c>disc_copies</c> tables are dumped from memory. Tables are dumped when <c>filesize(Log) > (filesize(Tab)/Dc_dump_limit)</c>. - Lower values reduces cpu overhead but increases disk space and - startup times. The default is 4.</p> + Lower values reduce CPU overhead but increase disk space + and startup times. Default is 4.</p> </item> <item> <p><c>-mnesia dir Directory</c>. The name of the directory - where all Mnesia data is stored. The name of the directory must - be unique for the current node. Two nodes may, under no - circumstances, share the same Mnesia directory. The results are - totally unpredictable.</p> + where all <c>Mnesia</c> data is stored. The directory name + must be unique for the current node. Two nodes must never + share the the same <c>Mnesia</c> directory. The results + are unpredictable.</p> </item> <item> <p><c>-mnesia dump_disc_copies_at_startup true | false</c>. @@ -3026,156 +2883,148 @@ raise(Name, Amount) -> </item> <item> <p><c>-mnesia dump_log_load_regulation true | false</c>. - Controls if the log dumps should be performed as fast as - possible or if the dumper should do its own load - regulation. This feature is temporary and will disappear in a - future release. The default is <c>false</c>. - </p> + Controls if log dumps are to be performed as fast as + possible, or if the dumper is to do its own load + regulation. Default is <c>false</c>.</p> + <p>This feature is temporary and will be removed in a + future release</p> </item> <item> <p><c>-mnesia dump_log_update_in_place true | false</c>. - Controls if log dumps are performed on a copy of - the original data file, or if the log dump is - performed on the original data file. The default is <c>true</c></p> + Controls if log dumps are performed on a copy of the + original data file, or if the log dump is performed + on the original data file. Default is <c>true</c></p> </item> <item> - <marker id=" dump_log_write_threshold"></marker> - <p><c>-mnesia dump_log_write_threshold Max</c>, where - <c>Max</c> is an integer which specifies the maximum number of writes - allowed to the transaction log before a new dump of the log - is performed. It defaults to 100 log writes. - </p> + <marker id=" dump_log_write_threshold"></marker> + <p><c>-mnesia dump_log_write_threshold Max</c>. + <c>Max</c> is an integer that specifies the maximum + number of writes allowed to the transaction log before + a new dump of the log is performed. Default is <c>100</c> + log writes.</p> </item> <item> - <marker id=" dump_log_time_threshold"></marker> - <p><c>-mnesia dump_log_time_threshold Max</c>, - where <c>Max</c> is an integer which - specifies the dump log interval in milliseconds. It defaults - to 3 minutes. If a dump has not been performed within - <c>dump_log_time_threshold</c> milliseconds, then a new dump is - performed regardless of how many writes have been - performed. - </p> + <marker id=" dump_log_time_threshold"></marker> + <p><c>-mnesia dump_log_time_threshold Max</c>. + <c>Max</c> is an integer that specifies the dump log + interval in milliseconds. Default is 3 minutes. If a + dump has not been performed within + <c>dump_log_time_threshold</c> milliseconds, a new dump + is performed regardless of the number of writes + performed.</p> </item> <item> - <p><c>-mnesia event_module Module</c>. The - name of the Mnesia event handler callback module. The default is - <c>mnesia_event</c>. - </p> + <p><c>-mnesia event_module Module</c>. The name of the + <c>Mnesia</c> event handler callback module. Default is + <c>mnesia_event</c>.</p> </item> <item> <p><c>-mnesia extra_db_nodes Nodes</c> specifies a list of - nodes, in addition to the ones found in the schema, with which - Mnesia should also establish contact. The default value - is the empty list <c>[]</c>. - </p> + nodes, in addition to the ones found in the schema, with + which <c>Mnesia</c> is also to establish contact. Default + is <c>[]</c> (empty list).</p> </item> <item> - <p><c>-mnesia fallback_error_function {UserModule, UserFunc}</c> - specifies a user supplied callback function - which will be called if a fallback is installed and mnesia - goes down on another node. Mnesia will call the function - with one argument the name of the dying node, e.g. - <c>UserModule:UserFunc(DyingNode)</c>. - Mnesia should be restarted or else - the database could be inconsistent. - The default behaviour is to terminate mnesia. - </p> + <p><c>-mnesia fallback_error_function {UserModule, UserFunc}</c>. + Specifies a user-supplied callback function, which is + called if a fallback is installed and <c>Mnesia</c> goes + down on another node. <c>Mnesia</c> calls the function + with one argument, the name of the dying node, for example, + <c>UserModule:UserFunc(DyingNode)</c>. <c>Mnesia</c> must + be restarted, otherwise the database can be inconsistent. + The default behavior is to terminate <c>Mnesia</c>.</p> </item> <item> <p><c>-mnesia max_wait_for_decision Timeout</c>. Specifies - how long Mnesia will wait for other nodes to share their - knowledge regarding the outcome of an unclear transaction. By - default the <c>Timeout</c> is set to the atom - <c>infinity</c>, which implies that if Mnesia upon startup - encounters a "heavyweight transaction" whose outcome is - unclear, the local Mnesia will wait until Mnesia is started - on some (in worst cases all) of the other nodes that were - involved in the interrupted transaction. This is a very rare - situation, but when/if it happens, Mnesia does not guess if - the transaction on the other nodes was committed or aborted. - Mnesia will wait until it knows the outcome and then act - accordingly. - </p> + how long <c>Mnesia</c> waits for other nodes to share their + knowledge about the outcome of an unclear transaction. By + default, <c>Timeout</c> is set to the atom <c>infinity</c>. + This implies that if <c>Mnesia</c> upon startup detects + a "heavyweight transaction" whose outcome is unclear, the + local <c>Mnesia</c> waits until <c>Mnesia</c> is started + on some (in the worst case all) of the other nodes that were + involved in the interrupted transaction. This is a rare + situation, but if it occurs, <c>Mnesia</c> does not guess if + the transaction on the other nodes was committed or + terminated. <c>Mnesia</c> waits until it knows the outcome + and then acts accordingly.</p> <p>If <c>Timeout</c> is set to an integer value in - milliseconds, Mnesia will force "heavyweight transactions" + milliseconds, <c>Mnesia</c> forces "heavyweight transactions" to be finished, even if the outcome of the transaction for the moment is unclear. After <c>Timeout</c> milliseconds, - Mnesia will commit/abort the transaction and continue with - the startup. This may lead to a situation where the - transaction is committed on some nodes and aborted on other - nodes. If the transaction was a schema transaction, the - inconsistency may be fatal. - </p> + <c>Mnesia</c> commits or terminates the transaction and + continues with the startup. This can lead to a situation + where the transaction is committed on some nodes and + terminated on other nodes. If the transaction is a + schema transaction, the inconsistency can be fatal.</p> </item> <item> - <p><c>-mnesia no_table_loaders NUMBER</c> specifies the number of - parallel table loaders during start. More loaders can be good if the - network latency is high or if many tables contains few records. - The default value is <c>2</c>. - </p> + <p><c>-mnesia no_table_loaders NUMBER</c>. Specifies the number + of parallel table loaders during start. More loaders can be + good if the network latency is high or if many tables + contain few records. Default is <c>2</c>.</p> </item> <item> - <p><c>-mnesia send_compressed Level</c> specifies the level of - compression to be used when copying a table from the local node to - another one. The default level is 0. - </p> - <p><c>Level</c> must be an integer in the interval [0, 9], with 0 - representing no compression and 9 representing maximum compression. - Before setting it to a non-zero value, make sure the remote nodes - understand this configuration. - </p> + <p><c>-mnesia send_compressed Level</c>. Specifies the level of + compression to be used when copying a table from the local + node to another one. Default is <c>0</c>.</p> + <p><c>Level</c> must be an integer in the interval + <c>[0, 9]</c>, where <c>0</c> means no compression and + <c>9</c> means maximum compression. Before setting it to a + non-zero value, ensure that the remote nodes + understand this configuration.</p> </item> <item> - <p><c>-mnesia schema_location Loc</c> controls where - Mnesia will look for its schema. The parameter - <c>Loc</c> may be one of the following atoms: </p> + <p><c>-mnesia schema_location Loc</c>. Controls where + <c>Mnesia</c> looks for its schema. Parameter + <c>Loc</c> can be one of the following atoms:</p> <taglist> <tag><c>disc</c></tag> <item> <p>Mandatory disc. The schema is assumed to be located - in the Mnesia directory. If the schema cannot be found, - Mnesia refuses to start. This is the old behavior. - </p> + in the <c>Mnesia</c> directory. If the schema cannot + be found, <c>Mnesia</c> refuses to start. This is the + old behavior.</p> </item> <tag><c>ram</c></tag> <item> <p>Mandatory RAM. The schema resides in RAM - only. At start-up, a tiny new schema is generated. This - default schema just contains the definition of the schema - table and only resides on the local node. Since no other - nodes are found in the default schema, the configuration - parameter <c>extra_db_nodes</c> must be used in - order to let the - node share its table definitions with other nodes. (The - <c>extra_db_nodes</c> parameter may also be used on disc based nodes.) - </p> + only. At startup, a tiny new schema is generated. This + default schema only contains the definition of the schema + table and only resides on the local node. Since no other + nodes are found in the default schema, configuration + parameter <c>extra_db_nodes</c> must be used to let the + node share its table definitions with other nodes.</p> + <p>Parameter <c>extra_db_nodes</c> can also be + used on disc based nodes.</p> </item> <tag><c>opt_disc</c></tag> <item> - <p>Optional disc. The schema may reside either on disc - or in RAM. If the schema is found on disc, Mnesia starts as a - disc based node and the storage type of the schema table is - <c>disc_copies</c>. If no schema is found on disc, Mnesia starts - as a disc-less node and the storage type of the schema table is - <c>ram_copies</c>. The default value for the application parameter - is <c>opt_disc</c>. - </p> + <p>Optional disc. The schema can reside on disc or in + RAM. If the schema is found on disc, <c>Mnesia</c> + starts as a disc-based node and the storage type of + the schema table is <c>disc_copies</c>. If no schema is + found on disc, <c>Mnesia</c> starts as a disc-less node + and the storage type of the schema table is + <c>ram_copies</c>. Default value for the application + parameter is <c>opt_disc</c>.</p> </item> </taglist> </item> </list> - <p>First the SASL application parameters are checked, then - the command line flags are checked, and finally, the default - value is chosen. - </p> + <p>First, the <c>SASL</c> application parameters are checked, + then the command-line flags are checked, and finally, the + default value is chosen.</p> </section> <section> <title>See Also</title> - <p>mnesia_registry(3), mnesia_session(3), qlc(3), - dets(3), ets(3), disk_log(3), application(3) - </p> + <p><seealso marker="kernel:application">application(3)</seealso>, + <seealso marker="stdlib:dets">dets(3)</seealso>, + <seealso marker="kernel:disk_log">disk_log(3)</seealso>, + <seealso marker="stdlib:ets">ets(3)</seealso>, + <seealso marker="mnesia:mnesia_registry">mnesia_registry(3)</seealso>, + <seealso marker="stdlib:qlc">qlc(3)</seealso></p> </section> </erlref> diff --git a/lib/mnesia/doc/src/mnesia_frag_hash.xml b/lib/mnesia/doc/src/mnesia_frag_hash.xml index 0d660925e7..fe6d847655 100644 --- a/lib/mnesia/doc/src/mnesia_frag_hash.xml +++ b/lib/mnesia/doc/src/mnesia_frag_hash.xml @@ -34,22 +34,23 @@ <file>mnesia_frag_hash.sgml</file> </header> <module>mnesia_frag_hash</module> - <modulesummary>Defines mnesia_frag_hash callback behaviour</modulesummary> + <modulesummary>Defines mnesia_frag_hash callback behavior</modulesummary> <description> - <p>The module <c>mnesia_frag_hash</c> defines a callback - behaviour for user defined hash functions of fragmented tables.</p> + <p>This module defines a callback behavior for user-defined hash + functions of fragmented tables.</p> <p>Which module that is selected to implement the <c>mnesia_frag_hash</c> - behaviour for a particular fragmented table is specified together + behavior for a particular fragmented table is specified together with the other <c>frag_properties</c>. The <c>hash_module</c> defines the module name. The <c>hash_state</c> defines the initial hash state.</p> - <p>It implements dynamic hashing which is a kind of hashing + <p>This module implements dynamic hashing, which is a kind of hashing that grows nicely when new fragments are added. It is well - suited for scalable hash tables</p> + suited for scalable hash tables.</p> </description> + <funcs> <func> <name>init_state(Tab, State) -> NewState | abort(Reason)</name> - <fsummary>Initiate the hash state for a new table</fsummary> + <fsummary>Initiates the hash state for a new table.</fsummary> <type> <v>Tab = atom()</v> <v>State = term()</v> @@ -57,21 +58,21 @@ <v>Reason = term()</v> </type> <desc> - <p>This function is invoked when a fragmented table is - created with <c>mnesia:create_table/2</c> or when a - normal (un-fragmented) table is converted to be a + <p>Starts when a fragmented table is + created with the function <c>mnesia:create_table/2</c> or + when a normal (unfragmented) table is converted to be a fragmented table with <c>mnesia:change_table_frag/2</c>.</p> - <p>Note that the <c>add_frag/2</c> function will be invoked - one time each for the rest of the fragments (all but number 1) + <p>Notice that the function <c>add_frag/2</c> is started + one time for each of the other fragments (except number 1) as a part of the table creation procedure.</p> - <p><c>State</c> is the initial value of the <c>hash_state</c> <c>frag_property</c>. The <c>NewState</c> will be stored as - <c>hash_state</c> among the other <c>frag_properties</c>. - </p> + <p><c>State</c> is the initial value of the <c>hash_state</c> + <c>frag_property</c>. <c>NewState</c> is stored as + <c>hash_state</c> among the other <c>frag_properties</c>.</p> </desc> </func> <func> <name>add_frag(State) -> {NewState, IterFrags, AdditionalLockFrags} | abort(Reason)</name> - <fsummary>This function is invoked when a new fragment is added to a fragmented table</fsummary> + <fsummary>Starts when a new fragment is added to a fragmented table.</fsummary> <type> <v>State = term()</v> <v>NewState = term()</v> @@ -80,27 +81,26 @@ <v>Reason = term()</v> </type> <desc> - <p>In order to scale well, it is a good idea ensure that the - records are evenly distributed over all fragments including + <p>To scale well, it is a good idea to ensure that the + records are evenly distributed over all fragments, including the new one.</p> - <p>The <c>NewState</c> will be stored as <c>hash_state</c> among the - other <c>frag_properties</c>. - </p> - <p>As a part of the <c>add_frag</c> procedure, Mnesia will iterate + <p><c>NewState</c> is stored as <c>hash_state</c> among the + other <c>frag_properties</c>.</p> + <p>As a part of the <c>add_frag</c> procedure, <c>Mnesia</c> iterates over all fragments corresponding to the <c>IterFrags</c> numbers - and invoke <c>key_to_frag_number(NewState,RecordKey)</c> for + and starts <c>key_to_frag_number(NewState,RecordKey)</c> for each record. If the new fragment differs from the old - fragment, the record will be moved to the new fragment.</p> + fragment, the record is moved to the new fragment.</p> <p>As the <c>add_frag</c> procedure is a part of a schema - transaction Mnesia will acquire a write locks on the - affected tables. That is both the fragments corresponding + transaction, <c>Mnesia</c> acquires write locks on the + affected tables. That is, both the fragments corresponding to <c>IterFrags</c> and those corresponding to <c>AdditionalLockFrags</c>.</p> </desc> </func> <func> <name>del_frag(State) -> {NewState, IterFrags, AdditionalLockFrags} | abort(Reason)</name> - <fsummary>This function is invoked when a fragment is deleted from a fragmented table</fsummary> + <fsummary>Starts when a fragment is deleted from a fragmented table.</fsummary> <type> <v>State = term()</v> <v>NewState = term()</v> @@ -109,39 +109,38 @@ <v>Reason = term()</v> </type> <desc> - <p>The <c>NewState</c> will be stored as <c>hash_state</c> among the - other <c>frag_properties</c>. - </p> - <p>As a part of the <c>del_frag</c> procedure, Mnesia will iterate + <p><c>NewState</c> is stored as <c>hash_state</c> among the + other <c>frag_properties</c>.</p> + <p>As a part of the <c>del_frag</c> procedure, <c>Mnesia</c> iterates over all fragments corresponding to the <c>IterFrags</c> numbers - and invoke <c>key_to_frag_number(NewState,RecordKey)</c> for + and starts <c>key_to_frag_number(NewState,RecordKey)</c> for each record. If the new fragment differs from the old - fragment, the record will be moved to the new fragment.</p> - <p>Note that all records in the last fragment must be moved to - another fragment as the entire fragment will be deleted.</p> + fragment, the record is moved to the new fragment.</p> + <p>Notice that all records in the last fragment must be moved to + another fragment, as the entire fragment is deleted.</p> <p>As the <c>del_frag</c> procedure is a part of a schema - transaction Mnesia will acquire a write locks on the - affected tables. That is both the fragments corresponding + transaction, <c>Mnesia</c> acquires write locks on the + affected tables. That is, both the fragments corresponding to <c>IterFrags</c> and those corresponding to <c>AdditionalLockFrags</c>.</p> </desc> </func> <func> <name>key_to_frag_number(State, Key) -> FragNum | abort(Reason)</name> - <fsummary>Resolves the key of a record into a fragment number</fsummary> + <fsummary>Resolves the key of a record into a fragment number.</fsummary> <type> <v>FragNum = integer()()</v> <v>Reason = term()</v> </type> <desc> - <p>This function is invoked whenever Mnesia needs to determine + <p>Starts whenever <c>Mnesia</c> needs to determine which fragment a certain record belongs to. It is typically - invoked at read, write and delete.</p> + started at <c>read</c>, <c>write</c>, and <c>delete</c>.</p> </desc> </func> <func> <name>match_spec_to_frag_numbers(State, MatchSpec) -> FragNums | abort(Reason)</name> - <fsummary>Resolves a MatchSpec into a list of fragment numbers</fsummary> + <fsummary>Resolves a <c>MatchSpec</c> into a list of fragment numbers.</fsummary> <type> <v>MatcSpec = ets_select_match_spec()</v> <v>FragNums = [FragNum]</v> @@ -149,17 +148,17 @@ <v>Reason = term()</v> </type> <desc> - <p>This function is invoked whenever Mnesia needs to determine - which fragments that needs to be searched for a MatchSpec. - It is typically invoked at select and match_object.</p> + <p>This function is called whenever <c>Mnesia</c> needs to determine + which fragments that need to be searched for a <c>MatchSpec</c>. + It is typically called by <c>select</c> and + <c>match_object</c>.</p> </desc> </func> </funcs> <section> <title>See Also</title> - <p>mnesia(3) - </p> + <p><seealso marker="mnesia:mnesia">mnesia(3)</seealso></p> </section> </erlref> diff --git a/lib/mnesia/doc/src/mnesia_registry.xml b/lib/mnesia/doc/src/mnesia_registry.xml index ad2b927315..c4b5bdc59a 100644 --- a/lib/mnesia/doc/src/mnesia_registry.xml +++ b/lib/mnesia/doc/src/mnesia_registry.xml @@ -34,71 +34,66 @@ <file>mnesia_registry.sgml</file> </header> <module>mnesia_registry</module> - <modulesummary>Dump support for registries in erl_interface. </modulesummary> + <modulesummary>Dump support for registries in erl_interface.</modulesummary> <description> - <p>The module <c>mnesia_registry</c> is usually part of - <c>erl_interface</c>, but for the time being, it is a part of the - Mnesia application. - </p> - <p><c>mnesia_registry</c> is mainly an module intended for - internal usage within OTP, but it has two functions that - are exported for public use. - </p> - <p>On C-nodes <c>erl_interface</c> has support for registry - tables. These reside in RAM on the C-node but they may also be - dumped into Mnesia tables. By default, the dumping of registry - tables via <c>erl_interface</c> causes a corresponding Mnesia - table to be created with <c>mnesia_registry:create_table/1</c> - if necessary. - </p> - <p>The tables that are created with these functions can be - administered as all other Mnesia tables. They may be included in - backups or replicas may be added etc. The tables are in fact - normal Mnesia tables owned by the user of the corresponding - <c>erl_interface</c> registries. - </p> + <p>This module is usually part of the <c>erl_interface</c> + application, but is currently part of the <c>Mnesia</c> + application.</p> + <p>This module is mainly intended for internal use within OTP, + but it has two functions that are exported for public use.</p> + <p>On C-nodes, <c>erl_interface</c> has support for registry + tables. These tables reside in RAM on the C-node, but can also + be dumped into <c>Mnesia</c> tables. By default, the dumping + of registry tables through <c>erl_interface</c> causes a + corresponding <c>Mnesia</c> table to be created with + <c>mnesia_registry:create_table/1</c>, if necessary.</p> + <p>Tables that are created with these functions can be + administered as all other <c>Mnesia</c> tables. They can be + included in backups, replicas can be added, and so on. + The tables are normal <c>Mnesia</c> tables owned by the user + of the corresponding <c>erl_interface</c> registries.</p> </description> + <funcs> <func> <name>create_table(Tab) -> ok | exit(Reason)</name> <fsummary>Creates a registry table in Mnesia.</fsummary> <desc> - <p>This is a wrapper function for - <c>mnesia:create_table/2</c> which creates a table (if there is no existing table) - with an appropriate set of <c>attributes</c>. The table will - only reside on the local node and its storage type will be - the same as the <c>schema</c> table on the local - node, ie. <c>{ram_copies,[node()]}</c> or - <c>{disc_copies,[node()]}</c>. - </p> - <p>It is this function that is used by <c>erl_interface</c> to - create the Mnesia table if it did not already exist.</p> + <p>A wrapper function for <c>mnesia:create_table/2</c>, + which creates a table (if there is no existing table) + with an appropriate set of <c>attributes</c>. The table + only resides on the local node and its storage type is + the same as the <c>schema</c> table on the local node, + that is, <c>{ram_copies,[node()]}</c> or + <c>{disc_copies,[node()]}</c>.</p> + <p>This function is used by <c>erl_interface</c> to + create the <c>Mnesia</c> table if it does not already + exist.</p> </desc> </func> <func> <name>create_table(Tab, TabDef) -> ok | exit(Reason)</name> - <fsummary>Creates a customized registry table in Mnesia. </fsummary> + <fsummary>Creates a customized registry table in Mnesia.</fsummary> <desc> - <p>This is a wrapper function for - <c>mnesia:create_table/2</c> which creates a table (if there is no existing table) - with an appropriate set of <c>attributes</c>. The attributes - and <c>TabDef</c> are forwarded to - <c>mnesia:create_table/2</c>. For example, if the table should - reside as <c>disc_only_copies</c> on all nodes a call would - look like:</p> + <p>A wrapper function for <c>mnesia:create_table/2</c>, + which creates a table (if there is no existing table) + with an appropriate set of <c>attributes</c>. The + attributes and <c>TabDef</c> are forwarded to + <c>mnesia:create_table/2</c>. For example, if the table + is to reside as <c>disc_only_copies</c> on all nodes, + a call looks as follows:</p> <code type="none"> TabDef = [{{disc_only_copies, node()|nodes()]}], - mnesia_registry:create_table(my_reg, TabDef) - </code> + mnesia_registry:create_table(my_reg, TabDef)</code> </desc> </func> </funcs> <section> <title>See Also</title> - <p>mnesia(3), erl_interface(3) - </p> + <p><seealso marker="erl_interface:index">erl_interface(3)</seealso>, + <seealso marker="mnesia:mnesia">mnesia(3)</seealso></p> </section> - + </erlref> diff --git a/lib/mnesia/doc/src/part.xml b/lib/mnesia/doc/src/part.xml index 2a16b0a791..ee458dcba0 100644 --- a/lib/mnesia/doc/src/part.xml +++ b/lib/mnesia/doc/src/part.xml @@ -29,12 +29,13 @@ <file>part.sgml</file> </header> <description> - <p><em>Mnesia</em> is a distributed DataBase Management - System(DBMS), appropriate for telecommunications applications and other - Erlang applications which require continuous operation and exhibit soft + <p>The <c>Mnesia</c> application is a distributed Database Management + System (DBMS), appropriate for telecommunications applications and other + Erlang applications, which require continuous operation and exhibit soft real-time properties.</p> </description> <xi:include href="Mnesia_chap1.xml"/> + <xi:include href="Mnesia_overview.xml"/> <xi:include href="Mnesia_chap2.xml"/> <xi:include href="Mnesia_chap3.xml"/> <xi:include href="Mnesia_chap4.xml"/> @@ -44,6 +45,5 @@ <xi:include href="Mnesia_App_A.xml"/> <xi:include href="Mnesia_App_B.xml"/> <xi:include href="Mnesia_App_C.xml"/> - <xi:include href="Mnesia_App_D.xml"/> </part> diff --git a/lib/mnesia/doc/src/ref_man.xml b/lib/mnesia/doc/src/ref_man.xml index e3c75be6e1..6cb91a5c40 100644 --- a/lib/mnesia/doc/src/ref_man.xml +++ b/lib/mnesia/doc/src/ref_man.xml @@ -32,10 +32,10 @@ <file>refman.sgml</file> </header> <description> - <p><em>Mnesia</em> is a distributed DataBase Management + <p>The <c>Mnesia</c> application is a distributed Database Management System (DBMS), appropriate for telecommunications applications and other - Erlang applications which require continuous operation and exhibit soft - real-time properties. </p> + Erlang applications, which require continuous operation and exhibit soft + real-time properties.</p> </description> <xi:include href="mnesia.xml"/> <xi:include href="mnesia_frag_hash.xml"/> diff --git a/lib/mnesia/src/mnesia_loader.erl b/lib/mnesia/src/mnesia_loader.erl index 65ea743fd3..492a061f76 100644 --- a/lib/mnesia/src/mnesia_loader.erl +++ b/lib/mnesia/src/mnesia_loader.erl @@ -209,8 +209,7 @@ do_get_network_copy(Tab, Reason, Ns, Storage, Cs) -> set({Tab, load_node}, Node), set({Tab, load_reason}, Reason), mnesia_controller:i_have_tab(Tab), - dbg_out("Table ~p copied from ~p to ~p (~b entries)~n", - [Tab, Node, node(), mnesia:table_info(Tab, size)]), + dbg_out("Table ~p copied from ~p to ~p~n", [Tab, Node, node()]), {loaded, ok}; Err = {error, _} when element(1, Reason) == dumper -> {not_loaded,Err}; diff --git a/lib/observer/src/observer_tv_wx.erl b/lib/observer/src/observer_tv_wx.erl index da4cb8e041..acfa6a2a99 100644 --- a/lib/observer/src/observer_tv_wx.erl +++ b/lib/observer/src/observer_tv_wx.erl @@ -176,10 +176,16 @@ handle_call(Event, From, _State) -> handle_cast(Event, _State) -> error({unhandled_cast, Event}). -handle_info(refresh_interval, State = #state{node=Node, grid=Grid, opt=Opt}) -> - Tables = get_tables(Node, Opt), - Tabs = update_grid(Grid, Opt, Tables), - {noreply, State#state{tabs=Tabs}}; +handle_info(refresh_interval, State = #state{node=Node, grid=Grid, opt=Opt, + tabs=OldTabs}) -> + case get_tables(Node, Opt) of + OldTabs -> + %% no change + {noreply, State}; + Tables -> + Tabs = update_grid(Grid, Opt, Tables), + {noreply, State#state{tabs=Tabs}} + end; handle_info({active, Node}, State = #state{parent=Parent, grid=Grid, opt=Opt, timer=Timer0}) -> diff --git a/lib/runtime_tools/doc/src/dbg.xml b/lib/runtime_tools/doc/src/dbg.xml index d31ccd834d..94556b7b51 100644 --- a/lib/runtime_tools/doc/src/dbg.xml +++ b/lib/runtime_tools/doc/src/dbg.xml @@ -167,7 +167,8 @@ Error: fun containing local erlang function calls ('is_atomm' called in guard)\ <item>If the <c>Item</c> is a <c>pid()</c>, the corresponding process is traced. The process may be a remote process (on another Erlang node). The node must be in the list of - traced nodes (<seealso marker="#n">see</seealso><c>n/1</c> and <c>tracer/0/2/3</c>).</item> + traced nodes (see <seealso marker="#n"><c>n/1</c></seealso> and + <c>tracer/0/2/3</c>).</item> <item>If the <c>Item</c> is the atom <c>all</c>, all processes in the system as well as all processes created hereafter are to be traced. This also affects all nodes added with the @@ -648,7 +649,7 @@ Error: fun containing local erlang function calls ('is_atomm' called in guard)\ <p>If <c>Nodename</c> is the local node, the error reason <c>cant_add_local_node</c> is returned. </p> - <p>If a trace port (<seealso marker="#trace_port">see</seealso><c>trace_port/2</c>) is + <p>If a trace port (see <seealso marker="#trace_port"><c>trace_port/2</c></seealso>) is running on the local node, remote nodes can not be traced with a tracer process. The error reason <c>cant_trace_remote_pid_to_local_port</c> is returned. A @@ -761,7 +762,7 @@ Error: fun containing local erlang function calls ('is_atomm' called in guard)\ type which is independent of the tracer on the trace control node.</p> </note> - <p>For details, <seealso marker="#tracer2">see</seealso><c>tracer/2</c>.</p> + <p>For details, see <seealso marker="#tracer2"><c>tracer/2</c></seealso>.</p> </desc> </func> <func> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index 5edd790400..c1235715cc 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -180,6 +180,14 @@ <p>Provides a fun to implement your own logging when a server disconnects the client.</p> </item> + <tag><c><![CDATA[{unexpectedfun, fun(Message:term(), Peer) -> report | skip }]]></c></tag> + <item> + <p>Provides a fun to implement your own logging or other action when an unexpected message arrives. + If the fun returns <c>report</c> the usual info report is issued but if <c>skip</c> is returned no + report is generated.</p> + <p><c>Peer</c> is in the format of <c>{Host,Port}</c>.</p> + </item> + <tag><c><![CDATA[{public_key_alg, 'ssh-rsa' | 'ssh-dss'}]]></c></tag> <item> <note> @@ -388,14 +396,14 @@ kex is implicit but public_key is set explicitly.</p> <tag><c><![CDATA[{auth_method_kb_interactive_data, PromptTexts}]]> <br/>where: - <br/>PromptTexts = kb_int_tuple() | fun(PeerName::string(), User::string(), Service::string()) -> kb_int_tuple() + <br/>PromptTexts = kb_int_tuple() | fun(Peer::{IP::tuple(),Port::integer()}, User::string(), Service::string()) -> kb_int_tuple() <br/>kb_int_tuple() = {Name::string(), Instruction::string(), Prompt::string(), Echo::boolean()}</c> </tag> <item> <p>Sets the text strings that the daemon sends to the client for presentation to the user when using <c>keyboar-interactive</c> authentication. If the fun/3 is used, it is called when the actual authentication occurs and may therefore return dynamic data like time, remote ip etc.</p> <p>The parameter <c>Echo</c> guides the client about need to hide the password.</p> <p>The default value is: - <c>{auth_method_kb_interactive_data, {"SSH server", "Enter password for \""++User++"\"", "pwd: ", false}></c></p> + <c>{auth_method_kb_interactive_data, {"SSH server", "Enter password for \""++User++"\"", "password: ", false}></c></p> </item> <tag><c><![CDATA[{user_passwords, [{string() = User, @@ -546,6 +554,14 @@ kex is implicit but public_key is set explicitly.</p> <p>Provides a fun to implement your own logging when a user disconnects from the server.</p> </item> + <tag><c><![CDATA[{unexpectedfun, fun(Message:term(), Peer) -> report | skip }]]></c></tag> + <item> + <p>Provides a fun to implement your own logging or other action when an unexpected message arrives. + If the fun returns <c>report</c> the usual info report is issued but if <c>skip</c> is returned no + report is generated.</p> + <p><c>Peer</c> is in the format of <c>{Host,Port}</c>.</p> + </item> + <tag><c><![CDATA[{ssh_msg_debug_fun, fun(ConnectionRef::ssh_connection_ref(), AlwaysDisplay::boolean(), Msg::binary(), LanguageTag::binary()) -> _}]]></c></tag> <item> <p>Provide a fun to implement your own logging of the SSH message SSH_MSG_DEBUG. The last three parameters are from the message, see RFC4253, section 11.3. The <c>ConnectionRef</c> is the reference to the connection on which the message arrived. The return value from the fun is not checked.</p> diff --git a/lib/ssh/doc/src/ssh_connection.xml b/lib/ssh/doc/src/ssh_connection.xml index 5422633dc3..e2b4e2ceb7 100644 --- a/lib/ssh/doc/src/ssh_connection.xml +++ b/lib/ssh/doc/src/ssh_connection.xml @@ -85,7 +85,7 @@ <tag><em>data_events()</em></tag> <item> <taglist> - <tag><c><![CDATA[{data, ssh_channel_id(), ssh_data_type_code(), binary() = Data}]]></c></tag> + <tag><c><![CDATA[{data, ssh_channel_id(), ssh_data_type_code(), Data :: binary()}]]></c></tag> <item><p>Data has arrived on the channel. This event is sent as a result of calling <seealso marker="ssh_connection#send-3"> ssh_connection:send/[3,4,5]</seealso>.</p></item> @@ -110,15 +110,15 @@ referred to are on OS-level and not something generated by an Erlang program.</p></item> - <tag><c><![CDATA[{exit_signal, ssh_channel_id(), string() = ExitSignal, string() = ErrorMsg, - string() = LanguageString}]]></c></tag> + <tag><c><![CDATA[{exit_signal, ssh_channel_id(), ExitSignal :: string(), ErrorMsg ::string(), + LanguageString :: string()}]]></c></tag> <item><p>A remote execution can terminate violently because of a signal. Then this message can be received. For details on valid string values, see <url href="http://www.ietf.org/rfc/rfc4254.txt">RFC 4254</url> Section 6.10, which shows a special case of these signals.</p></item> - <tag><c><![CDATA[{exit_status, ssh_channel_id(), integer() = ExitStatus}]]></c></tag> + <tag><c><![CDATA[{exit_status, ssh_channel_id(), ExitStatus :: integer()}]]></c></tag> <item><p>When the command running at the other end terminates, the following message can be sent to return the exit status of the command. A zero <c>exit_status</c> usually means that the command @@ -148,18 +148,18 @@ with the boolean value of <c>WantReply</c> as the second argument.</p> <taglist> - <tag><c><![CDATA[{env, ssh_channel_id(), boolean() = WantReply, - string() = Var, string() = Value}]]></c></tag> + <tag><c><![CDATA[{env, ssh_channel_id(), WantReply :: boolean(), + Var ::string(), Value :: string()}]]></c></tag> <item><p>Environment variables can be passed to the shell/command to be started later. This event is sent as a result of calling <seealso marker="ssh_connection#setenv-5"> ssh_connection:setenv/5</seealso>. </p></item> <tag><c><![CDATA[{pty, ssh_channel_id(), - boolean() = WantReply, {string() = Terminal, integer() = CharWidth, - integer() = RowHeight, integer() = PixelWidth, integer() = PixelHeight, - [{atom() | integer() = Opcode, - integer() = Value}] = TerminalModes}}]]></c></tag> + WantReply :: boolean(), {Terminal :: string(), CharWidth :: integer(), + RowHeight :: integer(), PixelWidth :: integer(), PixelHeight :: integer(), + TerminalModes :: [{Opcode :: atom() | integer(), + Value :: integer()}]}}]]></c></tag> <item><p>A pseudo-terminal has been requested for the session. <c>Terminal</c> is the value of the TERM environment variable value, that is, <c>vt100</c>. Zero dimension parameters must @@ -174,20 +174,20 @@ echo</c>. This event is sent as a result of calling <seealso marker="ssh_connection#ptty_alloc/4">ssh_connection:ptty_alloc/4</seealso>.</p></item> - <tag><c><![CDATA[{shell, boolean() = WantReply}]]></c></tag> + <tag><c><![CDATA[{shell, WantReply :: boolean()}]]></c></tag> <item><p>This message requests that the user default shell is started at the other end. This event is sent as a result of calling <seealso marker="ssh_connection#shell-2"> ssh_connection:shell/2</seealso>. </p></item> - <tag><c><![CDATA[{window_change, ssh_channel_id(), integer() = CharWidth, - integer() = RowHeight, integer() = PixWidth, integer() = PixHeight}]]></c></tag> + <tag><c><![CDATA[{window_change, ssh_channel_id(), CharWidth() :: integer(), + RowHeight :: integer(), PixWidth :: integer(), PixHeight :: integer()}]]></c></tag> <item><p>When the window (terminal) size changes on the client side, it <em>can</em> send a message to the server side to inform it of the new dimensions. No API function generates this event.</p></item> <tag><c><![CDATA[{exec, ssh_channel_id(), - boolean() = WantReply, string() = Cmd}]]></c></tag> + WantReply :: boolean(), Cmd :: string()}]]></c></tag> <item><p>This message requests that the server starts execution of the given command. This event is sent as a result of calling <seealso marker="ssh_connection#exec-4">ssh_connection:exec/4 </seealso>. @@ -256,7 +256,7 @@ <taglist> <tag><c>N x {ssh_cm, ssh_connection_ref(), - {data, ssh_channel_id(), ssh_data_type_code(), binary() = Data}}</c></tag> + {data, ssh_channel_id(), ssh_data_type_code(), Data :: binary()}}</c></tag> <item><p>The result of executing the command can be only one line or thousands of lines depending on the command.</p></item> @@ -265,12 +265,12 @@ <tag><c>0 or 1 x {ssh_cm, ssh_connection_ref(), {exit_signal, - ssh_channel_id(), string() = ExitSignal, string() = ErrorMsg, string() = LanguageString}}</c></tag> + ssh_channel_id(), ExitSignal :: string(), ErrorMsg :: string(), LanguageString :: string()}}</c></tag> <item><p>Not all systems send signals. For details on valid string values, see RFC 4254, Section 6.10</p></item> <tag><c>0 or 1 x {ssh_cm, ssh_connection_ref(), {exit_status, - ssh_channel_id(), integer() = ExitStatus}}</c></tag> + ssh_channel_id(), ExitStatus :: integer()}}</c></tag> <item><p>It is recommended by the SSH Connection Protocol to send this message, but that is not always the case.</p></item> diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index 826c585d65..86c042781c 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -345,6 +345,8 @@ handle_option([{connectfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{disconnectfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); +handle_option([{unexpectedfun, _} = Opt | Rest], SocketOptions, SshOptions) -> + handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{failfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{ssh_msg_debug_fun, _} = Opt | Rest], SocketOptions, SshOptions) -> @@ -450,7 +452,9 @@ handle_ssh_option({infofun, Value} = Opt) when is_function(Value) -> Opt; handle_ssh_option({connectfun, Value} = Opt) when is_function(Value) -> Opt; -handle_ssh_option({disconnectfun , Value} = Opt) when is_function(Value) -> +handle_ssh_option({disconnectfun, Value} = Opt) when is_function(Value) -> + Opt; +handle_ssh_option({unexpectedfun, Value} = Opt) when is_function(Value,2) -> Opt; handle_ssh_option({failfun, Value} = Opt) when is_function(Value) -> Opt; diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 94154c8a96..a02c87505d 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -128,8 +128,10 @@ user, service, userauth_quiet_mode, % boolean() - userauth_supported_methods , % - userauth_methods, + userauth_supported_methods, % string() eg "keyboard-interactive,password" + userauth_methods, % list( string() ) eg ["keyboard-interactive", "password"] + kb_tries_left = 0, % integer(), num tries left for "keyboard-interactive" + kb_data, userauth_preference, available_host_keys, authenticated = false diff --git a/lib/ssh/src/ssh_auth.erl b/lib/ssh/src/ssh_auth.erl index df9a97c8f8..020fb06530 100644 --- a/lib/ssh/src/ssh_auth.erl +++ b/lib/ssh/src/ssh_auth.erl @@ -169,7 +169,8 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, service = "ssh-connection", method = "password", data = <<?FALSE, ?UINT32(Sz), BinPwd:Sz/binary>>}, _, - #ssh{opts = Opts} = Ssh) -> + #ssh{opts = Opts, + userauth_supported_methods = Methods} = Ssh) -> Password = unicode:characters_to_list(BinPwd), case check_password(User, Password, Opts) of true -> @@ -178,7 +179,7 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, false -> {not_authorized, {User, {error,"Bad user or password"}}, ssh_transport:ssh_packet(#ssh_msg_userauth_failure{ - authentications = "", + authentications = Methods, partial_success = false}, Ssh)} end; @@ -191,7 +192,7 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, %% ?UINT32(Sz2), NewBinPwd:Sz2/binary >> }, _, - Ssh) -> + #ssh{userauth_supported_methods = Methods} = Ssh) -> %% Password change without us having sent SSH_MSG_USERAUTH_PASSWD_CHANGEREQ (because we never do) %% RFC 4252 says: %% SSH_MSG_USERAUTH_FAILURE without partial success - The password @@ -200,7 +201,7 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, {not_authorized, {User, {error,"Password change not supported"}}, ssh_transport:ssh_packet(#ssh_msg_userauth_failure{ - authentications = "", + authentications = Methods, partial_success = false}, Ssh)}; handle_userauth_request(#ssh_msg_userauth_request{user = User, @@ -216,7 +217,9 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, service = "ssh-connection", method = "publickey", data = Data}, - SessionId, #ssh{opts = Opts} = Ssh) -> + SessionId, + #ssh{opts = Opts, + userauth_supported_methods = Methods} = Ssh) -> <<?BYTE(HaveSig), ?UINT32(ALen), BAlg:ALen/binary, ?UINT32(KLen), KeyBlob:KLen/binary, SigWLen/binary>> = Data, Alg = binary_to_list(BAlg), @@ -231,7 +234,7 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, false -> {not_authorized, {User, undefined}, ssh_transport:ssh_packet(#ssh_msg_userauth_failure{ - authentications="publickey,password", + authentications = Methods, partial_success = false}, Ssh)} end; ?FALSE -> @@ -245,49 +248,60 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, service = "ssh-connection", method = "keyboard-interactive", data = _}, - _, #ssh{opts = Opts} = Ssh) -> - %% RFC4256 - %% The data field contains: - %% - language tag (deprecated). If =/=[] SHOULD use it however. We skip - %% it for simplicity. - %% - submethods. "... the user can give a hint of which actual methods - %% he wants to use. ...". It's a "MAY use" so we skip - %% it. It also needs an understanding between the client - %% and the server. - %% - %% "The server MUST reply with an SSH_MSG_USERAUTH_SUCCESS, - %% SSH_MSG_USERAUTH_FAILURE, or SSH_MSG_USERAUTH_INFO_REQUEST message." - Default = {"SSH server", - "Enter password for \""++User++"\"", - "pwd: ", - false}, - - {Name, Instruction, Prompt, Echo} = - case proplists:get_value(auth_method_kb_interactive_data, Opts) of - undefined -> - Default; - {_,_,_,_}=V -> - V; - F when is_function(F) -> - {_,PeerName} = Ssh#ssh.peer, - F(PeerName, User, "ssh-connection") - end, - EchoEnc = case Echo of - true -> <<?TRUE>>; - false -> <<?FALSE>> - end, - Msg = #ssh_msg_userauth_info_request{name = unicode:characters_to_list(Name), - instruction = unicode:characters_to_list(Instruction), - language_tag = "", - num_prompts = 1, - data = <<?STRING(unicode:characters_to_binary(Prompt)), - EchoEnc/binary - >> - }, - {not_authorized, {User, undefined}, - ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, - opts = [{max_kb_tries,3},{kb_userauth_info_msg,Msg}|Opts] - })}; + _, #ssh{opts = Opts, + kb_tries_left = KbTriesLeft, + userauth_supported_methods = Methods} = Ssh) -> + case KbTriesLeft of + N when N<1 -> + {not_authorized, {User, {authmethod, "keyboard-interactive"}}, + ssh_transport:ssh_packet( + #ssh_msg_userauth_failure{authentications = Methods, + partial_success = false}, Ssh)}; + + _ -> + %% RFC4256 + %% The data field contains: + %% - language tag (deprecated). If =/=[] SHOULD use it however. We skip + %% it for simplicity. + %% - submethods. "... the user can give a hint of which actual methods + %% he wants to use. ...". It's a "MAY use" so we skip + %% it. It also needs an understanding between the client + %% and the server. + %% + %% "The server MUST reply with an SSH_MSG_USERAUTH_SUCCESS, + %% SSH_MSG_USERAUTH_FAILURE, or SSH_MSG_USERAUTH_INFO_REQUEST message." + Default = {"SSH server", + "Enter password for \""++User++"\"", + "password: ", + false}, + + {Name, Instruction, Prompt, Echo} = + case proplists:get_value(auth_method_kb_interactive_data, Opts) of + undefined -> + Default; + {_,_,_,_}=V -> + V; + F when is_function(F) -> + {_,PeerName} = Ssh#ssh.peer, + F(PeerName, User, "ssh-connection") + end, + EchoEnc = case Echo of + true -> <<?TRUE>>; + false -> <<?FALSE>> + end, + Msg = #ssh_msg_userauth_info_request{name = unicode:characters_to_list(Name), + instruction = unicode:characters_to_list(Instruction), + language_tag = "", + num_prompts = 1, + data = <<?STRING(unicode:characters_to_binary(Prompt)), + EchoEnc/binary + >> + }, + {not_authorized, {User, undefined}, + ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, + kb_data = Msg + })} + end; handle_userauth_request(#ssh_msg_userauth_request{user = User, service = "ssh-connection", @@ -314,33 +328,37 @@ handle_userauth_info_request( handle_userauth_info_response(#ssh_msg_userauth_info_response{num_responses = 1, data = <<?UINT32(Sz), Password:Sz/binary>>}, - #ssh{opts = Opts0, - user = User} = Ssh) -> - NumTriesLeft = proplists:get_value(max_kb_tries, Opts0, 0) - 1, - Opts = lists:keydelete(max_kb_tries,1,Opts0), + #ssh{opts = Opts, + kb_tries_left = KbTriesLeft0, + kb_data = InfoMsg, + user = User, + userauth_supported_methods = Methods} = Ssh) -> + KbTriesLeft = KbTriesLeft0 - 1, case check_password(User, unicode:characters_to_list(Password), Opts) of true -> {authorized, User, ssh_transport:ssh_packet(#ssh_msg_userauth_success{}, Ssh)}; - false when NumTriesLeft > 0 -> + false when KbTriesLeft > 0 -> UserAuthInfoMsg = - (proplists:get_value(kb_userauth_info_msg,Opts)) - #ssh_msg_userauth_info_request{name = "", - instruction = - lists:concat( - ["Bad user or password, try again. ", - integer_to_list(NumTriesLeft), - " tries left."])}, + InfoMsg#ssh_msg_userauth_info_request{ + name = "", + instruction = + lists:concat( + ["Bad user or password, try again. ", + integer_to_list(KbTriesLeft), + " tries left."]) + }, {not_authorized, {User, undefined}, ssh_transport:ssh_packet(UserAuthInfoMsg, - Ssh#ssh{opts = [{max_kb_tries,NumTriesLeft}|Opts]})}; + Ssh#ssh{kb_tries_left = KbTriesLeft})}; false -> {not_authorized, {User, {error,"Bad user or password"}}, ssh_transport:ssh_packet(#ssh_msg_userauth_failure{ - authentications = "", + authentications = Methods, partial_success = false}, - Ssh#ssh{opts = lists:keydelete(kb_userauth_info_msg,1,Opts)} + Ssh#ssh{kb_data = undefined, + kb_tries_left = 0} )} end; @@ -483,22 +501,16 @@ keyboard_interact_fun(KbdInteractFun, Name, Instr, PromptInfos, NumPrompts) -> end. decode_public_key_v2(<<?UINT32(Len0), _:Len0/binary, - ?UINT32(Len1), BinE:Len1/binary, - ?UINT32(Len2), BinN:Len2/binary>> + ?UINT32(Len1), E:Len1/big-signed-integer-unit:8, + ?UINT32(Len2), N:Len2/big-signed-integer-unit:8>> ,"ssh-rsa") -> - E = ssh_bits:erlint(Len1, BinE), - N = ssh_bits:erlint(Len2, BinN), {ok, #'RSAPublicKey'{publicExponent = E, modulus = N}}; decode_public_key_v2(<<?UINT32(Len0), _:Len0/binary, - ?UINT32(Len1), BinP:Len1/binary, - ?UINT32(Len2), BinQ:Len2/binary, - ?UINT32(Len3), BinG:Len3/binary, - ?UINT32(Len4), BinY:Len4/binary>> + ?UINT32(Len1), P:Len1/big-signed-integer-unit:8, + ?UINT32(Len2), Q:Len2/big-signed-integer-unit:8, + ?UINT32(Len3), G:Len3/big-signed-integer-unit:8, + ?UINT32(Len4), Y:Len4/big-signed-integer-unit:8>> , "ssh-dss") -> - P = ssh_bits:erlint(Len1, BinP), - Q = ssh_bits:erlint(Len2, BinQ), - G = ssh_bits:erlint(Len3, BinG), - Y = ssh_bits:erlint(Len4, BinY), {ok, {Y, #'Dss-Parms'{p = P, q = Q, g = G}}}; decode_public_key_v2(_, _) -> diff --git a/lib/ssh/src/ssh_bits.erl b/lib/ssh/src/ssh_bits.erl index 8aaff93b9f..d5f8df6fe4 100644 --- a/lib/ssh/src/ssh_bits.erl +++ b/lib/ssh/src/ssh_bits.erl @@ -26,7 +26,7 @@ -include("ssh.hrl"). -export([encode/2]). --export([mpint/1, erlint/2, string/1, name_list/1]). +-export([mpint/1, string/1, name_list/1]). -export([random/1]). -define(name_list(X), @@ -145,11 +145,7 @@ enc(Xs, ['...'| []], _Offset) -> enc([], [],_) -> []. -erlint(Len, BinInt) -> - Sz = Len*8, - <<Int:Sz/big-signed-integer>> = BinInt, - Int. - + %% %% Create a binary with constant bytes %% diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index ab1fc93a1b..e6e5749e07 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -483,17 +483,22 @@ userauth(#ssh_msg_userauth_request{service = "ssh-connection", service = "ssh-connection", peer = {_, Address}} = Ssh0, opts = Opts, starter = Pid} = State) -> - case ssh_auth:handle_userauth_request(Msg, SessionId, Ssh0) of - {authorized, User, {Reply, Ssh}} -> - send_msg(Reply, State), - Pid ! ssh_connected, - connected_fun(User, Address, Method, Opts), - {next_state, connected, - next_packet(State#state{auth_user = User, ssh_params = Ssh})}; - {not_authorized, {User, Reason}, {Reply, Ssh}} -> - retry_fun(User, Address, Reason, Opts), - send_msg(Reply, State), - {next_state, userauth, next_packet(State#state{ssh_params = Ssh})} + case lists:member(Method, Ssh0#ssh.userauth_methods) of + true -> + case ssh_auth:handle_userauth_request(Msg, SessionId, Ssh0) of + {authorized, User, {Reply, Ssh}} -> + send_msg(Reply, State), + Pid ! ssh_connected, + connected_fun(User, Address, Method, Opts), + {next_state, connected, + next_packet(State#state{auth_user = User, ssh_params = Ssh})}; + {not_authorized, {User, Reason}, {Reply, Ssh}} -> + retry_fun(User, Address, Reason, Opts), + send_msg(Reply, State), + {next_state, userauth, next_packet(State#state{ssh_params = Ssh})} + end; + false -> + userauth(Msg#ssh_msg_userauth_request{method="none"}, State) end; userauth(#ssh_msg_userauth_info_request{} = Msg, @@ -984,15 +989,38 @@ handle_info({check_cache, _ , _}, #connection{channel_cache = Cache}} = State) -> {next_state, StateName, check_cache(State, Cache)}; -handle_info(UnexpectedMessage, StateName, #state{ssh_params = SshParams} = State) -> - Msg = lists:flatten(io_lib:format( - "Unexpected message '~p' received in state '~p'\n" - "Role: ~p\n" - "Peer: ~p\n" - "Local Address: ~p\n", [UnexpectedMessage, StateName, - SshParams#ssh.role, SshParams#ssh.peer, - proplists:get_value(address, SshParams#ssh.opts)])), - error_logger:info_report(Msg), +handle_info(UnexpectedMessage, StateName, #state{opts = Opts, + ssh_params = SshParams} = State) -> + case unexpected_fun(UnexpectedMessage, Opts, SshParams) of + report -> + Msg = lists:flatten( + io_lib:format( + "Unexpected message '~p' received in state '~p'\n" + "Role: ~p\n" + "Peer: ~p\n" + "Local Address: ~p\n", [UnexpectedMessage, StateName, + SshParams#ssh.role, SshParams#ssh.peer, + proplists:get_value(address, SshParams#ssh.opts)])), + error_logger:info_report(Msg); + + skip -> + ok; + + Other -> + Msg = lists:flatten( + io_lib:format("Call to fun in 'unexpectedfun' failed:~n" + "Return: ~p\n" + "Message: ~p\n" + "Role: ~p\n" + "Peer: ~p\n" + "Local Address: ~p\n", [Other, UnexpectedMessage, + SshParams#ssh.role, + element(2,SshParams#ssh.peer), + proplists:get_value(address, SshParams#ssh.opts)] + )), + + error_logger:error_report(Msg) + end, {next_state, StateName, State}. %%-------------------------------------------------------------------- @@ -1148,9 +1176,9 @@ init_ssh(client = Role, Vsn, Version, Options, Socket) -> }; init_ssh(server = Role, Vsn, Version, Options, Socket) -> - AuthMethods = proplists:get_value(auth_methods, Options, ?SUPPORTED_AUTH_METHODS), + AuthMethodsAsList = string:tokens(AuthMethods, ","), {ok, PeerAddr} = inet:peername(Socket), KeyCb = proplists:get_value(key_cb, Options, ssh_file), @@ -1161,6 +1189,8 @@ init_ssh(server = Role, Vsn, Version, Options, Socket) -> io_cb = proplists:get_value(io_cb, Options, ssh_io), opts = Options, userauth_supported_methods = AuthMethods, + userauth_methods = AuthMethodsAsList, + kb_tries_left = 3, peer = {undefined, PeerAddr}, available_host_keys = supported_host_keys(Role, KeyCb, Options) }. @@ -1706,6 +1736,15 @@ disconnect_fun(Reason, Opts) -> catch Fun(Reason) end. +unexpected_fun(UnexpectedMessage, Opts, #ssh{peer={_,Peer}}) -> + case proplists:get_value(unexpectedfun, Opts) of + undefined -> + report; + Fun -> + catch Fun(UnexpectedMessage, Peer) + end. + + check_cache(#state{opts = Opts} = State, Cache) -> %% Check the number of entries in Cache case proplists:get_value(size, ets:info(Cache)) of diff --git a/lib/ssh/src/ssh_info.erl b/lib/ssh/src/ssh_info.erl index 9c79d773a7..fc8f564bc3 100644 --- a/lib/ssh/src/ssh_info.erl +++ b/lib/ssh/src/ssh_info.erl @@ -79,7 +79,7 @@ print_clients(D) -> print_client(D, {undefined,Pid,supervisor,[ssh_connection_handler]}) -> {{Local,Remote},_Str} = ssh_connection_handler:get_print_info(Pid), - io:format(D, " Local=~s Remote=~s~n",[fmt_host_port(Local),fmt_host_port(Remote)]); + io:format(D, " Local=~s Remote=~s ConnectionRef=~p~n",[fmt_host_port(Local),fmt_host_port(Remote),Pid]); print_client(D, Other) -> io:format(D, " [[Other 1: ~p]]~n",[Other]). @@ -134,10 +134,11 @@ walk_sups(D, StartPid) -> io:format(D, "Start at ~p, ~s.~n",[StartPid,dead_or_alive(StartPid)]), walk_sups(D, children(StartPid), _Indent=?inc(0)). -walk_sups(D, [H={_,Pid,SupOrWorker,_}|T], Indent) -> +walk_sups(D, [H={_,Pid,_,_}|T], Indent) -> indent(D, Indent), io:format(D, '~200p ~p is ~s~n',[H,Pid,dead_or_alive(Pid)]), - case SupOrWorker of - supervisor -> walk_sups(D, children(Pid), ?inc(Indent)); + case H of + {_,_,supervisor,[ssh_connection_handler]} -> ok; + {_,Pid,supervisor,_} -> walk_sups(D, children(Pid), ?inc(Indent)); _ -> ok end, walk_sups(D, T, Indent); diff --git a/lib/ssh/src/ssh_message.erl b/lib/ssh/src/ssh_message.erl index 66e7717095..483c6cb4aa 100644 --- a/lib/ssh/src/ssh_message.erl +++ b/lib/ssh/src/ssh_message.erl @@ -421,8 +421,8 @@ decode(<<?BYTE(?SSH_MSG_USERAUTH_INFO_RESPONSE), ?UINT32(Num), Data/binary>>) -> decode(<<?BYTE(?SSH_MSG_KEXINIT), Cookie:128, Data/binary>>) -> decode_kex_init(Data, [Cookie, ssh_msg_kexinit], 10); -decode(<<?BYTE(?SSH_MSG_KEXDH_INIT), ?UINT32(Len), E:Len/binary>>) -> - #ssh_msg_kexdh_init{e = erlint(Len, E) +decode(<<?BYTE(?SSH_MSG_KEXDH_INIT), ?UINT32(Len), E:Len/big-signed-integer-unit:8>>) -> + #ssh_msg_kexdh_init{e = E }; decode(<<?BYTE(?SSH_MSG_KEX_DH_GEX_REQUEST), ?UINT32(Min), ?UINT32(N), ?UINT32(Max)>>) -> #ssh_msg_kex_dh_gex_request{ @@ -442,11 +442,11 @@ decode(<<?BYTE(?SSH_MSG_KEX_DH_GEX_GROUP), g = Generator }; decode(<<?BYTE(?SSH_MSG_KEXDH_REPLY), ?UINT32(Len0), Key:Len0/binary, - ?UINT32(Len1), F:Len1/binary, + ?UINT32(Len1), F:Len1/big-signed-integer-unit:8, ?UINT32(Len2), Hashsign:Len2/binary>>) -> #ssh_msg_kexdh_reply{ public_host_key = decode_host_key(Key), - f = erlint(Len1, F), + f = F, h_sig = decode_sign(Hashsign) }; @@ -514,10 +514,7 @@ decode_kex_init(<<?UINT32(Len), Data:Len/binary, Rest/binary>>, Acc, N) -> Names = string:tokens(unicode:characters_to_list(Data), ","), decode_kex_init(Rest, [Names | Acc], N -1). -erlint(MPIntSize, MPIntValue) -> - Bits = MPIntSize * 8, - <<Integer:Bits/integer>> = MPIntValue, - Integer. + decode_sign(<<?UINT32(Len), _Alg:Len/binary, ?UINT32(_), Signature/binary>>) -> Signature. @@ -525,18 +522,19 @@ decode_sign(<<?UINT32(Len), _Alg:Len/binary, ?UINT32(_), Signature/binary>>) -> decode_host_key(<<?UINT32(Len), Alg:Len/binary, Rest/binary>>) -> decode_host_key(Alg, Rest). -decode_host_key(<<"ssh-rsa">>, <<?UINT32(Len0), E:Len0/binary, - ?UINT32(Len1), N:Len1/binary>>) -> - #'RSAPublicKey'{publicExponent = erlint(Len0, E), - modulus = erlint(Len1, N)}; +decode_host_key(<<"ssh-rsa">>, <<?UINT32(Len0), E:Len0/big-signed-integer-unit:8, + ?UINT32(Len1), N:Len1/big-signed-integer-unit:8>>) -> + #'RSAPublicKey'{publicExponent = E, + modulus = N}; decode_host_key(<<"ssh-dss">>, - <<?UINT32(Len0), P:Len0/binary, - ?UINT32(Len1), Q:Len1/binary, - ?UINT32(Len2), G:Len2/binary, - ?UINT32(Len3), Y:Len3/binary>>) -> - {erlint(Len3, Y), #'Dss-Parms'{p = erlint(Len0, P), q = erlint(Len1, Q), - g = erlint(Len2, G)}}. + <<?UINT32(Len0), P:Len0/big-signed-integer-unit:8, + ?UINT32(Len1), Q:Len1/big-signed-integer-unit:8, + ?UINT32(Len2), G:Len2/big-signed-integer-unit:8, + ?UINT32(Len3), Y:Len3/big-signed-integer-unit:8>>) -> + {Y, #'Dss-Parms'{p = P, + q = Q, + g = G}}. encode_host_key(#'RSAPublicKey'{modulus = N, publicExponent = E}) -> ssh_bits:encode(["ssh-rsa", E, N], [string, mpint, mpint]); diff --git a/lib/ssh/test/Makefile b/lib/ssh/test/Makefile index 843b1d906d..50efc33f98 100644 --- a/lib/ssh/test/Makefile +++ b/lib/ssh/test/Makefile @@ -38,6 +38,7 @@ MODULES= \ ssh_sftp_SUITE \ ssh_sftpd_SUITE \ ssh_sftpd_erlclient_SUITE \ + ssh_upgrade_SUITE \ ssh_connection_SUITE \ ssh_echo_server \ ssh_peername_sockname_server \ diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 7e75d84d82..e62feb6857 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -62,6 +62,8 @@ all() -> ssh_msg_debug_fun_option_server, disconnectfun_option_server, disconnectfun_option_client, + unexpectedfun_option_server, + unexpectedfun_option_client, preferred_algorithms, id_string_no_opt_client, id_string_own_string_client, @@ -69,7 +71,8 @@ all() -> id_string_no_opt_server, id_string_own_string_server, id_string_random_server, - {group, hardening_tests} + {group, hardening_tests}, + ssh_info_print ]. groups() -> @@ -151,18 +154,36 @@ init_per_group(dir_options, Config) -> %% Make readable file: File_readable = filename:join(PrivDir, "file"), ok = file:write_file(File_readable, <<>>), + %% Check: case {file:read_file_info(Dir_unreadable), file:read_file_info(File_readable)} of - {{ok, #file_info{type=directory, access=Md}}, - {ok, #file_info{type=regular, access=Mf}}} when Md=/=read, Md=/=read_write -> - %% Save: - [{unreadable_dir, Dir_unreadable}, - {readable_file, File_readable} - | Config]; - X -> - ct:log("#file_info : ~p",[X]), - {skip, "File or dir mode settings failed"} + {{ok, Id=#file_info{type=directory, access=Md}}, + {ok, If=#file_info{type=regular, access=Mf}}} -> + AccessOK = + case {Md, Mf} of + {read, _} -> false; + {read_write, _} -> false; + {_, read} -> true; + {_, read_write} -> true; + _ -> false + end, + + case AccessOK of + true -> + %% Save: + [{unreadable_dir, Dir_unreadable}, + {readable_file, File_readable} + | Config]; + false -> + ct:log("File#file_info : ~p~n" + "Dir#file_info : ~p",[If,Id]), + {skip, "File or dir mode settings failed"} + end; + + NotDirFile -> + ct:log("{Dir,File} -> ~p",[NotDirFile]), + {skip, "File/Dir creation failed"} end; init_per_group(_, Config) -> Config. @@ -818,7 +839,7 @@ connectfun_disconnectfun_client(Config) -> {user_dir, UserDir}, {password, "morot"}, {failfun, fun ssh_test_lib:failfun/2}]), - ConnectionRef = + _ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user, "foo"}, {password, "morot"}, @@ -949,6 +970,88 @@ disconnectfun_option_client(Config) -> end. %%-------------------------------------------------------------------- +unexpectedfun_option_server(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + ConnFun = fun(_,_,_) -> Parent ! {connection_pid,self()} end, + UnexpFun = fun(Msg,Peer) -> + Parent ! {unexpected,Msg,Peer,self()}, + skip + end, + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}, + {connectfun, ConnFun}, + {unexpectedfun, UnexpFun}]), + _ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}]), + receive + {connection_pid,Server} -> + %% Beware, implementation knowledge: + Server ! unexpected_message, + receive + {unexpected, unexpected_message, {{_,_,_,_},_}, _} -> ok; + {unexpected, unexpected_message, Peer, _} -> ct:fail("Bad peer ~p",[Peer]); + M = {unexpected, _, _, _} -> ct:fail("Bad msg ~p",[M]) + after 3000 -> + ssh:stop_daemon(Pid), + {fail,timeout2} + end + after 3000 -> + ssh:stop_daemon(Pid), + {fail,timeout1} + end. + +%%-------------------------------------------------------------------- +unexpectedfun_option_client(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + UnexpFun = fun(Msg,Peer) -> + Parent ! {unexpected,Msg,Peer,self()}, + skip + end, + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}]), + ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}, + {unexpectedfun, UnexpFun}]), + %% Beware, implementation knowledge: + ConnectionRef ! unexpected_message, + + receive + {unexpected, unexpected_message, {{_,_,_,_},_}, ConnectionRef} -> + ok; + {unexpected, unexpected_message, Peer, ConnectionRef} -> + ct:fail("Bad peer ~p",[Peer]); + M = {unexpected, _, _, _} -> + ct:fail("Bad msg ~p",[M]) + after 3000 -> + ssh:stop_daemon(Pid), + {fail,timeout} + end. + +%%-------------------------------------------------------------------- known_hosts() -> [{doc, "check that known_hosts is updated correctly"}]. known_hosts(Config) when is_list(Config) -> @@ -1611,6 +1714,74 @@ max_sessions(Config, ParallelLogin, Connect0) when is_function(Connect0,2) -> end. %%-------------------------------------------------------------------- +ssh_info_print(Config) -> + %% Just check that ssh_print:info() crashes + PrivDir = ?config(priv_dir, Config), + PrintFile = filename:join(PrivDir,info), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + UnexpFun = fun(Msg,_Peer) -> + Parent ! {unexpected,Msg,self()}, + skip + end, + ConnFun = fun(_,_,_) -> Parent ! {connect,self()} end, + + {DaemonRef, Host, Port} = + ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {unexpectedfun, UnexpFun}, + {connectfun, ConnFun}, + {failfun, fun ssh_test_lib:failfun/2}]), + ClientConnRef1 = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {unexpectedfun, UnexpFun}, + {user_interaction, false}]), + ClientConnRef2 = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {unexpectedfun, UnexpFun}, + {user_interaction, false}]), + receive + {connect,DaemonConnRef} -> + ct:log("DaemonRef=~p, DaemonConnRef=~p, ClientConnRefs=~p",[DaemonRef, DaemonConnRef, + [ClientConnRef1,ClientConnRef2] + ]) + after 2000 -> + ok + end, + + {ok,D} = file:open(PrintFile, write), + ssh_info:print(D), + ok = file:close(D), + + {ok,Bin} = file:read_file(PrintFile), + ct:log("~s",[Bin]), + + receive + {unexpected, Msg, Pid} -> + ct:log("~p got unexpected msg ~p",[Pid,Msg]), + ct:log("process_info(~p) = ~n~p",[Pid,process_info(Pid)]), + ok = ssh:close(ClientConnRef1), + ok = ssh:close(ClientConnRef2), + ok = ssh:stop_daemon(DaemonRef), + {fail,"unexpected msg"} + after 1000 -> + ok = ssh:close(ClientConnRef1), + ok = ssh:close(ClientConnRef2), + ok = ssh:stop_daemon(DaemonRef) + end. + + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/ssh/test/ssh_upgrade_SUITE.erl b/lib/ssh/test/ssh_upgrade_SUITE.erl new file mode 100644 index 0000000000..861c7ab3dd --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE.erl @@ -0,0 +1,206 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/.2 +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(ssh_upgrade_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-record(state, { + config, + root_dir, + server, + client, + connection, + soft + }). + + +%%%================================================================ +%%% +%%% CommonTest callbacks +%%% +all() -> + [ + minor_upgrade, + major_upgrade + ]. + +init_per_suite(Config0) -> + catch crypto:stop(), + try {crypto:start(), erlang:system_info({wordsize, internal}) == + erlang:system_info({wordsize, external})} of + {ok, true} -> + case ct_release_test:init(Config0) of + {skip, Reason} -> + {skip, Reason}; + Config -> + ssh:start(), + Config + end; + {ok, false} -> + {skip, "Test server will not handle halfwordemulator correctly. Skip as halfwordemulator is deprecated"} + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(Config) -> + ct_release_test:cleanup(Config), + ssh:stop(), + crypto:stop(), + UserDir = ?config(priv_dir, Config), + ssh_test_lib:clean_rsa(UserDir). + +init_per_testcase(_TestCase, Config) -> + Config. +end_per_testcase(_TestCase, Config) -> + Config. + +%%%================================================================ +%%% +%%% Test cases +%%% +major_upgrade(Config) when is_list(Config) -> + ct_release_test:upgrade(ssh, major,{?MODULE, #state{config = Config}}, Config). + +minor_upgrade(Config) when is_list(Config) -> + ct_release_test:upgrade(ssh, minor,{?MODULE, #state{config = Config}}, Config). + +%%%================================================================ +%%% +%%% ct_release_test callbacks +%%% + +%%%---------------------------------------------------------------- +%%% Initialyze system before upgrade test starts. +%%% Called by ct_release_test:upgrade/4 +upgrade_init(CTData, State) -> + {ok, AppUp={_, _, Up, _Down}} = ct_release_test:get_appup(CTData, ssh), + ct:pal("AppUp: ~p", [AppUp]), + ct:pal("Up: ~p", [Up]), + case Soft = is_soft(Up) of + %% It is symmetrical, if upgrade is soft so is downgrade + true -> + setup_server_client(State#state{soft = Soft}); + false -> + State#state{soft = Soft} + end. + +%%%---------------------------------------------------------------- +%%% Check that upgrade was successful +%%% Called by ct_release_test:upgrade/4 +upgrade_upgraded(_, #state{soft=false} = State) -> + test_hard(State, "upgrade"); + +upgrade_upgraded(_, State) -> + test_soft(State, "upgrade1"). + +%%%---------------------------------------------------------------- +%%% Check that downgrade was successful. +%%% Called by ct_release_test:upgrade/4 +upgrade_downgraded(_, #state{soft=false} = State) -> + test_hard(State, "downgrade"); + +upgrade_downgraded(_, #state{soft=true} = State) -> + test_soft(State, "downgrade1"). + +%%%================================================================ +%%% +%%% Private functions +%%% + +is_soft([{restart_application, ssh}]) -> + false; +is_soft(_) -> + true. + + +test_hard(State0, FileName) -> + ct:pal("test_hard State0=~p, FileName=~p",[State0, FileName]), + State = setup_server_client(State0), + test_connection(FileName, random_contents(), State). + +test_soft(State0, FileName) -> + ct:pal("test_soft State0=~p, FileName=~p",[State0, FileName]), + State = test_connection(FileName, random_contents(), State0), + setup_server_client( close(State) ). + + +setup_server_client(#state{config=Config} = State) -> + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + + FtpRootDir = filename:join(PrivDir, "ftp_root"), + catch file:make_dir(FtpRootDir), + + SFTP = ssh_sftpd:subsystem_spec([{root,FtpRootDir},{cwd,FtpRootDir}]), + + {Server,Host,Port} = ssh_test_lib:daemon([{system_dir,DataDir}, + {user_passwords,[{"hej","hopp"}]}, + {subsystems,[SFTP]}]), + + {ok, ChannelPid, Connection} = + ssh_sftp:start_channel(Host, Port, [{user_interaction,false}, + {silently_accept_hosts,true}, + {user_dir,DataDir}, + {user,"hej"}, + {password,"hopp"}]), + State#state{server = Server, + client = ChannelPid, + connection = Connection}. + + +test_connection(FileName, FileContents, + #state{client = ChannelPid, + root_dir = FtpRootDir} = State) -> + ct:pal("test_connection Writing with ssh_sftp:write_file",[]), + case ssh_sftp:write_file(ChannelPid, FileName, FileContents) of + ok -> + case ssh_sftp:read_file(ChannelPid, FileName) of + {ok,FileContents} -> + State; + {ok,Unexpected} -> + ct:fail("Expected ~p but got ~p from sftp:read_file(~p,..) in RootDir ~p", + [FileContents,Unexpected,FileName,FtpRootDir] + ); + Other -> + ct:fail("ssh_sftp:read_file(~p,~p) -> ~p~n" + "ssh_sftp:list_dir(~p,\".\") -> ~p", + [ChannelPid,FileName,Other, + ChannelPid, catch ssh_sftp:list_dir(ChannelPid, ".")]) + end; + + Other -> + ct:fail("ssh_sftp:write_file(~p,~p,~p) -> ~p",[ChannelPid,FileName,FileContents,Other]) + end. + + +close(#state{server = Server, + connection = Connection} = State) -> + ssh:close(Connection), + ssh:stop_daemon(Server), + State#state{server = undefined, + client = undefined, + connection = undefined}. + + +random_contents() -> list_to_binary( random_chars(3) ). + +random_chars(N) -> [crypto:rand_uniform($a,$z) || _<-lists:duplicate(N,x)]. diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/id_dsa b/lib/ssh/test/ssh_upgrade_SUITE_data/id_dsa new file mode 100644 index 0000000000..d306f8b26e --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/id_dsa @@ -0,0 +1,13 @@ +-----BEGIN DSA PRIVATE KEY----- +MIIBvAIBAAKBgQDfi2flSTZZofwT4yQT0NikX/LGNT7UPeB/XEWe/xovEYCElfaQ +APFixXvEgXwoojmZ5kiQRKzLM39wBP0jPERLbnZXfOOD0PDnw0haMh7dD7XKVMod +/EigVgHf/qBdM2M8yz1s/rRF7n1UpLSypziKjkzCm7JoSQ2zbWIPdmBIXwIVAMgP +kpr7Sq3O7sHdb8D601DRjoExAoGAMOQxDfB2Fd8ouz6G96f/UOzRMI/Kdv8kYYKW +JIGY+pRYrLPyYzUeJznwZreOJgrczAX+luHnKFWJ2Dnk5CyeXk67Wsr7pJ/4MBMD +OKeIS0S8qoSBN8+Krp79fgA+yS3IfqbkJLtLu4EBaCX4mKQIX4++k44d4U5lc8pt ++9hlEI8CgYEAznKxx9kyC6bVo7LUYKaGhofRFt0SYFc5PVmT2VUGRs1R6+6DPD+e +uEO6IhFct7JFSRbP9p0JD4Uk+3zlZF+XX6b2PsZkeV8f/02xlNGUSmEzCSiNg1AX +Cy/WusYhul0MncWCHMcOZB5rIvU/aP5EJJtn3xrRaz6u0SThF6AnT34CFQC63czE +ZU8w8Q+H7z0j+a+70x2iAw== +-----END DSA PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/id_rsa b/lib/ssh/test/ssh_upgrade_SUITE_data/id_rsa new file mode 100644 index 0000000000..9d7e0dd5fb --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/id_rsa @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQD1OET+3O/Bvj/dtjxDTXmj1oiJt4sIph5kGy0RfjoPrZfaS+CU +DhakCmS6t2ivxWFgtpKWaoGMZMJqWj6F6ZsumyFl3FPBtujwY/35cgifrI9Ns4Tl +zR1uuengNBmV+WRQ5cd9F2qS6Z8aDQihzt0r8JUqLcK+VQbrmNzboCCQQwIDAQAB +AoGAPQEyqPTt8JUT7mRXuaacjFXiweAXhp9NEDpyi9eLOjtFe9lElZCrsUOkq47V +TGUeRKEm9qSodfTbKPoqc8YaBJGJPhUaTAcha+7QcDdfHBvIsgxvU7ePVnlpXRp3 +CCUEMPhlnx6xBoTYP+fRU0e3+xJIPVyVCqX1jAdUMkzfRoECQQD6ux7B1QJAIWyK +SGkbDUbBilNmzCFNgIpOP6PA+bwfi5d16diTpra5AX09keQABAo/KaP1PdV8Vg0p +z4P3A7G3AkEA+l+AKG6m0kQTTBMJDqOdVPYwe+5GxunMaqmhokpEbuGsrZBl5Dvd +WpcBjR7jmenrhKZRIuA+Fz5HPo/UQJPl1QJBAKxstDkeED8j/S2XoFhPKAJ+6t39 +sUVICVTIZQeXdmzHJXCcUSkw8+WEhakqw/3SyW0oaK2FSWQJFWJUZ+8eJj8CQEh3 +xeduB5kKnS9CvzdeghZqX6QvVosSdtlUmfUYW/BgH5PpHKTP8wTaeld3XldZTpMJ +dKiMkUw2+XYROVUrubUCQD+Na1LhULlpn4ISEtIEfqpdlUhxDgO15Wg8USmsng+x +ICliVOSQtwaZjm8kwaFt0W7XnpnDxbRs37vIEbIMWak= +-----END RSA PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/known_hosts b/lib/ssh/test/ssh_upgrade_SUITE_data/known_hosts new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/known_hosts @@ -0,0 +1 @@ + diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key new file mode 100644 index 0000000000..51ab6fbd88 --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key @@ -0,0 +1,13 @@ +-----BEGIN DSA PRIVATE KEY----- +MIIBuwIBAAKBgQCClaHzE2ul0gKSUxah5W0W8UiJLy4hXngKEqpaUq9SSdVdY2LK +wVfKH1gt5iuaf1FfzOhsIC9G/GLnjYttXZc92cv/Gfe3gR+s0ni2++MX+T++mE/Q +diltXv/Hp27PybS67SmiFW7I+RWnT2OKlMPtw2oUuKeztCe5UWjaj/y5FQIVAPLA +l9RpiU30Z87NRAHY3NTRaqtrAoGANMRxw8UfdtNVR0CrQj3AgPaXOGE4d+G4Gp4X +skvnCHycSVAjtYxebUkzUzt5Q6f/IabuLUdge3gXrc8BetvrcKbp+XZgM0/Vj2CF +Ymmy3in6kzGZq7Fw1sZaku6AOU8vLa5woBT2vAcHLLT1bLAzj7viL048T6MfjrOP +ef8nHvACgYBhDWFQJ1mf99sg92LalVq1dHLmVXb3PTJDfCO/Gz5NFmj9EZbAtdah +/XcF3DeRF+eEoz48wQF/ExVxSMIhLdL+o+ElpVhlM7Yii+T7dPhkQfEul6zZXu+U +ykSTXYUbtsfTNRFQGBW2/GfnEc0mnIxfn9v10NEWMzlq5z9wT9P0CgIVAN4wtL5W +Lv62jKcdskxNyz2NQoBx +-----END DSA PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key.pub b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key.pub new file mode 100644 index 0000000000..4dbb1305b0 --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key.pub @@ -0,0 +1,11 @@ +---- BEGIN SSH2 PUBLIC KEY ---- +AAAAB3NzaC1kc3MAAACBAIKVofMTa6XSApJTFqHlbRbxSIkvLiFeeAoSqlpSr1JJ1V1j +YsrBV8ofWC3mK5p/UV/M6GwgL0b8YueNi21dlz3Zy/8Z97eBH6zSeLb74xf5P76YT9B2 +KW1e/8enbs/JtLrtKaIVbsj5FadPY4qUw+3DahS4p7O0J7lRaNqP/LkVAAAAFQDywJfU +aYlN9GfOzUQB2NzU0WqrawAAAIA0xHHDxR9201VHQKtCPcCA9pc4YTh34bganheyS+cI +fJxJUCO1jF5tSTNTO3lDp/8hpu4tR2B7eBetzwF62+twpun5dmAzT9WPYIViabLeKfqT +MZmrsXDWxlqS7oA5Ty8trnCgFPa8BwcstPVssDOPu+IvTjxPox+Os495/yce8AAAAIBh +DWFQJ1mf99sg92LalVq1dHLmVXb3PTJDfCO/Gz5NFmj9EZbAtdah/XcF3DeRF+eEoz48 +wQF/ExVxSMIhLdL+o+ElpVhlM7Yii+T7dPhkQfEul6zZXu+UykSTXYUbtsfTNRFQGBW2 +/GfnEc0mnIxfn9v10NEWMzlq5z9wT9P0Cg== +---- END SSH2 PUBLIC KEY ---- diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key new file mode 100644 index 0000000000..79968bdd7d --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key @@ -0,0 +1,16 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDCZX+4FBDwZIh9y/Uxee1VJnEXlowpz2yDKwj8semM4q843337 +zbNfxHmladB1lpz2NqyxI175xMIJuDxogyZdsOxGnFAzAnthR4dqL/RWRWzjaxSB +6IAO9SPYVVlrpZ+1hsjLW79fwXK/yc8VdhRuWTeQiRgYY2ek8+OKbOqz4QIDAQAB +AoGANmvJzJO5hkLuvyDZHKfAnGTtpifcR1wtSa9DjdKUyn8vhKF0mIimnbnYQEmW +NUUb3gXCZLi9PvkpRSVRrASDOZwcjoU/Kvww163vBUVb2cOZfFhyn6o2Sk88Tt++ +udH3hdjpf9i7jTtUkUe+QYPsia+wgvvrmn4QrahLAH86+kECQQDx5gFeXTME3cnW +WMpFz3PPumduzjqgqMMWEccX4FtQkMX/gyGa5UC7OHFyh0N/gSWvPbRHa8A6YgIt +n8DO+fh5AkEAzbqX4DOn8NY6xJIi42q7l/2jIA0RkB6P7YugW5NblhqBZ0XDnpA5 +sMt+rz+K07u9XZtxgh1xi7mNfwY6lEAMqQJBAJBEauCKmRj35Z6OyeQku59SPsnY ++SJEREVvSNw2lH9SOKQQ4wPsYlTGbvKtNVZgAcen91L5MmYfeckYE/fdIZECQQCt +64zxsTnM1I8iFxj/gP/OYlJBikrKt8udWmjaghzvLMEw+T2DExJyb9ZNeT53+UMB +m6O+B/4xzU/djvp+0hbhAkAemIt+rA5kTmYlFndhpvzkSSM8a2EXsO4XIPgGWCTT +tQKS/tTly0ADMjN/TVy11+9d6zcqadNVuHXHGtR4W0GR +-----END RSA PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key.pub b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key.pub new file mode 100644 index 0000000000..75d2025c71 --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key.pub @@ -0,0 +1,5 @@ +---- BEGIN SSH2 PUBLIC KEY ---- +AAAAB3NzaC1yc2EAAAADAQABAAAAgQDCZX+4FBDwZIh9y/Uxee1VJnEXlowpz2yDKwj8 +semM4q843337zbNfxHmladB1lpz2NqyxI175xMIJuDxogyZdsOxGnFAzAnthR4dqL/RW +RWzjaxSB6IAO9SPYVVlrpZ+1hsjLW79fwXK/yc8VdhRuWTeQiRgYY2ek8+OKbOqz4Q== +---- END SSH2 PUBLIC KEY ---- diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 18d98e5efb..9122066787 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -650,6 +650,27 @@ fun(srp, Username :: string(), UserState :: term()) -> The option <c>sni_fun</c>, and <c>sni_hosts</c> are mutually exclusive.</p></item> + <tag><c>{client_renegotiation, boolean()}</c></tag> + <item>In protocols that support client-initiated renegotiation, the cost + of resources of such an operation is higher for the server than the + client. This can act as a vector for denial of service attacks. The SSL + application already takes measures to counter-act such attempts, + but client-initiated renegotiation can be stricly disabled by setting + this option to <c>false</c>. The default value is <c>true</c>. + Note that disabling renegotiation can result in long-lived connections + becoming unusable due to limits on the number of messages the underlying + cipher suite can encipher. + </item> + + <tag><c>{psk_identity, string()}</c></tag> + <item>Specifies the server identity hint the server presents to the client. + </item> + <tag><c>{log_alert, boolean()}</c></tag> + <item>If false, error reports will not be displayed.</item> + <tag><c>{honor_cipher_order, boolean()}</c></tag> + <item>If true, use the server's preference for cipher selection. If false + (the default), use the client's preference. + </item> </taglist> </section> diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index 610e2c4e41..0c73a49a04 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -514,6 +514,7 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions}, User, user_data_buffer = <<>>, session_cache_cb = SessionCacheCb, renegotiation = {false, first}, + allow_renegotiate = SSLOptions#ssl_options.client_renegotiation, start_or_recv_from = undefined, send_queue = queue:new(), protocol_cb = ?MODULE diff --git a/lib/ssl/src/ssl.appup.src b/lib/ssl/src/ssl.appup.src index d100e41930..1476336039 100644 --- a/lib/ssl/src/ssl.appup.src +++ b/lib/ssl/src/ssl.appup.src @@ -1,16 +1,14 @@ %% -*- erlang -*- {"%VSN%", [ - {<<"6.0">>, [{load_module, ssl_handshake, soft_purge, soft_purge, []}]}, - {<<"5\\.3\\.[1-7]($|\\..*)">>, [{restart_application, ssl}]}, - {<<"5\\.[0-2]($|\\..*)">>, [{restart_application, ssl}]}, + {<<"6\\..*">>, [{restart_application, ssl}]}, + {<<"5\\..*">>, [{restart_application, ssl}]}, {<<"4\\..*">>, [{restart_application, ssl}]}, {<<"3\\..*">>, [{restart_application, ssl}]} ], [ - {<<"6.0">>, [{load_module, ssl_handshake, soft_purge, soft_purge, []}]}, - {<<"5\\.3\\.[1-7]($|\\..*)">>, [{restart_application, ssl}]}, - {<<"5\\.[0-2]($|\\..*)">>, [{restart_application, ssl}]}, + {<<"6\\..*">>, [{restart_application, ssl}]}, + {<<"5\\..*">>, [{restart_application, ssl}]}, {<<"4\\..*">>, [{restart_application, ssl}]}, {<<"3\\..*">>, [{restart_application, ssl}]} ] diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 225a9be66f..f8ddfba7e3 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -685,6 +685,7 @@ handle_options(Opts0) -> reuse_session = handle_option(reuse_session, Opts, ReuseSessionFun), reuse_sessions = handle_option(reuse_sessions, Opts, true), secure_renegotiate = handle_option(secure_renegotiate, Opts, false), + client_renegotiation = handle_option(client_renegotiation, Opts, true), renegotiate_at = handle_option(renegotiate_at, Opts, ?DEFAULT_RENEGOTIATE_AT), hibernate_after = handle_option(hibernate_after, Opts, undefined), erl_dist = handle_option(erl_dist, Opts, false), @@ -715,7 +716,7 @@ handle_options(Opts0) -> depth, cert, certfile, key, keyfile, password, cacerts, cacertfile, dh, dhfile, user_lookup_fun, psk_identity, srp_identity, ciphers, - reuse_session, reuse_sessions, ssl_imp, + reuse_session, reuse_sessions, ssl_imp, client_renegotiation, cb_info, renegotiate_at, secure_renegotiate, hibernate_after, erl_dist, alpn_advertised_protocols, sni_hosts, sni_fun, alpn_preferred_protocols, next_protocols_advertised, @@ -857,6 +858,8 @@ validate_option(reuse_sessions, Value) when is_boolean(Value) -> validate_option(secure_renegotiate, Value) when is_boolean(Value) -> Value; +validate_option(client_renegotiation, Value) when is_boolean(Value) -> + Value; validate_option(renegotiate_at, Value) when is_integer(Value) -> erlang:min(Value, ?DEFAULT_RENEGOTIATE_AT); @@ -1226,6 +1229,8 @@ new_ssl_options([{renegotiate_at, Value} | Rest], #ssl_options{} = Opts, RecordC new_ssl_options(Rest, Opts#ssl_options{ renegotiate_at = validate_option(renegotiate_at, Value)}, RecordCB); new_ssl_options([{secure_renegotiate, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{secure_renegotiate = validate_option(secure_renegotiate, Value)}, RecordCB); +new_ssl_options([{client_renegotiation, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options(Rest, Opts#ssl_options{client_renegotiation = validate_option(client_renegotiation, Value)}, RecordCB); new_ssl_options([{hibernate_after, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{hibernate_after = validate_option(hibernate_after, Value)}, RecordCB); new_ssl_options([{alpn_advertised_protocols, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 8584e56d6c..47ee4d68fb 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -1573,7 +1573,9 @@ hash_algorithm(?SHA) -> sha; hash_algorithm(?SHA224) -> sha224; hash_algorithm(?SHA256) -> sha256; hash_algorithm(?SHA384) -> sha384; -hash_algorithm(?SHA512) -> sha512. +hash_algorithm(?SHA512) -> sha512; +hash_algorithm(Other) when is_integer(Other) andalso ((Other >= 7) and (Other =< 223)) -> unassigned; +hash_algorithm(Other) when is_integer(Other) andalso ((Other >= 224) and (Other =< 255)) -> Other. sign_algorithm(anon) -> ?ANON; sign_algorithm(rsa) -> ?RSA; @@ -1582,7 +1584,9 @@ sign_algorithm(ecdsa) -> ?ECDSA; sign_algorithm(?ANON) -> anon; sign_algorithm(?RSA) -> rsa; sign_algorithm(?DSA) -> dsa; -sign_algorithm(?ECDSA) -> ecdsa. +sign_algorithm(?ECDSA) -> ecdsa; +sign_algorithm(Other) when is_integer(Other) andalso ((Other >= 4) and (Other =< 223)) -> unassigned; +sign_algorithm(Other) when is_integer(Other) andalso ((Other >= 224) and (Other =< 255)) -> Other. hash_size(null) -> 0; diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index baeae68bc4..40eb3d0284 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -110,6 +110,7 @@ reuse_sessions :: boolean(), renegotiate_at, secure_renegotiate, + client_renegotiation, %% undefined if not hibernating, or number of ms of %% inactivity after which ssl_connection will go into %% hibernation diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 3304ffcddb..ed7ccb3d70 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -392,6 +392,7 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions, Tracker}, Us user_data_buffer = <<>>, session_cache_cb = SessionCacheCb, renegotiation = {false, first}, + allow_renegotiate = SSLOptions#ssl_options.client_renegotiation, start_or_recv_from = undefined, send_queue = queue:new(), protocol_cb = ?MODULE, diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index 3566a8a0a5..399ca7d0ee 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -31,8 +31,6 @@ %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [ {group, 'tlsv1.2'}, diff --git a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl index ae76f5849e..ee79a9d641 100644 --- a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl @@ -30,8 +30,6 @@ %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [{group, 'tlsv1.2'}, {group, 'tlsv1.1'}, diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index e1a36dbbd4..44a5f192e1 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -45,9 +45,6 @@ %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- - -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [ {group, basic}, @@ -162,7 +159,8 @@ renegotiate_tests() -> client_no_wrap_sequence_number, server_no_wrap_sequence_number, renegotiate_dos_mitigate_active, - renegotiate_dos_mitigate_passive]. + renegotiate_dos_mitigate_passive, + renegotiate_dos_mitigate_absolute]. cipher_tests() -> [cipher_suites, @@ -2998,8 +2996,36 @@ renegotiate_dos_mitigate_passive(Config) when is_list(Config) -> ssl_test_lib:close(Client). %%-------------------------------------------------------------------- +renegotiate_dos_mitigate_absolute() -> + [{doc, "Mitigate DOS computational attack by not allowing client to initiate renegotiation"}]. +renegotiate_dos_mitigate_absolute(Config) when is_list(Config) -> + ServerOpts = ?config(server_opts, Config), + ClientOpts = ?config(client_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, [{client_renegotiation, false} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, + renegotiate_rejected, + []}}, + {options, ClientOpts}]), + + ssl_test_lib:check_result(Client, ok, Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- tcp_error_propagation_in_active_mode() -> - [{doc,"Test that process recives {ssl_error, Socket, closed} when tcp error ocurres"}]. + [{doc,"Test that process recives {ssl_error, Socket, closed} when tcp error occurs"}]. tcp_error_propagation_in_active_mode(Config) when is_list(Config) -> ClientOpts = ?config(client_opts, Config), ServerOpts = ?config(server_opts, Config), @@ -3433,12 +3459,12 @@ renegotiate_reuse_session(Socket, Data) -> renegotiate(Socket, Data). renegotiate_immediately(Socket) -> - receive + receive {ssl, Socket, "Hello world"} -> ok; %% Handle 1/n-1 splitting countermeasure Rizzo/Duong-Beast {ssl, Socket, "H"} -> - receive + receive {ssl, Socket, "ello world"} -> ok end @@ -3450,6 +3476,26 @@ renegotiate_immediately(Socket) -> ct:log("Renegotiated again"), ssl:send(Socket, "Hello world"), ok. + +renegotiate_rejected(Socket) -> + receive + {ssl, Socket, "Hello world"} -> + ok; + %% Handle 1/n-1 splitting countermeasure Rizzo/Duong-Beast + {ssl, Socket, "H"} -> + receive + {ssl, Socket, "ello world"} -> + ok + end + end, + {error, renegotiation_rejected} = ssl:renegotiate(Socket), + {error, renegotiation_rejected} = ssl:renegotiate(Socket), + ct:sleep(?RENEGOTIATION_DISABLE_TIME +1), + {error, renegotiation_rejected} = ssl:renegotiate(Socket), + ct:log("Failed to renegotiate again"), + ssl:send(Socket, "Hello world"), + ok. + new_config(PrivDir, ServerOpts0) -> CaCertFile = proplists:get_value(cacertfile, ServerOpts0), diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index dab7a941db..9ace41acf8 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -37,9 +37,6 @@ %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- - -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [{group, active}, {group, passive}, diff --git a/lib/ssl/test/ssl_cipher_SUITE.erl b/lib/ssl/test/ssl_cipher_SUITE.erl index 3433f9a445..4cf80e1f43 100644 --- a/lib/ssl/test/ssl_cipher_SUITE.erl +++ b/lib/ssl/test/ssl_cipher_SUITE.erl @@ -34,9 +34,6 @@ %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- - -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [aes_decipher_good, aes_decipher_fail, padding_test]. diff --git a/lib/ssl/test/ssl_crl_SUITE.erl b/lib/ssl/test/ssl_crl_SUITE.erl index c6bf8898ad..79e1910b18 100644 --- a/lib/ssl/test/ssl_crl_SUITE.erl +++ b/lib/ssl/test/ssl_crl_SUITE.erl @@ -31,10 +31,6 @@ %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- - -suite() -> - [{ct_hooks,[ts_install_cth]}]. - all() -> [ {group, check_true}, diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl index 1a1b2af8d4..5a6556ffb8 100644 --- a/lib/ssl/test/ssl_dist_SUITE.erl +++ b/lib/ssl/test/ssl_dist_SUITE.erl @@ -38,10 +38,6 @@ %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- - -suite() -> - [{ct_hooks,[ts_install_cth]}]. - all() -> [basic, payload, plain_options, plain_verify_options]. diff --git a/lib/ssl/test/ssl_handshake_SUITE.erl b/lib/ssl/test/ssl_handshake_SUITE.erl index d4433393a1..7ee5f50b58 100644 --- a/lib/ssl/test/ssl_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_handshake_SUITE.erl @@ -31,8 +31,6 @@ %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [decode_hello_handshake, decode_single_hello_extension_correctly, decode_supported_elliptic_curves_hello_extension_correctly, diff --git a/lib/ssl/test/ssl_npn_handshake_SUITE.erl b/lib/ssl/test/ssl_npn_handshake_SUITE.erl index 8e95679306..fa55898835 100644 --- a/lib/ssl/test/ssl_npn_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_npn_handshake_SUITE.erl @@ -29,8 +29,6 @@ %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [{group, 'tlsv1.2'}, {group, 'tlsv1.1'}, diff --git a/lib/ssl/test/ssl_npn_hello_SUITE.erl b/lib/ssl/test/ssl_npn_hello_SUITE.erl index 68ff9172e9..288bca621e 100644 --- a/lib/ssl/test/ssl_npn_hello_SUITE.erl +++ b/lib/ssl/test/ssl_npn_hello_SUITE.erl @@ -32,9 +32,6 @@ %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- - -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [encode_and_decode_npn_client_hello_test, encode_and_decode_npn_server_hello_test, diff --git a/lib/ssl/test/ssl_packet_SUITE.erl b/lib/ssl/test/ssl_packet_SUITE.erl index d50498f547..12a9cad127 100644 --- a/lib/ssl/test/ssl_packet_SUITE.erl +++ b/lib/ssl/test/ssl_packet_SUITE.erl @@ -44,9 +44,6 @@ %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- - -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [ {group, 'tlsv1.2'}, diff --git a/lib/ssl/test/ssl_payload_SUITE.erl b/lib/ssl/test/ssl_payload_SUITE.erl index f95eae51b7..aa0cc70315 100644 --- a/lib/ssl/test/ssl_payload_SUITE.erl +++ b/lib/ssl/test/ssl_payload_SUITE.erl @@ -29,8 +29,6 @@ %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [ {group, 'tlsv1.2'}, diff --git a/lib/ssl/test/ssl_session_cache_SUITE.erl b/lib/ssl/test/ssl_session_cache_SUITE.erl index 36d086338e..3501d7e120 100644 --- a/lib/ssl/test/ssl_session_cache_SUITE.erl +++ b/lib/ssl/test/ssl_session_cache_SUITE.erl @@ -41,8 +41,6 @@ %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [session_cleanup, session_cache_process_list, diff --git a/lib/ssl/test/ssl_sni_SUITE.erl b/lib/ssl/test/ssl_sni_SUITE.erl index b059ff991b..1785dc4b86 100644 --- a/lib/ssl/test/ssl_sni_SUITE.erl +++ b/lib/ssl/test/ssl_sni_SUITE.erl @@ -28,7 +28,6 @@ %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- -suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [no_sni_header, sni_match, diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index a3bfdf8893..f35c0502ae 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -778,7 +778,12 @@ send_selected_port(_,_,_) -> rsa_suites(CounterPart) -> ECC = is_sane_ecc(CounterPart), - lists:filter(fun({rsa, _, _}) -> + FIPS = is_fips(CounterPart), + lists:filter(fun({rsa, des_cbc, sha}) when FIPS == true -> + false; + ({dhe_rsa, des_cbc, sha}) when FIPS == true -> + false; + ({rsa, _, _}) -> true; ({dhe_rsa, _, _}) -> true; @@ -1090,6 +1095,25 @@ is_sane_ecc(crypto) -> is_sane_ecc(_) -> true. +is_fips(openssl) -> + VersionStr = os:cmd("openssl version"), + case re:split(VersionStr, "fips") of + [_] -> + false; + _ -> + true + end; +is_fips(crypto) -> + [{_,_, Bin}] = crypto:info_lib(), + case re:split(Bin, <<"fips">>) of + [_] -> + false; + _ -> + true + end; +is_fips(_) -> + false. + cipher_restriction(Config0) -> case is_sane_ecc(openssl) of false -> diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index aca34cb6e9..463ab8088c 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -37,8 +37,6 @@ %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- -suite() -> [{ct_hooks,[ts_install_cth]}]. - all() -> [ {group, basic}, @@ -1036,7 +1034,7 @@ erlang_client_openssl_server_alpn(Config) when is_list(Config) -> erlang_server_alpn_openssl_client(Config) when is_list(Config) -> Data = "From openssl to erlang", start_erlang_server_and_openssl_client_with_opts(Config, - [{alpn_advertised_protocols, [<<"spdy/2">>]}], + [{alpn_preferred_protocols, [<<"spdy/2">>]}], "", Data, fun(Server, OpensslPort) -> true = port_command(OpensslPort, Data), diff --git a/lib/stdlib/doc/src/Makefile b/lib/stdlib/doc/src/Makefile index a4a2ed9931..d41f91250e 100644 --- a/lib/stdlib/doc/src/Makefile +++ b/lib/stdlib/doc/src/Makefile @@ -102,7 +102,7 @@ XML_REF3_FILES = \ XML_REF6_FILES = stdlib_app.xml XML_PART_FILES = part.xml part_notes.xml part_notes_history.xml -XML_CHAPTER_FILES = io_protocol.xml unicode_usage.xml notes.xml notes_history.xml +XML_CHAPTER_FILES = io_protocol.xml unicode_usage.xml notes.xml notes_history.xml assert_hrl.xml BOOK_FILES = book.xml diff --git a/lib/stdlib/doc/src/array.xml b/lib/stdlib/doc/src/array.xml index b03a2fa0cc..af23cd95d9 100644 --- a/lib/stdlib/doc/src/array.xml +++ b/lib/stdlib/doc/src/array.xml @@ -93,9 +93,6 @@ the default value cannot be confused with the values of set entries.</p> </datatype> <datatype> <name name="array" n_vars="0"/> - <desc> - <p><c>array()</c> is equivalent to <c>array(term())</c>.</p> - </desc> </datatype> <datatype> <name name="array_indx"/> diff --git a/lib/stdlib/doc/src/assert_hrl.xml b/lib/stdlib/doc/src/assert_hrl.xml new file mode 100644 index 0000000000..d812ee16dc --- /dev/null +++ b/lib/stdlib/doc/src/assert_hrl.xml @@ -0,0 +1,160 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE fileref SYSTEM "fileref.dtd"> + +<fileref> + <header> + <copyright> + <year>2012</year><year>2015</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>assert.hrl</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + </header> + <file>assert.hrl</file> + <filesummary>Assert Macros</filesummary> + <description> + <p>The include file <c>assert.hrl</c> provides macros for inserting + assertions in your program code.</p> + <p>These macros are defined in the Stdlib include file + <c>assert.hrl</c>. Include the following directive in the module + from which the function is called:</p> + <code type="none"> +-include_lib("stdlib/include/assert.hrl").</code> + <p>When an assertion succeeds, the assert macro yields the atom + <c>ok</c>. When an assertion fails, an exception of type <c>error</c> is + instead generated. The associated error term will have the form + <c>{Macro, Info}</c>, where <c>Macro</c> is the name of the macro, for + example <c>assertEqual</c>, and <c>Info</c> will be a list of tagged + values such as <c>[{module, M}, {line, L}, ...]</c> giving more + information about the location and cause of the exception. All entries + in the <c>Info</c> list are optional, and you should not rely + programatically on any of them being present.</p> + + <p>If the macro <c>NOASSERT</c> is defined when the <c>assert.hrl</c> + include file is read by the compiler, the macros will be defined as + equivalent to the atom <c>ok</c>. The test will not be performed, and + there will be no cost at runtime.</p> + + <p>For example, using <c>erlc</c> to compile your modules, the following + will disable all assertions:</p> + <code type="none"> +erlc -DNOASSERT=true *.erl</code> + <p>(The value of <c>NOASSERT</c> does not matter, only the fact that it + is defined.)</p> + <p>A few other macros also have effect on the enabling or disabling of + assertions:</p> + <list type="bulleted"> + <item>If <c>NODEBUG</c> is defined, it implies <c>NOASSERT</c>, unless + <c>DEBUG</c> is also defined, which is assumed to take + precedence.</item> + <item>If <c>ASSERT</c> is defined, it overrides <c>NOASSERT</c>, that + is, the assertions will remain enabled.</item> + </list> + <p>If you prefer, you can thus use only <c>DEBUG</c>/<c>NODEBUG</c> as + the main flags to control the behaviour of the assertions (which is + useful if you have other compiler conditionals or debugging macros + controlled by those flags), or you can use <c>ASSERT</c>/<c>NOASSERT</c> + to control only the assert macros.</p> + + </description> + + <section> + </section> + + <section> + <title>Macros</title> + <taglist> + <tag><c>assert(BoolExpr)</c></tag> + <item><p>Tests that <c>BoolExpr</c> completes normally returning + <c>true</c>.</p> + </item> + + <tag><c>assertNot(BoolExpr)</c></tag> + <item><p>Tests that <c>BoolExpr</c> completes normally returning + <c>false</c>.</p> + </item> + + <tag><c>assertMatch(GuardedPattern, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> completes normally yielding a value + that matches <c>GuardedPattern</c>. For example: + <code type="none"> + ?assertMatch({bork, _}, f())</code></p> + <p>Note that a guard <c>when ...</c> can be included: + <code type="none"> + ?assertMatch({bork, X} when X > 0, f())</code></p> + </item> + + <tag><c>assertNotMatch(GuardedPattern, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> completes normally yielding a value + that does not match <c>GuardedPattern</c>.</p> + <p>As in <c>assertMatch</c>, <c>GuardedPattern</c> can have a + <c>when</c> part.</p> + </item> + + <tag><c>assertEqual(ExpectedValue, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> completes normally yielding a value + that is exactly equal to <c>ExpectedValue</c>.</p> + </item> + + <tag><c>assertNotEqual(ExpectedValue, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> completes normally yielding a value + that is not exactly equal to <c>ExpectedValue</c>.</p> + </item> + + <tag><c>assertException(Class, Term, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> completes abnormally with an exception + of type <c>Class</c> and with the associated <c>Term</c>. The + assertion fails if <c>Expr</c> raises a different exception or if it + completes normally returning any value.</p> + <p>Note that both <c>Class</c> and <c>Term</c> can be guarded + patterns, as in <c>assertMatch</c>.</p> + </item> + + <tag><c>assertNotException(Class, Term, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> does not evaluate abnormally with an + exception of type <c>Class</c> and with the associated <c>Term</c>. + The assertion succeeds if <c>Expr</c> raises a different exception or + if it completes normally returning any value.</p> + <p>As in <c>assertException</c>, both <c>Class</c> and <c>Term</c> + can be guarded patterns.</p> + </item> + + <tag><c>assertError(Term, Expr)</c></tag> + <item><p>Equivalent to <c>assertException(error, Term, + Expr)</c></p> + </item> + + <tag><c>assertExit(Term, Expr)</c></tag> + <item><p>Equivalent to <c>assertException(exit, Term, Expr)</c></p> + </item> + + <tag><c>assertThrow(Term, Expr)</c></tag> + <item><p>Equivalent to <c>assertException(throw, Term, Expr)</c></p> + </item> + + </taglist> + </section> + + <section> + <title>SEE ALSO</title> + <p><seealso marker="compiler:compile">compile(3)</seealso></p> + <p><seealso marker="erts:erlc">erlc(3)</seealso></p> + </section> +</fileref> diff --git a/lib/stdlib/doc/src/dict.xml b/lib/stdlib/doc/src/dict.xml index 0771682a25..b456b97578 100644 --- a/lib/stdlib/doc/src/dict.xml +++ b/lib/stdlib/doc/src/dict.xml @@ -46,9 +46,6 @@ </datatype> <datatype> <name name="dict" n_vars="0"/> - <desc> - <p><c>dict()</c> is equivalent to <c>dict(term(), term())</c>.</p> - </desc> </datatype> </datatypes> <funcs> diff --git a/lib/stdlib/doc/src/gb_sets.xml b/lib/stdlib/doc/src/gb_sets.xml index 405bae5698..99e92d8680 100644 --- a/lib/stdlib/doc/src/gb_sets.xml +++ b/lib/stdlib/doc/src/gb_sets.xml @@ -120,9 +120,6 @@ </datatype> <datatype> <name name="set" n_vars="0"/> - <desc> - <p><c>set()</c> is equivalent to <c>set(term())</c>.</p> - </desc> </datatype> <datatype> <name name="iter" n_vars="1"/> @@ -130,9 +127,6 @@ </datatype> <datatype> <name name="iter" n_vars="0"/> - <desc> - <p><c>iter()</c> is equivalent to <c>iter(term())</c>.</p> - </desc> </datatype> </datatypes> <funcs> diff --git a/lib/stdlib/doc/src/gb_trees.xml b/lib/stdlib/doc/src/gb_trees.xml index 82167e1083..99ca2d6a9a 100644 --- a/lib/stdlib/doc/src/gb_trees.xml +++ b/lib/stdlib/doc/src/gb_trees.xml @@ -64,9 +64,6 @@ </datatype> <datatype> <name name="tree" n_vars="0"/> - <desc> - <p><c>tree()</c> is equivalent to <c>tree(term(), term())</c>.</p> - </desc> </datatype> <datatype> <name name="iter" n_vars="2"/> @@ -74,9 +71,6 @@ </datatype> <datatype> <name name="iter" n_vars="0"/> - <desc> - <p><c>iter()</c> is equivalent to <c>iter(term(), term())</c>.</p> - </desc> </datatype> </datatypes> <funcs> diff --git a/lib/stdlib/doc/src/queue.xml b/lib/stdlib/doc/src/queue.xml index 9c994154d4..f689412988 100644 --- a/lib/stdlib/doc/src/queue.xml +++ b/lib/stdlib/doc/src/queue.xml @@ -95,9 +95,6 @@ </datatype> <datatype> <name name="queue" n_vars="0"/> - <desc> - <p><c>queue()</c> is equivalent to <c>queue(term())</c>.</p> - </desc> </datatype> </datatypes> diff --git a/lib/stdlib/doc/src/ref_man.xml b/lib/stdlib/doc/src/ref_man.xml index eee4a68ca1..cae62612aa 100644 --- a/lib/stdlib/doc/src/ref_man.xml +++ b/lib/stdlib/doc/src/ref_man.xml @@ -35,6 +35,7 @@ </description> <xi:include href="stdlib_app.xml"/> <xi:include href="array.xml"/> + <xi:include href="assert_hrl.xml"/> <xi:include href="base64.xml"/> <xi:include href="beam_lib.xml"/> <xi:include href="binary.xml"/> diff --git a/lib/stdlib/doc/src/sets.xml b/lib/stdlib/doc/src/sets.xml index 4a31648f8f..eecddb7fd4 100644 --- a/lib/stdlib/doc/src/sets.xml +++ b/lib/stdlib/doc/src/sets.xml @@ -50,9 +50,6 @@ </datatype> <datatype> <name name="set" n_vars="0"/> - <desc> - <p><c>set()</c> is equivalent to <c>set(term())</c>.</p> - </desc> </datatype> </datatypes> <funcs> diff --git a/lib/stdlib/doc/src/unicode_usage.xml b/lib/stdlib/doc/src/unicode_usage.xml index 29b8940c62..eb3a556e10 100644 --- a/lib/stdlib/doc/src/unicode_usage.xml +++ b/lib/stdlib/doc/src/unicode_usage.xml @@ -850,8 +850,9 @@ Eshell V5.10.1 (abort with ^G) expected to be in Unicode.</p> <p>If Unicode file names are enabled, the calls to <seealso marker="kernel:os#getenv/0"><c>os:getenv/0</c></seealso>, - <seealso marker="kernel:os#getenv/1"><c>os:getenv/1</c></seealso> and - <seealso marker="kernel:os#putenv/2"><c>os:putenv/2</c></seealso> + <seealso marker="kernel:os#getenv/1"><c>os:getenv/1</c></seealso>, + <seealso marker="kernel:os#putenv/2"><c>os:putenv/2</c></seealso> and + <seealso marker="kernel:os#unsetenv/1"><c>os:unsetenv/1</c></seealso> will handle Unicode strings. On Unix-like platforms, the built-in functions will translate environment variables in UTF-8 to/from Unicode strings, possibly with code points > 255. On Windows the diff --git a/lib/stdlib/include/assert.hrl b/lib/stdlib/include/assert.hrl new file mode 100644 index 0000000000..239d19a6dc --- /dev/null +++ b/lib/stdlib/include/assert.hrl @@ -0,0 +1,260 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright (C) 2004-2014 Richard Carlsson, Mickaël Rémond +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-ifndef(ASSERT_HRL). +-define(ASSERT_HRL, true). + +%% Asserts are enabled unless NOASSERT is defined, and ASSERT can be used to +%% override it: if both ASSERT and NOASSERT are defined, then ASSERT takes +%% precedence, and NOASSERT will become undefined. +%% +%% Furthermore, if NODEBUG is defined, it implies NOASSERT, unless DEBUG or +%% ASSERT are defined. +%% +%% If asserts are disabled, all assert macros are defined to be the atom +%% 'ok'. If asserts are enabled, all assert macros are defined to yield 'ok' +%% as the result if the test succeeds, and raise an error exception if the +%% test fails. The error term will then have the form {Name, Info} where +%% Name is the name of the macro and Info is a list of tagged tuples. + +%% allow NODEBUG to imply NOASSERT, unless DEBUG +-ifdef(NODEBUG). +-ifndef(DEBUG). +-ifndef(NOASSERT). +-define(NOASSERT, true). +-endif. +-endif. +-endif. + +%% allow ASSERT to override NOASSERT +-ifdef(ASSERT). +-undef(NOASSERT). +-endif. + +%% Assert macros must not depend on any non-kernel or stdlib libraries. +%% +%% We must use fun-call wrappers ((fun () -> ... end)()) to avoid +%% exporting local variables, and furthermore we only use variable names +%% prefixed with "__", that hopefully will not be bound outside the fun. +%% It is not possible to nest assert macros. + +-ifdef(NOASSERT). +-define(assert(BoolExpr),ok). +-else. +%% The assert macro is written the way it is so as not to cause warnings +%% for clauses that cannot match, even if the expression is a constant. +-define(assert(BoolExpr), + begin + ((fun () -> + case (BoolExpr) of + true -> ok; + __V -> erlang:error({assert, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??BoolExpr)}, + {expected, true}, + case __V of false -> {value, __V}; + _ -> {not_boolean,__V} + end]}) + end + end)()) + end). +-endif. + +%% This is the inverse case of assert, for convenience. +-ifdef(NOASSERT). +-define(assertNot(BoolExpr),ok). +-else. +-define(assertNot(BoolExpr), + begin + ((fun () -> + case (BoolExpr) of + false -> ok; + __V -> erlang:error({assert, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??BoolExpr)}, + {expected, false}, + case __V of true -> {value, __V}; + _ -> {not_boolean,__V} + end]}) + end + end)()) + end). +-endif. + +%% This is mostly a convenience which gives more detailed reports. +%% Note: Guard is a guarded pattern, and can not be used for value. +-ifdef(NOASSERT). +-define(assertMatch(Guard, Expr), ok). +-else. +-define(assertMatch(Guard, Expr), + begin + ((fun () -> + case (Expr) of + Guard -> ok; + __V -> erlang:error({assertMatch, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {pattern, (??Guard)}, + {value, __V}]}) + end + end)()) + end). +-endif. + +%% This is the inverse case of assertMatch, for convenience. +-ifdef(NOASSERT). +-define(assertNotMatch(Guard, Expr), ok). +-else. +-define(assertNotMatch(Guard, Expr), + begin + ((fun () -> + __V = (Expr), + case __V of + Guard -> erlang:error({assertNotMatch, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {pattern, (??Guard)}, + {value, __V}]}); + _ -> ok + end + end)()) + end). +-endif. + +%% This is a convenience macro which gives more detailed reports when +%% the expected LHS value is not a pattern, but a computed value +-ifdef(NOASSERT). +-define(assertEqual(Expect, Expr), ok). +-else. +-define(assertEqual(Expect, Expr), + begin + ((fun (__X) -> + case (Expr) of + __X -> ok; + __V -> erlang:error({assertEqual, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {expected, __X}, + {value, __V}]}) + end + end)(Expect)) + end). +-endif. + +%% This is the inverse case of assertEqual, for convenience. +-ifdef(NOASSERT). +-define(assertNotEqual(Unexpected, Expr), ok). +-else. +-define(assertNotEqual(Unexpected, Expr), + begin + ((fun (__X) -> + case (Expr) of + __X -> erlang:error({assertNotEqual, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {value, __X}]}); + _ -> ok + end + end)(Unexpected)) + end). +-endif. + +%% Note: Class and Term are patterns, and can not be used for value. +%% Term can be a guarded pattern, but Class cannot. +-ifdef(NOASSERT). +-define(assertException(Class, Term, Expr), ok). +-else. +-define(assertException(Class, Term, Expr), + begin + ((fun () -> + try (Expr) of + __V -> erlang:error({assertException, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {pattern, + "{ "++(??Class)++" , "++(??Term) + ++" , [...] }"}, + {unexpected_success, __V}]}) + catch + Class:Term -> ok; + __C:__T -> + erlang:error({assertException, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {pattern, + "{ "++(??Class)++" , "++(??Term) + ++" , [...] }"}, + {unexpected_exception, + {__C, __T, + erlang:get_stacktrace()}}]}) + end + end)()) + end). +-endif. + +-define(assertError(Term, Expr), ?assertException(error, Term, Expr)). +-define(assertExit(Term, Expr), ?assertException(exit, Term, Expr)). +-define(assertThrow(Term, Expr), ?assertException(throw, Term, Expr)). + +%% This is the inverse case of assertException, for convenience. +%% Note: Class and Term are patterns, and can not be used for value. +%% Both Class and Term can be guarded patterns. +-ifdef(NOASSERT). +-define(assertNotException(Class, Term, Expr), ok). +-else. +-define(assertNotException(Class, Term, Expr), + begin + ((fun () -> + try (Expr) of + _ -> ok + catch + __C:__T -> + case __C of + Class -> + case __T of + Term -> + erlang:error({assertNotException, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {pattern, + "{ "++(??Class)++" , " + ++(??Term)++" , [...] }"}, + {unexpected_exception, + {__C, __T, + erlang:get_stacktrace() + }}]}); + _ -> ok + end; + _ -> ok + end + end + end)()) + end). +-endif. + +-endif. % ASSERT_HRL diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile index 55bda60da5..344a5dc099 100644 --- a/lib/stdlib/src/Makefile +++ b/lib/stdlib/src/Makefile @@ -122,6 +122,7 @@ MODULES= \ zip HRL_FILES= \ + ../include/assert.hrl \ ../include/erl_compile.hrl \ ../include/erl_bits.hrl \ ../include/ms_transform.hrl \ diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl index 10d2ccea45..f98a587c55 100644 --- a/lib/stdlib/src/array.erl +++ b/lib/stdlib/src/array.erl @@ -164,7 +164,7 @@ elements :: elements(_) %% the tuple tree }). --opaque array() :: array(term()). +-type array() :: array(term()). -opaque array(Type) :: #array{default :: Type, elements :: elements(Type)}. diff --git a/lib/stdlib/src/dict.erl b/lib/stdlib/src/dict.erl index 5a9f63c5e2..d2af9554a1 100644 --- a/lib/stdlib/src/dict.erl +++ b/lib/stdlib/src/dict.erl @@ -70,7 +70,7 @@ }). --opaque dict() :: dict(_, _). +-type dict() :: dict(_, _). -opaque dict(Key, Value) :: #dict{segs :: segs(Key, Value)}. diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index e328e065e3..274bb2a782 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -125,22 +125,19 @@ top_type_100 -> type_200 : '$1'. top_type_100 -> type_200 '|' top_type_100 : lift_unions('$1','$3'). type_200 -> type_300 '..' type_300 : {type, ?anno('$1'), range, - [skip_paren('$1'), - skip_paren('$3')]}. + ['$1', '$3']}. type_200 -> type_300 : '$1'. -type_300 -> type_300 add_op type_400 : ?mkop2(skip_paren('$1'), - '$2', skip_paren('$3')). +type_300 -> type_300 add_op type_400 : ?mkop2('$1', '$2', '$3'). type_300 -> type_400 : '$1'. -type_400 -> type_400 mult_op type_500 : ?mkop2(skip_paren('$1'), - '$2', skip_paren('$3')). +type_400 -> type_400 mult_op type_500 : ?mkop2('$1', '$2', '$3'). type_400 -> type_500 : '$1'. -type_500 -> prefix_op type : ?mkop1('$1', skip_paren('$2')). +type_500 -> prefix_op type : ?mkop1('$1', '$2'). type_500 -> type : '$1'. -type -> '(' top_type ')' : {paren_type, ?anno('$2'), ['$2']}. +type -> '(' top_type ')' : '$2'. type -> var : '$1'. type -> atom : '$1'. type -> atom '(' ')' : build_gen_type('$1'). @@ -524,6 +521,7 @@ Erlang code. -export([normalise/1,abstract/1,tokens/1,tokens/2]). -export([abstract/2]). -export([inop_prec/1,preop_prec/1,func_prec/0,max_prec/0]). +-export([type_inop_prec/1,type_preop_prec/1]). -export([map_anno/2, fold_anno/3, mapfold_anno/3, new_anno/1, anno_to_term/1, anno_from_term/1]). -export([set_line/2,get_attribute/2,get_attributes/1]). @@ -671,11 +669,6 @@ lift_unions(T1, {type, _Aa, union, List}) -> lift_unions(T1, T2) -> {type, ?anno(T1), union, [T1, T2]}. -skip_paren({paren_type,_A,[Type]}) -> - skip_paren(Type); -skip_paren(Type) -> - Type. - build_gen_type({atom, Aa, tuple}) -> {type, Aa, tuple, any}; build_gen_type({atom, Aa, map}) -> @@ -687,7 +680,7 @@ build_gen_type({atom, Aa, Name}) -> build_bin_type([{var, _, '_'}|Left], Int) -> build_bin_type(Left, Int); build_bin_type([], Int) -> - skip_paren(Int); + Int; build_bin_type([{var, Aa, _}|_], _) -> ret_err(Aa, "Bad binary type"). @@ -807,8 +800,7 @@ record_fields([{typed,Expr,TypeInfo}|Fields]) -> {atom, Aa, _} -> case has_undefined(TypeInfo) of false -> - TypeInfo2 = maybe_add_paren(TypeInfo), - lift_unions(abstract2(undefined, Aa), TypeInfo2); + lift_unions(abstract2(undefined, Aa), TypeInfo); true -> TypeInfo end @@ -822,18 +814,11 @@ has_undefined({atom,_,undefined}) -> true; has_undefined({ann_type,_,[_,T]}) -> has_undefined(T); -has_undefined({paren_type,_,[T]}) -> - has_undefined(T); has_undefined({type,_,union,Ts}) -> lists:any(fun has_undefined/1, Ts); has_undefined(_) -> false. -maybe_add_paren({ann_type,A,T}) -> - {paren_type,A,[{ann_type,A,T}]}; -maybe_add_paren(T) -> - T. - term(Expr) -> try normalise(Expr) catch _:_R -> ret_err(?anno(Expr), "bad attribute") @@ -1099,6 +1084,39 @@ func_prec() -> {800,700}. max_prec() -> 900. +-type prec() :: non_neg_integer(). + +-type type_inop() :: '::' | '|' | '..' | '+' | '-' | 'bor' | 'bxor' + | 'bsl' | 'bsr' | '*' | '/' | 'div' | 'rem' | 'band'. + +-type type_preop() :: '+' | '-' | 'bnot' | '#'. + +-spec type_inop_prec(type_inop()) -> {prec(), prec(), prec()}. + +type_inop_prec('=') -> {150,100,100}; +type_inop_prec('::') -> {160,150,150}; +type_inop_prec('|') -> {180,170,170}; +type_inop_prec('..') -> {300,200,300}; +type_inop_prec('+') -> {400,400,500}; +type_inop_prec('-') -> {400,400,500}; +type_inop_prec('bor') -> {400,400,500}; +type_inop_prec('bxor') -> {400,400,500}; +type_inop_prec('bsl') -> {400,400,500}; +type_inop_prec('bsr') -> {400,400,500}; +type_inop_prec('*') -> {500,500,600}; +type_inop_prec('/') -> {500,500,600}; +type_inop_prec('div') -> {500,500,600}; +type_inop_prec('rem') -> {500,500,600}; +type_inop_prec('band') -> {500,500,600}; +type_inop_prec('#') -> {800,700,800}. + +-spec type_preop_prec(type_preop()) -> {prec(), prec()}. + +type_preop_prec('+') -> {600,700}; +type_preop_prec('-') -> {600,700}; +type_preop_prec('bnot') -> {600,700}; +type_preop_prec('#') -> {700,800}. + %%% [Experimental]. The parser just copies the attributes of the %%% scanner tokens to the abstract format. This design decision has %%% been hidden to some extent: use set_line() and get_attribute() to diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl index 623a29f923..6da585b72e 100644 --- a/lib/stdlib/src/erl_pp.erl +++ b/lib/stdlib/src/erl_pp.erl @@ -27,7 +27,8 @@ -import(lists, [append/1,foldr/3,mapfoldl/3,reverse/1,reverse/2]). -import(io_lib, [write/1,format/2]). --import(erl_parse, [inop_prec/1,preop_prec/1,func_prec/0,max_prec/0]). +-import(erl_parse, [inop_prec/1,preop_prec/1,func_prec/0,max_prec/0, + type_inop_prec/1, type_preop_prec/1]). -define(MAXLINE, 72). @@ -271,49 +272,64 @@ typeattr(Tag, {TypeName,Type,Args}, _Opts) -> {first,leaf("-"++atom_to_list(Tag)++" "), typed(call({atom,a0(),TypeName}, Args, 0, options(none)), Type)}. -ltype({ann_type,_Line,[V,T]}) -> - typed(lexpr(V, options(none)), T); -ltype({paren_type,_Line,[T]}) -> - [$(,ltype(T),$)]; -ltype({type,_Line,union,Ts}) -> - {seq,[],[],[' |'],ltypes(Ts)}; -ltype({type,_Line,list,[T]}) -> +ltype(T) -> + ltype(T, 0). + +ltype({ann_type,_Line,[V,T]}, Prec) -> + {_L,P,_R} = type_inop_prec('::'), + E = typed(lexpr(V, options(none)), T), + maybe_paren(P, Prec, E); +ltype({paren_type,_Line,[T]}, P) -> + %% Generated before Erlang/OTP 18. + ltype(T, P); +ltype({type,_Line,union,Ts}, Prec) -> + {_L,P,R} = type_inop_prec('|'), + E = {seq,[],[],[' |'],ltypes(Ts, R)}, + maybe_paren(P, Prec, E); +ltype({type,_Line,list,[T]}, _) -> {seq,$[,$],$,,[ltype(T)]}; -ltype({type,_Line,nonempty_list,[T]}) -> +ltype({type,_Line,nonempty_list,[T]}, _) -> {seq,$[,$],[$,],[ltype(T),leaf("...")]}; -ltype({type,Line,nil,[]}) -> - lexpr({nil,Line}, 0, options(none)); -ltype({type,Line,map,any}) -> +ltype({type,Line,nil,[]}, _) -> + lexpr({nil,Line}, options(none)); +ltype({type,Line,map,any}, _) -> simple_type({atom,Line,map}, []); -ltype({type,_Line,map,Pairs}) -> - map_type(Pairs); -ltype({type,Line,tuple,any}) -> +ltype({type,_Line,map,Pairs}, Prec) -> + {P,_R} = type_preop_prec('#'), + E = map_type(Pairs), + maybe_paren(P, Prec, E); +ltype({type,Line,tuple,any}, _) -> simple_type({atom,Line,tuple}, []); -ltype({type,_Line,tuple,Ts}) -> - tuple_type(Ts, fun ltype/1); -ltype({type,_Line,record,[{atom,_,N}|Fs]}) -> - record_type(N, Fs); -ltype({type,_Line,range,[_I1,_I2]=Es}) -> - expr_list(Es, '..', fun lexpr/2, options(none)); -ltype({type,_Line,binary,[I1,I2]}) -> +ltype({type,_Line,tuple,Ts}, _) -> + tuple_type(Ts, fun ltype/2); +ltype({type,_Line,record,[{atom,_,N}|Fs]}, Prec) -> + {P,_R} = type_preop_prec('#'), + E = record_type(N, Fs), + maybe_paren(P, Prec, E); +ltype({type,_Line,range,[_I1,_I2]=Es}, Prec) -> + {_L,P,R} = type_inop_prec('..'), + F = fun(E, Opts) -> lexpr(E, R, Opts) end, + E = expr_list(Es, '..', F, options(none)), + maybe_paren(P, Prec, E); +ltype({type,_Line,binary,[I1,I2]}, _) -> binary_type(I1, I2); % except binary() -ltype({type,_Line,'fun',[]}) -> +ltype({type,_Line,'fun',[]}, _) -> leaf("fun()"); -ltype({type,_,'fun',[{type,_,any},_]}=FunType) -> +ltype({type,_,'fun',[{type,_,any},_]}=FunType, _) -> [fun_type(['fun',$(], FunType),$)]; -ltype({type,_Line,'fun',[{type,_,product,_},_]}=FunType) -> +ltype({type,_Line,'fun',[{type,_,product,_},_]}=FunType, _) -> [fun_type(['fun',$(], FunType),$)]; -ltype({type,Line,T,Ts}) -> +ltype({type,Line,T,Ts}, _) -> %% Compatibility. Before 18.0. simple_type({atom,Line,T}, Ts); -ltype({user_type,Line,T,Ts}) -> +ltype({user_type,Line,T,Ts}, _) -> simple_type({atom,Line,T}, Ts); -ltype({remote_type,Line,[M,F,Ts]}) -> +ltype({remote_type,Line,[M,F,Ts]}, _) -> simple_type({remote,Line,M,F}, Ts); -ltype({atom,_,T}) -> +ltype({atom,_,T}, _) -> leaf(write(T)); -ltype(E) -> - lexpr(E, 0, options(none)). +ltype(E, P) -> + lexpr(E, P, options(none)). binary_type(I1, I2) -> B = [[] || {integer,_,0} <- [I1]] =:= [], @@ -327,42 +343,37 @@ map_type(Fs) -> {first,[$#],map_pair_types(Fs)}. map_pair_types(Fs) -> - tuple_type(Fs, fun map_pair_type/1). + tuple_type(Fs, fun map_pair_type/2). -map_pair_type({type,_Line,map_field_assoc,[Ktype,Vtype]}) -> - map_assoc_typed(ltype(Ktype), Vtype). +map_pair_type({type,_Line,map_field_assoc,[Ktype,Vtype]}, Prec) -> + map_assoc_typed(ltype(Ktype), Vtype, Prec). -map_assoc_typed(B, {type,_,union,Ts}) -> - {first,[B,$\s],{seq,[],[],[],map_assoc_union_type(Ts)}}; -map_assoc_typed(B, Type) -> - {list,[{cstep,[B," =>"],ltype(Type)}]}. +map_assoc_typed(B, {type,_,union,Ts}, Prec) -> + {first,[B,$\s],{seq,[],[],[],map_assoc_union_type(Ts, Prec)}}; +map_assoc_typed(B, Type, Prec) -> + {list,[{cstep,[B," =>"],ltype(Type, Prec)}]}. -map_assoc_union_type([T|Ts]) -> - [[leaf("=> "),ltype(T)] | ltypes(Ts, fun union_elem/1)]. +map_assoc_union_type([T|Ts], Prec) -> + [[leaf("=> "),ltype(T)] | ltypes(Ts, fun union_elem/2, Prec)]. record_type(Name, Fields) -> {first,[record_name(Name)],field_types(Fields)}. field_types(Fs) -> - tuple_type(Fs, fun field_type/1). + tuple_type(Fs, fun field_type/2). -field_type({type,_Line,field_type,[Name,Type]}) -> +field_type({type,_Line,field_type,[Name,Type]}, _Prec) -> typed(lexpr(Name, options(none)), Type). -typed(B, {type,_,union,Ts}) -> - %% Special layout for :: followed by union. - {first,[B,$\s],{seq,[],[],[],union_type(Ts)}}; typed(B, Type) -> - {list,[{cstep,[B,' ::'],ltype(Type)}]}. + {_L,_P,R} = type_inop_prec('::'), + {list,[{cstep,[B,' ::'],ltype(Type, R)}]}. -union_type([T|Ts]) -> - [[leaf(":: "),ltype(T)] | ltypes(Ts, fun union_elem/1)]. - -union_elem(T) -> - [leaf(" | "),ltype(T)]. +union_elem(T, Prec) -> + [leaf(" | "),ltype(T, Prec)]. tuple_type(Ts, F) -> - {seq,${,$},[$,],ltypes(Ts, F)}. + {seq,${,$},[$,],ltypes(Ts, F, 0)}. specattr(SpecKind, {FuncSpec,TypeSpecs}) -> Func = case FuncSpec of @@ -399,16 +410,16 @@ type_args({type,_line,product,Ts}) -> targs(Ts). simple_type(Tag, Types) -> - {first,lexpr(Tag, 0, options(none)),targs(Types)}. + {first,lexpr(Tag, options(none)),targs(Types)}. targs(Ts) -> - {seq,$(,$),[$,],ltypes(Ts)}. + {seq,$(,$),[$,],ltypes(Ts, 0)}. -ltypes(Ts) -> - ltypes(Ts, fun ltype/1). +ltypes(Ts, Prec) -> + ltypes(Ts, fun ltype/2, Prec). -ltypes(Ts, F) -> - [F(T) || T <- Ts]. +ltypes(Ts, F, Prec) -> + [F(T, Prec) || T <- Ts]. attr(Name, Args) -> call({var,a0(),format("-~s", [Name])}, Args, 0, options(none)). diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl index 0e2d59d0c3..bec64861c3 100644 --- a/lib/stdlib/src/ets.erl +++ b/lib/stdlib/src/ets.erl @@ -1308,18 +1308,30 @@ create_tab(I, TabArg) -> {name, Name} = lists:keyfind(name, 1, I), {type, Type} = lists:keyfind(type, 1, I), {protection, P} = lists:keyfind(protection, 1, I), - {named_table, Val} = lists:keyfind(named_table, 1, I), {keypos, _Kp} = Keypos = lists:keyfind(keypos, 1, I), {size, Sz} = lists:keyfind(size, 1, I), - Comp = case lists:keyfind(compressed, 1, I) of - {compressed, true} -> [compressed]; - {compressed, false} -> []; - false -> [] - end, + L1 = [Type, P, Keypos], + L2 = case lists:keyfind(named_table, 1, I) of + {named_table, true} -> [named_table | L1]; + {named_table, false} -> L1 + end, + L3 = case lists:keyfind(compressed, 1, I) of + {compressed, true} -> [compressed | L2]; + {compressed, false} -> L2; + false -> L2 + end, + L4 = case lists:keyfind(write_concurrency, 1, I) of + {write_concurrency, _}=Wcc -> [Wcc | L3]; + _ -> L3 + end, + L5 = case lists:keyfind(read_concurrency, 1, I) of + {read_concurrency, _}=Rcc -> [Rcc | L4]; + false -> L4 + end, case TabArg of [] -> try - Tab = ets:new(Name, [Type, P, Keypos] ++ named_table(Val) ++ Comp), + Tab = ets:new(Name, L5), {ok, Tab, Sz} catch _:_ -> throw(cannot_create_table) @@ -1328,8 +1340,6 @@ create_tab(I, TabArg) -> {ok, TabArg, Sz} end. -named_table(true) -> [named_table]; -named_table(false) -> []. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% tabfile_info/1 reads the head information in an ets table dumped to diff --git a/lib/stdlib/src/gb_sets.erl b/lib/stdlib/src/gb_sets.erl index d3fbd542f7..d099737d8f 100644 --- a/lib/stdlib/src/gb_sets.erl +++ b/lib/stdlib/src/gb_sets.erl @@ -203,11 +203,10 @@ -export_type([set/0, set/1, iter/0, iter/1]). -type gb_set_node(Element) :: 'nil' | {Element, _, _}. --type gb_set_node() :: gb_set_node(_). -opaque set(Element) :: {non_neg_integer(), gb_set_node(Element)}. --opaque set() :: set(_). +-type set() :: set(_). -opaque iter(Element) :: [gb_set_node(Element)]. --opaque iter() :: [gb_set_node()]. +-type iter() :: iter(_). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/stdlib/src/gb_trees.erl b/lib/stdlib/src/gb_trees.erl index 259e8f718b..2cbfd8fd2a 100644 --- a/lib/stdlib/src/gb_trees.erl +++ b/lib/stdlib/src/gb_trees.erl @@ -160,11 +160,10 @@ -type gb_tree_node(K, V) :: 'nil' | {K, V, gb_tree_node(K, V), gb_tree_node(K, V)}. --type gb_tree_node() :: gb_tree_node(_, _). -opaque tree(Key, Value) :: {non_neg_integer(), gb_tree_node(Key, Value)}. --opaque tree() :: tree(_, _). +-type tree() :: tree(_, _). -opaque iter(Key, Value) :: [gb_tree_node(Key, Value)]. --opaque iter() :: [gb_tree_node()]. +-type iter() :: iter(_, _). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 0340015c35..62807e2e54 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -634,6 +634,19 @@ obsolete_1(ssl, negotiated_next_protocol, 1) -> obsolete_1(ssl, connection_info, 1) -> {deprecated, "deprecated; use connection_information/[1,2] instead"}; +obsolete_1(httpd_conf, check_enum, 2) -> + {deprecated, "deprecated; use lists:member/2 instead"}; +obsolete_1(httpd_conf, clean, 1) -> + {deprecated, "deprecated; use sting:strip/1 instead or possible the re module"}; +obsolete_1(httpd_conf, custom_clean, 3) -> + {deprecated, "deprecated; use sting:strip/3 instead or possible the re module"}; +obsolete_1(httpd_conf, is_directory, 1) -> + {deprecated, "deprecated; use filelib:is_dir/1 instead"}; +obsolete_1(httpd_conf, is_file, 1) -> + {deprecated, "deprecated; use filelib:is_file/1 instead"}; +obsolete_1(httpd_conf, make_integer, 1) -> + {deprecated, "deprecated; use erlang:list_to_integer/1 instead"}; + obsolete_1(_, _, _) -> no. diff --git a/lib/stdlib/src/queue.erl b/lib/stdlib/src/queue.erl index 472d503b99..d2e6848258 100644 --- a/lib/stdlib/src/queue.erl +++ b/lib/stdlib/src/queue.erl @@ -48,7 +48,7 @@ -opaque queue(Item) :: {list(Item), list(Item)}. --opaque queue() :: queue(_). +-type queue() :: queue(_). %% Creation, inspection and conversion diff --git a/lib/stdlib/src/sets.erl b/lib/stdlib/src/sets.erl index 167a676281..041d281148 100644 --- a/lib/stdlib/src/sets.erl +++ b/lib/stdlib/src/sets.erl @@ -70,7 +70,7 @@ segs :: segs(_) % Segments }). --opaque set() :: set(_). +-type set() :: set(_). -opaque set(Element) :: #set{segs :: segs(Element)}. diff --git a/lib/stdlib/src/supervisor.erl b/lib/stdlib/src/supervisor.erl index 1d7396adee..ac51bb3b79 100644 --- a/lib/stdlib/src/supervisor.erl +++ b/lib/stdlib/src/supervisor.erl @@ -393,6 +393,15 @@ handle_call({start_child, EArgs}, _From, State) when ?is_simple(State) -> {reply, What, State} end; +handle_call({start_child, ChildSpec}, _From, State) -> + case check_childspec(ChildSpec) of + {ok, Child} -> + {Resp, NState} = handle_start_child(Child, State), + {reply, Resp, NState}; + What -> + {reply, {error, What}, State} + end; + %% terminate_child for simple_one_for_one can only be done with pid handle_call({terminate_child, Name}, _From, State) when not is_pid(Name), ?is_simple(State) -> @@ -411,20 +420,10 @@ handle_call({terminate_child, Name}, _From, State) -> {reply, {error, not_found}, State} end; -%%% The requests delete_child and restart_child are invalid for -%%% simple_one_for_one supervisors. -handle_call({_Req, _Data}, _From, State) when ?is_simple(State) -> +%% restart_child request is invalid for simple_one_for_one supervisors +handle_call({restart_child, _Name}, _From, State) when ?is_simple(State) -> {reply, {error, simple_one_for_one}, State}; -handle_call({start_child, ChildSpec}, _From, State) -> - case check_childspec(ChildSpec) of - {ok, Child} -> - {Resp, NState} = handle_start_child(Child, State), - {reply, Resp, NState}; - What -> - {reply, {error, What}, State} - end; - handle_call({restart_child, Name}, _From, State) -> case get_child(Name, State) of {value, Child} when Child#child.pid =:= undefined -> @@ -446,6 +445,10 @@ handle_call({restart_child, Name}, _From, State) -> {reply, {error, not_found}, State} end; +%% delete_child request is invalid for simple_one_for_one supervisors +handle_call({delete_child, _Name}, _From, State) when ?is_simple(State) -> + {reply, {error, simple_one_for_one}, State}; + handle_call({delete_child, Name}, _From, State) -> case get_child(Name, State) of {value, Child} when Child#child.pid =:= undefined -> diff --git a/lib/stdlib/test/Makefile b/lib/stdlib/test/Makefile index 61eb34d565..d4ab674486 100644 --- a/lib/stdlib/test/Makefile +++ b/lib/stdlib/test/Makefile @@ -107,7 +107,8 @@ RELSYSDIR = $(RELEASE_PATH)/stdlib_test ERL_MAKE_FLAGS += ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/test_server/include \ - -I$(ERL_TOP)/lib/kernel/include + -I$(ERL_TOP)/lib/kernel/include \ + -I$(ERL_TOP)/lib/stdlib/include EBIN = . diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index 1d63c8e17e..afeeb5bfd4 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -1149,7 +1149,7 @@ otp_11100(Config) when is_list(Config) -> {a,{type,A1,range,[{integer,A1,1},{foo,bar}]},[]}}), "-type foo(INVALID-FORM:{foo,bar}:) :: A.\n" = pf({attribute,A1,type,{foo,{var,A1,'A'},[{foo,bar}]}}), - "-type foo() :: (INVALID-FORM:{foo,bar}: :: []).\n" = + "-type foo() :: INVALID-FORM:{foo,bar}: :: [].\n" = pf({attribute,A1,type, {foo,{paren_type,A1, [{ann_type,A1,[{foo,bar},{type,A1,nil,[]}]}]}, diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index fff6b11a38..f47c2c518d 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -117,6 +117,7 @@ init_per_testcase(Case, Config) -> start_spawn_logger(), wait_for_test_procs(), %% Ensure previous case cleaned up Dog=test_server:timetrap(test_server:minutes(20)), + put('__ETS_TEST_CASE__', Case), [{watchdog, Dog}, {test_case, Case} | Config]. end_per_testcase(_Func, Config) -> @@ -216,8 +217,9 @@ memory_check_summary(_Config) -> ets_test_spawn_logger ! {self(), get_failed_memchecks}, receive {get_failed_memchecks, FailedMemchecks} -> ok end, io:format("Failed memchecks: ~p\n",[FailedMemchecks]), - if FailedMemchecks > 3 -> - ct:fail("Too many failed (~p) memchecks", [FailedMemchecks]); + NoFailedMemchecks = length(FailedMemchecks), + if NoFailedMemchecks > 3 -> + ct:fail("Too many failed (~p) memchecks", [NoFailedMemchecks]); true -> ok end @@ -4088,8 +4090,11 @@ tab2file(Config) when is_list(Config) -> tab2file_do(FName, Opts) -> %% Write an empty ets table to a file, read back and check properties. - ?line Tab = ets_new(ets_SUITE_foo_tab, [named_table, set, private, - {keypos, 2}]), + ?line Tab = ets_new(ets_SUITE_foo_tab, [named_table, set, public, + {keypos, 2}, + compressed, + {write_concurrency,true}, + {read_concurrency,true}]), catch file:delete(FName), Res = ets:tab2file(Tab, FName, Opts), true = ets:delete(Tab), @@ -4097,10 +4102,14 @@ tab2file_do(FName, Opts) -> % ?line EtsMem = etsmem(), ?line {ok, Tab2} = ets:file2tab(FName), - ?line private = ets:info(Tab2, protection), + public = ets:info(Tab2, protection), ?line true = ets:info(Tab2, named_table), ?line 2 = ets:info(Tab2, keypos), ?line set = ets:info(Tab2, type), + true = ets:info(Tab2, compressed), + Smp = erlang:system_info(smp_support), + Smp = ets:info(Tab2, read_concurrency), + Smp = ets:info(Tab2, write_concurrency), ?line true = ets:delete(Tab2), ?line verify_etsmem(EtsMem). @@ -4319,7 +4328,7 @@ tabfile_ext4(Config) when is_list(Config) -> {error,Y} = ets:file2tab(FName,[{verify,true}]), ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), {X,Y} - end || N <- lists:seq(400,500) ], + end || N <- lists:seq(500,600) ], io:format("~p~n",[Res]), file:delete(FName), ok. @@ -5921,7 +5930,7 @@ verify_etsmem({MemInfo,AllTabs}) -> io:format("Actual: ~p", [MemInfo2]), io:format("Changed tables before: ~p\n",[AllTabs -- AllTabs2]), io:format("Changed tables after: ~p\n", [AllTabs2 -- AllTabs]), - ets_test_spawn_logger ! failed_memcheck, + ets_test_spawn_logger ! {failed_memcheck, get('__ETS_TEST_CASE__')}, {comment, "Failed memory check"} end. @@ -5972,8 +5981,8 @@ spawn_logger(Procs, FailedMemchecks) -> From ! test_procs_synced, spawn_logger([From], FailedMemchecks); - failed_memcheck -> - spawn_logger(Procs, FailedMemchecks+1); + {failed_memcheck, TestCase} -> + spawn_logger(Procs, [TestCase|FailedMemchecks]); {Pid, get_failed_memchecks} -> Pid ! {get_failed_memchecks, FailedMemchecks}, @@ -5993,7 +6002,7 @@ start_spawn_logger() -> case whereis(ets_test_spawn_logger) of Pid when is_pid(Pid) -> true; _ -> register(ets_test_spawn_logger, - spawn_opt(fun () -> spawn_logger([], 0) end, + spawn_opt(fun () -> spawn_logger([], []) end, [{priority, max}])) end. diff --git a/lib/stdlib/test/stdlib_SUITE.erl b/lib/stdlib/test/stdlib_SUITE.erl index 206eb4fd74..8ab30eb62b 100644 --- a/lib/stdlib/test/stdlib_SUITE.erl +++ b/lib/stdlib/test/stdlib_SUITE.erl @@ -30,7 +30,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [app_test, appup_test, {group,upgrade}]. + [app_test, appup_test, assert_test, {group,upgrade}]. groups() -> [{upgrade,[minor_upgrade,major_upgrade]}]. @@ -185,3 +185,68 @@ upgrade_upgraded(_CtData,State) -> State. upgrade_downgraded(_CtData,State) -> State. + + +-include_lib("stdlib/include/assert.hrl"). +-include_lib("stdlib/include/assert.hrl"). % test repeated inclusion +assert_test(suite) -> + []; +assert_test(doc) -> + ["Assert macros test."]; +assert_test(_Config) -> + ok = ?assert(true), + {'EXIT',{{assert, _},_}} = (catch ?assert(false)), + {'EXIT',{{assert, Info1},_}} = (catch ?assert(0)), + {not_boolean,0} = lists:keyfind(not_boolean,1,Info1), + + ok = ?assertNot(false), + {'EXIT',{{assert, _},_}} = (catch ?assertNot(true)), + {'EXIT',{{assert, Info2},_}} = (catch ?assertNot(0)), + {not_boolean,0} = lists:keyfind(not_boolean,1,Info2), + + ok = ?assertMatch({foo,_}, {foo,bar}), + {'EXIT',{{assertMatch,_},_}} = + (catch ?assertMatch({foo,_}, {foo})), + + ok = ?assertMatch({foo,N} when N > 0, {foo,1}), + {'EXIT',{{assertMatch,_},_}} = + (catch ?assertMatch({foo,N} when N > 0, {foo,0})), + + ok = ?assertNotMatch({foo,_}, {foo,bar,baz}), + {'EXIT',{{assertNotMatch,_},_}} = + (catch ?assertNotMatch({foo,_}, {foo,baz})), + + ok = ?assertNotMatch({foo,N} when N > 0, {foo,0}), + {'EXIT',{{assertNotMatch,_},_}} = + (catch ?assertNotMatch({foo,N} when N > 0, {foo,1})), + + ok = ?assertEqual(1.0, 1.0), + {'EXIT',{{assertEqual,_},_}} = (catch ?assertEqual(1, 1.0)), + + ok = ?assertNotEqual(1, 1.0), + {'EXIT',{{assertNotEqual,_},_}} = (catch ?assertNotEqual(1.0, 1.0)), + + ok = ?assertException(error, badarith, 1/0), + ok = ?assertException(exit, foo, exit(foo)), + ok = ?assertException(throw, foo, throw(foo)), + ok = ?assertException(throw, {foo,_}, throw({foo,bar})), + ok = ?assertException(throw, {foo,N} when N > 0, throw({foo,1})), + {'EXIT',{{assertException,Why1},_}} = + (catch ?assertException(error, badarith, 0/1)), + true = lists:keymember(unexpected_success,1,Why1), + {'EXIT',{{assertException,Why2},_}} = + (catch ?assertException(error, badarith, 1/length(0))), + true = lists:keymember(unexpected_exception,1,Why2), + {'EXIT',{{assertException,Why3},_}} = + (catch ?assertException(throw, {foo,N} when N > 0, throw({foo,0}))), + true = lists:keymember(unexpected_exception,1,Why3), + + ok = ?assertNotException(throw, {foo,baz}, throw({foo,bar})), + {'EXIT',{{assertNotException,Why4},_}} = + (catch ?assertNotException(throw, {foo,bar}, throw({foo,bar}))), + true = lists:keymember(unexpected_exception,1,Why4), + + ok = ?assertError(badarith, 1/0), + ok = ?assertExit(foo, exit(foo)), + ok = ?assertThrow(foo, throw(foo)), + ok. diff --git a/lib/stdlib/test/supervisor_SUITE.erl b/lib/stdlib/test/supervisor_SUITE.erl index 015b09f35e..31d4b44f30 100644 --- a/lib/stdlib/test/supervisor_SUITE.erl +++ b/lib/stdlib/test/supervisor_SUITE.erl @@ -39,7 +39,8 @@ sup_start_ignore_temporary_child_start_child_simple/1, sup_start_ignore_permanent_child_start_child_simple/1, sup_start_error_return/1, sup_start_fail/1, - sup_start_map/1, sup_start_map_faulty_specs/1, + sup_start_map/1, sup_start_map_simple/1, + sup_start_map_faulty_specs/1, sup_stop_infinity/1, sup_stop_timeout/1, sup_stop_brutal_kill/1, child_adm/1, child_adm_simple/1, child_specs/1, extra_return/1, sup_flags/1]). @@ -103,7 +104,7 @@ groups() -> sup_start_ignore_permanent_child_start_child_simple, sup_start_error_return, sup_start_fail]}, {sup_start_map, [], - [sup_start_map, sup_start_map_faulty_specs]}, + [sup_start_map, sup_start_map_simple, sup_start_map_faulty_specs]}, {sup_stop, [], [sup_stop_infinity, sup_stop_timeout, sup_stop_brutal_kill]}, @@ -323,6 +324,30 @@ sup_start_map(Config) when is_list(Config) -> terminate(Pid, shutdown). %%------------------------------------------------------------------------- +%% Tests that the supervisor process starts correctly with map +%% startspec, and that the full childspec can be read when using +%% simple_one_for_one strategy. +sup_start_map_simple(Config) when is_list(Config) -> + process_flag(trap_exit, true), + SupFlags = #{strategy=>simple_one_for_one}, + ChildSpec = #{id=>undefined, + start=>{supervisor_1, start_child, []}, + restart=>temporary}, + {ok, Pid} = start_link({ok, {SupFlags, [ChildSpec]}}), + + {ok, Child1} = supervisor:start_child(Pid, []), + {ok, Child2} = supervisor:start_child(Pid, []), + {ok, Child3} = supervisor:start_child(Pid, []), + + Spec = ChildSpec#{type=>worker, shutdown=>5000, modules=>[supervisor_1]}, + + {ok, Spec} = supervisor:get_childspec(Pid, Child1), + {ok, Spec} = supervisor:get_childspec(Pid, Child2), + {ok, Spec} = supervisor:get_childspec(Pid, Child3), + {error,not_found} = supervisor:get_childspec(Pid, self()), + terminate(Pid, shutdown). + +%%------------------------------------------------------------------------- %% Tests that the supervisor produces good error messages when start- %% and child specs are faulty. sup_start_map_faulty_specs(Config) when is_list(Config) -> diff --git a/lib/tools/emacs/erlang-skels-old.el b/lib/tools/emacs/erlang-skels-old.el index b88d7bcc4b..c8229300c8 100644 --- a/lib/tools/emacs/erlang-skels-old.el +++ b/lib/tools/emacs/erlang-skels-old.el @@ -838,7 +838,7 @@ Please see the function `tempo-define-template'.") "Config." n n (erlang-skel-separator 2) - "%% Function: end_per_suite(Config) -> void()" n + "%% Function: end_per_suite(Config) -> term()" n "%%" n "%% Config = [tuple()]" n "%% A list of key/value pairs, holding the test case configuration." n @@ -867,7 +867,7 @@ Please see the function `tempo-define-template'.") "Config." n n (erlang-skel-separator 2) - "%% Function: end_per_testcase(TestCase, Config) -> void()" n + "%% Function: end_per_testcase(TestCase, Config) -> term()" n "%%" n "%% TestCase = atom()" n "%% Name of the test case that is finished." n @@ -993,7 +993,7 @@ Please see the function `tempo-define-template'.") "Config." n n (erlang-skel-separator 2) - "%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}" n + "%% Function: end_per_suite(Config0) -> term() | {save_config,Config1}" n "%%" n "%% Config0 = Config1 = [tuple()]" n "%% A list of key/value pairs, holding the test case configuration." n @@ -1021,7 +1021,7 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator 2) "%% Function: end_per_group(GroupName, Config0) ->" n - "%% void() | {save_config,Config1}" n + "%% term() | {save_config,Config1}" n "%%" n "%% GroupName = atom()" n "%% Name of the test case group that is finished." n @@ -1054,7 +1054,7 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator 2) "%% Function: end_per_testcase(TestCase, Config0) ->" n - "%% void() | {save_config,Config1} | {fail,Reason}" n + "%% term() | {save_config,Config1} | {fail,Reason}" n "%%" n "%% TestCase = atom()" n "%% Name of the test case that is finished." n @@ -1175,7 +1175,7 @@ Please see the function `tempo-define-template'.") "Config." n n (erlang-skel-separator 2) - "%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}" n + "%% Function: end_per_suite(Config0) -> term() | {save_config,Config1}" n "%% Config0 = Config1 = [tuple()]" n (erlang-skel-separator 2) "end_per_suite(_Config) ->" n > @@ -1193,7 +1193,7 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator 2) "%% Function: end_per_group(GroupName, Config0) ->" n - "%% void() | {save_config,Config1}" n + "%% term() | {save_config,Config1}" n "%% GroupName = atom()" n "%% Config0 = Config1 = [tuple()]" n (erlang-skel-separator 2) @@ -1212,7 +1212,7 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator 2) "%% Function: end_per_testcase(TestCase, Config0) ->" n - "%% void() | {save_config,Config1} | {fail,Reason}" n + "%% term() | {save_config,Config1} | {fail,Reason}" n "%% TestCase = atom()" n "%% Config0 = Config1 = [tuple()]" n "%% Reason = term()" n diff --git a/lib/tools/emacs/erlang-skels.el b/lib/tools/emacs/erlang-skels.el index 8d2c02e455..78b1e7ad19 100644 --- a/lib/tools/emacs/erlang-skels.el +++ b/lib/tools/emacs/erlang-skels.el @@ -1234,7 +1234,7 @@ Please see the function `tempo-define-template'.") "Config." n n (erlang-skel-separator-start 2) - "%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}" n + "%% @spec end_per_suite(Config0) -> term() | {save_config,Config1}" n "%% Config0 = Config1 = [tuple()]" n (erlang-skel-separator-end 2) "end_per_suite(_Config) ->" n > @@ -1252,7 +1252,7 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @spec end_per_group(GroupName, Config0) ->" n - "%% void() | {save_config,Config1}" n + "%% term() | {save_config,Config1}" n "%% GroupName = atom()" n "%% Config0 = Config1 = [tuple()]" n (erlang-skel-separator-end 2) @@ -1271,7 +1271,7 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @spec end_per_testcase(TestCase, Config0) ->" n - "%% void() | {save_config,Config1} | {fail,Reason}" n + "%% term() | {save_config,Config1} | {fail,Reason}" n "%% TestCase = atom()" n "%% Config0 = Config1 = [tuple()]" n "%% Reason = term()" n @@ -1412,7 +1412,7 @@ Please see the function `tempo-define-template'.") "%% A list of key/value pairs, holding configuration data for the group." n "%%" n "%% @spec end_per_group(GroupName, Config0) ->" n - "%% void() | {save_config,Config1}" n + "%% term() | {save_config,Config1}" n (erlang-skel-separator-end 2) "end_per_group(_GroupName, _Config) ->" n > "ok." n n @@ -1446,7 +1446,7 @@ Please see the function `tempo-define-template'.") "%% A list of key/value pairs, holding the test case configuration." n "%%" n "%% @spec end_per_testcase(TestCase, Config0) ->" n - "%% void() | {save_config,Config1} | {fail,Reason}" n + "%% term() | {save_config,Config1} | {fail,Reason}" n (erlang-skel-separator-end 2) "end_per_testcase(_TestCase, _Config) ->" n > "ok." n n diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 3610356355..ca194703bb 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -880,10 +880,10 @@ resulting regexp is surrounded by \\_< and \\_>." "dt_restore_tag" "dt_spread_tag" "dunlink" + "convert_time_unit" "external_size" "finish_after_on_load" "finish_loading" - "flush_monitor_message" "format_cpu_topology" "fun_info" "fun_info_mfa" @@ -913,6 +913,7 @@ resulting regexp is surrounded by \\_< and \\_>." "memory" "module_info" "monitor_node" + "monotonic_time" "nif_error" "phash" "phash2" @@ -946,13 +947,17 @@ resulting regexp is surrounded by \\_< and \\_>." "system_info" "system_monitor" "system_profile" + "system_time" "trace" "trace_delivered" "trace_info" "trace_pattern" + "time_offset" + "timestamp" "universaltime" "universaltime_to_localtime" "universaltime_to_posixtime" + "unique_integer" "yield") "Erlang built-in functions (BIFs) that needs erlang: prefix")) diff --git a/lib/tools/src/cover.erl b/lib/tools/src/cover.erl index 71e17e0ba1..9ec5e809bc 100644 --- a/lib/tools/src/cover.erl +++ b/lib/tools/src/cover.erl @@ -782,7 +782,7 @@ main_process_loop(State) -> {From, {{analyse_to_file, Opts},Module}} -> S = try Loaded = is_loaded(Module, State), - spawn(fun() -> + spawn_link(fun() -> ?SPAWN_DBG(analyse_to_file,{Module,Opts}), do_parallel_analysis_to_file( Module, Opts, Loaded, From, State) @@ -1017,14 +1017,24 @@ load_compiled([{Module,File,Binary,InitialTable}|Compiled],Acc) -> %% Make sure the #bump{} records are available *before* the %% module is loaded. insert_initial_data(InitialTable), - NewAcc = - case code:load_binary(Module, ?TAG, Binary) of - {module,Module} -> - add_compiled(Module, File, Acc); - _ -> - do_clear(Module), - Acc - end, + Sticky = case code:is_sticky(Module) of + true -> + code:unstick_mod(Module), + true; + false -> + false + end, + NewAcc = case code:load_binary(Module, ?TAG, Binary) of + {module,Module} -> + add_compiled(Module, File, Acc); + _ -> + do_clear(Module), + Acc + end, + case Sticky of + true -> code:stick_mod(Module); + false -> ok + end, load_compiled(Compiled,NewAcc); load_compiled([],Acc) -> Acc. @@ -2143,7 +2153,13 @@ find_source(Module, File0) -> throw_file(filename:join([BeamDir, "..", "src", Base])), %% Not in ../src: look for source path in compile info, but %% first look relative the beam directory. - Info = lists:keyfind(source, 1, Module:module_info(compile)), + Info = + try lists:keyfind(source, 1, Module:module_info(compile)) + catch error:undef -> + %% The module might have been imported + %% and the beam not available + throw({beam, File0}) + end, false == Info andalso throw({beam, File0}), %% stripped {source, SrcFile} = Info, throw_file(splice(BeamDir, SrcFile)), %% below ../src diff --git a/lib/tools/test/cover_SUITE.erl b/lib/tools/test/cover_SUITE.erl index 368fa6c3d1..bc85f3c045 100644 --- a/lib/tools/test/cover_SUITE.erl +++ b/lib/tools/test/cover_SUITE.erl @@ -29,7 +29,8 @@ export_import/1, otp_5031/1, eif/1, otp_5305/1, otp_5418/1, otp_6115/1, otp_7095/1, otp_8188/1, otp_8270/1, otp_8273/1, otp_8340/1, - otp_10979_hanging_node/1, compile_beam_opts/1, eep37/1]). + otp_10979_hanging_node/1, compile_beam_opts/1, eep37/1, + analyse_no_beam/1]). -export([do_coverage/1]). @@ -52,7 +53,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> NoStartStop = [eif,otp_5305,otp_5418,otp_7095,otp_8273, - otp_8340,otp_8188,compile_beam_opts,eep37], + otp_8340,otp_8188,compile_beam_opts,eep37, + analyse_no_beam], StartStop = [start, compile, analyse, misc, stop, distribution, reconnect, die_and_reconnect, dont_reconnect_after_stop, stop_node_after_disconnect, @@ -1687,6 +1689,43 @@ compile_beam_opts(Config) when is_list(Config) -> ok = file:set_cwd(Cwd), ok. +analyse_no_beam(doc) -> + ["Don't crash if beam is not available"]; +analyse_no_beam(suite) -> []; +analyse_no_beam(Config) when is_list(Config) -> + {ok, Cwd} = file:get_cwd(), + ok = file:set_cwd(?config(data_dir, Config)), + + code:purge(t), + code:delete(t), + + {ok,_} = file:copy("compile_beam/t.erl", "t.erl"), + {ok,t} = compile:file(t, [debug_info]), + {module,t} = code:load_file(t), + {ok,t} = cover:compile_beam(t), + t:f(), + ok = cover:export("t.coverdata"), + + code:purge(t), + code:delete(t), + + %% this is just so that cover realises (without stopping) + %% that this module is not cover compiled any more + {error, {not_cover_compiled,t}} = cover:analyse(t), + + %% source and beam not available any more + ok = file:delete("t.erl"), + ok = file:delete("t.beam"), + + ok = cover:import("t.coverdata"), + + {error,{no_source_code_found,t}} = cover:analyse_to_file(t), + {result,[],[{no_source_code_found,t}]} = cover:analyse_to_file([t]), + + ok = file:delete("t.coverdata"), + ok = file:set_cwd(Cwd), + ok. + %%--Auxiliary------------------------------------------------------------ analyse_expr(Expr, Config) -> diff --git a/lib/tools/test/cover_SUITE_data/compile_beam/t.erl b/lib/tools/test/cover_SUITE_data/compile_beam/t.erl new file mode 100644 index 0000000000..96dc2f4209 --- /dev/null +++ b/lib/tools/test/cover_SUITE_data/compile_beam/t.erl @@ -0,0 +1,6 @@ +-module(t). + +-export([f/0]). + +f() -> + ok. diff --git a/system/doc/programming_examples/funs.xmlsrc b/system/doc/programming_examples/funs.xmlsrc index d4c32bc854..80877f0164 100644 --- a/system/doc/programming_examples/funs.xmlsrc +++ b/system/doc/programming_examples/funs.xmlsrc @@ -111,7 +111,9 @@ foreach(fun(Pid) -> Pid ! M end, L)</code> <section> <title>Syntax of Funs</title> - <p>Funs are written with the following syntax:</p> + <p>Funs are written with the following syntax (see <seealso + marker="doc/reference_manual:expressions#funs">Fun Expressions + </seealso> for full description):</p> <code type="none"> F = fun (Arg1, Arg2, ... ArgN) -> ... |