diff options
Diffstat (limited to 'lib/mnesia')
103 files changed, 3497 insertions, 1267 deletions
diff --git a/lib/mnesia/Makefile b/lib/mnesia/Makefile index 0333f73a27..810433c4d0 100644 --- a/lib/mnesia/Makefile +++ b/lib/mnesia/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2009. All Rights Reserved. +# Copyright Ericsson AB 1996-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/lib/mnesia/doc/misc/Makefile b/lib/mnesia/doc/misc/Makefile index 4d364649b6..29e2682967 100644 --- a/lib/mnesia/doc/misc/Makefile +++ b/lib/mnesia/doc/misc/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2009. All Rights Reserved. +# Copyright Ericsson AB 1996-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/lib/mnesia/doc/src/Makefile b/lib/mnesia/doc/src/Makefile index af85e1bc59..39f2b28637 100644 --- a/lib/mnesia/doc/src/Makefile +++ b/lib/mnesia/doc/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2012. All Rights Reserved. +# Copyright Ericsson AB 1997-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/lib/mnesia/doc/src/Mnesia_App_A.xmlsrc b/lib/mnesia/doc/src/Mnesia_App_A.xmlsrc index 7bc0688887..22b6fa9f39 100644 --- a/lib/mnesia/doc/src/Mnesia_App_A.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_App_A.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc b/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc index 1b76a31ef0..45bd2824e5 100644 --- a/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1998</year><year>2013</year> + <year>1998</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc b/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc index 34dafc772b..42e25edc5d 100644 --- a/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2002</year><year>2013</year> + <year>2002</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/Mnesia_chap1.xml b/lib/mnesia/doc/src/Mnesia_chap1.xml index fd614a655b..035e934ed2 100644 --- a/lib/mnesia/doc/src/Mnesia_chap1.xml +++ b/lib/mnesia/doc/src/Mnesia_chap1.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -32,38 +32,38 @@ <rev>C</rev> <file>Mnesia_chap1.xml</file> </header> - <p>The <c>Mnesia</c> application provides a heavy duty real-time + <p>The Mnesia application provides a heavy duty real-time distributed database.</p> <section> <title>Scope</title> <p>This User's Guide describes how to - build <c>Mnesia</c> database applications, and how to integrate - and use the <c>Mnesia</c> database management system with + build Mnesia database applications, and how to integrate + and use the Mnesia database management system with OTP. Programming constructs are described, and numerous programming examples are included to illustrate the use of - <c>Mnesia</c>.</p> + Mnesia.</p> <p>This User's Guide is organized as follows:</p> <list type="bulleted"> <item><seealso marker="Mnesia_overview">Mnesia</seealso> provides an introduction to - <c>Mnesia</c>. + Mnesia. </item> <item><seealso marker="Mnesia_chap2">Getting Started</seealso> - introduces <c>Mnesia</c> with an example database. Examples + introduces Mnesia with an example database. Examples are included how to start an Erlang session, specify a - <c>Mnesia</c> database directory, initialize a database - schema, start <c>Mnesia</c>, and create tables. Initial + Mnesia database directory, initialize a database + schema, start Mnesia, and create tables. Initial prototyping of record definitions is also discussed. </item> <item><seealso marker="Mnesia_chap3">Build a Mnesia Database</seealso> more formally describes the steps - introduced in the previous section, namely the <c>Mnesia</c> - functions that define a database schema, start <c>Mnesia</c>, + introduced in the previous section, namely the Mnesia + functions that define a database schema, start Mnesia, and create the required tables. </item> <item><seealso marker="Mnesia_chap4">Transactions and Other Access Contexts</seealso> - describes the transactions properties that make <c>Mnesia</c> into + describes the transactions properties that make Mnesia into a fault tolerant, real-time distributed database management system. This section also describes the concept of locking to ensure consistency in tables, and "dirty @@ -76,16 +76,16 @@ features include indexing, checkpoints, distribution and fault tolerance, disc-less nodes, replication manipulation, local content tables, concurrency, and object-based programming in - <c>Mnesia</c>. + Mnesia. </item> <item><seealso marker="Mnesia_chap7">Mnesia System Information</seealso> describes the files contained in the - <c>Mnesia</c> database directory, database configuration data, + Mnesia database directory, database configuration data, core and table dumps, as well as the important subject of backup, fall-back, and disaster recovery principles. </item> <item><seealso marker="Mnesia_chap8">Combine Mnesia with - SNMP</seealso> is a short section that outlines <c>Mnesia</c> + SNMP</seealso> is a short section that outlines Mnesia integrated with SNMP. </item> <item><seealso marker="Mnesia_App_A">Appendix A: Backup diff --git a/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc index ad2ba7784e..37389ce5ae 100644 --- a/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -317,7 +317,7 @@ sex = male, phone = 98108, room_no = {221, 015}}, - insert_emp(Me, 'B/SFR', [Erlang, mnesia, otp]).</code> + insert_emp(Emp, 'B/SFR', [Erlang, mnesia, otp]).</code> <note><p>For information about Funs, see "Fun Expressions" in section <c>Erlang Reference Manual</c> in System Documentation..</p> diff --git a/lib/mnesia/doc/src/Mnesia_chap3.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap3.xmlsrc index 849b5f4281..8f1a4366ee 100644 --- a/lib/mnesia/doc/src/Mnesia_chap3.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap3.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc index db48811d76..b8d86adbf1 100644 --- a/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc index f32f9eb590..a83d1d77d2 100644 --- a/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc index fa8fa6bbb0..ffbbdadec0 100644 --- a/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/Mnesia_chap8.xml b/lib/mnesia/doc/src/Mnesia_chap8.xml index 8a4ed8a23d..4a2eed84d7 100644 --- a/lib/mnesia/doc/src/Mnesia_chap8.xml +++ b/lib/mnesia/doc/src/Mnesia_chap8.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -51,11 +51,11 @@ </item> </list> <p>All these approaches have different advantages and - disadvantages. <c>Mnesia</c> applications can easily be opened to + disadvantages. Mnesia applications can easily be opened to the SNMP protocol. A direct 1-to-1 mapping can be established - between <c>Mnesia</c> tables and SNMP tables. This means - that a <c>Mnesia</c> table can be configured to be <em>both</em> - a <c>Mnesia</c> table and an SNMP table. A number of functions to + between Mnesia tables and SNMP tables. This means + that a Mnesia table can be configured to be <em>both</em> + a Mnesia table and an SNMP table. A number of functions to control this behavior are described in the Reference Manual.</p> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_overview.xml b/lib/mnesia/doc/src/Mnesia_overview.xml index b0d67d3e1a..63f2309284 100644 --- a/lib/mnesia/doc/src/Mnesia_overview.xml +++ b/lib/mnesia/doc/src/Mnesia_overview.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -39,14 +39,14 @@ high level of fault tolerance that is required in many nonstop systems, combined with requirements on the DBMS to run in the same address space as the application, have led us to implement a new - DBMS, called <c>Mnesia</c>.</p> - <p><c>Mnesia</c> is implemented in, and tightly connected to Erlang. + DBMS, called Mnesia.</p> + <p>Mnesia is implemented in, and tightly connected to Erlang. It provides the functionality that is necessary for the implementation of fault tolerant telecommunications systems.</p> - <p><c>Mnesia</c> is a multiuser distributed DBMS specially made for + <p>Mnesia is a multiuser distributed DBMS specially made for industrial telecommunications applications written in Erlang, which is also the intended target language. - <c>Mnesia</c> tries to address all the data + Mnesia tries to address all the data management issues required for typical telecommunications systems. It has a number of features that are not normally found in traditional databases.</p> @@ -54,7 +54,7 @@ from the features provided by traditional DBMSs. The applications now implemented in Erlang need a mixture of a broad range of features, which generally are not satisfied by traditional DBMSs. - <c>Mnesia</c> is designed with requirements like the following in + Mnesia is designed with requirements like the following in mind:</p> <list type="ordered"> <item>Fast real-time key/value lookup @@ -71,9 +71,9 @@ <item>Complex objects </item> </list> - <p><c>Mnesia</c> is designed with the typical data management problems - of telecommunications applications in mind. This sets <c>Mnesia</c> - apart from most other DBMS. Hence <c>Mnesia</c> + <p>Mnesia is designed with the typical data management problems + of telecommunications applications in mind. This sets Mnesia + apart from most other DBMS. Hence Mnesia combines many concepts found in traditional databases such as transactions and queries with concepts found in data management systems for telecommunications applications, for example:</p> @@ -86,7 +86,7 @@ suspending it. </item> </list> - <p><c>Mnesia</c> is also interesting because of its tight coupling to + <p>Mnesia is also interesting because of its tight coupling to Erlang, thus almost turning Erlang into a database programming language. This has many benefits, the foremost is that the impedance mismatch between the data format used by the DBMS @@ -97,7 +97,7 @@ <title>Mnesia Database Management System (DBMS)</title> <section> <title>Features</title> - <p><c>Mnesia</c> contains the following features that combine to + <p>Mnesia contains the following features that combine to produce a fault-tolerant, distributed DBMS written in Erlang: </p> <list type="bulleted"> @@ -118,7 +118,7 @@ functions can be called within one transaction. </item> <item>Several transactions can run concurrently, and their execution - is fully synchronized by the DBMS. <c>Mnesia</c> ensures that no + is fully synchronized by the DBMS. Mnesia ensures that no two processes manipulate data simultaneously. </item> <item>Transactions can be assigned the property of being executed on @@ -132,29 +132,29 @@ <section> <title>Add-On Application</title> - <p>Query List Comprehension (QLC) can be used with <c>Mnesia</c> + <p>Query List Comprehension (QLC) can be used with Mnesia to produce specialized functions that enhance the operational - ability of <c>Mnesia</c>. QLC has its own documentation as part + ability of Mnesia. QLC has its own documentation as part of the OTP documentation set. The main features of QLC - when used with <c>Mnesia</c> are as follows:</p> + when used with Mnesia are as follows:</p> <list type="bulleted"> - <item>QLC can optimize the query compiler for the <c>Mnesia</c> + <item>QLC can optimize the query compiler for the Mnesia DBMS, essentially making the DBMS more efficient. </item> <item>QLC can be used as a database programming - language for <c>Mnesia</c>. It includes a notation called "list + language for Mnesia. It includes a notation called "list comprehensions" and can be used to make complex database queries over a set of tables. </item> </list> <p>For information about QLC, see the <seealso marker="stdlib:qlc">qlc</seealso> manual page - in <c>STDLIB</c>.</p> + in STDLIB.</p> </section> <section> <title>When to Use Mnesia</title> - <p>Use <c>Mnesia</c> with the following types of applications:</p> + <p>Use Mnesia with the following types of applications:</p> <list type="bulleted"> <item>Applications that need to replicate data. </item> @@ -166,7 +166,7 @@ <item>Applications that use soft real-time characteristics. </item> </list> - <p><c>Mnesia</c> is not as appropriate with the + <p>Mnesia is not as appropriate with the following types of applications:</p> <list type="bulleted"> <item>Programs that process plain text or binary data files. @@ -176,14 +176,14 @@ library module <c>dets</c>, which is a disc-based version of the module <c>ets</c>. For information about <c>dets</c>, see the <seealso marker="stdlib:dets">dets</seealso> - manual page in <c>STDLIB</c>. + manual page in STDLIB. </item> <item>Applications that need disc logging facilities. Those applications can use the module <c>disk_log</c> by preference. For information about <c>disk_log</c>, see the <seealso marker="kernel:disk_log">disk_log</seealso> - manual page in <c>Kernel</c>. + manual page in Kernel. </item> <item>Hard real-time systems. </item> diff --git a/lib/mnesia/doc/src/book.xml b/lib/mnesia/doc/src/book.xml index 4c638bffc0..c4714a012c 100644 --- a/lib/mnesia/doc/src/book.xml +++ b/lib/mnesia/doc/src/book.xml @@ -4,7 +4,7 @@ <book xmlns:xi="http://www.w3.org/2001/XInclude"> <header titlestyle="normal"> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/company.erl b/lib/mnesia/doc/src/company.erl index 0cc9d249cc..20e3235347 100644 --- a/lib/mnesia/doc/src/company.erl +++ b/lib/mnesia/doc/src/company.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2012. All Rights Reserved. +%% Copyright Ericsson AB 1999-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/doc/src/company_o.erl b/lib/mnesia/doc/src/company_o.erl index 9b75d3fb3c..7300e9d4bb 100644 --- a/lib/mnesia/doc/src/company_o.erl +++ b/lib/mnesia/doc/src/company_o.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2012. All Rights Reserved. +%% Copyright Ericsson AB 1999-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/doc/src/mnesia.xml b/lib/mnesia/doc/src/mnesia.xml index f441ba4c63..621b6047ee 100644 --- a/lib/mnesia/doc/src/mnesia.xml +++ b/lib/mnesia/doc/src/mnesia.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2014</year> + <year>1996</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -37,7 +37,7 @@ <description> <p>The following are some of the most important and attractive - capabilities provided by <c>Mnesia</c>:</p> + capabilities provided by Mnesia:</p> <list type="bulleted"> <item>A relational/object hybrid data model that is suitable for telecommunications applications. @@ -62,15 +62,15 @@ reconfigured at runtime without stopping the system. </item> </list> - <p>This Reference Manual describes the <c>Mnesia</c> API. This - includes functions that define and manipulate <c>Mnesia</c> + <p>This Reference Manual describes the Mnesia API. This + includes functions that define and manipulate Mnesia tables.</p> <p>All functions in this Reference Manual can be used in any combination with queries using the list comprehension notation. For information about the query notation, see the <seealso marker="stdlib:qlc">qlc</seealso> - manual page in <c>STDLIB</c>.</p> - <p>Data in <c>Mnesia</c> is organized as a set of tables. Each table + manual page in STDLIB.</p> + <p>Data in Mnesia is organized as a set of tables. Each table has a name that must be an atom. Each table is made up of Erlang records. The user is responsible for the record definitions. Each table also has a set of properties. The @@ -121,14 +121,14 @@ <item> <p><c>index</c>. This is a list of attribute names, or integers, which specify the tuple positions on which - <c>Mnesia</c> is to build and maintain an extra index + Mnesia is to build and maintain an extra index table.</p> </item> <item> <p><c>local_content</c>. When an application requires tables whose contents are local to each node, <c>local_content</c> tables can be used. The table name - is known to all <c>Mnesia</c> nodes, but its content is + is known to all Mnesia nodes, but its content is unique on each node. This means that access to such a table must be done locally. Set field <c>local_content</c> to <c>true</c> to enable the <c>local_content</c> @@ -143,7 +143,7 @@ avoid inconsistencies because of network splits.</p> </item> <item> - <p><c>snmp</c>. Each (set-based) <c>Mnesia</c> table can be + <p><c>snmp</c>. Each (set-based) Mnesia table can be automatically turned into a Simple Network Management Protocol (SNMP) ordered table as well. This property specifies the types of the SNMP keys.</p> @@ -174,7 +174,7 @@ copy of each modified record during the transaction. During iteration, that is, <c>mnesia:fold[lr]/4</c>, <c>mnesia:next/2</c>, <c>mnesia:prev/2</c>, and - <c>mnesia:snmp_get_next_index/2</c>, <c>Mnesia</c> + <c>mnesia:snmp_get_next_index/2</c>, Mnesia compensates for every written or deleted record, which can reduce the performance.</p> <p>If possible, avoid writing or deleting records in the same @@ -188,7 +188,7 @@ <desc> <p>Makes the transaction silently return the tuple <c>{aborted, Reason}</c>. - Termination of a <c>Mnesia</c> transaction means that + Termination of a Mnesia transaction means that an exception is thrown to an enclosing <c>catch</c>. Thus, the expression <c>catch mnesia:abort(x)</c> does not terminate the transaction.</p> @@ -324,7 +324,7 @@ <c>mnesia:ets</c>. Argument <c>AccessMod</c> is the name of a callback module, which implements the <c>mnesia_access</c> behavior.</p> - <p><c>Mnesia</c> forwards calls to the following functions:</p> + <p>Mnesia forwards calls to the following functions:</p> <list type="bulleted"> <item>mnesia:lock/2 (read_lock_table/1, write_lock_table/1) </item> @@ -391,15 +391,15 @@ </item> </list> <p><c>ActivityId</c> is a record that represents the identity - of the enclosing <c>Mnesia</c> activity. The first field + of the enclosing Mnesia activity. The first field (obtained with <c>element(1, ActivityId)</c>) contains an atom, which can be interpreted as the activity type: <c>ets</c>, <c>async_dirty</c>, <c>sync_dirty</c>, or <c>tid</c>. <c>tid</c> means that the activity is a transaction. The structure of the rest of the identity - record is internal to <c>Mnesia</c>.</p> + record is internal to Mnesia.</p> <p><c>Opaque</c> is an opaque data structure that is internal - to <c>Mnesia</c>.</p> + to Mnesia.</p> </desc> </func> <func> @@ -458,7 +458,7 @@ mnesia:add_table_index(person, age)</code> <desc> <marker id="async_dirty"></marker> <p>Calls the <c>Fun</c> in a context that is not protected by - a transaction. The <c>Mnesia</c> function calls performed in + a transaction. The Mnesia function calls performed in the <c>Fun</c> are mapped to the corresponding dirty functions. This still involves logging, replication, and subscriptions, but there is no locking, local transaction @@ -467,7 +467,7 @@ mnesia:add_table_index(person, age)</code> for normal <c>mnesia:dirty_*</c> operations, the operations are performed semi-asynchronously. For details, see <c>mnesia:activity/4</c> and the User's Guide.</p> - <p>The <c>Mnesia</c> tables can be manipulated without + <p>The Mnesia tables can be manipulated without using transactions. This has some serious disadvantages, but is considerably faster, as the transaction manager is not involved and no locks are set. A dirty operation does, @@ -480,7 +480,7 @@ mnesia:add_table_index(person, age)</code> read records dirty than within a transaction.</p> <p>Depending on the application, it can be a good idea to use the dirty functions for certain operations. Almost all - <c>Mnesia</c> functions that can be called within + Mnesia functions that can be called within transactions have a dirty equivalent, which is much more efficient.</p> <p>However, notice that there is a risk that the database can @@ -497,7 +497,7 @@ mnesia:add_table_index(person, age)</code> <fsummary>Backs up all tables in the database.</fsummary> <desc> <marker id="backup"></marker> - <p>Activates a new checkpoint covering all <c>Mnesia</c> tables, + <p>Activates a new checkpoint covering all Mnesia tables, including the schema, with maximum degree of redundancy, and performs a backup using <c>backup_checkpoint/2/3</c>. The default value of the backup callback module <c>BackupMod</c> @@ -529,16 +529,16 @@ mnesia:add_table_index(person, age)</code> <taglist> <tag><c>extra_db_nodes</c></tag> <item> - <p><c>Value</c> is a list of nodes that <c>Mnesia</c> + <p><c>Value</c> is a list of nodes that Mnesia is to try to connect to. <c>ReturnValue</c> is those - nodes in <c>Value</c> that <c>Mnesia</c> is connected + nodes in <c>Value</c> that Mnesia is connected to.</p> <p>Notice that this function must only be used to connect to newly started RAM nodes (N.D.R.S.N.) with an empty schema. If, for example, this function is used after the network has been partitioned, it can lead to inconsistent tables.</p> - <p>Notice that <c>Mnesia</c> can be connected to other + <p>Notice that Mnesia can be connected to other nodes than those returned in <c>ReturnValue</c>.</p> </item> <tag><c>dc_dump_limit</c></tag> @@ -548,7 +548,7 @@ mnesia:add_table_index(person, age)</code> Configuration Parameters</seealso>. <c>ReturnValue</c> is the new value. Notice that this configuration parameter is not persistent. It is lost when - <c>Mnesia</c> has stopped.</p> + Mnesia has stopped.</p> </item> </taglist> </desc> @@ -562,9 +562,9 @@ mnesia:add_table_index(person, age)</code> <c>read_write</c> but it can also be set to the atom <c>read_only</c>. If <c>AccessMode</c> is set to <c>read_only</c>, updates to the table cannot be - performed. At startup, <c>Mnesia</c> always loads + performed. At startup, Mnesia always loads <c>read_only</c> tables locally regardless of when and if - <c>Mnesia</c> is terminated on other nodes.</p> + Mnesia is terminated on other nodes.</p> </desc> </func> <func> @@ -620,13 +620,13 @@ mnesia:change_table_copy_type(person, node(), disc_copies)</code> <desc> <marker id="create_schema"></marker> <p>Creates a new database on disc. Various files are - created in the local <c>Mnesia</c> directory of each node. + created in the local Mnesia directory of each node. Notice that the directory must be unique for each node. Two nodes must never share the same directory. If possible, use a local disc device to improve performance.</p> <p><c>mnesia:create_schema/1</c> fails if any of the Erlang nodes given as <c>DiscNodes</c> are not alive, if - <c>Mnesia</c> is running on any of the nodes, or if any + Mnesia is running on any of the nodes, or if any of the nodes already have a schema. Use <c>mnesia:delete_schema/1</c> to get rid of old faulty schemas.</p> @@ -638,10 +638,10 @@ mnesia:change_table_copy_type(person, node(), disc_copies)</code> </func> <func> <name>create_table(Name, TabDef) -> {atomic, ok} | {aborted, Reason}</name> - <fsummary>Creates a <c>Mnesia</c> table called <c>Name</c>with properties as described by argument <c>TabDef</c>.</fsummary> + <fsummary>Creates a Mnesia table called <c>Name</c>with properties as described by argument <c>TabDef</c>.</fsummary> <desc> <marker id="create_table"></marker> - <p>Creates a <c>Mnesia</c> table called + <p>Creates a Mnesia table called <c>Name</c> according to argument <c>TabDef</c>. This list must be a list of <c>{Item, Value}</c> tuples, where the following values are allowed:</p> @@ -652,8 +652,8 @@ mnesia:change_table_copy_type(person, node(), disc_copies)</code> set to the atom <c>read_only</c>. If <c>AccessMode</c> is set to <c>read_only</c>, updates to the table cannot be performed.</p> - <p>At startup, <c>Mnesia</c> always loads <c>read_only</c> - table locally regardless of when and if <c>Mnesia</c> is + <p>At startup, Mnesia always loads <c>read_only</c> + table locally regardless of when and if Mnesia is terminated on other nodes. This argument returns the access mode of the table. The access mode can be <c>read_only</c> or <c>read_write</c>.</p> @@ -693,7 +693,7 @@ mnesia:change_table_copy_type(person, node(), disc_copies)</code> <item> <p><c>{index, Intlist}</c>, where <c>Intlist</c> is a list of attribute names (atoms) or - record fields for which <c>Mnesia</c> is to build and + record fields for which Mnesia is to build and maintain an extra index table. The <c>qlc</c> query compiler <em>may</em> be able to optimize queries if there are indexes available.</p> @@ -735,10 +735,10 @@ mnesia:change_table_copy_type(person, node(), disc_copies)</code> <c>mnesia:create_table/2</c>, the table is immediately accessible by SNMP. Therefore applications that use SNMP to manipulate and control the system can be - designed easily, since <c>Mnesia</c> provides a + designed easily, since Mnesia provides a direct mapping between the logical tables that make up an SNMP control application and the physical data that - makes up a <c>Mnesia</c> table.</p> + makes up a Mnesia table.</p> </item> <item> <p><c>{storage_properties, [{Backend, Properties}]</c> @@ -746,7 +746,7 @@ mnesia:change_table_copy_type(person, node(), disc_copies)</code> <c>Backend</c> can currently be <c>ets</c> or <c>dets</c>. <c>Properties</c> is a list of options sent to the back end storage during table creation. <c>Properties</c> - cannot contain properties already used by <c>Mnesia</c>, + cannot contain properties already used by Mnesia, such as <c>type</c> or <c>named_table</c>.</p> <p>For example:</p> <code type="none"> @@ -776,7 +776,7 @@ mnesia:create_table(table, [{ram_copies, [node()]}, {disc_only_copies, nodes()}, mnesia:create_table(person, [{ram_copies, [N1, N2]}, {attributes, record_info(fields, person)}]).</code> - <p>If it is required that <c>Mnesia</c> must build and + <p>If it is required that Mnesia must build and maintain an extra index table on attribute <c>address</c> of all the <c>person</c> records that are inserted in the table, the following code would be issued:</p> @@ -819,8 +819,8 @@ mnesia:create_table(person, When the last replica is deleted with this function, the table disappears entirely.</p> <p>This function can also be used to delete a replica of - the table named <c>schema</c>. The <c>Mnesia</c> node is - then removed. Notice that <c>Mnesia</c> must be + the table named <c>schema</c>. The Mnesia node is + then removed. Notice that Mnesia must be stopped on the node first.</p> </desc> </func> @@ -891,9 +891,9 @@ mnesia:create_table(person, <c>mnesia:create_schema/1</c>. <c>mnesia:delete_schema/1</c> fails if any of the Erlang nodes given as <c>DiscNodes</c> are not alive, or if - <c>Mnesia</c> is running on any of the nodes.</p> + Mnesia is running on any of the nodes.</p> <p>After the database is deleted, it can still be possible - to start <c>Mnesia</c> as a disc-less node. This depends + to start Mnesia as a disc-less node. This depends on how configuration parameter <c>schema_location</c> is set.</p> <warning> @@ -1100,7 +1100,7 @@ mnesia:create_table(person, <name>dirty_update_counter(Tab, Key, Incr) -> NewVal | exit({aborted, Reason})</name> <fsummary>Dirty update of a counter record.</fsummary> <desc> - <p><c>Mnesia</c> has no special counter records. However, + <p>Mnesia has no special counter records. However, records of the form <c>{Tab, Key, Integer}</c> can be used as (possibly disc-resident) counters when <c>Tab</c> is a <c>set</c>. This function updates a counter with a positive @@ -1147,7 +1147,7 @@ mnesia:create_table(person, <desc> <marker id="dump_log"></marker> <p>Performs a user-initiated dump of the local log file. - This is usually not necessary, as <c>Mnesia</c> by default + This is usually not necessary, as Mnesia by default manages this automatically. See configuration parameters <seealso marker="#dump_log_time_threshold">dump_log_time_threshold</seealso> and @@ -1172,7 +1172,7 @@ mnesia:create_table(person, <fsummary>Dumps local tables into a text file.</fsummary> <desc> <marker id="dump_to_textfile"></marker> - <p>Dumps all local tables of a <c>Mnesia</c> system into a + <p>Dumps all local tables of a Mnesia system into a text file, which can be edited (by a normal text editor) and then be reloaded with <c>mnesia:load_textfile/1</c>. Only use this function for @@ -1182,10 +1182,10 @@ mnesia:create_table(person, </func> <func> <name>error_description(Error) -> String</name> - <fsummary>Returns a string describing a particular <c>Mnesia</c> error.</fsummary> + <fsummary>Returns a string describing a particular Mnesia error.</fsummary> <desc> <marker id="error_description"></marker> - <p>All <c>Mnesia</c> transactions, including all the schema + <p>All Mnesia transactions, including all the schema update functions, either return value <c>{atomic, Val}</c> or the tuple <c>{aborted, Reason}</c>. <c>Reason</c> can be either of the atoms in the following list. The @@ -1264,8 +1264,8 @@ mnesia:create_table(person, <desc> <marker id="ets"></marker> <p>Calls the <c>Fun</c> in a raw context that is not protected by - a transaction. The <c>Mnesia</c> function call is performed in - the <c>Fun</c> and performed directly on the local <c>ets</c> + a transaction. The Mnesia function call is performed in + the <c>Fun</c> and performed directly on the local ETS tables on the assumption that the local storage type is <c>ram_copies</c> and the tables are not replicated to other nodes. Subscriptions are not triggered and checkpoints are @@ -1321,13 +1321,13 @@ mnesia:create_table(person, <fsummary>Forces a table to be loaded into the system.</fsummary> <desc> <marker id="force_load_table"></marker> - <p>The <c>Mnesia</c> algorithm for table load can lead to a + <p>The Mnesia algorithm for table load can lead to a situation where a table cannot be loaded. This situation - occurs when a node is started and <c>Mnesia</c> concludes, or + occurs when a node is started and Mnesia concludes, or suspects, that another copy of the table was active after this local copy became inactive because of a system crash.</p> <p>If this situation is not acceptable, this function can be - used to override the strategy of the <c>Mnesia</c> table + used to override the strategy of the Mnesia table load algorithm. This can lead to a situation where some transaction effects are lost with an inconsistent database as result, but for some applications high availability is more @@ -1402,9 +1402,9 @@ mnesia:create_table(person, <desc> <marker id="info"></marker> <p>Prints system information on the terminal. - This function can be used even if <c>Mnesia</c> is not + This function can be used even if Mnesia is not started. However, more information is displayed if - <c>Mnesia</c> is started.</p> + Mnesia is started.</p> </desc> </func> <func> @@ -1431,7 +1431,7 @@ mnesia:create_table(person, <p>Installs a backup as fallback. The fallback is used to restore the database at the next startup. Installation of fallbacks requires Erlang to be operational on all the - involved nodes, but it does not matter if <c>Mnesia</c> + involved nodes, but it does not matter if Mnesia is running or not. The installation of the fallback fails if the local node is not one of the disc-resident nodes in the backup.</p> @@ -1465,14 +1465,14 @@ mnesia:create_table(person, <p><c>{mnesia_dir, AlternateDir}</c>. This argument is only valid if the scope of the installation is <c>local</c>. Normally the installation - of a fallback is targeted to the <c>Mnesia</c> directory, + of a fallback is targeted to the Mnesia directory, as configured with configuration parameter <c>-mnesia dir</c>. But by explicitly supplying an <c>AlternateDir</c>, the fallback is installed there - regardless of the <c>Mnesia</c> directory configuration + regardless of the Mnesia directory configuration parameter setting. After installation of a fallback on - an alternative <c>Mnesia</c> directory, that directory - is fully prepared for use as an active <c>Mnesia</c> + an alternative Mnesia directory, that directory + is fully prepared for use as an active Mnesia directory.</p> <p>This is a dangerous feature that must be used with care. By unintentional mixing of directories, @@ -1509,7 +1509,7 @@ mnesia:create_table(person, <marker id="load_textfile"></marker> <p>Loads a series of definitions and data found in the text file (generated with <c>mnesia:dump_to_textfile/1</c>) - into <c>Mnesia</c>. This function also starts <c>Mnesia</c> + into Mnesia. This function also starts Mnesia and possibly creates a new schema. This function is intended for educational purposes only. It is recommended to use other functions to deal with real backups.</p> @@ -1558,7 +1558,7 @@ mnesia:create_table(person, </taglist> <p>Conflicting lock requests are automatically queued if there is no risk of a deadlock. Otherwise the transaction must be - terminated and executed again. <c>Mnesia</c> does this + terminated and executed again. Mnesia does this automatically as long as the upper limit of the maximum <c>retries</c> is not reached. For details, see <c>mnesia:transaction/3</c>.</p> @@ -1726,19 +1726,19 @@ mnesia:create_table(person, </func> <func> <name>report_event(Event) -> ok</name> - <fsummary>Reports a user event to the <c>Mnesia</c> event handler.</fsummary> + <fsummary>Reports a user event to the Mnesia event handler.</fsummary> <desc> <marker id="report_event"></marker> - <p>When tracing a system of <c>Mnesia</c> applications it is - useful to be able to interleave <c>Mnesia</c> own events with + <p>When tracing a system of Mnesia applications it is + useful to be able to interleave Mnesia own events with application-related events that give information about the application context.</p> <p>Whenever the application begins a - new and demanding <c>Mnesia</c> task, or if it enters a new + new and demanding Mnesia task, or if it enters a new interesting phase in its execution, it can be a good idea to use <c>mnesia:report_event/1</c>. <c>Event</c> can be any term and generates a <c>{mnesia_user, Event}</c> event - for any processes that subscribe to <c>Mnesia</c> system + for any processes that subscribe to Mnesia system events.</p> </desc> </func> @@ -1748,7 +1748,7 @@ mnesia:create_table(person, <desc> <marker id="restore"></marker> <p>With this function, tables can be restored online from a - backup without restarting <c>Mnesia</c>. + backup without restarting Mnesia. <c>Opaque</c> is forwarded to the backup module. <c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> @@ -1873,7 +1873,7 @@ mnesia:create_table(person, <p>For a complete description of <c>select</c>, see the <seealso marker="erts:index">ERTS</seealso> User's Guide and the <seealso marker="stdlib:ets">ets</seealso> manual page in - <c>STDLIB</c>.</p> + STDLIB.</p> <p>For example, to find the names of all male persons older than 30 in table <c>Tab</c>:</p> <code type="none"> @@ -1920,10 +1920,10 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]),</code> </func> <func> <name>set_debug_level(Level) -> OldLevel</name> - <fsummary>Changes the internal debug level of <c>Mnesia</c>.</fsummary> + <fsummary>Changes the internal debug level of Mnesia.</fsummary> <desc> <marker id="set_debug_level"></marker> - <p>Changes the internal debug level of <c>Mnesia</c>. + <p>Changes the internal debug level of Mnesia. For details, see <seealso marker="#configuration_parameters">Section Configuration Parameters</seealso>.</p> @@ -1934,7 +1934,7 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]),</code> <fsummary>Sets the master nodes for all tables.</fsummary> <desc> <marker id="set_master_nodes_1"></marker> - <p>For each table <c>Mnesia</c> determines its replica nodes + <p>For each table Mnesia determines its replica nodes (<c>TabNodes</c>) and starts <c>mnesia:set_master_nodes(Tab, TabMasterNodes)</c>. where <c>TabMasterNodes</c> is the intersection of @@ -1952,16 +1952,16 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]),</code> that can have caused an inconsistent database, it can use the function <c>mnesia:set_master_nodes(Tab, MasterNodes)</c> to define from which nodes each table is to be loaded. - At startup, the <c>Mnesia</c> normal table load algorithm is + At startup, the Mnesia normal table load algorithm is bypassed and the table is loaded from one of the master nodes - defined for the table, regardless of when and if <c>Mnesia</c> + defined for the table, regardless of when and if Mnesia terminated on other nodes. <c>MasterNodes</c> can only contain nodes where the table has a replica. If the <c>MasterNodes</c> list is empty, the master node recovery mechanism for the particular table is reset, and the normal load mechanism is used at the next restart.</p> <p>The master node setting is always local. It can be - changed regardless if <c>Mnesia</c> is started or not.</p> + changed regardless if Mnesia is started or not.</p> <p>The database can also become inconsistent if configuration parameter <c>max_wait_for_decision</c> is used or if <c>mnesia:force_load_table/1</c> is used.</p> @@ -1976,7 +1976,7 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]),</code> </func> <func> <name>snmp_get_mnesia_key(Tab, RowIndex) -> {ok, Key} | undefined</name> - <fsummary>Gets the corresponding <c>Mnesia</c> key from an SNMP index.</fsummary> + <fsummary>Gets the corresponding Mnesia key from an SNMP index.</fsummary> <type> <v>Tab ::= atom()</v> <v>RowIndex ::= [integer()]</v> @@ -1984,7 +1984,7 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]),</code> <v>key() ::= integer() | string() | [integer()]</v> </type> <desc> - <p>Transforms an SNMP index to the corresponding <c>Mnesia</c> + <p>Transforms an SNMP index to the corresponding Mnesia key. If the SNMP table has multiple keys, the key is a tuple of the key columns.</p> </desc> @@ -2020,7 +2020,7 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]),</code> </func> <func> <name>snmp_open_table(Tab, SnmpStruct) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Organizes a <c>Mnesia</c> table as an SNMP table.</fsummary> + <fsummary>Organizes a Mnesia table as an SNMP table.</fsummary> <type> <v>Tab ::= atom()</v> <v>SnmpStruct ::= [{key, type()}]</v> @@ -2029,14 +2029,14 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]),</code> </type> <desc> <p>A direct one-to-one mapping can be established between - <c>Mnesia</c> tables and SNMP tables. Many telecommunication + Mnesia tables and SNMP tables. Many telecommunication applications are controlled and monitored by the SNMP - protocol. This connection between <c>Mnesia</c> and SNMP + protocol. This connection between Mnesia and SNMP makes it simple and convenient to achieve this mapping.</p> <p>Argument <c>SnmpStruct</c> is a list of SNMP information. Currently, the only information needed is information about the key types in the table. Multiple - keys cannot be handled in <c>Mnesia</c>, but many SNMP + keys cannot be handled in Mnesia, but many SNMP tables have multiple keys. Therefore, the following convention is used: if a table has multiple keys, these must always be stored as a tuple of the keys. Information about @@ -2069,39 +2069,39 @@ mnesia:create_table(employee, <p>When a table is SNMP ordered, modifications are more expensive than usual, O(logN). Also, more memory is used.</p> <p>Notice that only the lexicographical SNMP ordering is - implemented in <c>Mnesia</c>, not the actual SNMP monitoring.</p> + implemented in Mnesia, not the actual SNMP monitoring.</p> </desc> </func> <func> <name>start() -> ok | {error, Reason}</name> - <fsummary>Starts a local <c>Mnesia</c> system.</fsummary> + <fsummary>Starts a local Mnesia system.</fsummary> <desc> <marker id="start"></marker> - <p>The startup procedure for a set of <c>Mnesia</c> nodes is a - fairly complicated operation. A <c>Mnesia</c> system consists - of a set of nodes, with <c>Mnesia</c> started locally on all + <p>The startup procedure for a set of Mnesia nodes is a + fairly complicated operation. A Mnesia system consists + of a set of nodes, with Mnesia started locally on all participating nodes. Normally, each node has a directory where - all the <c>Mnesia</c> files are written. This directory is - referred to as the <c>Mnesia</c> directory. <c>Mnesia</c> can + all the Mnesia files are written. This directory is + referred to as the Mnesia directory. Mnesia can also be started on disc-less nodes. For more information about disc-less nodes, see <c>mnesia:create_schema/1</c> and the User's Guide.</p> - <p>The set of nodes that makes up a <c>Mnesia</c> system is kept - in a schema. <c>Mnesia</c> nodes can be added to or removed + <p>The set of nodes that makes up a Mnesia system is kept + in a schema. Mnesia nodes can be added to or removed from the schema. The initial schema is normally created on disc with the function <c>mnesia:create_schema/1</c>. On disc-less nodes, a tiny default schema is generated each time - <c>Mnesia</c> is started. During the startup procedure, - <c>Mnesia</c> exchanges schema information between the nodes + Mnesia is started. During the startup procedure, + Mnesia exchanges schema information between the nodes to verify that the table definitions are compatible.</p> <p>Each schema has a unique cookie, which can be regarded as a unique schema identifier. The cookie must be the same on all - nodes where <c>Mnesia</c> is supposed to run. For details, + nodes where Mnesia is supposed to run. For details, see the User's Guide.</p> - <p>The schema file and all other files that <c>Mnesia</c> - needs are kept in the <c>Mnesia</c> directory. The + <p>The schema file and all other files that Mnesia + needs are kept in the Mnesia directory. The command-line option <c>-mnesia dir Dir</c> can be used to - specify the location of this directory to the <c>Mnesia</c> + specify the location of this directory to the Mnesia system. If no such command-line option is found, the name of the directory defaults to <c>Mnesia.Node</c>.</p> <p><c>application:start(mnesia)</c> can also be used.</p> @@ -2109,10 +2109,10 @@ mnesia:create_table(employee, </func> <func> <name>stop() -> stopped</name> - <fsummary>Stops <c>Mnesia</c> locally.</fsummary> + <fsummary>Stops Mnesia locally.</fsummary> <desc> <marker id="stop"></marker> - <p>Stops <c>Mnesia</c> locally on the current node.</p> + <p>Stops Mnesia locally on the current node.</p> <p><c>application:stop(mnesia)</c> can also be used.</p> </desc> </func> @@ -2132,7 +2132,7 @@ mnesia:create_table(employee, <desc> <marker id="sync_dirty"></marker> <p>Calls the <c>Fun</c> in a context that is not protected by - a transaction. The <c>Mnesia</c> function calls performed in + a transaction. The Mnesia function calls performed in the <c>Fun</c> are mapped to the corresponding dirty functions. It is performed in almost the same context as <c>mnesia:async_dirty/1,2</c>. The difference is that the @@ -2167,10 +2167,10 @@ mnesia:create_table(employee, </func> <func> <name>system_info(InfoKey) -> Info | exit({aborted, Reason})</name> - <fsummary>Returns information about the <c>Mnesia</c> system.</fsummary> + <fsummary>Returns information about the Mnesia system.</fsummary> <desc> <marker id="system_info"></marker> - <p>Returns information about the <c>Mnesia</c> system, such as + <p>Returns information about the Mnesia system, such as transaction statistics, <c>db_nodes</c>, and configuration parameters. The valid keys are as follows:</p> <list type="bulleted"> @@ -2188,7 +2188,7 @@ mnesia:create_table(employee, </item> <item> <p><c>auto_repair</c>. Returns <c>true</c> or <c>false</c> - to indicate if <c>Mnesia</c> is configured to start the + to indicate if Mnesia is configured to start the auto-repair facility on corrupted disc files.</p> </item> <item> @@ -2209,20 +2209,20 @@ mnesia:create_table(employee, in the list of nodes if they explicitly have been added to the schema, for example, with <c>mnesia:add_table_copy/3</c>. The function can be - started even if <c>Mnesia</c> is not yet running.</p> + started even if Mnesia is not yet running.</p> </item> <item> <p><c>debug</c>. Returns the current debug level of - <c>Mnesia</c>.</p> + Mnesia.</p> </item> <item> - <p><c>directory</c>. Returns the name of the <c>Mnesia</c> - directory. It can be called even if <c>Mnesia</c> is + <p><c>directory</c>. Returns the name of the Mnesia + directory. It can be called even if Mnesia is not yet running.</p> </item> <item> <p><c>dump_log_load_regulation</c>. Returns a boolean that - tells if <c>Mnesia</c> is configured to regulate the + tells if Mnesia is configured to regulate the dumper process load.</p> <p>This feature is temporary and will be removed in future releases.</p> @@ -2233,9 +2233,9 @@ mnesia:create_table(employee, </item> <item> <p><c>dump_log_update_in_place</c>. Returns a boolean that - tells if <c>Mnesia</c> is configured to perform the - updates in the <c>dets</c> files directly, or if the - updates are to be performed in a copy of the <c>dets</c> + tells if Mnesia is configured to perform the + updates in the Dets files directly, or if the + updates are to be performed in a copy of the Dets files.</p> </item> <item> @@ -2253,13 +2253,13 @@ mnesia:create_table(employee, </item> <item> <p><c>held_locks</c>. Returns a list of all - locks held by the local <c>Mnesia</c> lock manager.</p> + locks held by the local Mnesia lock manager.</p> </item> <item> <p><c>is_running</c>. Returns <c>yes</c> or <c>no</c> to - indicate if <c>Mnesia</c> is running. It can + indicate if Mnesia is running. It can also return <c>starting</c> or <c>stopping</c>. Can be - called even if <c>Mnesia</c> is not yet running.</p> + called even if Mnesia is not yet running.</p> </item> <item> <p><c>local_tables</c>. Returns a list @@ -2272,7 +2272,7 @@ mnesia:create_table(employee, </item> <item> <p><c>log_version</c>. Returns the version - number of the <c>Mnesia</c> transaction log format.</p> + number of the Mnesia transaction log format.</p> </item> <item> <p><c>master_node_tables</c>. Returns a @@ -2280,25 +2280,25 @@ mnesia:create_table(employee, </item> <item> <p><c>protocol_version</c>. Returns the version number of - the <c>Mnesia</c> inter-process communication protocol.</p> + the Mnesia inter-process communication protocol.</p> </item> <item> <p><c>running_db_nodes</c>. Returns a list of nodes where - <c>Mnesia</c> currently is running. This function can be - called even if <c>Mnesia</c> is not yet running, but it + Mnesia currently is running. This function can be + called even if Mnesia is not yet running, but it then has slightly different semantics.</p> - <p>If <c>Mnesia</c> is down on the local node, the function + <p>If Mnesia is down on the local node, the function returns those other <c>db_nodes</c> and <c>extra_db_nodes</c> that for the moment are operational.</p> - <p>If <c>Mnesia</c> is started, the function returns - those nodes that <c>Mnesia</c> on the local node is fully - connected to. Only those nodes that <c>Mnesia</c> has + <p>If Mnesia is started, the function returns + those nodes that Mnesia on the local node is fully + connected to. Only those nodes that Mnesia has exchanged schema information with are included as <c>running_db_nodes</c>. After the merge of schemas, the - local <c>Mnesia</c> system is fully operable and + local Mnesia system is fully operable and applications can perform access of remote replicas. - Before the schema merge, <c>Mnesia</c> only operates + Before the schema merge, Mnesia only operates locally. Sometimes there are more nodes included in the <c>running_db_nodes</c> list than all <c>db_nodes</c> and <c>extra_db_nodes</c> together.</p> @@ -2322,17 +2322,17 @@ mnesia:create_table(employee, <item> <p><c>transaction_failures</c>. Returns a number that indicates how many transactions have - failed since <c>Mnesia</c> was started.</p> + failed since Mnesia was started.</p> </item> <item> <p><c>transaction_commits</c>. Returns a number that indicates how many transactions have - terminated successfully since <c>Mnesia</c> was started.</p> + terminated successfully since Mnesia was started.</p> </item> <item> <p><c>transaction_restarts</c>. Returns a number that indicates how many transactions have been - restarted since <c>Mnesia</c> was started.</p> + restarted since Mnesia was started.</p> </item> <item> <p><c>transaction_log_writes</c>. @@ -2342,12 +2342,12 @@ mnesia:create_table(employee, </item> <item> <p><c>use_dir</c>. Returns a boolean that indicates if - the <c>Mnesia</c> directory is used or not. Can be - started even if <c>Mnesia</c> is not yet running.</p> + the Mnesia directory is used or not. Can be + started even if Mnesia is not yet running.</p> </item> <item> <p><c>version</c>. Returns the current - version number of <c>Mnesia</c>.</p> + version number of Mnesia.</p> </item> </list> </desc> @@ -2356,17 +2356,16 @@ mnesia:create_table(employee, <name>table(Tab [,[Option]]) -> QueryHandle</name> <fsummary>Return a QLC query handle.</fsummary> <desc> - <marker id="qlc_table"></marker> <marker id="table"></marker> <p>Returns a Query List Comprehension (QLC) query handle, see the <seealso marker="stdlib:qlc">qlc(3)</seealso> - manual page in <c>STDLIB</c>. The module <c>qlc</c> - implements a query language that can use <c>Mnesia</c> + manual page in STDLIB. The module <c>qlc</c> + implements a query language that can use Mnesia tables as sources of data. Calling <c>mnesia:table/1,2</c> is the means to make the <c>mnesia</c> table <c>Tab</c> usable to QLC.</p> - <p><c>Option</c> can contain <c>Mnesia</c> - options or QLC options. <c>Mnesia</c> recognizes the + <p><c>Option</c> can contain Mnesia + options or QLC options. Mnesia recognizes the following options (any other option is forwarded to QLC).</p> <list type="bulleted"> @@ -2375,7 +2374,7 @@ mnesia:create_table(employee, </item> <item><c>{n_objects,Number}</c>, where <c>n_objects</c> specifies (roughly) the number of objects returned - from <c>Mnesia</c> to QLC. Queries to remote tables + from Mnesia to QLC. Queries to remote tables can need a larger chunk to reduce network overhead. By default, <c>100</c> objects at a time are returned. </item> @@ -2414,7 +2413,7 @@ mnesia:create_table(employee, <desc> <marker id="table_info"></marker> <p>The <c>table_info/2</c> function takes two arguments. - The first is the name of a <c>Mnesia</c> table. + The first is the name of a Mnesia table. The second is one of the following keys:</p> <list type="bulleted"> <item> @@ -2466,7 +2465,7 @@ mnesia:create_table(employee, </item> <item> <p><c>load_node</c>. Returns the name of - the node that <c>Mnesia</c> loaded the table from. The + the node that Mnesia loaded the table from. The structure of the returned value is unspecified, but can be useful for debugging purposes.</p> </item> @@ -2477,7 +2476,7 @@ mnesia:create_table(employee, </item> <item> <p><c>load_reason</c>. Returns the - reason of why <c>Mnesia</c> decided to load the table. + reason of why Mnesia decided to load the table. The structure of the returned value is unspecified, but can be useful for debugging purposes.</p> </item> @@ -2621,7 +2620,7 @@ raise(Name, Amount) -> several processes running on different nodes can concurrently execute the function <c>raise/2</c> without interfering with each other.</p> - <p>Since <c>Mnesia</c> detects deadlocks, a transaction can be + <p>Since Mnesia detects deadlocks, a transaction can be restarted any number of times. This function attempts a restart as specified in <c>Retries</c>. <c>Retries</c> must be an integer greater than 0 or the atom <c>infinity</c>. @@ -2644,7 +2643,7 @@ raise(Name, Amount) -> <p><c>NewAttributeList</c> and <c>NewRecordName</c> specify the attributes and the new record type of the converted table. Table name always remains unchanged. If - <c>record_name</c> is changed, only the <c>Mnesia</c> + <c>record_name</c> is changed, only the Mnesia functions that use table identifiers work, for example, <c>mnesia:write/3</c> works, but not <c>mnesia:write/1</c>.</p> </desc> @@ -2707,7 +2706,7 @@ raise(Name, Amount) -> distributed operation that is either performed on all nodes with disc resident schema, or none. Uninstallation of fallbacks requires Erlang to be operational on all - involved nodes, but it does not matter if <c>Mnesia</c> is + involved nodes, but it does not matter if Mnesia is running or not. Which nodes that are considered as disc-resident nodes is determined from the schema information in the local fallback.</p> @@ -2793,28 +2792,28 @@ raise(Name, Amount) -> <section> <title>Configuration Parameters</title> <marker id="configuration_parameters"></marker> - <p><c>Mnesia</c> reads the following application configuration + <p>Mnesia reads the following application configuration parameters:</p> <list type="bulleted"> <item> <p><c>-mnesia access_module Module</c>. The name of the - <c>Mnesia</c> activity access callback module. Default is + Mnesia activity access callback module. Default is <c>mnesia</c>.</p> </item> <item> <p><c>-mnesia auto_repair true | false</c>. This flag - controls if <c>Mnesia</c> automatically tries to repair + controls if Mnesia automatically tries to repair files that have not been properly closed. Default is <c>true</c>.</p> </item> <item> <p><c>-mnesia backup_module Module</c>. The name of the - <c>Mnesia</c> backup callback module. Default is + Mnesia backup callback module. Default is <c>mnesia_backup</c>.</p> </item> <item> <p><c>-mnesia debug Level</c>. Controls the debug level - of <c>Mnesia</c>. The possible values are as follows:</p> + of Mnesia. The possible values are as follows:</p> <taglist> <tag><c>none</c></tag> <item> @@ -2826,7 +2825,7 @@ raise(Name, Amount) -> events generate <c>{mnesia_info, Format, Args}</c> system events. Processes can subscribe to these events with <c>mnesia:subscribe/1</c>. The events are always sent to - the <c>Mnesia</c> event handler.</p> + the Mnesia event handler.</p> </item> <tag><c>debug</c></tag> <item> @@ -2835,15 +2834,15 @@ raise(Name, Amount) -> <c>{mnesia_info, Format, Args}</c> system events. Processes can subscribe to these events with <c>mnesia:subscribe/1</c>. The events are always sent to - the <c>Mnesia</c> event handler. On this debug level, - the <c>Mnesia</c> event handler starts subscribing to + the Mnesia event handler. On this debug level, + the Mnesia event handler starts subscribing to updates in the schema table.</p> </item> <tag><c>trace</c></tag> <item> <p>Activates all events at the debug level. On this - level, the <c>Mnesia</c> event handler starts subscribing - to updates on all <c>Mnesia</c> tables. This level is + level, the Mnesia event handler starts subscribing + to updates on all Mnesia tables. This level is intended only for debugging small toy systems, as many large events can be generated.</p> </item> @@ -2857,7 +2856,7 @@ raise(Name, Amount) -> </item> <item> <p><c>-mnesia core_dir Directory</c>. The name of the - directory where <c>Mnesia</c> core files is stored, or + directory where Mnesia core files is stored, or false. Setting it implies that also RAM-only nodes generate a core file if a crash occurs.</p> </item> @@ -2871,9 +2870,9 @@ raise(Name, Amount) -> </item> <item> <p><c>-mnesia dir Directory</c>. The name of the directory - where all <c>Mnesia</c> data is stored. The directory name + where all Mnesia data is stored. The directory name must be unique for the current node. Two nodes must never - share the the same <c>Mnesia</c> directory. The results + share the the same Mnesia directory. The results are unpredictable.</p> </item> <item> @@ -2916,44 +2915,44 @@ raise(Name, Amount) -> </item> <item> <p><c>-mnesia event_module Module</c>. The name of the - <c>Mnesia</c> event handler callback module. Default is + Mnesia event handler callback module. Default is <c>mnesia_event</c>.</p> </item> <item> <p><c>-mnesia extra_db_nodes Nodes</c> specifies a list of nodes, in addition to the ones found in the schema, with - which <c>Mnesia</c> is also to establish contact. Default + which Mnesia is also to establish contact. Default is <c>[]</c> (empty list).</p> </item> <item> <p><c>-mnesia fallback_error_function {UserModule, UserFunc}</c>. Specifies a user-supplied callback function, which is - called if a fallback is installed and <c>Mnesia</c> goes - down on another node. <c>Mnesia</c> calls the function + called if a fallback is installed and Mnesia goes + down on another node. Mnesia calls the function with one argument, the name of the dying node, for example, - <c>UserModule:UserFunc(DyingNode)</c>. <c>Mnesia</c> must + <c>UserModule:UserFunc(DyingNode)</c>. Mnesia must be restarted, otherwise the database can be inconsistent. - The default behavior is to terminate <c>Mnesia</c>.</p> + The default behavior is to terminate Mnesia.</p> </item> <item> <p><c>-mnesia max_wait_for_decision Timeout</c>. Specifies - how long <c>Mnesia</c> waits for other nodes to share their + how long Mnesia waits for other nodes to share their knowledge about the outcome of an unclear transaction. By default, <c>Timeout</c> is set to the atom <c>infinity</c>. - This implies that if <c>Mnesia</c> upon startup detects + This implies that if Mnesia upon startup detects a "heavyweight transaction" whose outcome is unclear, the - local <c>Mnesia</c> waits until <c>Mnesia</c> is started + local Mnesia waits until Mnesia is started on some (in the worst case all) of the other nodes that were involved in the interrupted transaction. This is a rare - situation, but if it occurs, <c>Mnesia</c> does not guess if + situation, but if it occurs, Mnesia does not guess if the transaction on the other nodes was committed or - terminated. <c>Mnesia</c> waits until it knows the outcome + terminated. Mnesia waits until it knows the outcome and then acts accordingly.</p> <p>If <c>Timeout</c> is set to an integer value in - milliseconds, <c>Mnesia</c> forces "heavyweight transactions" + milliseconds, Mnesia forces "heavyweight transactions" to be finished, even if the outcome of the transaction for the moment is unclear. After <c>Timeout</c> milliseconds, - <c>Mnesia</c> commits or terminates the transaction and + Mnesia commits or terminates the transaction and continues with the startup. This can lead to a situation where the transaction is committed on some nodes and terminated on other nodes. If the transaction is a @@ -2977,14 +2976,14 @@ raise(Name, Amount) -> </item> <item> <p><c>-mnesia schema_location Loc</c>. Controls where - <c>Mnesia</c> looks for its schema. Parameter + Mnesia looks for its schema. Parameter <c>Loc</c> can be one of the following atoms:</p> <taglist> <tag><c>disc</c></tag> <item> <p>Mandatory disc. The schema is assumed to be located - in the <c>Mnesia</c> directory. If the schema cannot - be found, <c>Mnesia</c> refuses to start. This is the + in the Mnesia directory. If the schema cannot + be found, Mnesia refuses to start. This is the old behavior.</p> </item> <tag><c>ram</c></tag> @@ -3002,10 +3001,10 @@ raise(Name, Amount) -> <tag><c>opt_disc</c></tag> <item> <p>Optional disc. The schema can reside on disc or in - RAM. If the schema is found on disc, <c>Mnesia</c> + RAM. If the schema is found on disc, Mnesia starts as a disc-based node and the storage type of the schema table is <c>disc_copies</c>. If no schema is - found on disc, <c>Mnesia</c> starts as a disc-less node + found on disc, Mnesia starts as a disc-less node and the storage type of the schema table is <c>ram_copies</c>. Default value for the application parameter is <c>opt_disc</c>.</p> @@ -3013,7 +3012,7 @@ raise(Name, Amount) -> </taglist> </item> </list> - <p>First, the <c>SASL</c> application parameters are checked, + <p>First, the SASL application parameters are checked, then the command-line flags are checked, and finally, the default value is chosen.</p> </section> diff --git a/lib/mnesia/doc/src/mnesia_frag_hash.xml b/lib/mnesia/doc/src/mnesia_frag_hash.xml index 7e532634d0..51b32129b6 100644 --- a/lib/mnesia/doc/src/mnesia_frag_hash.xml +++ b/lib/mnesia/doc/src/mnesia_frag_hash.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2002</year> - <year>2013</year> + <year>2016</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -87,13 +87,13 @@ the new one.</p> <p><c>NewState</c> is stored as <c>hash_state</c> among the other <c>frag_properties</c>.</p> - <p>As a part of the <c>add_frag</c> procedure, <c>Mnesia</c> iterates + <p>As a part of the <c>add_frag</c> procedure, Mnesia iterates over all fragments corresponding to the <c>IterFrags</c> numbers and starts <c>key_to_frag_number(NewState,RecordKey)</c> for each record. If the new fragment differs from the old fragment, the record is moved to the new fragment.</p> <p>As the <c>add_frag</c> procedure is a part of a schema - transaction, <c>Mnesia</c> acquires write locks on the + transaction, Mnesia acquires write locks on the affected tables. That is, both the fragments corresponding to <c>IterFrags</c> and those corresponding to <c>AdditionalLockFrags</c>.</p> @@ -112,7 +112,7 @@ <desc> <p><c>NewState</c> is stored as <c>hash_state</c> among the other <c>frag_properties</c>.</p> - <p>As a part of the <c>del_frag</c> procedure, <c>Mnesia</c> iterates + <p>As a part of the <c>del_frag</c> procedure, Mnesia iterates over all fragments corresponding to the <c>IterFrags</c> numbers and starts <c>key_to_frag_number(NewState,RecordKey)</c> for each record. If the new fragment differs from the old @@ -120,7 +120,7 @@ <p>Notice that all records in the last fragment must be moved to another fragment, as the entire fragment is deleted.</p> <p>As the <c>del_frag</c> procedure is a part of a schema - transaction, <c>Mnesia</c> acquires write locks on the + transaction, Mnesia acquires write locks on the affected tables. That is, both the fragments corresponding to <c>IterFrags</c> and those corresponding to <c>AdditionalLockFrags</c>.</p> @@ -134,7 +134,7 @@ <v>Reason = term()</v> </type> <desc> - <p>Starts whenever <c>Mnesia</c> needs to determine + <p>Starts whenever Mnesia needs to determine which fragment a certain record belongs to. It is typically started at <c>read</c>, <c>write</c>, and <c>delete</c>.</p> </desc> @@ -149,7 +149,7 @@ <v>Reason = term()</v> </type> <desc> - <p>This function is called whenever <c>Mnesia</c> needs to determine + <p>This function is called whenever Mnesia needs to determine which fragments that need to be searched for a <c>MatchSpec</c>. It is typically called by <c>select</c> and <c>match_object</c>.</p> diff --git a/lib/mnesia/doc/src/mnesia_registry.xml b/lib/mnesia/doc/src/mnesia_registry.xml index b2529f08bd..a76f716981 100644 --- a/lib/mnesia/doc/src/mnesia_registry.xml +++ b/lib/mnesia/doc/src/mnesia_registry.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>1998</year> - <year>2013</year> + <year>2016</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -38,20 +38,20 @@ <modulesummary>Dump support for registries in erl_interface.</modulesummary> <description> <p>This module is usually part of the <c>erl_interface</c> - application, but is currently part of the <c>Mnesia</c> + application, but is currently part of the Mnesia application.</p> <p>This module is mainly intended for internal use within OTP, but it has two functions that are exported for public use.</p> <p>On C-nodes, <c>erl_interface</c> has support for registry tables. These tables reside in RAM on the C-node, but can also - be dumped into <c>Mnesia</c> tables. By default, the dumping + be dumped into Mnesia tables. By default, the dumping of registry tables through <c>erl_interface</c> causes a - corresponding <c>Mnesia</c> table to be created with + corresponding Mnesia table to be created with <c>mnesia_registry:create_table/1</c>, if necessary.</p> <p>Tables that are created with these functions can be - administered as all other <c>Mnesia</c> tables. They can be + administered as all other Mnesia tables. They can be included in backups, replicas can be added, and so on. - The tables are normal <c>Mnesia</c> tables owned by the user + The tables are normal Mnesia tables owned by the user of the corresponding <c>erl_interface</c> registries.</p> </description> @@ -68,7 +68,7 @@ that is, <c>{ram_copies,[node()]}</c> or <c>{disc_copies,[node()]}</c>.</p> <p>This function is used by <c>erl_interface</c> to - create the <c>Mnesia</c> table if it does not already + create the Mnesia table if it does not already exist.</p> </desc> </func> diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml index 18bb110104..9f59759cb6 100644 --- a/lib/mnesia/doc/src/notes.xml +++ b/lib/mnesia/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1996</year><year>2013</year> + <year>1996</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -39,7 +39,90 @@ thus constitutes one section in this document. The title of each section is the version number of Mnesia.</p> - <section><title>Mnesia 4.13.4</title> + <section><title>Mnesia 4.14.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed crash in checkpoint handling when table was deleted + during backup.</p> + <p> + Own Id: OTP-14167</p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.14.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A continuation returned by mnesia:select/[14] should be + reusable in different, non-transactional activities.</p> + <p> + Own Id: OTP-13944 Aux Id: PR-1184 </p> + </item> + <item> + <p> + Fixed crash when calling block_table multiple times. + Could happen when having locks for a long time and + restarting mnesia.</p> + <p> + Own Id: OTP-13970 Aux Id: Seq-13198 </p> + </item> + <item> + <p> + Change mnesia_tm process to have off-heap messages since + mnesia_tm can be the receiver of many non-synchronized + message from other nodes.</p> + <p> + Own Id: OTP-14074</p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.14.1</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> Correct some minor documentation issues. </p> + <p> + Own Id: OTP-13891</p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.14</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Added experimental external backend plugin api. This adds + the possibility for the user to write other storage + backends for data, for example by using shared memory or + ram-cached disk storage.</p> + <p> + The plugin api may change in future versions after being + battle tested.</p> + <p> + Own Id: OTP-13058</p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.13.4</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/mnesia/doc/src/notes_history.xml b/lib/mnesia/doc/src/notes_history.xml index b1c050a8af..2300a1ecac 100644 --- a/lib/mnesia/doc/src/notes_history.xml +++ b/lib/mnesia/doc/src/notes_history.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2013</year> + <year>2004</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/part.xml b/lib/mnesia/doc/src/part.xml index 3406da4a34..d3ffe93937 100644 --- a/lib/mnesia/doc/src/part.xml +++ b/lib/mnesia/doc/src/part.xml @@ -4,7 +4,7 @@ <part xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,7 +30,7 @@ <file>part.sgml</file> </header> <description> - <p>The <c>Mnesia</c> application is a distributed Database Management + <p>The Mnesia application is a distributed Database Management System (DBMS), appropriate for telecommunications applications and other Erlang applications, which require continuous operation and exhibit soft real-time properties.</p> diff --git a/lib/mnesia/doc/src/part_notes.xml b/lib/mnesia/doc/src/part_notes.xml index e1f475a40d..1e4e4bc60a 100644 --- a/lib/mnesia/doc/src/part_notes.xml +++ b/lib/mnesia/doc/src/part_notes.xml @@ -4,7 +4,7 @@ <part xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/part_notes_history.xml b/lib/mnesia/doc/src/part_notes_history.xml index fec251cea6..a1c6f5aef0 100644 --- a/lib/mnesia/doc/src/part_notes_history.xml +++ b/lib/mnesia/doc/src/part_notes_history.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2004</year> - <year>2013</year> + <year>2016</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> diff --git a/lib/mnesia/doc/src/ref_man.xml b/lib/mnesia/doc/src/ref_man.xml index 370267e3b8..7fb71b9c45 100644 --- a/lib/mnesia/doc/src/ref_man.xml +++ b/lib/mnesia/doc/src/ref_man.xml @@ -4,7 +4,7 @@ <application xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -33,7 +33,7 @@ <file>refman.sgml</file> </header> <description> - <p>The <c>Mnesia</c> application is a distributed Database Management + <p>The Mnesia application is a distributed Database Management System (DBMS), appropriate for telecommunications applications and other Erlang applications, which require continuous operation and exhibit soft real-time properties.</p> diff --git a/lib/mnesia/examples/Makefile b/lib/mnesia/examples/Makefile index 0410ea2e8e..5580f0bdcb 100644 --- a/lib/mnesia/examples/Makefile +++ b/lib/mnesia/examples/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2012. All Rights Reserved. +# Copyright Ericsson AB 1996-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/lib/mnesia/examples/bench/bench.erl b/lib/mnesia/examples/bench/bench.erl index 5a414edcf3..e049ab97c1 100644 --- a/lib/mnesia/examples/bench/bench.erl +++ b/lib/mnesia/examples/bench/bench.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2009. All Rights Reserved. +%% Copyright Ericsson AB 2001-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/examples/bench/bench.hrl b/lib/mnesia/examples/bench/bench.hrl index 5f19513fb2..f312d76bd1 100644 --- a/lib/mnesia/examples/bench/bench.hrl +++ b/lib/mnesia/examples/bench/bench.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2009. All Rights Reserved. +%% Copyright Ericsson AB 2001-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/examples/bench/bench_generate.erl b/lib/mnesia/examples/bench/bench_generate.erl index e838f07fbb..c0b33e40f2 100644 --- a/lib/mnesia/examples/bench/bench_generate.erl +++ b/lib/mnesia/examples/bench/bench_generate.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2009. All Rights Reserved. +%% Copyright Ericsson AB 2001-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/examples/bench/bench_populate.erl b/lib/mnesia/examples/bench/bench_populate.erl index b35cc61b5a..c7bf2c52d6 100644 --- a/lib/mnesia/examples/bench/bench_populate.erl +++ b/lib/mnesia/examples/bench/bench_populate.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2009. All Rights Reserved. +%% Copyright Ericsson AB 2001-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/examples/bench/bench_trans.erl b/lib/mnesia/examples/bench/bench_trans.erl index 85641b29d4..73934d4fb5 100644 --- a/lib/mnesia/examples/bench/bench_trans.erl +++ b/lib/mnesia/examples/bench/bench_trans.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2009. All Rights Reserved. +%% Copyright Ericsson AB 2001-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/examples/mnesia_meter.erl b/lib/mnesia/examples/mnesia_meter.erl index 73f35a4a46..c278dfde93 100644 --- a/lib/mnesia/examples/mnesia_meter.erl +++ b/lib/mnesia/examples/mnesia_meter.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/examples/mnesia_tpcb.erl b/lib/mnesia/examples/mnesia_tpcb.erl index c6eda1c448..fb39ee321d 100644 --- a/lib/mnesia/examples/mnesia_tpcb.erl +++ b/lib/mnesia/examples/mnesia_tpcb.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/include/Makefile b/lib/mnesia/include/Makefile index fec963885c..6eed4d716f 100644 --- a/lib/mnesia/include/Makefile +++ b/lib/mnesia/include/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1998-2012. All Rights Reserved. +# Copyright Ericsson AB 1998-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/Makefile b/lib/mnesia/src/Makefile index dae0df76f3..08a00e6aba 100644 --- a/lib/mnesia/src/Makefile +++ b/lib/mnesia/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2013. All Rights Reserved. +# Copyright Ericsson AB 1996-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -43,6 +43,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/mnesia-$(VSN) # ---------------------------------------------------- MODULES= \ mnesia \ + mnesia_backend_type \ mnesia_backup \ mnesia_bup \ mnesia_checkpoint \ @@ -50,6 +51,7 @@ MODULES= \ mnesia_controller \ mnesia_dumper\ mnesia_event \ + mnesia_ext_sup \ mnesia_frag \ mnesia_frag_hash \ mnesia_frag_old_hash \ diff --git a/lib/mnesia/src/mnesia.app.src b/lib/mnesia/src/mnesia.app.src index c78a7cba1e..006ad4bac1 100644 --- a/lib/mnesia/src/mnesia.app.src +++ b/lib/mnesia/src/mnesia.app.src @@ -3,6 +3,7 @@ {vsn, "%VSN%"}, {modules, [ mnesia, + mnesia_backend_type, mnesia_backup, mnesia_bup, mnesia_checkpoint, @@ -10,6 +11,7 @@ mnesia_controller, mnesia_dumper, mnesia_event, + mnesia_ext_sup, mnesia_frag, mnesia_frag_hash, mnesia_frag_old_hash, diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 2fdc7afacd..34efeca4fa 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -82,7 +82,8 @@ system_info/0, % Not for public use %% Database mgt - create_schema/1, delete_schema/1, + create_schema/1, create_schema/2, delete_schema/1, + add_backend_type/2, backup/1, backup/2, traverse_backup/4, traverse_backup/6, install_fallback/1, install_fallback/2, uninstall_fallback/0, uninstall_fallback/1, @@ -197,6 +198,9 @@ e_has_var(X, Pos) -> %% Start and stop start() -> + start([]). + +start_() -> {Time , Res} = timer:tc(application, start, [?APPLICATION, temporary]), Secs = Time div 1000000, @@ -232,7 +236,7 @@ patched_start([{Env, Val} | Tail]) when is_atom(Env) -> patched_start([Head | _]) -> {error, {bad_type, Head}}; patched_start([]) -> - start(). + start_(). stop() -> case application:stop(?APPLICATION) of @@ -297,6 +301,7 @@ ms() -> %% Keep these last in the list, so %% mnesia_sup kills these last + mnesia_ext_sup, mnesia_monitor, mnesia_event ]. @@ -556,22 +561,16 @@ write(_Tid, _Ts, Tab, Val, LockKind) -> abort({bad_type, Tab, Val, LockKind}). write_to_store(Tab, Store, Oid, Val) -> - case ?catch_val({Tab, record_validation}) of - {RecName, Arity, Type} - when tuple_size(Val) == Arity, RecName == element(1, Val) -> - case Type of - bag -> - ?ets_insert(Store, {Oid, Val, write}); - _ -> - ?ets_delete(Store, Oid), - ?ets_insert(Store, {Oid, Val, write}) - end, - ok; - {'EXIT', _} -> - abort({no_exists, Tab}); - _ -> - abort({bad_type, Val}) - end. + {_, _, Type} = mnesia_lib:validate_record(Tab, Val), + Oid = {Tab, element(2, Val)}, + case Type of + bag -> + ?ets_insert(Store, {Oid, Val, write}); + _ -> + ?ets_delete(Store, Oid), + ?ets_insert(Store, {Oid, Val, write}) + end, + ok. delete({Tab, Key}) -> delete(Tab, Key, write); @@ -1410,8 +1409,14 @@ select_cont(Tid,_,State=#mnesia_select{tid=Tid,written=[]}) -> select_state(dirty_sel_cont(State),State); select_cont(Tid,_Ts,State=#mnesia_select{tid=Tid}) -> trans_select(dirty_sel_cont(State), State); -select_cont(_Tid2,_,#mnesia_select{tid=_Tid1}) -> % Missmatching tids +select_cont(Tid2,_,#mnesia_select{tid=_Tid1}) + when element(1,Tid2) == tid -> % Mismatching tids abort(wrong_transaction); +select_cont(Tid,Ts,State=#mnesia_select{}) -> + % Repair mismatching tids in non-transactional contexts + RepairedState = State#mnesia_select{tid = Tid, written = [], + spec = undefined, type = undefined}, + select_cont(Tid,Ts,RepairedState); select_cont(_,_,Cont) -> abort({badarg, Cont}). @@ -1548,16 +1553,9 @@ dirty_write(Tab, Val) -> do_dirty_write(SyncMode, Tab, Val) when is_atom(Tab), Tab /= schema, is_tuple(Val), tuple_size(Val) > 2 -> - case ?catch_val({Tab, record_validation}) of - {RecName, Arity, _Type} - when tuple_size(Val) == Arity, RecName == element(1, Val) -> - Oid = {Tab, element(2, Val)}, - mnesia_tm:dirty(SyncMode, {Oid, Val, write}); - {'EXIT', _} -> - abort({no_exists, Tab}); - _ -> - abort({bad_type, Val}) - end; + {_, _, _} = mnesia_lib:validate_record(Tab, Val), + Oid = {Tab, element(2, Val)}, + mnesia_tm:dirty(SyncMode, {Oid, Val, write}); do_dirty_write(_SyncMode, Tab, Val) -> abort({bad_type, Tab, Val}). @@ -1609,8 +1607,8 @@ dirty_update_counter(Tab, Key, Incr) -> do_dirty_update_counter(SyncMode, Tab, Key, Incr) when is_atom(Tab), Tab /= schema, is_integer(Incr) -> - case ?catch_val({Tab, record_validation}) of - {RecName, 3, set} -> + case mnesia_lib:validate_key(Tab, Key) of + {RecName, 3, Type} when Type == set; Type == ordered_set -> Oid = {Tab, Key}, mnesia_tm:dirty(SyncMode, {Oid, {RecName, Incr}, update_counter}); _ -> @@ -1884,9 +1882,10 @@ any_table_info(Tab, Item) when is_atom(Tab) -> [] -> abort({no_exists, Tab, Item}); Props -> - lists:map(fun({setorbag, Type}) -> {type, Type}; - (Prop) -> Prop end, - Props) + Rename = fun ({setorbag, Type}) -> {type, Type}; + (Prop) -> Prop + end, + lists:sort(lists:map(Rename, Props)) end; name -> Tab; @@ -1910,6 +1909,8 @@ raw_table_info(Tab, Item) -> info_reply(?ets_info(Tab, Item), Tab, Item); disc_only_copies -> info_reply(dets:info(Tab, Item), Tab, Item); + {ext, Alias, Mod} -> + info_reply(catch Mod:info(Alias, Tab, Item), Tab, Item); unknown -> bad_info_reply(Tab, Item) end @@ -2022,15 +2023,26 @@ display_tab_info() -> MasterTabs = mnesia_recover:get_master_node_tables(), io:format("master node tables = ~p~n", [lists:sort(MasterTabs)]), + case get_backend_types() of + [] -> ok; + Ts -> list_backend_types(Ts, "backend types = ") + end, + + case get_index_plugins() of + [] -> ok; + Ps -> list_index_plugins(Ps, "index plugins = ") + end, + Tabs = system_info(tables), - {Unknown, Ram, Disc, DiscOnly} = - lists:foldl(fun storage_count/2, {[], [], [], []}, Tabs), + {Unknown, Ram, Disc, DiscOnly, Ext} = + lists:foldl(fun storage_count/2, {[], [], [], [], []}, Tabs), io:format("remote = ~p~n", [lists:sort(Unknown)]), io:format("ram_copies = ~p~n", [lists:sort(Ram)]), io:format("disc_copies = ~p~n", [lists:sort(Disc)]), io:format("disc_only_copies = ~p~n", [lists:sort(DiscOnly)]), + [io:format("~-19s= ~p~n", [atom_to_list(A), Ts]) || {A,Ts} <- Ext], Rfoldl = fun(T, Acc) -> Rpat = @@ -2038,7 +2050,7 @@ display_tab_info() -> read_only -> lists:sort([{A, read_only} || A <- val({T, active_replicas})]); read_write -> - table_info(T, where_to_commit) + [fix_wtc(W) || W <- table_info(T, where_to_commit)] end, case lists:keysearch(Rpat, 1, Acc) of {value, {_Rpat, Rtabs}} -> @@ -2051,12 +2063,60 @@ display_tab_info() -> Rdisp = fun({Rpat, Rtabs}) -> io:format("~p = ~p~n", [Rpat, Rtabs]) end, lists:foreach(Rdisp, lists:sort(Repl)). -storage_count(T, {U, R, D, DO}) -> +get_backend_types() -> + case ?catch_val({schema, user_property, mnesia_backend_types}) of + {'EXIT', _} -> + []; + {mnesia_backend_types, Ts} -> + lists:sort(Ts) + end. + +get_index_plugins() -> + case ?catch_val({schema, user_property, mnesia_index_plugins}) of + {'EXIT', _} -> + []; + {mnesia_index_plugins, Ps} -> + lists:sort(Ps) + end. + + +list_backend_types([{A,M} | T] = Ts, Legend) -> + Indent = [$\s || _ <- Legend], + W = integer_to_list( + lists:foldl(fun({Alias,_}, Wa) -> + erlang:max(Wa, length(atom_to_list(Alias))) + end, 0, Ts)), + io:fwrite(Legend ++ "~-" ++ W ++ "s - ~s~n", + [atom_to_list(A), atom_to_list(M)]), + [io:fwrite(Indent ++ "~-" ++ W ++ "s - ~s~n", + [atom_to_list(A1), atom_to_list(M1)]) + || {A1,M1} <- T]. + +list_index_plugins([{N,M,F} | T] = Ps, Legend) -> + Indent = [$\s || _ <- Legend], + W = integer_to_list( + lists:foldl(fun({N1,_,_}, Wa) -> + erlang:max(Wa, length(pp_ix_name(N1))) + end, 0, Ps)), + io:fwrite(Legend ++ "~-" ++ W ++ "s - ~s:~s~n", + [pp_ix_name(N), atom_to_list(M), atom_to_list(F)]), + [io:fwrite(Indent ++ "~-" ++ W ++ "s - ~s:~s~n", + [pp_ix_name(N1), atom_to_list(M1), atom_to_list(F1)]) + || {N1,M1,F1} <- T]. + +pp_ix_name(N) -> + lists:flatten(io_lib:fwrite("~w", [N])). + +fix_wtc({N, {ext,A,_}}) -> {N, A}; +fix_wtc({N,A}) when is_atom(A) -> {N, A}. + +storage_count(T, {U, R, D, DO, Ext}) -> case table_info(T, storage_type) of - unknown -> {[T | U], R, D, DO}; - ram_copies -> {U, [T | R], D, DO}; - disc_copies -> {U, R, [T | D], DO}; - disc_only_copies -> {U, R, D, [T | DO]} + unknown -> {[T | U], R, D, DO, Ext}; + ram_copies -> {U, [T | R], D, DO, Ext}; + disc_copies -> {U, R, [T | D], DO, Ext}; + disc_only_copies -> {U, R, D, [T | DO], Ext}; + {ext, A, _} -> {U, R, D, DO, orddict:append(A, T, Ext)} end. system_info(Item) -> @@ -2071,9 +2131,10 @@ system_info2(all) -> system_info2(db_nodes) -> DiscNs = ?catch_val({schema, disc_copies}), RamNs = ?catch_val({schema, ram_copies}), + ExtNs = ?catch_val({schema, external_copies}), if - is_list(DiscNs), is_list(RamNs) -> - DiscNs ++ RamNs; + is_list(DiscNs), is_list(RamNs), is_list(ExtNs) -> + DiscNs ++ RamNs ++ ExtNs; true -> case mnesia_schema:read_nodes() of {ok, Nodes} -> Nodes; @@ -2177,6 +2238,7 @@ system_info2(access_module) -> mnesia_monitor:get_env(access_module); system_info2(auto_repair) -> mnesia_monitor:get_env(auto_repair); system_info2(is_running) -> mnesia_lib:is_running(); system_info2(backup_module) -> mnesia_monitor:get_env(backup_module); +system_info2(backend_types) -> mnesia_schema:backend_types(); system_info2(event_module) -> mnesia_monitor:get_env(event_module); system_info2(debug) -> mnesia_monitor:get_env(debug); system_info2(dump_log_load_regulation) -> mnesia_monitor:get_env(dump_log_load_regulation); @@ -2213,6 +2275,7 @@ system_info_items(yes) -> [ access_module, auto_repair, + backend_types, backup_module, checkpoints, db_nodes, @@ -2306,11 +2369,17 @@ load_mnesia_or_abort() -> %% Database mgt create_schema(Ns) -> - mnesia_bup:create_schema(Ns). + create_schema(Ns, []). + +create_schema(Ns, Properties) -> + mnesia_bup:create_schema(Ns, Properties). delete_schema(Ns) -> mnesia_schema:delete_schema(Ns). +add_backend_type(Alias, Module) -> + mnesia_schema:add_backend_type(Alias, Module). + backup(Opaque) -> mnesia_log:backup(Opaque). diff --git a/lib/mnesia/src/mnesia.hrl b/lib/mnesia/src/mnesia.hrl index 72940b4729..0716dd87c8 100644 --- a/lib/mnesia/src/mnesia.hrl +++ b/lib/mnesia/src/mnesia.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -68,6 +68,7 @@ ram_copies = [], % [Node] disc_copies = [], % [Node] disc_only_copies = [], % [Node] + external_copies = [], % [{{Alias,Mod},[Node]}] load_order = 0, % Integer access_mode = read_write, % read_write | read_only majority = false, % true | false @@ -103,7 +104,7 @@ ram_copies = [], disc_copies = [], disc_only_copies = [], - snmp = [], + ext = [], schema_ops = [] }). diff --git a/lib/mnesia/src/mnesia_backend_type.erl b/lib/mnesia/src/mnesia_backend_type.erl new file mode 100644 index 0000000000..4791b82a01 --- /dev/null +++ b/lib/mnesia/src/mnesia_backend_type.erl @@ -0,0 +1,115 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Behaviour definition for mnesia backend types. +%% + +%%%header_doc_include + +-module(mnesia_backend_type). + +-export([behaviour_info/1]). + +%%%header_doc_include + +%%%impl_doc_include + +%% Note that mnesia considers all callbacks mandatory!! +%% +behaviour_info(callbacks) -> + [ + {add_aliases, 1}, % (Aliases) -> ok + {check_definition, 4}, % (TypeAlias, Tab, Nodes, Properties) -> ok + {close_table, 2}, % (TypeAlias, Tab) -> ok + {create_table, 3}, % (TypeAlias, Tab, Properties) -> ok + {delete, 3}, % (TypeAlias, Tab, Key) -> true | ok + {delete_table, 2}, % (TypeAlias, Tab) -> ok + {first, 2}, % (TypeAlias, Tab) -> Key::Term | '$end_of_table' + {fixtable, 3}, % (TypeAlias, Tab, Bool) -> ok | true + {last, 2}, % (TypeAlias, Tab) -> Key::Term | '$end_of_table' + {index_is_consistent,3}, % (TypeAlias, IxTag, Bool) -> ok + {init_backend, 0}, % () -> ok + {info, 3}, % (TypeAlias, Tab, Item) -> Term + {insert, 3}, % (TypeAlias, Tab, Object) -> ok + {lookup, 3}, % (TypeAlias, Tab, Key) -> [Objects] + {is_index_consistent,2}, % (TypeAlias, IxTag) -> Bool + {load_table, 4}, % (TypeAlias, Tab, Reason, CsList) -> ok + {match_delete, 3}, % (TypeAlias, Tab, Pattern) -> ok + {next, 3}, % (TypeAlias, Tab, Key) -> Key::Term | '$end_of_table' + {prev, 3}, % (TypeAlias, Tab, Key) -> Key::Term | '$end_of_table' + {receiver_first_message, 4}, % (Sender, FirstMsg, Alias, Tab) -> {Size, State} + {receive_data, 5}, % (Data, Alias, Name, Sender, State) -> {more, State} | {{more, Msg}, State} + {receive_done, 4}, % (Alias, Tab, Sender, State) -> ok + {real_suffixes, 0}, % () -> [FileSuffix] + {remove_aliases, 1}, % (Aliases) -> ok + {repair_continuation, 2}, % (Continuation, MatchSpec) -> Continuation + {select, 1}, % (Continuation) -> {[Match], Continuation'} | '$end_of_table' + {select, 3}, % (TypeAlias, Tab, Pattern) -> {[Match], Continuation'} | '$end_of_table' + {select, 4}, % (TypeAlias, Tab, MatchSpec, Limit) {[Match], Continuation'} | '$end_of_table' + {sender_init, 4}, % (TypeAlias, Tab, LoadReason, Pid) -> + % {standard, Init(), Chunk()} | {Init(), Chunk()} + {semantics, 2}, % (TypeAlias, storage | types | index_fun | index_types) -> + % ram_copies | disc_copies, set | ordered_set | bag, fun(), ordered | bag + {slot, 3}, % (TypeAlias, Tab, Pos) -> '$end_of_table' | Objects | {error, Reason} + {sync_close_table, 2}, % (TypeAlias, Tab) -> ok + {tmp_suffixes, 0}, % () -> [FileSuffix] + {update_counter, 4}, % (TypeAlias, Tab, Counter, Val) -> NewVal + {validate_key, 6}, % (TypeAlias, Tab, RecName, Arity, Type, Key) -> {RecName, Arity, Type} + {validate_record, 6} % (TypeAlias, Tab, RecName, Arity, Type, Obj) -> {RecName, Arity, Type} + ]. + +%%%impl_doc_include + +%% -type tab() :: atom(). +%% -type alias() :: atom(). +%% -type rec_name() :: atom(). +%% -type type() :: set | bag | ordered_set. +%% -type proplist() :: [{atom(), any()}]. +%% -type key() :: any(). +%% -type db_object() :: tuple(). + +%% -type matchspec() :: ets:match_spec(). +%% -type limit() :: integer() | infinity. + +%% -type cont_fun() :: any(). +%% -type cont() :: '$end_of_table' | cont_fun(). + +%% -callback check_definition(alias(), tab(), [node()], proplist()) -> ok. +%% -callback create_table(alias(), tab(), proplist()) -> tab(). +%% -callback load_table(alias(), tab(), any()) -> ok. +%% -callback delete_table(alias(), tab()) -> ok. +%% -callback first(alias(), tab()) -> key(). +%% -callback last(alias(), tab()) -> key(). +%% -callback next(alias(), tab(), key()) -> key(). +%% -callback prev(alias(), tab(), key()) -> key(). +%% -callback insert(alias(), tab(), db_object()) -> ok. +%% -callback lookup(alias(), tab(), key()) -> [db_object()]. +%% -callback delete(alias(), tab(), key()) -> ok. +%% -callback update_counter(alias(), tab(), key(), integer()) -> integer(). +%% -callback select(cont()) -> {list(), cont()}. +%% -callback select(alias(), tab(), matchspec()) -> list() | '$end_of_table'. +%% -callback select(alias(), tab(), matchspec(), limit()) -> {list(), cont()}. +%% -callback slot(alias(), tab(), integer()) -> key(). +%% -callback validate_key(alias(), tab(), rec_name(), arity(), type(), key()) -> +%% {rec_name(), arity(), type()}. +%% -callback validate_record(alias(),tab(),rec_name(),arity(),type(),db_obj()) -> +%% {rec_name(), arity(), type()}. +%% -callback repair_continuation(cont(), matchspec()) -> cont(). diff --git a/lib/mnesia/src/mnesia_backup.erl b/lib/mnesia/src/mnesia_backup.erl index 78107bda56..719e010c17 100644 --- a/lib/mnesia/src/mnesia_backup.erl +++ b/lib/mnesia/src/mnesia_backup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_bup.erl b/lib/mnesia/src/mnesia_bup.erl index 8b1143a352..3e55deb958 100644 --- a/lib/mnesia/src/mnesia_bup.erl +++ b/lib/mnesia/src/mnesia_bup.erl @@ -28,6 +28,7 @@ fallback_exists/0, tm_fallback_start/1, create_schema/1, + create_schema/2, install_fallback/1, install_fallback/2, uninstall_fallback/0, @@ -46,7 +47,7 @@ uninstall_fallback_master/2, local_uninstall_fallback/2, do_traverse_backup/7, - trav_apply/4 + trav_apply/5 ]). -include("mnesia.hrl"). @@ -81,7 +82,8 @@ iterate(Mod, Fun, Opaque, Acc) -> R = #restore{bup_module = Mod, bup_data = Opaque}, try read_schema_section(R) of {R2, {Header, Schema, Rest}} -> - try iter(R2, Header, Schema, Fun, Acc, Rest) of + Ext = get_ext_types(Schema), + try iter(R2, Header, Schema, Ext, Fun, Acc, Rest) of {ok, R3, Res} -> close_read(R3), {ok, Res} @@ -96,17 +98,32 @@ iterate(Mod, Fun, Opaque, Acc) -> Err end. -iter(R, Header, Schema, Fun, Acc, []) -> +get_ext_types(Schema) -> + try + List = lookup_schema(schema, Schema), + case lists:keyfind(user_properties, 1, List) of + {_, Props} -> + proplists:get_value( + mnesia_backend_types, Props, []); + false -> + [] + end + catch + throw:{error, {"Cannot lookup",_}} -> + [] + end. + +iter(R, Header, Schema, Ext, Fun, Acc, []) -> case safe_apply(R, read, [R#restore.bup_data]) of {R2, []} -> - Res = Fun([], Header, Schema, Acc), + Res = Fun([], Header, Schema, Ext, Acc), {ok, R2, Res}; {R2, BupItems} -> - iter(R2, Header, Schema, Fun, Acc, BupItems) + iter(R2, Header, Schema, Ext, Fun, Acc, BupItems) end; -iter(R, Header, Schema, Fun, Acc, BupItems) -> - Acc2 = Fun(BupItems, Header, Schema, Acc), - iter(R, Header, Schema, Fun, Acc2, []). +iter(R, Header, Schema, Ext, Fun, Acc, BupItems) -> + Acc2 = Fun(BupItems, Header, Schema, Ext, Acc), + iter(R, Header, Schema, Ext, Fun, Acc2, []). -spec safe_apply(#restore{}, atom(), list()) -> tuple(). safe_apply(R, write, [_, Items]) when Items =:= [] -> @@ -262,7 +279,7 @@ convert_0_1(Schema) -> Cs = mnesia_schema:list2cs(List), convert_0_1(Schema2, [], Cs); false -> - List = mnesia_schema:get_initial_schema(disc_copies, [node()]), + List = mnesia_schema:get_initial_schema(disc_copies, [node()], []), Cs = mnesia_schema:list2cs(List), convert_0_1(Schema, [], Cs) end. @@ -310,16 +327,19 @@ schema2bup({schema, Tab, TableDef}) -> %% Create schema on the given nodes %% Requires that old schemas has been deleted %% Returns ok | {error, Reason} -create_schema([]) -> - create_schema([node()]); -create_schema(Ns) when is_list(Ns) -> +create_schema(Nodes) -> + create_schema(Nodes, []). + +create_schema([], Props) -> + create_schema([node()], Props); +create_schema(Ns, Props) when is_list(Ns), is_list(Props) -> case is_set(Ns) of true -> - create_schema(Ns, mnesia_schema:ensure_no_schema(Ns)); + create_schema(Ns, mnesia_schema:ensure_no_schema(Ns), Props); false -> {error, {combine_error, Ns}} end; -create_schema(Ns) -> +create_schema(Ns, _Props) -> {error, {badarg, Ns}}. is_set(List) when is_list(List) -> @@ -327,7 +347,7 @@ is_set(List) when is_list(List) -> is_set(_) -> false. -create_schema(Ns, ok) -> +create_schema(Ns, ok, Props) -> %% Ensure that we access the intended Mnesia %% directory. This function may not be called %% during startup since it will cause the @@ -346,7 +366,7 @@ create_schema(Ns, ok) -> Str = mk_str(), File = mnesia_lib:dir(Str), file:delete(File), - try make_initial_backup(Ns, File, Mod) of + try make_initial_backup(Ns, File, Mod, Props) of {ok, _Res} -> case do_install_fallback(File, Mod) of ok -> @@ -363,9 +383,9 @@ create_schema(Ns, ok) -> {error, Reason} -> {error, Reason} end; -create_schema(_Ns, {error, Reason}) -> +create_schema(_Ns, {error, Reason}, _) -> {error, Reason}; -create_schema(_Ns, Reason) -> +create_schema(_Ns, Reason, _) -> {error, Reason}. mk_str() -> @@ -373,7 +393,10 @@ mk_str() -> lists:concat([node()] ++ Now ++ ".TMP"). make_initial_backup(Ns, Opaque, Mod) -> - Orig = mnesia_schema:get_initial_schema(disc_copies, Ns), + make_initial_backup(Ns, Opaque, Mod, []). + +make_initial_backup(Ns, Opaque, Mod, Props) -> + Orig = mnesia_schema:get_initial_schema(disc_copies, Ns, Props), Modded = proplists:delete(storage_properties, proplists:delete(majority, Orig)), Schema = [{schema, schema, Modded}], O2 = do_apply(Mod, open_write, [Opaque], Opaque), @@ -486,15 +509,15 @@ install_fallback_master(ClientPid, FA) -> State = {start, FA}, Opaque = FA#fallback_args.opaque, Mod = FA#fallback_args.module, - Res = iterate(Mod, fun restore_recs/4, Opaque, State), + Res = iterate(Mod, fun restore_recs/5, Opaque, State), unlink(ClientPid), ClientPid ! {self(), Res}, exit(shutdown). -restore_recs(_, _, _, stop) -> +restore_recs(_, _, _, _, stop) -> throw({error, "restore_recs already stopped"}); -restore_recs(Recs, Header, Schema, {start, FA}) -> +restore_recs(Recs, Header, Schema, Ext, {start, FA}) -> %% No records in backup Schema2 = convert_schema(Header#log_header.log_version, Schema), CreateList = lookup_schema(schema, Schema2), @@ -505,19 +528,19 @@ restore_recs(Recs, Header, Schema, {start, FA}) -> Args = [self(), FA], Pids = [spawn_link(N, ?MODULE, fallback_receiver, Args) || N <- Ns], send_fallback(Pids, {start, Header, Schema2}), - Res = restore_recs(Recs, Header, Schema2, Pids), + Res = restore_recs(Recs, Header, Schema2, Ext, Pids), global:del_lock({{mnesia_table_lock, schema}, self()}, Ns), Res catch _:Reason -> throw({error, {"Bad schema in restore_recs", Reason}}) end; -restore_recs([], _Header, _Schema, Pids) -> +restore_recs([], _Header, _Schema, _Ext, Pids) -> send_fallback(Pids, swap), send_fallback(Pids, stop), stop; -restore_recs(Recs, _, _, Pids) -> +restore_recs(Recs, _, _, _, Pids) -> send_fallback(Pids, {records, Recs}), Pids. @@ -716,7 +739,7 @@ do_fallback_start(true, false) -> BupFile = fallback_bup(), Mod = mnesia_backup, LocalTabs = ?ets_new_table(mnesia_local_tables, [set, public, {keypos, 2}]), - case iterate(Mod, fun restore_tables/4, BupFile, {start, LocalTabs}) of + case iterate(Mod, fun restore_tables/5, BupFile, {start, LocalTabs}) of {ok, _Res} -> ?SAFE(dets:close(schema)), TmpSchema = mnesia_lib:tab2tmp(schema), @@ -737,23 +760,24 @@ do_fallback_start(true, false) -> {error, {"Cannot start from fallback", Reason}} end. -restore_tables(All=[Rec | Recs], Header, Schema, State={local, LocalTabs, LT}) -> +restore_tables(All=[Rec | Recs], Header, Schema, Ext, + State={local, LocalTabs, LT}) -> Tab = element(1, Rec), if Tab =:= LT#local_tab.name -> Key = element(2, Rec), (LT#local_tab.add)(Tab, Key, Rec, LT), - restore_tables(Recs, Header, Schema, State); + restore_tables(Recs, Header, Schema, Ext, State); true -> NewState = {new, LocalTabs}, - restore_tables(All, Header, Schema, NewState) + restore_tables(All, Header, Schema, Ext, NewState) end; -restore_tables(All=[Rec | Recs], Header, Schema, {new, LocalTabs}) -> +restore_tables(All=[Rec | Recs], Header, Schema, Ext, {new, LocalTabs}) -> Tab = element(1, Rec), case ?ets_lookup(LocalTabs, Tab) of [] -> State = {not_local, LocalTabs, Tab}, - restore_tables(Recs, Header, Schema, State); + restore_tables(Recs, Header, Schema, Ext, State); [LT] when is_record(LT, local_tab) -> State = {local, LocalTabs, LT}, case LT#local_tab.opened of @@ -762,38 +786,39 @@ restore_tables(All=[Rec | Recs], Header, Schema, {new, LocalTabs}) -> (LT#local_tab.open)(Tab, LT), ?ets_insert(LocalTabs,LT#local_tab{opened=true}) end, - restore_tables(All, Header, Schema, State) + restore_tables(All, Header, Schema, Ext, State) end; -restore_tables(All=[Rec | Recs], Header, Schema, S = {not_local, LocalTabs, PrevTab}) -> +restore_tables(All=[Rec | Recs], Header, Schema, Ext, + S = {not_local, LocalTabs, PrevTab}) -> Tab = element(1, Rec), if Tab =:= PrevTab -> - restore_tables(Recs, Header, Schema, S); + restore_tables(Recs, Header, Schema, Ext, S); true -> State = {new, LocalTabs}, - restore_tables(All, Header, Schema, State) + restore_tables(All, Header, Schema, Ext, State) end; -restore_tables(Recs, Header, Schema, {start, LocalTabs}) -> +restore_tables(Recs, Header, Schema, Ext, {start, LocalTabs}) -> Dir = mnesia_lib:dir(), OldDir = filename:join([Dir, "OLD_DIR"]), mnesia_schema:purge_dir(OldDir, []), mnesia_schema:purge_dir(Dir, [fallback_name()]), - init_dat_files(Schema, LocalTabs), + init_dat_files(Schema, Ext, LocalTabs), State = {new, LocalTabs}, - restore_tables(Recs, Header, Schema, State); -restore_tables([], _Header, _Schema, State) -> + restore_tables(Recs, Header, Schema, Ext, State); +restore_tables([], _Header, _Schema, _Ext, State) -> State. %% Creates all neccessary dat files and inserts %% the table definitions in the schema table %% %% Returns a list of local_tab tuples for all local tables -init_dat_files(Schema, LocalTabs) -> +init_dat_files(Schema, Ext, LocalTabs) -> TmpFile = mnesia_lib:tab2tmp(schema), Args = [{file, TmpFile}, {keypos, 2}, {type, set}], case dets:open_file(schema, Args) of % Assume schema lock {ok, _} -> - create_dat_files(Schema, LocalTabs), + create_dat_files(Schema, Ext, LocalTabs), ok = dets:close(schema), LocalTab = #local_tab{name = schema, storage_type = disc_copies, @@ -808,10 +833,10 @@ init_dat_files(Schema, LocalTabs) -> throw({error, {"Cannot open file", schema, Args, Reason}}) end. -create_dat_files([{schema, schema, TabDef} | Tail], LocalTabs) -> +create_dat_files([{schema, schema, TabDef} | Tail], Ext, LocalTabs) -> ok = dets:insert(schema, {schema, schema, TabDef}), - create_dat_files(Tail, LocalTabs); -create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) -> + create_dat_files(Tail, Ext, LocalTabs); +create_dat_files([{schema, Tab, TabDef} | Tail], Ext, LocalTabs) -> TmpFile = mnesia_lib:tab2tmp(Tab), DatFile = mnesia_lib:tab2dat(Tab), DclFile = mnesia_lib:tab2dcl(Tab), @@ -824,56 +849,21 @@ create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) -> mnesia_lib:dets_sync_close(Tab), file:delete(TmpFile), - Cs = mnesia_schema:list2cs(TabDef), + Cs = mnesia_schema:list2cs(TabDef, Ext), ok = dets:insert(schema, {schema, Tab, TabDef}), RecName = Cs#cstruct.record_name, Storage = mnesia_lib:cs_to_storage_type(node(), Cs), + delete_ext(Storage, Tab), + Semantics = mnesia_lib:semantics(Storage, storage), if - Storage =:= unknown -> + Semantics =:= undefined -> ok = dets:delete(schema, {schema, Tab}), - create_dat_files(Tail, LocalTabs); - Storage =:= disc_only_copies -> - Args = [{file, TmpFile}, {keypos, 2}, - {type, mnesia_lib:disk_type(Tab, Cs#cstruct.type)}], - Open = fun(T, LT) when T =:= LT#local_tab.name -> - case mnesia_lib:dets_sync_open(T, Args) of - {ok, _} -> - ok; - {error, Reason} -> - throw({error, {"Cannot open file", T, Args, Reason}}) - end - end, - Add = fun(T, Key, Rec, LT) when T =:= LT#local_tab.name -> - case Rec of - {_T, Key} -> - ok = dets:delete(T, Key); - (Rec) when T =:= RecName -> - ok = dets:insert(Tab, Rec); - (Rec) -> - Rec2 = setelement(1, Rec, RecName), - ok = dets:insert(T, Rec2) - end - end, - Close = fun(T, LT) when T =:= LT#local_tab.name -> - mnesia_lib:dets_sync_close(T) - end, - Swap = fun(T, LT) when T =:= LT#local_tab.name -> - Expunge(), - case LT#local_tab.opened of - true -> - Close(T,LT); - false -> - Open(T,LT), - Close(T,LT) - end, - case file:rename(TmpFile, DatFile) of - ok -> - ok; - {error, Reason} -> - mnesia_lib:fatal("Cannot rename file ~p -> ~p: ~p~n", - [TmpFile, DatFile, Reason]) - end - end, + create_dat_files(Tail, Ext, LocalTabs); + Semantics =:= disc_only_copies -> + Open = disc_only_open_fun(Storage, Cs), + Add = disc_only_add_fun(Storage, Cs), + Close = disc_only_close_fun(Storage), + Swap = disc_only_swap_fun(Storage, Expunge, Open, Close), LocalTab = #local_tab{name = Tab, storage_type = Storage, open = Open, @@ -883,8 +873,8 @@ create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) -> record_name = RecName, opened = false}, ?ets_insert(LocalTabs, LocalTab), - create_dat_files(Tail, LocalTabs); - Storage =:= ram_copies; Storage =:= disc_copies -> + create_dat_files(Tail, Ext, LocalTabs); + Semantics =:= ram_copies; Storage =:= disc_copies -> Open = fun(T, LT) when T =:= LT#local_tab.name -> mnesia_log:open_log({?MODULE, T}, mnesia_log:dcl_log_header(), @@ -945,18 +935,97 @@ create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) -> opened = false }, ?ets_insert(LocalTabs, LocalTab), - create_dat_files(Tail, LocalTabs) + create_dat_files(Tail, Ext, LocalTabs); + true -> + error({unknown_semantics, [{semantics, Semantics}, + {tabdef, TabDef}, + {ext, Ext}]}) end; -create_dat_files([{schema, Tab} | Tail], LocalTabs) -> +create_dat_files([{schema, Tab} | Tail], Ext, LocalTabs) -> ?ets_delete(LocalTabs, Tab), ok = dets:delete(schema, {schema, Tab}), TmpFile = mnesia_lib:tab2tmp(Tab), mnesia_lib:dets_sync_close(Tab), file:delete(TmpFile), - create_dat_files(Tail, LocalTabs); -create_dat_files([], _LocalTabs) -> + create_dat_files(Tail, Ext, LocalTabs); +create_dat_files([], _Ext, _LocalTabs) -> ok. +delete_ext({ext, Alias, Mod}, Tab) -> + Mod:close_table(Alias, Tab), + Mod:delete_table(Alias, Tab), + ok; +delete_ext(_, _) -> + ok. + + +disc_only_open_fun(disc_only_copies, #cstruct{name = Tab} =Cs) -> + TmpFile = mnesia_lib:tab2tmp(Tab), + Args = [{file, TmpFile}, {keypos, 2}, + {type, mnesia_lib:disk_type(Tab, Cs#cstruct.type)}], + fun(T, LT) when T =:= LT#local_tab.name -> + case mnesia_lib:dets_sync_open(T, Args) of + {ok, _} -> + ok; + {error, Reason} -> + throw({error, {"Cannot open file", T, Args, Reason}}) + end + end; +disc_only_open_fun({ext,Alias,Mod}, Cs) -> + fun(T, LT) when T =:= LT#local_tab.name -> + ok = Mod:load_table(Alias, T, restore, mnesia_schema:cs2list(Cs)) + end. + +disc_only_add_fun(Storage, #cstruct{name = Tab, + record_name = RecName}) -> + fun(T, Key, Rec, #local_tab{name = T}) when T =:= Tab-> + case Rec of + {_T, Key} -> + ok = mnesia_lib:db_erase(Storage, T, Key); + (Rec) when T =:= RecName -> + ok = mnesia_lib:db_put(Storage, T, Rec); + (Rec) -> + ok = mnesia_lib:db_put(Storage, T, + setelement(1, Rec, RecName)) + end + end. + +disc_only_close_fun(disc_only_copies) -> + fun(T, LT) when T =:= LT#local_tab.name -> + mnesia_lib:dets_sync_close(T) + end; +disc_only_close_fun({ext, Alias, Mod}) -> + fun(T, _LT) -> + Mod:sync_close_table(Alias, T) + end. + + +disc_only_swap_fun(disc_only_copies, Expunge, Open, Close) -> + fun(T, LT) when T =:= LT#local_tab.name -> + TmpFile = mnesia_lib:tab2tmp(T), + DatFile = mnesia_lib:tab2dat(T), + Expunge(), + case LT#local_tab.opened of + true -> + Close(T,LT); + false -> + Open(T,LT), + Close(T,LT) + end, + case file:rename(TmpFile, DatFile) of + ok -> + ok; + {error, Reason} -> + mnesia_lib:fatal("Cannot rename file ~p -> ~p: ~p~n", + [TmpFile, DatFile, Reason]) + end + end; +disc_only_swap_fun({ext, _Alias, _Mod}, _Expunge, _Open, Close) -> + fun(T, #local_tab{name = T} = LT) -> + Close(T, LT) + end. + + uninstall_fallback() -> uninstall_fallback([{scope, global}]). @@ -1133,7 +1202,7 @@ do_traverse_backup(ClientPid, Source, SourceMod, Target, TargetMod, Fun, Acc) -> end, A = {start, Fun, Acc, TargetMod, Iter}, Res = - case iterate(SourceMod, fun trav_apply/4, Source, A) of + case iterate(SourceMod, fun trav_apply/5, Source, A) of {ok, {iter, _, Acc2, _, Iter2}} when TargetMod =/= read_only -> try do_apply(TargetMod, commit_write, [Iter2], Iter2), @@ -1152,7 +1221,7 @@ do_traverse_backup(ClientPid, Source, SourceMod, Target, TargetMod, Fun, Acc) -> unlink(ClientPid), ClientPid ! {iter_done, self(), Res}. -trav_apply(Recs, _Header, _Schema, {iter, Fun, Acc, Mod, Iter}) -> +trav_apply(Recs, _Header, _Schema, _Ext, {iter, Fun, Acc, Mod, Iter}) -> {NewRecs, Acc2} = filter_foldl(Fun, Acc, Recs), if Mod =/= read_only, NewRecs =/= [] -> @@ -1161,7 +1230,7 @@ trav_apply(Recs, _Header, _Schema, {iter, Fun, Acc, Mod, Iter}) -> true -> {iter, Fun, Acc2, Mod, Iter} end; -trav_apply(Recs, Header, Schema, {start, Fun, Acc, Mod, Iter}) -> +trav_apply(Recs, Header, Schema, Ext, {start, Fun, Acc, Mod, Iter}) -> Iter2 = if Mod =/= read_only -> @@ -1169,8 +1238,9 @@ trav_apply(Recs, Header, Schema, {start, Fun, Acc, Mod, Iter}) -> true -> Iter end, - TravAcc = trav_apply(Schema, Header, Schema, {iter, Fun, Acc, Mod, Iter2}), - trav_apply(Recs, Header, Schema, TravAcc). + TravAcc = trav_apply(Schema, Header, Schema, Ext, + {iter, Fun, Acc, Mod, Iter2}), + trav_apply(Recs, Header, Schema, Ext, TravAcc). filter_foldl(Fun, Acc, [Head|Tail]) -> case Fun(Head, Acc) of diff --git a/lib/mnesia/src/mnesia_checkpoint.erl b/lib/mnesia/src/mnesia_checkpoint.erl index 1407bc0557..fc626940b4 100644 --- a/lib/mnesia/src/mnesia_checkpoint.erl +++ b/lib/mnesia/src/mnesia_checkpoint.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013 +%% Copyright Ericsson AB 1996-2016 %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -675,6 +675,16 @@ tab2retainer({Tab, Name}) -> FlatName = lists:flatten(io_lib:write(Name)), mnesia_lib:dir(lists:concat([?MODULE, "_", Tab, "_", FlatName, ".RET"])). +retainer_create(_Cp, R, Tab, Name, Ext = {ext, Alias, Mod}) -> + T = {Tab, retainer, Name}, + P = mnesia_schema:cs2list(val({Tab, cstruct})), + Mod:delete_table(Alias, T), + ok = Mod:create_table(Alias, T, P), + Cs = val({Tab, cstruct}), + Mod:load_table(Alias, T, {retainer, create_table}, + mnesia_schema:cs2list(Cs)), + dbg_out("Checkpoint retainer created ~p ~p~n", [Name, Tab]), + R#retainer{store = {Ext, T}, really_retain = true}; retainer_create(_Cp, R, Tab, Name, disc_only_copies) -> Fname = tab2retainer({Tab, Name}), file:delete(Fname), @@ -734,15 +744,23 @@ traverse_dcd({Cont, Recs}, Log, Fun) -> %% trashed data?? traverse_dcd(eof, _Log, _Fun) -> ok. +retainer_get({{ext, Alias, Mod}, Store}, Key) -> + Mod:lookup(Alias, Store, Key); retainer_get({ets, Store}, Key) -> ?ets_lookup(Store, Key); retainer_get({dets, Store}, Key) -> dets:lookup(Store, Key). +retainer_put({{ext, Alias, Mod}, Store}, Val) -> + Mod:insert(Alias, Store, Val); retainer_put({ets, Store}, Val) -> ?ets_insert(Store, Val); retainer_put({dets, Store}, Val) -> dets:insert(Store, Val). +retainer_first({{ext, Alias, Mod}, Store}) -> + Mod:first(Alias, Store); retainer_first({ets, Store}) -> ?ets_first(Store); retainer_first({dets, Store}) -> dets:first(Store). +retainer_next({{ext, Alias, Mod}, Store}, Key) -> + Mod:next(Alias, Store, Key); retainer_next({ets, Store}, Key) -> ?ets_next(Store, Key); retainer_next({dets, Store}, Key) -> dets:next(Store, Key). @@ -761,11 +779,16 @@ retainer_next({dets, Store}, Key) -> dets:next(Store, Key). retainer_fixtable(Tab, Bool) when is_atom(Tab) -> mnesia_lib:db_fixtable(val({Tab, storage_type}), Tab, Bool); +retainer_fixtable({Ext = {ext, _, _}, Tab}, Bool) -> + mnesia_lib:db_fixtable(Ext, Tab, Bool); retainer_fixtable({ets, Tab}, Bool) -> mnesia_lib:db_fixtable(ram_copies, Tab, Bool); retainer_fixtable({dets, Tab}, Bool) -> mnesia_lib:db_fixtable(disc_only_copies, Tab, Bool). +retainer_delete({{ext, Alias, Mod}, Store}) -> + Mod:close_table(Alias, Store), + Mod:delete_table(Alias, Store); retainer_delete({ets, Store}) -> ?ets_delete_table(Store); retainer_delete({dets, Store}) -> @@ -886,7 +909,7 @@ retainer_loop(Cp = #checkpoint_args{name=Name}) -> retainer_loop(Cp2); {From, {iter_end, Iter}} -> - retainer_fixtable(Iter#iter.oid_tab, false), + ?SAFE(retainer_fixtable(Iter#iter.oid_tab, false)), Iters = Cp#checkpoint_args.iterators -- [Iter], reply(From, Name, ok), retainer_loop(Cp#checkpoint_args{iterators = Iters}); @@ -948,7 +971,8 @@ do_stop(Cp) -> unset({checkpoint, Name}), lists:foreach(fun deactivate_tab/1, Cp#checkpoint_args.retainers), Iters = Cp#checkpoint_args.iterators, - lists:foreach(fun(I) -> retainer_fixtable(I#iter.oid_tab, false) end, Iters). + [?SAFE(retainer_fixtable(Tab, false)) || #iter{main_tab=Tab} <- Iters], + ok. deactivate_tab(R) -> Name = R#retainer.cp_name, @@ -1128,7 +1152,7 @@ do_change_copy(Cp, Tab, FromType, ToType) -> Cp#checkpoint_args{retainers = Rs, nodes = writers(Rs)}. check_iter(From, Iter) when Iter#iter.pid == From -> - retainer_fixtable(Iter#iter.oid_tab, false), + ?SAFE(retainer_fixtable(Iter#iter.oid_tab, false)), false; check_iter(_From, _Iter) -> true. diff --git a/lib/mnesia/src/mnesia_checkpoint_sup.erl b/lib/mnesia/src/mnesia_checkpoint_sup.erl index 680cd86182..ae3f77f37e 100644 --- a/lib/mnesia/src/mnesia_checkpoint_sup.erl +++ b/lib/mnesia/src/mnesia_checkpoint_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl index 69ccc1d2c9..17b47c059e 100644 --- a/lib/mnesia/src/mnesia_controller.erl +++ b/lib/mnesia/src/mnesia_controller.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -385,6 +385,8 @@ force_load_table(Tab) when is_atom(Tab), Tab /= schema -> do_force_load_table(Tab); disc_only_copies -> do_force_load_table(Tab); + {ext, _, _} -> + do_force_load_table(Tab); unknown -> set({Tab, load_by_force}, true), cast({force_load_updated, Tab}), @@ -1533,8 +1535,8 @@ update_whereabouts(Tab, Node, State) -> Storage == unknown -> %% No own copy, continue to read remotely add_active_replica(Tab, Node), - NodeST = mnesia_lib:storage_type_at_node(Node, Tab), - ReadST = mnesia_lib:storage_type_at_node(Read, Tab), + NodeST = mnesia_lib:semantics(mnesia_lib:storage_type_at_node(Node, Tab), storage), + ReadST = mnesia_lib:semantics(mnesia_lib:storage_type_at_node(Read, Tab), storage), if %% Avoid reading from disc_only_copies NodeST == disc_only_copies -> ignore; @@ -1588,6 +1590,7 @@ last_consistent_replica(Tab, Downs) -> Ram = Cs#cstruct.ram_copies, Disc = Cs#cstruct.disc_copies, DiscOnly = Cs#cstruct.disc_only_copies, + Ext = Cs#cstruct.external_copies, BetterCopies0 = mnesia_lib:remote_copy_holders(Cs) -- Downs, BetterCopies = BetterCopies0 -- Ram, AccessMode = Cs#cstruct.access_mode, @@ -1620,7 +1623,7 @@ last_consistent_replica(Tab, Downs) -> false; Storage == ram_copies -> if - Disc == [], DiscOnly == [] -> + Disc == [], DiscOnly == [], Ext == [] -> %% Nobody has copy on disc {true, {Tab, ram_only}}; true -> @@ -1700,9 +1703,10 @@ add_active_replica(Tab, Node, Cs = #cstruct{}) -> block_table(Tab) -> Var = {Tab, where_to_commit}, - Old = val(Var), - New = {blocked, Old}, - set(Var, New). % where_to_commit + case is_tab_blocked(val(Var)) of + {true, _} -> ok; + {false, W2C} -> set(Var, mark_blocked_tab(true, W2C)) + end. unblock_table(Tab) -> call({unblock_table, Tab}). @@ -1865,6 +1869,11 @@ info([Tab | Tail]) -> dets:info(Tab, size), dets:info(Tab, file_size), "bytes on disc"); + {ext, Alias, Mod} -> + info_format(Tab, + Mod:info(Alias, Tab, size), + Mod:info(Alias, Tab, memory), + "words of mem"); _ -> info_format(Tab, ?ets_info(Tab, size), diff --git a/lib/mnesia/src/mnesia_dumper.erl b/lib/mnesia/src/mnesia_dumper.erl index 0e954d869d..eb02a585a6 100644 --- a/lib/mnesia/src/mnesia_dumper.erl +++ b/lib/mnesia/src/mnesia_dumper.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -38,6 +38,8 @@ needs_dump_ets/1, raw_dump_table/2, raw_named_dump_table/2, + dump_to_logfile/2, + load_from_logfile/3, start_regulator/0, opt_dump_log/1, update/3, @@ -227,11 +229,11 @@ insert_rec(Rec, InPlace, InitBy, LogV) when is_record(Rec, commit) -> D = Rec#commit.decision, case mnesia_recover:wait_for_decision(D, InitBy) of {Tid, committed} -> - do_insert_rec(Tid, Rec, InPlace, InitBy, LogV); + do_insert_rec(Tid, mnesia_tm:new_cr_format(Rec), InPlace, InitBy, LogV); {Tid, aborted} -> case InitBy of startup -> - mnesia_schema:undo_prepare_commit(Tid, Rec); + mnesia_schema:undo_prepare_commit(Tid, mnesia_tm:new_cr_format(Rec)); _ -> ok end @@ -271,15 +273,30 @@ do_insert_rec(Tid, Rec, InPlace, InitBy, LogV) -> end end, D = Rec#commit.disc_copies, + ExtOps = commit_ext(Rec), insert_ops(Tid, disc_copies, D, InPlace, InitBy, LogV), + [insert_ops(Tid, Ext, Ops, InPlace, InitBy, LogV) || + {Ext, Ops} <- ExtOps, + storage_semantics(Ext) == disc_copies], case InitBy of startup -> DO = Rec#commit.disc_only_copies, - insert_ops(Tid, disc_only_copies, DO, InPlace, InitBy, LogV); + insert_ops(Tid, disc_only_copies, DO, InPlace, InitBy, LogV), + [insert_ops(Tid, Ext, Ops, InPlace, InitBy, LogV) || + {Ext, Ops} <- ExtOps, storage_semantics(Ext) == disc_only_copies]; _ -> ignore end. +commit_ext(#commit{ext = []}) -> []; +commit_ext(#commit{ext = Ext}) -> + case lists:keyfind(ext_copies, 1, Ext) of + {_, C} -> + lists:foldl(fun({Ext0, Op}, D) -> + orddict:append(Ext0, Op, D) + end, orddict:new(), C); + false -> [] + end. update(_Tid, [], _DumperMode) -> dumped; @@ -330,14 +347,15 @@ insert_ops(Tid, Storage, [Op | Ops], InPlace, InitBy, Ver) when Ver < "4.3" -> %% Normal ops disc_insert(_Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy) -> - case open_files(Tab, Storage, InPlace, InitBy) of + Semantics = storage_semantics(Storage), + case open_files(Tab, Semantics, Storage, InPlace, InitBy) of true -> - case Storage of + case Semantics of disc_copies when Tab /= schema -> mnesia_log:append({?MODULE,Tab}, {{Tab, Key}, Val, Op}), ok; _ -> - dets_insert(Op,Tab,Key,Val) + dets_insert(Op,Tab,Key,Val,Storage) end; false -> ignore @@ -349,34 +367,37 @@ disc_insert(_Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy) -> %% Otherwise we will get a double increment. %% This is perfect but update_counter is a dirty op. -dets_insert(Op,Tab,Key,Val) -> +dets_insert(Op,Tab,Key,Val, Storage0) -> + Storage = if Tab == schema -> disc_only_copies; + true -> Storage0 + end, case Op of write -> dets_updated(Tab,Key), - ok = dets:insert(Tab, Val); + ok = mnesia_lib:db_put(Storage, Tab, Val); delete -> dets_updated(Tab,Key), - ok = dets:delete(Tab, Key); + ok = mnesia_lib:db_erase(Storage, Tab, Key); update_counter -> case dets_incr_counter(Tab,Key) of true -> {RecName, Incr} = Val, - try _ = dets:update_counter(Tab, Key, Incr) + try _ = mnesia_lib:db_update_counter(Storage, Tab, Key, Incr) catch error:_ when Incr < 0 -> Zero = {RecName, Key, 0}, - ok = dets:insert(Tab, Zero); + ok = mnesia_lib:db_put(Storage, Tab, Zero); error:_ -> Init = {RecName, Key, Incr}, - ok = dets:insert(Tab, Init) + ok = mnesia_lib:db_put(Storage, Tab, Init) end; false -> ok end; delete_object -> dets_updated(Tab,Key), - ok = dets:delete_object(Tab, Val); + mnesia_lib:db_match_erase(Storage, Tab, Val); clear_table -> dets_cleared(Tab), - ok = dets:delete_all_objects(Tab) + ok = mnesia_lib:db_match_erase(Storage, Tab, '_') end. dets_updated(Tab,Key) -> @@ -431,23 +452,29 @@ insert(_Tid, _Storage, _Tab, _Key, [], _Op, _InPlace, _InitBy) -> ok; insert(Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy) -> + Semantics = storage_semantics(Storage), Item = {{Tab, Key}, Val, Op}, case InitBy of startup -> disc_insert(Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy); - _ when Storage == ram_copies -> + _ when Semantics == ram_copies -> mnesia_tm:do_update_op(Tid, Storage, Item), Snmp = mnesia_tm:prepare_snmp(Tab, Key, [Item]), mnesia_tm:do_snmp(Tid, Snmp); - _ when Storage == disc_copies -> + _ when Semantics == disc_copies -> disc_insert(Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy), mnesia_tm:do_update_op(Tid, Storage, Item), Snmp = mnesia_tm:prepare_snmp(Tab, Key, [Item]), mnesia_tm:do_snmp(Tid, Snmp); - _ when Storage == disc_only_copies -> + _ when Semantics == disc_only_copies -> + mnesia_tm:do_update_op(Tid, Storage, Item), + Snmp = mnesia_tm:prepare_snmp(Tab, Key, [Item]), + mnesia_tm:do_snmp(Tid, Snmp); + + _ when element(1, Storage) == ext -> mnesia_tm:do_update_op(Tid, Storage, Item), Snmp = mnesia_tm:prepare_snmp(Tab, Key, [Item]), mnesia_tm:do_snmp(Tid, Snmp); @@ -456,6 +483,9 @@ insert(Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy) -> ignore end. +disc_delete_table(Tab, {ext, Alias, Mod}) -> + Mod:close_table(Alias, Tab), + Mod:delete_table(Alias, Tab); disc_delete_table(Tab, Storage) -> case mnesia_monitor:use_dir() of true -> @@ -485,11 +515,14 @@ disc_delete_table(Tab, Storage) -> ok end. -disc_delete_indecies(_Tab, _Cs, Storage) when Storage /= disc_only_copies -> - ok; -disc_delete_indecies(Tab, Cs, disc_only_copies) -> - Indecies = Cs#cstruct.index, - mnesia_index:del_transient(Tab, Indecies, disc_only_copies). +disc_delete_indecies(Tab, Cs, Storage) -> + case storage_semantics(Storage) of + disc_only_copies -> + Indecies = Cs#cstruct.index, + mnesia_index:del_transient(Tab, Indecies, Storage); + _ -> + ok + end. insert_op(Tid, Storage, {{Tab, Key}, Val, Op}, InPlace, InitBy) -> %% Propagate to disc only @@ -515,6 +548,8 @@ insert_op(Tid, _, {op, change_table_copy_type, N, FromS, ToS, TabDef}, InPlace, Cs = mnesia_schema:list2cs(TabDef), Val = mnesia_schema:insert_cstruct(Tid, Cs, true), % Update ram only {schema, Tab, _} = Val, + FromSem = storage_semantics(FromS), + ToSem = storage_semantics(ToS), case lists:member(N, val({current, db_nodes})) of true when InitBy /= startup -> mnesia_controller:add_active_replica(Tab, N, Cs); @@ -527,62 +562,118 @@ insert_op(Tid, _, {op, change_table_copy_type, N, FromS, ToS, TabDef}, InPlace, Dat = mnesia_lib:tab2dat(Tab), Dcd = mnesia_lib:tab2dcd(Tab), Dcl = mnesia_lib:tab2dcl(Tab), + Logtmp = mnesia_lib:tab2logtmp(Tab), case {FromS, ToS} of - {ram_copies, disc_copies} when Tab == schema -> - ok = ensure_rename(Dmp, Dat); - {ram_copies, disc_copies} -> - file:delete(Dcl), - ok = ensure_rename(Dmp, Dcd); - {disc_copies, ram_copies} when Tab == schema -> - mnesia_lib:set(use_dir, false), - mnesia_monitor:unsafe_close_dets(Tab), - ok = file:delete(Dat); - {disc_copies, ram_copies} -> - _ = file:delete(Dcl), - _ = file:delete(Dcd), - ok; - {ram_copies, disc_only_copies} -> - ok = ensure_rename(Dmp, Dat), - true = open_files(Tab, disc_only_copies, InPlace, InitBy), - %% ram_delete_table must be done before init_indecies, - %% it uses info which is reset in init_indecies, - %% it doesn't matter, because init_indecies don't use - %% the ram replica of the table when creating the disc - %% index; Could be improved :) - mnesia_schema:ram_delete_table(Tab, FromS), - PosList = Cs#cstruct.index, - mnesia_index:init_indecies(Tab, disc_only_copies, PosList); - {disc_only_copies, ram_copies} -> - mnesia_monitor:unsafe_close_dets(Tab), - disc_delete_indecies(Tab, Cs, disc_only_copies), - case InitBy of - startup -> - ignore; + {{ext,_FromAlias,_FromMod},{ext,ToAlias,ToMod}} -> + disc_delete_table(Tab, FromS), + ok = ToMod:delete_table(ToAlias, Tab), + ok = mnesia_monitor:unsafe_create_external(Tab, ToAlias, ToMod, TabDef), + ok = ToMod:load_table(ToAlias, Tab, {dumper,change_table_copy_type}, TabDef), + ok = load_from_logfile(ToS, Tab, Logtmp), + file:delete(Logtmp), + restore_indexes(Tab, ToS, Cs); + + {_,{ext,ToAlias,ToMod}} -> + case FromSem of + ram_copies -> + mnesia_schema:ram_delete_table(Tab, FromS); _ -> - mnesia_controller:get_disc_copy(Tab), - ok + if FromSem == disc_copies -> + mnesia_schema:ram_delete_table( + Tab, FromS); + true -> ok + end, + disc_delete_table(Tab, FromS) end, - disc_delete_table(Tab, disc_only_copies); - {disc_copies, disc_only_copies} -> - ok = ensure_rename(Dmp, Dat), - true = open_files(Tab, disc_only_copies, InPlace, InitBy), - mnesia_schema:ram_delete_table(Tab, FromS), - PosList = Cs#cstruct.index, - mnesia_index:init_indecies(Tab, disc_only_copies, PosList), - _ = file:delete(Dcl), - _ = file:delete(Dcd), - ok; - {disc_only_copies, disc_copies} -> - mnesia_monitor:unsafe_close_dets(Tab), - disc_delete_indecies(Tab, Cs, disc_only_copies), - case InitBy of - startup -> - ignore; - _ -> - mnesia_log:ets2dcd(Tab), - mnesia_controller:get_disc_copy(Tab), - disc_delete_table(Tab, disc_only_copies) - end + + ok = ToMod:delete_table(ToAlias, Tab), + ok = mnesia_monitor:unsafe_create_external(Tab, ToAlias, ToMod, TabDef), + ok = ToMod:load_table(ToAlias, Tab, {dumper,change_table_copy_type}, TabDef), + ok = load_from_logfile(ToS, Tab, Logtmp), + file:delete(Logtmp), + restore_indexes(Tab, ToS, Cs); + + {{ext,_FromAlias,_FromMod} = FromS, ToS} -> + disc_delete_table(Tab, FromS), + case ToS of + ram_copies -> + change_disc_to_ram( + Tab, Cs, FromS, ToS, Logtmp, InitBy); + disc_copies -> + Args = [{keypos, 2}, public, named_table, + Cs#cstruct.type], + mnesia_monitor:mktab(Tab, Args), + ok = load_from_logfile(ToS, Tab, Logtmp), + file:delete(Logtmp); + disc_only_copies -> + %% ok = ensure_rename(Dmp, Dat), + true = open_files(Tab, ToS, InPlace, InitBy), + ok = load_from_logfile(ToS, Tab, Logtmp), + file:delete(Logtmp) + end, + restore_indexes(Tab, ToS, Cs); + + _NoneAreExt -> + + case {FromSem, ToSem} of + {ram_copies, disc_copies} when Tab == schema -> + ok = ensure_rename(Dmp, Dat); + {ram_copies, disc_copies} -> + file:delete(Dcl), + ok = ensure_rename(Dmp, Dcd); + {disc_copies, ram_copies} when Tab == schema -> + mnesia_lib:set(use_dir, false), + mnesia_monitor:unsafe_close_dets(Tab), + ok = file:delete(Dat); + {disc_copies, ram_copies} -> + _ = file:delete(Dcl), + _ = file:delete(Dcd), + ok; + {ram_copies, disc_only_copies} -> + ok = ensure_rename(Dmp, Dat), + true = open_files(Tab, ToS, InPlace, InitBy), + %% ram_delete_table must be done before + %% init_indecies, it uses info which is reset + %% in init_indecies, it doesn't matter, because + %% init_indecies don't use the ram replica of + %% the table when creating the disc index; + %% Could be improved :) + mnesia_schema:ram_delete_table(Tab, FromS), + restore_indexes(Tab, ToS, Cs); + {disc_only_copies, ram_copies} when FromS == disc_only_copies -> + mnesia_monitor:unsafe_close_dets(Tab), + disc_delete_indecies(Tab, Cs, FromS), + case InitBy of + startup -> + ignore; + _ -> + mnesia_controller:get_disc_copy(Tab), + ok + end, + disc_delete_table(Tab, FromS); + {disc_only_copies, ram_copies} when element(1, FromS) == ext -> + change_disc_to_ram( + Tab, Cs, FromS, ToS, Logtmp, InitBy); + {disc_copies, disc_only_copies} -> + ok = ensure_rename(Dmp, Dat), + true = open_files(Tab, ToS, InPlace, InitBy), + mnesia_schema:ram_delete_table(Tab, FromS), + restore_indexes(Tab, ToS, Cs), + _ = file:delete(Dcl), + _ = file:delete(Dcd), + ok; + {disc_only_copies, disc_copies} -> + mnesia_monitor:unsafe_close_dets(Tab), + disc_delete_indecies(Tab, Cs, disc_only_copies), + case InitBy of + startup -> + ignore; + _ -> + mnesia_log:ets2dcd(Tab), + mnesia_controller:get_disc_copy(Tab), + disc_delete_table(Tab, disc_only_copies) + end + end end; true -> ignore @@ -607,6 +698,9 @@ insert_op(Tid, _, {op, restore_recreate, TabDef}, InPlace, InitBy) -> Tab = Cs#cstruct.name, Type = Cs#cstruct.type, Storage = mnesia_lib:cs_to_storage_type(node(), Cs), + Semantics = if Storage==unknown -> unknown; + true -> storage_semantics(Storage) + end, %% Delete all possibly existing files and tables disc_delete_table(Tab, Storage), disc_delete_indecies(Tab, Cs, Storage), @@ -625,13 +719,13 @@ insert_op(Tid, _, {op, restore_recreate, TabDef}, InPlace, InitBy) -> %% And create new ones.. if - (InitBy == startup) or (Storage == unknown) -> + (InitBy == startup) or (Semantics == unknown) -> ignore; - Storage == ram_copies -> + Semantics == ram_copies -> EtsProps = proplists:get_value(ets, StorageProps, []), Args = [{keypos, 2}, public, named_table, Type | EtsProps], mnesia_monitor:mktab(Tab, Args); - Storage == disc_copies -> + Semantics == disc_copies -> EtsProps = proplists:get_value(ets, StorageProps, []), Args = [{keypos, 2}, public, named_table, Type | EtsProps], mnesia_monitor:mktab(Tab, Args), @@ -640,7 +734,7 @@ insert_op(Tid, _, {op, restore_recreate, TabDef}, InPlace, InitBy) -> {repair, false}, {mode, read_write}], {ok, Log} = mnesia_monitor:open_log(FArg), mnesia_monitor:unsafe_close_log(Log); - Storage == disc_only_copies -> + Storage == disc_only_copies -> % note: Storage, not Semantics File = mnesia_lib:tab2dat(Tab), file:delete(File), DetsProps = proplists:get_value(dets, StorageProps, []), @@ -649,7 +743,10 @@ insert_op(Tid, _, {op, restore_recreate, TabDef}, InPlace, InitBy) -> {keypos, 2}, {repair, mnesia_monitor:get_env(auto_repair)} | DetsProps ], - mnesia_monitor:open_dets(Tab, Args) + mnesia_monitor:open_dets(Tab, Args); + element(1,Storage) == ext -> + {ext, Alias, Mod} = Storage, + Mod:create_table(Alias, Tab, []) end, insert_op(Tid, ignore, {op, create_table, TabDef}, InPlace, InitBy); @@ -659,9 +756,10 @@ insert_op(Tid, _, {op, create_table, TabDef}, InPlace, InitBy) -> Tab = Cs#cstruct.name, Storage = mnesia_lib:cs_to_storage_type(node(), Cs), StorageProps = Cs#cstruct.storage_properties, + Semantics = storage_semantics(Storage), case InitBy of startup -> - case Storage of + case Semantics of unknown -> ignore; ram_copies -> @@ -679,20 +777,10 @@ insert_op(Tid, _, {op, create_table, TabDef}, InPlace, InitBy) -> read_write), mnesia_log:unsafe_close_log(temp) end; - _ -> + disc_only_copies -> DetsProps = proplists:get_value(dets, StorageProps, []), - Args = [{file, mnesia_lib:tab2dat(Tab)}, - {type, mnesia_lib:disk_type(Tab, Cs#cstruct.type)}, - {keypos, 2}, - {repair, mnesia_monitor:get_env(auto_repair)} - | DetsProps ], - case mnesia_monitor:open_dets(Tab, Args) of - {ok, _} -> - mnesia_monitor:unsafe_close_dets(Tab); - {error, Error} -> - exit({"Failed to create dets table", Error}) - end + try_create_disc_only_copy(Storage, Tab, Cs, DetsProps) end; _ -> Copies = mnesia_lib:copy_holders(Cs), @@ -715,7 +803,7 @@ insert_op(Tid, _, {op, create_table, TabDef}, InPlace, InitBy) -> false -> mnesia_lib:set({Tab, where_to_read}, node()) end, - case Storage of + case Semantics of ram_copies -> ignore; _ -> @@ -838,7 +926,7 @@ insert_op(Tid, _, {op, del_table_copy, Storage, Node, TabDef}, InPlace, InitBy) insert_cstruct(Tid, Cs, true, InPlace, InitBy); Tab /= schema -> mnesia_controller:del_active_replica(Tab, Node), - mnesia_lib:del({Tab, Storage}, Node), + mnesia_lib:del({Tab, storage_alias(Storage)}, Node), if Node == node() -> case Cs#cstruct.local_content of @@ -896,9 +984,10 @@ insert_op(Tid, _, {op, add_index, Pos, TabDef}, InPlace, InitBy) -> Cs = mnesia_schema:list2cs(TabDef), Tab = insert_cstruct(Tid, Cs, true, InPlace, InitBy), Storage = mnesia_lib:cs_to_storage_type(node(), Cs), + Semantics = storage_semantics(Storage), case InitBy of - startup when Storage == disc_only_copies -> - true = open_files(Tab, Storage, InPlace, InitBy), + startup when Semantics == disc_only_copies -> + true = open_files(Tab, Semantics, Storage, InPlace, InitBy), mnesia_index:init_indecies(Tab, Storage, [Pos]); startup -> ignore; @@ -919,6 +1008,8 @@ insert_op(Tid, _, {op, del_index, Pos, TabDef}, InPlace, InitBy) -> mnesia_index:del_index_table(Tab, Storage, Pos); startup -> ignore; + _ when element(1, Storage) == ext -> + mnesia_index:del_index_table(Tab, Storage, Pos); _ -> mnesia_index:del_index_table(Tab, Storage, Pos) end, @@ -958,19 +1049,68 @@ insert_op(Tid, _, {op, change_table_frag, _Change, TabDef}, InPlace, InitBy) -> Cs = mnesia_schema:list2cs(TabDef), insert_cstruct(Tid, Cs, true, InPlace, InitBy). -open_files(Tab, Storage, UpdateInPlace, InitBy) - when Storage /= unknown, Storage /= ram_copies -> + +storage_semantics({ext, Alias, Mod}) -> + Mod:semantics(Alias, storage); +storage_semantics(Storage) when is_atom(Storage) -> + Storage. + +storage_alias({ext, Alias, _}) -> + Alias; +storage_alias(Storage) when is_atom(Storage) -> + Storage. + +change_disc_to_ram(Tab, Cs, FromS, ToS, Logtmp, InitBy) -> + disc_delete_indecies(Tab, Cs, FromS), + case InitBy of + startup -> + ignore; + _ -> + %% ram table will already have been created + Tab = ets:info(Tab, name), %% assertion + load_from_logfile(ToS, Tab, Logtmp), + PosList = Cs#cstruct.index, + mnesia_index:init_indecies(Tab, ToS, PosList) + end, + disc_delete_table(Tab, FromS). + + +try_create_disc_only_copy({ext,Alias,Mod}, Tab, Cs, _) -> + Mod:create_table(Alias, Tab, mnesia_schema:cs2list(Cs)); +try_create_disc_only_copy(disc_only_copies, Tab, Cs, DetsProps) -> + Args = [{file, mnesia_lib:tab2dat(Tab)}, + {type, mnesia_lib:disk_type(Tab, Cs#cstruct.type)}, + {keypos, 2}, + {repair, mnesia_monitor:get_env(auto_repair)} + | DetsProps], + case mnesia_monitor:open_dets(Tab, Args) of + {ok, _} -> + mnesia_monitor:unsafe_close_dets(Tab); + {error, Error} -> + exit({"Failed to create dets table", Error}) + end. + +restore_indexes(Tab, ToS, Cs) -> + PosList = Cs#cstruct.index, + mnesia_index:init_indecies(Tab, ToS, PosList). + + +open_files(Tab, Storage, UpdateInPlace, InitBy) -> + open_files(Tab, storage_semantics(Storage), Storage, UpdateInPlace, InitBy). + +open_files(Tab, Semantics, Storage, UpdateInPlace, InitBy) + when Storage /= unknown, Semantics /= ram_copies -> case get({?MODULE, Tab}) of undefined -> case ?catch_val({Tab, setorbag}) of {'EXIT', _} -> false; Type -> - case Storage of - disc_copies when Tab /= schema -> + Cs = val({Tab, cstruct}), + if Semantics == disc_copies, Tab /= schema -> Bool = open_disc_copies(Tab, InitBy), Bool; - _ -> + Storage == disc_only_copies; Tab == schema -> Props = val({Tab, storage_properties}), DetsProps = proplists:get_value(dets, Props, []), Fname = prepare_open(Tab, UpdateInPlace), @@ -981,6 +1121,12 @@ open_files(Tab, Storage, UpdateInPlace, InitBy) | DetsProps], {ok, _} = mnesia_monitor:open_dets(Tab, Args), put({?MODULE, Tab}, {opened_dumper, dat}), + true; + element(1, Storage) == ext -> + {ext, Alias, Mod} = Storage, + Mod:load_table(Alias, Tab, InitBy, + mnesia_schema:cs2list(Cs)), + put({?MODULE, Tab}, {opened_dumper, ext}), true end end; @@ -989,7 +1135,7 @@ open_files(Tab, Storage, UpdateInPlace, InitBy) {opened_dumper, _} -> true end; -open_files(_Tab, _Storage, _UpdateInPlace, _InitBy) -> +open_files(_Tab, _Semantics, _Storage, _UpdateInPlace, _InitBy) -> false. open_disc_copies(Tab, InitBy) -> @@ -1076,12 +1222,13 @@ close_files(InPlace, Outcome, InitBy, [{{?MODULE, Tab}, already_dumped} | Tail]) close_files(InPlace, Outcome, InitBy, Tail); close_files(InPlace, Outcome, InitBy, [{{?MODULE, Tab}, {opened_dumper, Type}} | Tail]) -> erase({?MODULE, Tab}), - case val({Tab, storage_type}) of + Storage = val({Tab, storage_type}), + case storage_semantics(Storage) of disc_only_copies when InitBy /= startup -> ignore; - disc_copies when Tab /= schema -> + disc_copies when Storage /= unknown, Tab /= schema -> mnesia_log:close_log({?MODULE,Tab}); - Storage -> + _ -> do_close(InPlace, Outcome, Tab, Type, Storage) end, close_files(InPlace, Outcome, InitBy, Tail); @@ -1093,11 +1240,16 @@ close_files(_, _, _InitBy, []) -> %% If storage is unknown during close clean up files, this can happen if timing %% is right and dirty_write conflicts with schema operations. +do_close(_, _, Tab, ext, {ext,Alias,Mod}) -> + Mod:close_table(Alias, Tab); do_close(_, _, Tab, dcl, unknown) -> mnesia_log:close_log({?MODULE,Tab}), file:delete(mnesia_lib:tab2dcl(Tab)); do_close(_, _, Tab, dcl, _) -> %% To be safe, can it happen? mnesia_log:close_log({?MODULE,Tab}); +do_close(_, _, _Tab, ext, unknown) -> + %% Not sure what to do here, but let's not crash + ok; do_close(InPlace, Outcome, Tab, dat, Storage) -> mnesia_monitor:close_dets(Tab), @@ -1148,7 +1300,8 @@ temp_set_master_nodes() -> Tabs = val({schema, local_tables}), Masters = [{Tab, (val({Tab, disc_copies}) ++ val({Tab, ram_copies}) ++ - val({Tab, disc_only_copies})) -- [node()]} + val({Tab, disc_only_copies}) ++ + external_copies(Tab)) -- [node()]} || Tab <- Tabs], %% UseDir = false since we don't want to remember these %% masternodes and we are running (really soon anyway) since we want this @@ -1156,6 +1309,13 @@ temp_set_master_nodes() -> mnesia_recover:log_master_nodes(Masters, false, yes), ok. +external_copies(Tab) -> + case ?catch_val({Tab, external_copies}) of + {'EXIT',_} -> []; + Ext -> + lists:concat([Ns || {_, Ns} <- Ext]) + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Raw dump of table. Dumper must have unique access to the ets table. @@ -1207,6 +1367,59 @@ raw_named_dump_table(Tab, Ftype) -> raw_dump_table(DetsRef, EtsRef) -> dets:from_ets(DetsRef, EtsRef). +dump_to_logfile(Storage, Tab) -> + case mnesia_monitor:use_dir() of + true -> + Logtmp = mnesia_lib:tab2logtmp(Tab), + file:delete(Logtmp), + case disk_log:open([{name, make_ref()}, + {file, Logtmp}, + {repair, false}, + {linkto, self()}]) of + {ok, Fd} -> + mnesia_lib:db_fixtable(Storage, Tab, true), + try do_dump_to_logfile(Storage, Tab, Fd) + after + mnesia_lib:db_fixtable(Storage, Tab, false) + end; + {error, _} = Error -> + Error + end; + false -> + {error, {has_no_disc, node()}} + end. + +do_dump_to_logfile(Storage, Tab, Fd) -> + Pat = [{'_',[],['$_']}], + log_terms(mnesia_lib:db_select_init(Storage, Tab, Pat, 100), Storage, Tab, Pat, Fd). + +log_terms({Objs, Cont}, Storage, Tab, Pat, Fd) -> + ok = disk_log:alog_terms(Fd, Objs), + log_terms(mnesia_lib:db_select_cont(Storage, Cont, '_'), Storage, Tab, Pat, Fd); +log_terms('$end_of_table', _, _, _, Fd) -> + disk_log:close(Fd). + +load_from_logfile(Storage, Tab, F) -> + case disk_log:open([{name, make_ref()}, + {file, F}, + {repair, true}, + {linkto, self()}]) of + {ok, Fd} -> + chunk_from_log(disk_log:chunk(Fd, start), Fd, Storage, Tab); + {repaired, Fd, _, _} -> + chunk_from_log(disk_log:chunk(Fd, start), Fd, Storage, Tab); + {error, _} = E -> + E + end. + +chunk_from_log({Cont, Terms}, Fd, Storage, Tab) -> + _ = [mnesia_lib:db_put(Storage, Tab, T) || T <- Terms], + chunk_from_log(disk_log:chunk(Fd, Cont), Fd, Storage, Tab); +chunk_from_log(eof, _, _, _) -> + ok. + + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Load regulator %% diff --git a/lib/mnesia/src/mnesia_event.erl b/lib/mnesia/src/mnesia_event.erl index 58806793e5..6f7531245f 100644 --- a/lib/mnesia/src/mnesia_event.erl +++ b/lib/mnesia/src/mnesia_event.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2014. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -114,7 +114,8 @@ handle_table_event({Oper, Record, TransId}, State) -> handle_system_event({mnesia_checkpoint_activated, _Checkpoint}, State) -> {ok, State}; -handle_system_event({mnesia_checkpoint_deactivated, _Checkpoint}, State) -> +handle_system_event({mnesia_checkpoint_deactivated, Checkpoint}, State) -> + report_error("Checkpoint '~p' has been deactivated, last table copy deleted.\n",[Checkpoint]), {ok, State}; handle_system_event({mnesia_up, Node}, State) -> diff --git a/lib/mnesia/src/mnesia_ext_sup.erl b/lib/mnesia/src/mnesia_ext_sup.erl new file mode 100644 index 0000000000..3e6c72161c --- /dev/null +++ b/lib/mnesia/src/mnesia_ext_sup.erl @@ -0,0 +1,59 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% This module implements a supervisor for external (plug-in) processes + +-module(mnesia_ext_sup). +-behaviour(supervisor). + +-export([start/0, + init/1, + start_proc/4, start_proc/5, + stop_proc/1]). + +-define(SHUTDOWN, 120000). % 2 minutes + +start() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +start_proc(Name, M, F, A) -> + start_proc(Name, M, F, A, []). + +start_proc(Name, M, F, A, Opts) -> + [Restart, Shutdown, Type, Modules] = + [proplists:get_value(K, Opts, Default) + || {K, Default} <- [{restart, transient}, + {shutdown, ?SHUTDOWN}, + {type, worker}, + {modules, [M]}]], + case supervisor:start_child( + ?MODULE, {Name, {M,F,A}, Restart, Shutdown, Type, Modules}) of + {error, already_present} -> + supervisor:restart_child(?MODULE, Name); + Other -> + Other + end. + +stop_proc(Name) -> + supervisor:terminate_child(?MODULE, Name). + +init(_) -> + {ok, {{one_for_all, 10, 60}, []}}. diff --git a/lib/mnesia/src/mnesia_frag.erl b/lib/mnesia/src/mnesia_frag.erl index cbb0dfb3be..c6e812b36d 100644 --- a/lib/mnesia/src/mnesia_frag.erl +++ b/lib/mnesia/src/mnesia_frag.erl @@ -1,7 +1,7 @@ %%% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2011. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -183,6 +183,8 @@ table_info2(ActivityId, Opaque, Tab, Frag, Item) -> length(val({Tab, disc_copies})); n_disc_only_copies -> length(val({Tab, disc_only_copies})); + n_external_copies -> + length(val({Tab, external_copies})); frag_names -> frag_names(Tab); @@ -498,13 +500,13 @@ expand_cstruct(Cs, Mode) -> %% Verify keys ValidKeys = [foreign_key, n_fragments, node_pool, n_ram_copies, n_disc_copies, n_disc_only_copies, - hash_module, hash_state], + n_external_copies, hash_module, hash_state], Keys = mnesia_schema:check_keys(Tab, Props, ValidKeys), mnesia_schema:check_duplicates(Tab, Keys), %% Pick fragmentation props ForeignKey = mnesia_schema:pick(Tab, foreign_key, Props, undefined), - {ForeignKey2, N, Pool, DefaultNR, DefaultND, DefaultNDO} = + {ForeignKey2, N, Pool, DefaultNR, DefaultND, DefaultNDO, DefaultNExt} = pick_props(Tab, Cs, ForeignKey), %% Verify node_pool @@ -518,6 +520,7 @@ expand_cstruct(Cs, Mode) -> NR = mnesia_schema:pick(Tab, n_ram_copies, Props, 0), ND = mnesia_schema:pick(Tab, n_disc_copies, Props, 0), NDO = mnesia_schema:pick(Tab, n_disc_only_copies, Props, 0), + NExt = mnesia_schema:pick(Tab, n_external_copies, Props, 0), PosInt = fun(I) when is_integer(I), I >= 0 -> true; (_I) -> false @@ -528,6 +531,8 @@ expand_cstruct(Cs, Mode) -> {bad_type, Tab, {n_disc_copies, ND}}), mnesia_schema:verify(true, PosInt(NDO), {bad_type, Tab, {n_disc_only_copies, NDO}}), + mnesia_schema:verify(true, PosInt(NExt), + {bad_type, Tab, {n_external_copies, NDO}}), %% Verify n_fragments Cs2 = verify_n_fragments(N, Cs, Mode), @@ -542,13 +547,13 @@ expand_cstruct(Cs, Mode) -> hash_module = HashMod, hash_state = HashState2}, if - NR == 0, ND == 0, NDO == 0 -> - do_expand_cstruct(Cs2, FH, N, Pool, DefaultNR, DefaultND, DefaultNDO, Mode); + NR == 0, ND == 0, NDO == 0, NExt == 0 -> + do_expand_cstruct(Cs2, FH, N, Pool, DefaultNR, DefaultND, DefaultNDO, DefaultNExt, Mode); true -> - do_expand_cstruct(Cs2, FH, N, Pool, NR, ND, NDO, Mode) + do_expand_cstruct(Cs2, FH, N, Pool, NR, ND, NDO, NExt, Mode) end. -do_expand_cstruct(Cs, FH, N, Pool, NR, ND, NDO, Mode) -> +do_expand_cstruct(Cs, FH, N, Pool, NR, ND, NDO, NExt, Mode) -> Tab = Cs#cstruct.name, LC = Cs#cstruct.local_content, @@ -562,14 +567,15 @@ do_expand_cstruct(Cs, FH, N, Pool, NR, ND, NDO, Mode) -> %% Add empty fragments CommonProps = [{base_table, Tab}], Cs2 = Cs#cstruct{frag_properties = lists:sort(CommonProps)}, - expand_frag_cstructs(N, NR, ND, NDO, Cs2, Pool, Pool, FH, Mode). + expand_frag_cstructs(N, NR, ND, NDO, NExt, Cs2, Pool, Pool, FH, Mode). verify_n_fragments(N, Cs, Mode) when is_integer(N), N >= 1 -> case Mode of create -> Cs#cstruct{ram_copies = [], disc_copies = [], - disc_only_copies = []}; + disc_only_copies = [], + external_copies = []}; activate -> Reason = {combine_error, Cs#cstruct.name, {n_fragments, N}}, mnesia_schema:verify(1, N, Reason), @@ -607,7 +613,8 @@ pick_props(Tab, Cs, {ForeignTab, Attr}) -> DefaultNR = length(val({ForeignTab, ram_copies})), DefaultND = length(val({ForeignTab, disc_copies})), DefaultNDO = length(val({ForeignTab, disc_only_copies})), - {Key, N, Pool, DefaultNR, DefaultND, DefaultNDO}; + DefaultNExt = length(val({ForeignTab, external_copies})), + {Key, N, Pool, DefaultNR, DefaultND, DefaultNDO, DefaultNExt}; pick_props(Tab, Cs, undefined) -> Props = Cs#cstruct.frag_properties, DefaultN = 1, @@ -617,22 +624,23 @@ pick_props(Tab, Cs, undefined) -> DefaultNR = 1, DefaultND = 0, DefaultNDO = 0, - {undefined, N, Pool, DefaultNR, DefaultND, DefaultNDO}; + DefaultNExt = 0, + {undefined, N, Pool, DefaultNR, DefaultND, DefaultNDO, DefaultNExt}; pick_props(Tab, _Cs, BadKey) -> mnesia:abort({bad_type, Tab, {foreign_key, BadKey}}). -expand_frag_cstructs(N, NR, ND, NDO, CommonCs, Dist, Pool, FH, Mode) +expand_frag_cstructs(N, NR, ND, NDO, NExt, CommonCs, Dist, Pool, FH, Mode) when N > 1, Mode == create -> Frag = n_to_frag_name(CommonCs#cstruct.name, N), Cs = CommonCs#cstruct{name = Frag}, - {Cs2, RevModDist, RestDist} = set_frag_nodes(NR, ND, NDO, Cs, Dist, []), + {Cs2, RevModDist, RestDist} = set_frag_nodes(NR, ND, NDO, NExt, Cs, Dist, []), ModDist = lists:reverse(RevModDist), Dist2 = rearrange_dist(Cs, ModDist, RestDist, Pool), %% Adjusts backwards, but it doesn't matter. {FH2, _FromFrags, _AdditionalWriteFrags} = adjust_before_split(FH), - CsList = expand_frag_cstructs(N - 1, NR, ND, NDO, CommonCs, Dist2, Pool, FH2, Mode), + CsList = expand_frag_cstructs(N - 1, NR, ND, NDO, NExt, CommonCs, Dist2, Pool, FH2, Mode), [Cs2 | CsList]; -expand_frag_cstructs(1, NR, ND, NDO, CommonCs, Dist, Pool, FH, Mode) -> +expand_frag_cstructs(1, NR, ND, NDO, NExt, CommonCs, Dist, Pool, FH, Mode) -> BaseProps = CommonCs#cstruct.frag_properties ++ [{foreign_key, FH#frag_state.foreign_key}, {hash_module, FH#frag_state.hash_module}, @@ -645,25 +653,29 @@ expand_frag_cstructs(1, NR, ND, NDO, CommonCs, Dist, Pool, FH, Mode) -> activate -> [BaseCs]; create -> - {BaseCs2, _, _} = set_frag_nodes(NR, ND, NDO, BaseCs, Dist, []), + {BaseCs2, _, _} = set_frag_nodes(NR, ND, NDO, NExt, BaseCs, Dist, []), [BaseCs2] end. -set_frag_nodes(NR, ND, NDO, Cs, [Head | Tail], Acc) when NR > 0 -> +set_frag_nodes(NR, ND, NDO, NExt, Cs, [Head | Tail], Acc) when NR > 0 -> Pos = #cstruct.ram_copies, {Cs2, Head2} = set_frag_node(Cs, Pos, Head), - set_frag_nodes(NR - 1, ND, NDO, Cs2, Tail, [Head2 | Acc]); -set_frag_nodes(NR, ND, NDO, Cs, [Head | Tail], Acc) when ND > 0 -> + set_frag_nodes(NR - 1, ND, NDO, NExt, Cs2, Tail, [Head2 | Acc]); +set_frag_nodes(NR, ND, NDO, NExt, Cs, [Head | Tail], Acc) when ND > 0 -> Pos = #cstruct.disc_copies, {Cs2, Head2} = set_frag_node(Cs, Pos, Head), - set_frag_nodes(NR, ND - 1, NDO, Cs2, Tail, [Head2 | Acc]); -set_frag_nodes(NR, ND, NDO, Cs, [Head | Tail], Acc) when NDO > 0 -> + set_frag_nodes(NR, ND - 1, NDO, NExt, Cs2, Tail, [Head2 | Acc]); +set_frag_nodes(NR, ND, NDO, NExt, Cs, [Head | Tail], Acc) when NDO > 0 -> Pos = #cstruct.disc_only_copies, {Cs2, Head2} = set_frag_node(Cs, Pos, Head), - set_frag_nodes(NR, ND, NDO - 1, Cs2, Tail, [Head2 | Acc]); -set_frag_nodes(0, 0, 0, Cs, RestDist, ModDist) -> + set_frag_nodes(NR, ND, NDO - 1, NExt, Cs2, Tail, [Head2 | Acc]); +set_frag_nodes(NR, ND, NDO, NExt, Cs, [Head | Tail], Acc) when NExt > 0 -> + Pos = #cstruct.external_copies, + {Cs2, Head2} = set_frag_node(Cs, Pos, Head), + set_frag_nodes(NR, ND, NDO, NExt - 1, Cs2, Tail, [Head2 | Acc]); +set_frag_nodes(0, 0, 0, 0, Cs, RestDist, ModDist) -> {Cs, ModDist, RestDist}; -set_frag_nodes(_, _, _, Cs, [], _) -> +set_frag_nodes(_, _, _, _, Cs, [], _) -> mnesia:abort({combine_error, Cs#cstruct.name, "Too few nodes in node_pool"}). set_frag_node(Cs, Pos, Head) -> @@ -840,13 +852,15 @@ make_add_frag(Tab, SortedNs) -> NR = length(Cs#cstruct.ram_copies), ND = length(Cs#cstruct.disc_copies), NDO = length(Cs#cstruct.disc_only_copies), + NExt = length(Cs#cstruct.external_copies), NewCs = Cs#cstruct{name = NewFrag, frag_properties = [{base_table, Tab}], ram_copies = [], disc_copies = [], - disc_only_copies = []}, + disc_only_copies = [], + external_copies = []}, - {NewCs2, _, _} = set_frag_nodes(NR, ND, NDO, NewCs, SortedNs, []), + {NewCs2, _, _} = set_frag_nodes(NR, ND, NDO, NExt, NewCs, SortedNs, []), [NewOp] = mnesia_schema:make_create_table(NewCs2), SplitOps = split(Tab, FH2, FromIndecies, FragNames, []), @@ -1319,7 +1333,8 @@ count_frag([Frag | Frags], Dist) -> Dist2 = incr_nodes(val({Frag, ram_copies}), Dist), Dist3 = incr_nodes(val({Frag, disc_copies}), Dist2), Dist4 = incr_nodes(val({Frag, disc_only_copies}), Dist3), - count_frag(Frags, Dist4); + Dist5 = incr_nodes(val({Frag, external_copies}), Dist4), + count_frag(Frags, Dist5); count_frag([], Dist) -> Dist. diff --git a/lib/mnesia/src/mnesia_frag_hash.erl b/lib/mnesia/src/mnesia_frag_hash.erl index 2e4ebb98b0..ae4105382e 100644 --- a/lib/mnesia/src/mnesia_frag_hash.erl +++ b/lib/mnesia/src/mnesia_frag_hash.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2011. All Rights Reserved. +%% Copyright Ericsson AB 2002-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_frag_old_hash.erl b/lib/mnesia/src/mnesia_frag_old_hash.erl index 707be6d121..b246c76236 100644 --- a/lib/mnesia/src/mnesia_frag_old_hash.erl +++ b/lib/mnesia/src/mnesia_frag_old_hash.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% Copyright Ericsson AB 2002-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_index.erl b/lib/mnesia/src/mnesia_index.erl index 0c882c0df6..c79f790973 100644 --- a/lib/mnesia/src/mnesia_index.erl +++ b/lib/mnesia/src/mnesia_index.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -23,8 +23,8 @@ -module(mnesia_index). -export([read/5, - add_index/5, - delete_index/3, + add_index/6, + delete_index/4, del_object_index/5, clear_index/4, dirty_match_object/3, @@ -39,19 +39,21 @@ get_index_table/3, tab2filename/2, - tab2tmp_filename/2, init_index/2, init_indecies/3, del_transient/2, del_transient/3, - del_index_table/3]). + del_index_table/3, + + index_info/2, + ext_index_instances/1]). -import(mnesia_lib, [val/1, verbose/2]). -include("mnesia.hrl"). -record(index, {setorbag, pos_list}). -%% read an object list throuh its index table +%% read an object list through its index table %% we assume that table Tab has index on attribute number Pos read(Tid, Store, Tab, IxKey, Pos) -> @@ -64,69 +66,103 @@ read(Tid, Store, Tab, IxKey, Pos) -> ResList end. -add_index(Index, Tab, Key, Obj, Old) -> - add_index2(Index#index.pos_list, Index#index.setorbag, Tab, Key, Obj, Old). - -add_index2([{Pos, Ixt} |Tail], bag, Tab, K, Obj, OldRecs) -> - db_put(Ixt, {element(Pos, Obj), K}), - add_index2(Tail, bag, Tab, K, Obj, OldRecs); -add_index2([{Pos, Ixt} |Tail], Type, Tab, K, Obj, OldRecs0) -> +ext_index_instances(Tab) -> + #index{pos_list = PosL} = val({Tab, index_info}), + lists:foldr( + fun({_, {{ext,Alias,Mod}, Tag}}, Acc) -> + [{Alias, Mod, Tag}|Acc]; + (_, Acc) -> + Acc + end, [], PosL). + + +add_index(#index{pos_list = PosL, setorbag = SorB}, + Storage, Tab, Key, Obj, Old) -> + add_index2(PosL, SorB, Storage, Tab, Key, Obj, Old). + +add_index2([{{Pos,Type}, Ixt} |Tail], bag, Storage, Tab, K, Obj, OldRecs) -> + ValsF = index_vals_f(Storage, Tab, Pos), + Vals = ValsF(Obj), + put_index_vals(Type, Ixt, Vals, K), + add_index2(Tail, bag, Storage, Tab, K, Obj, OldRecs); +add_index2([{{Pos, Type}, Ixt} |Tail], SorB, Storage, Tab, K, Obj, OldRecs0) -> %% Remove old tuples in index if Tab is updated + ValsF = index_vals_f(Storage, Tab, Pos), NewVals = ValsF(Obj), OldRecs1 = case OldRecs0 of - undefined -> mnesia_lib:db_get(Tab, K); + undefined -> mnesia_lib:db_get(Storage, Tab, K); _ -> OldRecs0 end, - IdxVal = element(Pos, Obj), - case [Old || Old <- OldRecs1, element(Pos, Old) =/= IdxVal] of - [] when OldRecs1 =:= [] -> %% Write - db_put(Ixt, {element(Pos, Obj), K}), - add_index2(Tail, Type, Tab, K, Obj, OldRecs0); + IdxVal = ValsF(Obj), + case [Old || Old <- OldRecs1, ValsF(Old) =/= IdxVal] of + [] when OldRecs1 =:= [] -> % Write + put_index_vals(Type, Ixt, NewVals, K), + add_index2(Tail, SorB, Storage, Tab, K, Obj, OldRecs1); [] -> %% when OldRecs1 =/= [] Update without modifying index field - add_index2(Tail, Type, Tab, K, Obj, OldRecs0); + add_index2(Tail, SorB, Storage, Tab, K, Obj, OldRecs1); OldRecs -> %% Update - db_put(Ixt, {element(Pos, Obj), K}), - del_ixes(Ixt, OldRecs, Pos, K), - add_index2(Tail, Type, Tab, K, Obj, OldRecs0) + put_index_vals(Type, Ixt, NewVals, K), + [del_ixes(Type, Ixt, ValsF, OldObj, K) || OldObj <- OldRecs], + add_index2(Tail, SorB, Storage, Tab, K, Obj, OldRecs1) end; -add_index2([], _, _Tab, _K, _Obj, _) -> ok. +add_index2([], _, _, _Tab, _K, _Obj, _) -> ok. + +delete_index(Index, Storage, Tab, K) -> + delete_index2(Index#index.pos_list, Storage, Tab, K). -delete_index(Index, Tab, K) -> - delete_index2(Index#index.pos_list, Tab, K). +delete_index2([{{Pos, Type}, Ixt} | Tail], Storage, Tab, K) -> + DelObjs = mnesia_lib:db_get(Storage, Tab, K), + ValsF = index_vals_f(Storage, Tab, Pos), + [del_ixes(Type, Ixt, ValsF, Obj, K) || Obj <- DelObjs], + delete_index2(Tail, Storage, Tab, K); +delete_index2([], _Storage, _Tab, _K) -> ok. -delete_index2([{Pos, Ixt} | Tail], Tab, K) -> - DelObjs = mnesia_lib:db_get(Tab, K), - del_ixes(Ixt, DelObjs, Pos, K), - delete_index2(Tail, Tab, K); -delete_index2([], _Tab, _K) -> ok. +put_index_vals(ordered, Ixt, Vals, K) -> + [db_put(Ixt, {{V, K}}) || V <- Vals]; +put_index_vals(bag, Ixt, Vals, K) -> + [db_put(Ixt, {V, K}) || V <- Vals]. -del_ixes(_Ixt, [], _Pos, _L) -> ok; -del_ixes(Ixt, [Obj | Tail], Pos, Key) -> - db_match_erase(Ixt, {element(Pos, Obj), Key}), - del_ixes(Ixt, Tail, Pos, Key). +del_ixes(bag, Ixt, ValsF, Obj, Key) -> + Vals = ValsF(Obj), + [db_match_erase(Ixt, {V, Key}) || V <- Vals]; +del_ixes(ordered, Ixt, ValsF, Obj, Key) -> + Vals = ValsF(Obj), + [db_erase(Ixt, {V,Key}) || V <- Vals]. -del_object_index(Index, Tab, K, Obj, Old) -> - del_object_index2(Index#index.pos_list, Index#index.setorbag, Tab, K, Obj, Old). +del_object_index(#index{pos_list = PosL, setorbag = SorB}, Storage, Tab, K, Obj) -> + del_object_index2(PosL, SorB, Storage, Tab, K, Obj). -del_object_index2([], _, _Tab, _K, _Obj, _Old) -> ok; -del_object_index2([{Pos, Ixt} | Tail], SoB, Tab, K, Obj, Old) -> +del_object_index2([], _, _Storage, _Tab, _K, _Obj) -> ok; +del_object_index2([{{Pos, Type}, Ixt} | Tail], SoB, Storage, Tab, K, Obj) -> + ValsF = index_vals_f(Storage, Tab, Pos), case SoB of bag -> - del_object_bag(Tab, K, Obj, Pos, Ixt, Old); + del_object_bag(Type, ValsF, Tab, K, Obj, Ixt); _ -> %% If set remove the tuple in index table - del_ixes(Ixt, [Obj], Pos, K) + del_ixes(Type, Ixt, ValsF, Obj, K) end, - del_object_index2(Tail, SoB, Tab, K, Obj, Old). - -del_object_bag(Tab, Key, Obj, Pos, Ixt, undefined) -> - IxKey = element(Pos, Obj), - Old = [X || X <- mnesia_lib:db_get(Tab, Key), element(Pos, X) =:= IxKey], - del_object_bag(Tab, Key, Obj, Pos, Ixt, Old); -%% If Tab type is bag we need remove index identifier if the object being -%% deleted was the last one -del_object_bag(_Tab, Key, Obj, Pos, Ixt, Old) when Old =:= [Obj] -> - del_ixes(Ixt, [Obj], Pos, Key); -del_object_bag(_Tab, _Key, _Obj, _Pos, _Ixt, _Old) -> ok. + del_object_index2(Tail, SoB, Storage, Tab, K, Obj). + +del_object_bag(Type, ValsF, Tab, Key, Obj, Ixt) -> + IxKeys = ValsF(Obj), + Found = [{X, ValsF(X)} || X <- mnesia_lib:db_get(Tab, Key)], + del_object_bag_(IxKeys, Found, Type, Tab, Key, Obj, Ixt). + +del_object_bag_([IxK|IxKs], Found, Type, Tab, Key, Obj, Ixt) -> + case [X || {X, Ixes} <- Found, lists:member(IxK, Ixes)] of + [Old] when Old =:= Obj -> + case Type of + bag -> + db_match_erase(Ixt, {IxK, Key}); + ordered -> + db_erase(Ixt, {{IxK, Key}}) + end; + _ -> + ok + end, + del_object_bag_(IxKs, Found, Type, Tab, Key, Obj, Ixt); +del_object_bag_([], _, _, _, _, _, _) -> + ok. clear_index(Index, Tab, K, Obj) -> clear_index2(Index#index.pos_list, Tab, K, Obj). @@ -138,8 +174,9 @@ clear_index2([{_Pos, Ixt} | Tail], Tab, K, Obj) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -dirty_match_object(Tab, Pat, Pos) -> +dirty_match_object(Tab, Pat, Pos) when is_integer(Pos) -> %% Assume that we are on the node where the replica is + %% Cannot use index plugins here, as they don't map to match patterns case element(2, Pat) of '_' -> IxKey = element(Pos, Pat), @@ -161,7 +198,7 @@ realkeys(Tab, Pos, IxKey) -> Index = get_index_table(Tab, Pos), db_get(Index, IxKey). % a list on the form [{IxKey, RealKey1} , .... -dirty_select(Tab, Spec, Pos) -> +dirty_select(Tab, Spec, Pos) when is_integer(Pos) -> %% Assume that we are on the node where the replica is %% Returns the records without applying the match spec %% The actual filtering is handled by the caller @@ -171,59 +208,80 @@ dirty_select(Tab, Spec, Pos) -> lists:append([mnesia_lib:db_get(StorageType, Tab, Key) || {_,Key} <- RealKeys]). dirty_read(Tab, IxKey, Pos) -> - ResList = mnesia:dirty_rpc(Tab, ?MODULE, dirty_read2, - [Tab, IxKey, Pos]), - case val({Tab, setorbag}) of - bag -> - %% Remove all tuples which don't include Ixkey - mnesia_lib:key_search_all(IxKey, Pos, ResList); - _ -> - ResList - end. + mnesia:dirty_rpc(Tab, ?MODULE, dirty_read2, + [Tab, IxKey, Pos]). dirty_read2(Tab, IxKey, Pos) -> - Ix = get_index_table(Tab, Pos), - Keys = db_match(Ix, {IxKey, '$1'}), - r_keys(Keys, Tab, []). - -r_keys([[H]|T],Tab,Ack) -> - V = mnesia_lib:db_get(Tab, H), - r_keys(T, Tab, V ++ Ack); -r_keys([], _, Ack) -> - Ack. + #index{pos_list = PosL} = val({Tab, index_info}), + Storage = val({Tab, storage_type}), + {Type, Ixt} = pick_index(PosL, Tab, Pos), + Pat = case Type of + ordered -> [{{{IxKey, '$1'}}, [], ['$1']}]; + bag -> [{{IxKey, '$1'}, [], ['$1']}] + end, + Keys = db_select(Ixt, Pat), + ValsF = index_vals_f(Storage, Tab, Pos), + lists:reverse( + lists:foldl( + fun(K, Acc) -> + lists:foldl( + fun(Obj, Acc1) -> + case lists:member(IxKey, ValsF(Obj)) of + true -> [Obj|Acc1]; + false -> Acc1 + end + end, Acc, mnesia_lib:db_get(Storage, Tab, K)) + end, [], Keys)). + +pick_index([{{{Pfx,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> + {IxType, Ixt}; +pick_index([{{Pos,IxType}, Ixt}|_], _Tab, Pos) -> + {IxType, Ixt}; +pick_index([_|T], Tab, Pos) -> + pick_index(T, Tab, Pos); +pick_index([], Tab, Pos) -> + mnesia:abort({no_exist, Tab, {index, Pos}}). + %%%%%%% Creation, Init and deletion routines for index tables %% We can have several indexes on the same table %% this can be a fairly costly operation if table is *very* large -tab2filename(Tab, Pos) -> +tab2filename(Tab, {A}) when is_atom(A) -> + mnesia_lib:dir(Tab) ++ "_-" ++ atom_to_list(A) ++ "-.DAT"; +tab2filename(Tab, T) when is_tuple(T) -> + tab2filename(Tab, element(1, T)); +tab2filename(Tab, Pos) when is_integer(Pos) -> mnesia_lib:dir(Tab) ++ "_" ++ integer_to_list(Pos) ++ ".DAT". -tab2tmp_filename(Tab, Pos) -> - mnesia_lib:dir(Tab) ++ "_" ++ integer_to_list(Pos) ++ ".TMP". - init_index(Tab, Storage) -> - PosList = val({Tab, index}), + Cs = val({Tab, cstruct}), + PosList = Cs#cstruct.index, init_indecies(Tab, Storage, PosList). init_indecies(Tab, Storage, PosList) -> case Storage of unknown -> ignore; + {ext, Alias, Mod} -> + init_ext_index(Tab, Storage, Alias, Mod, PosList); disc_only_copies -> - init_disc_index(Tab, PosList); + init_disc_index(Tab, Storage, PosList); ram_copies -> - make_ram_index(Tab, PosList); + make_ram_index(Tab, Storage, PosList); disc_copies -> - make_ram_index(Tab, PosList) + make_ram_index(Tab, Storage, PosList) end. %% works for both ram and disc indexes del_index_table(_, unknown, _) -> ignore; -del_index_table(Tab, Storage, Pos) -> +del_index_table(Tab, Storage, {_} = Pos) -> + delete_transient_index(Tab, Pos, Storage), + mnesia_lib:del({Tab, index}, Pos); +del_index_table(Tab, Storage, Pos) when is_integer(Pos) -> delete_transient_index(Tab, Pos, Storage), mnesia_lib:del({Tab, index}, Pos). @@ -236,13 +294,24 @@ del_transient(Tab, [Pos | Tail], Storage) -> delete_transient_index(Tab, Pos, Storage), del_transient(Tab, Tail, Storage). +delete_transient_index(Tab, Pos, {ext, Alias, Mod}) -> + PosInfo = case Pos of + _ when is_integer(Pos) -> + Cs = val({Tab, cstruct}), + lists:keyfind(Pos, 1, Cs#cstruct.index); + {P, T} -> {P, T} + end, + Tag = {Tab, index, PosInfo}, + Mod:close_table(Alias, Tag), + Mod:delete_table(Alias, Tag), + del_index_info(Tab, Pos), + mnesia_lib:unset({Tab, {index, Pos}}); delete_transient_index(Tab, Pos, disc_only_copies) -> Tag = {Tab, index, Pos}, mnesia_monitor:unsafe_close_dets(Tag), _ = file:delete(tab2filename(Tab, Pos)), del_index_info(Tab, Pos), %% Uses val(..) mnesia_lib:unset({Tab, {index, Pos}}); - delete_transient_index(Tab, Pos, _Storage) -> Ixt = val({Tab, {index, Pos}}), ?ets_delete_table(Ixt), @@ -252,11 +321,12 @@ delete_transient_index(Tab, Pos, _Storage) -> %%%%% misc functions for the index create/init/delete functions above %% assuming that the file exists. -init_disc_index(_Tab, []) -> +init_disc_index(_Tab, _Storage, []) -> done; -init_disc_index(Tab, [Pos | Tail]) when is_integer(Pos) -> +init_disc_index(Tab, disc_only_copies, [{Pos,_Pref} | Tail]) -> + PosInfo = {Pos, bag}, Fn = tab2filename(Tab, Pos), - IxTag = {Tab, index, Pos}, + IxTag = {Tab, index, PosInfo}, _ = file:delete(Fn), Args = [{file, Fn}, {keypos, 1}, {type, bag}], mnesia_monitor:open_dets(IxTag, Args), @@ -269,16 +339,59 @@ init_disc_index(Tab, [Pos | Tail]) when is_integer(Pos) -> mnesia_lib:db_fixtable(Storage, Tab, true), ok = dets:init_table(IxTag, create_fun(Init, Tab, Pos)), mnesia_lib:db_fixtable(Storage, Tab, false), - mnesia_lib:set({Tab, {index, Pos}}, IxTag), - add_index_info(Tab, val({Tab, setorbag}), {Pos, {dets, IxTag}}), - init_disc_index(Tab, Tail). + mnesia_lib:set({Tab, {index, PosInfo}}, IxTag), + add_index_info(Tab, val({Tab, setorbag}), {PosInfo, {dets, IxTag}}), + init_disc_index(Tab, Storage, Tail). + +init_ext_index(_, _, _, _, []) -> + done; +init_ext_index(Tab, Storage, Alias, Mod, [{Pos,Type} | Tail]) -> + PosInfo = {Pos, Type}, + IxTag = {Tab, index, PosInfo}, + CS = val({Tab, cstruct}), + CsList = mnesia_schema:cs2list(CS), + _Res = mnesia_monitor:unsafe_create_external(IxTag, Alias, Mod, CsList), + Mod:load_table(Alias, IxTag, init_index, CsList), + case Mod:is_index_consistent(Alias, IxTag) of + false -> + Mod:index_is_consistent(Alias, IxTag, false), + Mod:match_delete(Alias, IxTag, '_'), + IxValsF = index_vals_f(Storage, Tab, Pos), + IxObjF = case Type of + bag -> fun(IxVal, Key) -> {IxVal, Key} end; + ordered -> fun(IxVal, Key) -> {{IxVal, Key}} end + end, + mnesia_lib:db_fixtable(Storage, Tab, true), + mnesia_lib:db_foldl( + Storage, + fun(Rec, Acc) -> + Key = element(2, Rec), + lists:foreach( + fun(V) -> + IxObj = IxObjF(V, Key), + Mod:insert(Alias, IxTag, IxObj) + end, IxValsF(Rec)), + Acc + end, ok, Tab, + [{'_', [], ['$_']}], 100), + Mod:index_is_consistent(Alias, IxTag, true); + true -> + ignore + end, + + mnesia_lib:set({Tab, {index, PosInfo}}, IxTag), + + add_index_info(Tab, val({Tab, setorbag}), {PosInfo, {Storage, IxTag}}), + init_ext_index(Tab, Storage, Alias, Mod, Tail). create_fun(Cont, Tab, Pos) -> + IxF = index_vals_f(disc_only_copies, Tab, Pos), fun(read) -> Data = case Cont of {start, KeysPerChunk} -> - mnesia_lib:db_init_chunk(disc_only_copies, Tab, KeysPerChunk); + mnesia_lib:db_init_chunk( + disc_only_copies, Tab, KeysPerChunk); '$end_of_table' -> '$end_of_table'; _Else -> @@ -288,56 +401,82 @@ create_fun(Cont, Tab, Pos) -> '$end_of_table' -> end_of_input; {Recs, Next} -> - IdxElems = [{element(Pos, Obj), element(2, Obj)} || Obj <- Recs], + IdxElems = lists:flatmap( + fun(Obj) -> + PrimK = element(2, Obj), + [{V, PrimK} || V <- IxF(Obj)] + end, Recs), {IdxElems, create_fun(Next, Tab, Pos)} end; (close) -> ok end. -make_ram_index(_, []) -> +make_ram_index(_, _, []) -> done; -make_ram_index(Tab, [Pos | Tail]) -> - add_ram_index(Tab, Pos), - make_ram_index(Tab, Tail). +make_ram_index(Tab, Storage, [Pos | Tail]) -> + add_ram_index(Tab, Storage, Pos), + make_ram_index(Tab, Storage, Tail). -add_ram_index(Tab, Pos) when is_integer(Pos) -> - verbose("Creating index for ~w ~n", [Tab]), +add_ram_index(Tab, Storage, {Pos, _Pref}) -> + Type = ordered, + verbose("Creating index for ~w ~p ~p~n", [Tab, Pos, Type]), SetOrBag = val({Tab, setorbag}), - IndexType = case SetOrBag of - set -> duplicate_bag; - ordered_set -> duplicate_bag; - bag -> bag - end, - Index = mnesia_monitor:mktab(mnesia_index, [IndexType, public]), + IxValsF = index_vals_f(Storage, Tab, Pos), + IxFun = fun(Val, Key) -> {{Val, Key}} end, + Index = mnesia_monitor:mktab(mnesia_index, [ordered_set, public]), Insert = fun(Rec, _Acc) -> - true = ?ets_insert(Index, {element(Pos, Rec), element(2, Rec)}) + PrimK = element(2, Rec), + true = ?ets_insert( + Index, [IxFun(V, PrimK) + || V <- IxValsF(Rec)]) end, mnesia_lib:db_fixtable(ram_copies, Tab, true), - true = ets:foldl(Insert, true, Tab), + true = mnesia_lib:db_foldl(Storage, Insert, true, Tab), mnesia_lib:db_fixtable(ram_copies, Tab, false), mnesia_lib:set({Tab, {index, Pos}}, Index), - add_index_info(Tab, SetOrBag, {Pos, {ram, Index}}); -add_ram_index(_Tab, snmp) -> + add_index_info(Tab, SetOrBag, {{Pos, Type}, {ram, Index}}); +add_ram_index(_Tab, _, snmp) -> ok. -add_index_info(Tab, Type, IxElem) -> +index_info(SetOrBag, PosList) -> + IxPlugins = mnesia_schema:index_plugins(), + ExpPosList = lists:map( + fun({{P,Type},Ixt} = PI) -> + case P of + {_} = IxN -> + {_, M, F} = + lists:keyfind(IxN, 1, IxPlugins), + {{{IxN,M,F}, Type}, Ixt}; + _ -> + PI + end + end, PosList), + #index{setorbag = SetOrBag, pos_list = ExpPosList}. + +add_index_info(Tab, SetOrBag, IxElem) -> Commit = val({Tab, commit_work}), case lists:keysearch(index, 1, Commit) of false -> - Index = #index{setorbag = Type, - pos_list = [IxElem]}, - %% Check later if mnesia_tm is sensative about the order + IndexInfo = index_info(SetOrBag, [IxElem]), + %% Check later if mnesia_tm is sensitive about the order + mnesia_lib:set({Tab, index_info}, IndexInfo), + mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), mnesia_lib:set({Tab, commit_work}, - mnesia_lib:sort_commit([Index | Commit])); + mnesia_lib:sort_commit([IndexInfo | Commit])); {value, Old} -> %% We could check for consistency here Index = Old#index{pos_list = [IxElem | Old#index.pos_list]}, + mnesia_lib:set({Tab, index_info}, Index), + mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end. +index_positions(#index{pos_list = PL}) -> + [P || {{P,_},_} <- PL]. + del_index_info(Tab, Pos) -> Commit = val({Tab, commit_work}), case lists:keysearch(index, 1, Commit) of @@ -345,13 +484,21 @@ del_index_info(Tab, Pos) -> %% Something is wrong ignore skip; {value, Old} -> - case lists:keydelete(Pos, 1, Old#index.pos_list) of + case lists:filter(fun({P,_}) -> + element(1,P)=/=Pos + end, + Old#index.pos_list) of [] -> + IndexInfo = index_info(Old#index.setorbag,[]), + mnesia_lib:set({Tab, index_info}, IndexInfo), + mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), NewC = lists:keydelete(index, 1, Commit), mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)); New -> Index = Old#index{pos_list = New}, + mnesia_lib:set({Tab, index_info}, Index), + mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) @@ -360,33 +507,69 @@ del_index_info(Tab, Pos) -> db_put({ram, Ixt}, V) -> true = ?ets_insert(Ixt, V); +db_put({{ext, _, _} = Ext, Ixt}, V) -> + mnesia_lib:db_put(Ext, Ixt, V); db_put({dets, Ixt}, V) -> ok = dets:insert(Ixt, V). -db_get({ram, Ixt}, K) -> - ?ets_lookup(Ixt, K); +db_get({ram, _}=Ixt, IxKey) -> + Pat = [{{{IxKey, '$1'}}, [], [{element, 1, '$_'}]}], + db_select(Ixt, Pat); +db_get({{ext,_,_} = _Storage, {_,_,{_,Type}}} = Ixt, IxKey) -> + Pat = case Type of + ordered -> [{{{IxKey, '$1'}}, [], [{element, 1, '$_'}]}]; + bag -> [{{IxKey, '_'}, [], ['$_']}] + end, + db_select(Ixt, Pat); db_get({dets, Ixt}, K) -> dets:lookup(Ixt, K). +db_erase({ram, Ixt}, K) -> + ?ets_delete(Ixt, K); +db_erase({{ext,_,_} = Ext, Ixt}, K) -> + mnesia_lib:db_erase(Ext, Ixt, K); +db_erase({dets, Ixt}, K) -> + dets:delete(Ixt, K). + db_match_erase({ram, Ixt}, Pat) -> true = ?ets_match_delete(Ixt, Pat); +db_match_erase({{ext,_,_} = Ext, Ixt}, Pat) -> + mnesia_lib:db_match_erase(Ext, Ixt, Pat); db_match_erase({dets, Ixt}, Pat) -> ok = dets:match_delete(Ixt, Pat). -db_match({ram, Ixt}, Pat) -> - ?ets_match(Ixt, Pat); -db_match({dets, Ixt}, Pat) -> - dets:match(Ixt, Pat). +db_select({ram, Ixt}, Pat) -> + ets:select(Ixt, Pat); +db_select({{ext,_,_} = Ext, Ixt}, Pat) -> + mnesia_lib:db_select(Ext, Ixt, Pat); +db_select({dets, Ixt}, Pat) -> + dets:select(Ixt, Pat). + get_index_table(Tab, Pos) -> get_index_table(Tab, val({Tab, storage_type}), Pos). -get_index_table(Tab, ram_copies, Pos) -> - {ram, val({Tab, {index, Pos}})}; -get_index_table(Tab, disc_copies, Pos) -> - {ram, val({Tab, {index, Pos}})}; -get_index_table(Tab, disc_only_copies, Pos) -> - {dets, val({Tab, {index, Pos}})}; -get_index_table(_Tab, unknown, _Pos) -> - unknown. - +get_index_table(Tab, _Storage, Pos) -> + #index{pos_list = PosL} = val({Tab, index_info}), + {_IxType, Ixt} = pick_index(PosL, Tab, Pos), + Ixt. + +index_vals_f(Storage, Tab, {_} = Pos) -> + index_vals_f(Storage, Tab, + lists:keyfind(Pos, 1, mnesia_schema:index_plugins())); +index_vals_f(_Storage, Tab, {Pos,M,F}) -> + fun(Obj) -> + M:F(Tab, Pos, Obj) + end; +index_vals_f(Storage, Tab, Pos) when is_integer(Pos) -> + case mnesia_lib:semantics(Storage, index_fun) of + undefined -> + fun(Obj) -> + [element(Pos, Obj)] + end; + F when is_function(F, 4) -> + {ext, Alias, _Mod} = Storage, + fun(Obj) -> + F(Alias, Tab, Pos, Obj) + end + end. diff --git a/lib/mnesia/src/mnesia_kernel_sup.erl b/lib/mnesia/src/mnesia_kernel_sup.erl index 1583e5e1f7..c9af5c460a 100644 --- a/lib/mnesia/src/mnesia_kernel_sup.erl +++ b/lib/mnesia/src/mnesia_kernel_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_late_loader.erl b/lib/mnesia/src/mnesia_late_loader.erl index bd21bdc347..e273329ffc 100644 --- a/lib/mnesia/src/mnesia_late_loader.erl +++ b/lib/mnesia/src/mnesia_late_loader.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2009. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_lib.erl b/lib/mnesia/src/mnesia_lib.erl index 0f1354f43e..10e232c800 100644 --- a/lib/mnesia/src/mnesia_lib.erl +++ b/lib/mnesia/src/mnesia_lib.erl @@ -54,6 +54,7 @@ db_erase_tab/2, db_first/1, db_first/2, + db_foldl/3, db_foldl/4, db_foldl/6, db_last/1, db_last/2, db_fixtable/3, @@ -135,6 +136,7 @@ set_local_content_whereabouts/1, set_remote_where_to_read/1, set_remote_where_to_read/2, + semantics/2, show/1, show/2, sort_commit/1, @@ -144,6 +146,7 @@ tab2tmp/1, tab2dcd/1, tab2dcl/1, + tab2logtmp/1, to_list/1, union/2, uniq/1, @@ -151,6 +154,8 @@ unset/1, %% update_counter/2, val/1, + validate_key/2, + validate_record/2, vcore/0, vcore/1, verbose/2, @@ -319,15 +324,41 @@ tab2dcd(Tab) -> %% Disc copies data tab2dcl(Tab) -> %% Disc copies log dir(lists:concat([Tab, ".DCL"])). +tab2logtmp(Tab) -> %% Disc copies log + dir(lists:concat([Tab, ".LOGTMP"])). + storage_type_at_node(Node, Tab) -> search_key(Node, [{disc_copies, val({Tab, disc_copies})}, {ram_copies, val({Tab, ram_copies})}, - {disc_only_copies, val({Tab, disc_only_copies})}]). + {disc_only_copies, val({Tab, disc_only_copies})}| + wrap_external(val({Tab, external_copies}))]). cs_to_storage_type(Node, Cs) -> search_key(Node, [{disc_copies, Cs#cstruct.disc_copies}, {ram_copies, Cs#cstruct.ram_copies}, - {disc_only_copies, Cs#cstruct.disc_only_copies}]). + {disc_only_copies, Cs#cstruct.disc_only_copies} | + wrap_external(Cs#cstruct.external_copies)]). + +-define(native(T), T==ram_copies; T==disc_copies; T==disc_only_copies). + +semantics({ext,Alias,Mod}, Item) -> + Mod:semantics(Alias, Item); +semantics({Alias,Mod}, Item) -> + Mod:semantics(Alias, Item); +semantics(Type, storage) when ?native(Type) -> + Type; +semantics(Type, types) when ?native(Type) -> + [set, ordered_set, bag]; +semantics(disc_only_copies, index_types) -> + [bag]; +semantics(Type, index_types) when ?native(Type) -> + [bag, ordered]; +semantics(_, _) -> + undefined. + + +wrap_external(L) -> + [{{ext,Alias,Mod},Ns} || {{Alias,Mod},Ns} <- L]. schema_cs_to_storage_type(Node, Cs) -> case cs_to_storage_type(Node, Cs) of @@ -335,7 +366,6 @@ schema_cs_to_storage_type(Node, Cs) -> Other -> Other end. - search_key(Key, [{Val, List} | Tail]) -> case lists:member(Key, List) of true -> Val; @@ -344,6 +374,33 @@ search_key(Key, [{Val, List} | Tail]) -> search_key(_Key, []) -> unknown. +validate_key(Tab, Key) -> + case ?catch_val({Tab, record_validation}) of + {RecName, Arity, Type} -> + {RecName, Arity, Type}; + {RecName, Arity, Type, Alias, Mod} -> + %% external type + Mod:validate_key(Alias, Tab, RecName, Arity, Type, Key); + {'EXIT', _} -> + mnesia:abort({no_exists, Tab}) + end. + + +validate_record(Tab, Obj) -> + case ?catch_val({Tab, record_validation}) of + {RecName, Arity, Type} + when tuple_size(Obj) == Arity, RecName == element(1, Obj) -> + {RecName, Arity, Type}; + {RecName, Arity, Type, Alias, Mod} + when tuple_size(Obj) == Arity, RecName == element(1, Obj) -> + %% external type + Mod:validate_record(Alias, Tab, RecName, Arity, Type, Obj); + {'EXIT', _} -> + mnesia:abort({no_exists, Tab}); + _ -> + mnesia:abort({bad_type, Obj}) + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% ops, we've got some global variables here :-) @@ -410,9 +467,9 @@ pr_other(Var) -> no -> {node_not_running, node()}; _ -> {no_exists, Var} end, - verbose("~p (~p) val(mnesia_gvar, ~w) -> ~p ~n", + verbose("~p (~p) val(mnesia_gvar, ~w) -> ~p ~p ~n", [self(), process_info(self(), registered_name), - Var, Why]), + Var, Why, erlang:get_stacktrace()]), mnesia:abort(Why). %% Some functions for list valued variables @@ -549,10 +606,16 @@ read_counter(Name) -> ?ets_lookup_element(mnesia_stats, Name, 2). cs_to_nodes(Cs) -> + ext_nodes(Cs#cstruct.external_copies) ++ Cs#cstruct.disc_only_copies ++ Cs#cstruct.disc_copies ++ Cs#cstruct.ram_copies. +ext_nodes(Ext) -> + lists:flatmap(fun({_, Ns}) -> + Ns + end, Ext). + overload_types() -> [mnesia_tm, mnesia_dump_log]. @@ -748,7 +811,7 @@ view(File) -> true -> view(File, dat); false -> - case suffix([".LOG", ".BUP", ".ETS"], File) of + case suffix([".LOG", ".BUP", ".ETS", ".LOGTMP"], File) of true -> view(File, log); false -> @@ -922,20 +985,7 @@ random_time(Retries, _Counter0) -> UpperLimit = 500, Dup = Retries * Retries, MaxIntv = trunc(UpperLimit * (1-(50/((Dup)+50)))), - - case get(random_seed) of - undefined -> - _ = random:seed(erlang:unique_integer(), - erlang:monotonic_time(), - erlang:unique_integer()), - Time = Dup + random:uniform(MaxIntv), - %% dbg_out("---random_test rs ~w max ~w val ~w---~n", [Retries, MaxIntv, Time]), - Time; - _ -> - Time = Dup + random:uniform(MaxIntv), - %% dbg_out("---random_test rs ~w max ~w val ~w---~n", [Retries, MaxIntv, Time]), - Time - end. + Dup + rand:uniform(MaxIntv). report_system_event(Event0) -> Event = {mnesia_system_event, Event0}, @@ -1057,18 +1107,24 @@ db_get(Tab, Key) -> db_get(val({Tab, storage_type}), Tab, Key). db_get(ram_copies, Tab, Key) -> ?ets_lookup(Tab, Key); db_get(disc_copies, Tab, Key) -> ?ets_lookup(Tab, Key); -db_get(disc_only_copies, Tab, Key) -> dets:lookup(Tab, Key). +db_get(disc_only_copies, Tab, Key) -> dets:lookup(Tab, Key); +db_get({ext, Alias, Mod}, Tab, Key) -> + Mod:lookup(Alias, Tab, Key). db_init_chunk(Tab) -> db_init_chunk(val({Tab, storage_type}), Tab, 1000). db_init_chunk(Tab, N) -> db_init_chunk(val({Tab, storage_type}), Tab, N). +db_init_chunk({ext, Alias, Mod}, Tab, N) -> + Mod:select(Alias, Tab, [{'_', [], ['$_']}], N); db_init_chunk(disc_only_copies, Tab, N) -> dets:select(Tab, [{'_', [], ['$_']}], N); db_init_chunk(_, Tab, N) -> ets:select(Tab, [{'_', [], ['$_']}], N). +db_chunk({ext, _Alias, Mod}, State) -> + Mod:select(State); db_chunk(disc_only_copies, State) -> dets:select(State); db_chunk(_, State) -> @@ -1079,7 +1135,9 @@ db_put(Tab, Val) -> db_put(ram_copies, Tab, Val) -> ?ets_insert(Tab, Val), ok; db_put(disc_copies, Tab, Val) -> ?ets_insert(Tab, Val), ok; -db_put(disc_only_copies, Tab, Val) -> dets:insert(Tab, Val). +db_put(disc_only_copies, Tab, Val) -> dets:insert(Tab, Val); +db_put({ext, Alias, Mod}, Tab, Val) -> + Mod:insert(Alias, Tab, Val). db_match_object(Tab, Pat) -> db_match_object(val({Tab, storage_type}), Tab, Pat). @@ -1088,12 +1146,36 @@ db_match_object(Storage, Tab, Pat) -> try case Storage of disc_only_copies -> dets:match_object(Tab, Pat); + {ext, Alias, Mod} -> Mod:select(Alias, Tab, [{Pat, [], ['$_']}]); _ -> ets:match_object(Tab, Pat) end after db_fixtable(Storage, Tab, false) end. +db_foldl(Fun, Acc, Tab) -> + db_foldl(val({Tab, storage_type}), Fun, Acc, Tab). + +db_foldl(Storage, Fun, Acc, Tab) -> + Limit = mnesia_monitor:get_env(fold_chunk_size), + db_foldl(Storage, Fun, Acc, Tab, [{'_', [], ['$_']}], Limit). + +db_foldl(ram_copies, Fun, Acc, Tab, Pat, Limit) -> + mnesia_lib:db_fixtable(ram_copies, Tab, true), + try select_foldl(db_select_init(ram_copies, Tab, Pat, Limit), + Fun, Acc, ram_copies) + after + mnesia_lib:db_fixtable(ram_copies, Tab, false) + end; +db_foldl(Storage, Fun, Acc, Tab, Pat, Limit) -> + select_foldl(mnesia_lib:db_select_init(Storage, Tab, Pat, Limit), Fun, Acc, Storage). + +select_foldl({Objs, Cont}, Fun, Acc, Storage) -> + select_foldl(mnesia_lib:db_select_cont(Storage, Cont, []), + Fun, lists:foldl(Fun, Acc, Objs), Storage); +select_foldl('$end_of_table', _, Acc, _) -> + Acc. + db_select(Tab, Pat) -> db_select(val({Tab, storage_type}), Tab, Pat). @@ -1102,17 +1184,33 @@ db_select(Storage, Tab, Pat) -> try case Storage of disc_only_copies -> dets:select(Tab, Pat); + {ext, Alias, Mod} -> Mod:select(Alias, Tab, Pat); _ -> ets:select(Tab, Pat) end after db_fixtable(Storage, Tab, false) end. +db_select_init({ext, Alias, Mod}, Tab, Pat, Limit) -> + case Mod:select(Alias, Tab, Pat, Limit) of + {Matches, Continuation} when is_list(Matches) -> + {Matches, {Alias, Continuation}}; + R -> + R + end; db_select_init(disc_only_copies, Tab, Pat, Limit) -> dets:select(Tab, Pat, Limit); db_select_init(_, Tab, Pat, Limit) -> ets:select(Tab, Pat, Limit). +db_select_cont({ext, Alias, Mod}, Cont0, Ms) -> + Cont = Mod:repair_continuation(Cont0, Ms), + case Mod:select(Cont) of + {Matches, Continuation} when is_list(Matches) -> + {Matches, {Alias, Continuation}}; + R -> + R + end; db_select_cont(disc_only_copies, Cont0, Ms) -> Cont = dets:repair_continuation(Cont0, Ms), dets:select(Cont); @@ -1129,13 +1227,18 @@ db_fixtable(disc_copies, Tab, Bool) -> db_fixtable(dets, Tab, Bool) -> dets:safe_fixtable(Tab, Bool); db_fixtable(disc_only_copies, Tab, Bool) -> - dets:safe_fixtable(Tab, Bool). + dets:safe_fixtable(Tab, Bool); +db_fixtable({ext, Alias, Mod}, Tab, Bool) -> + Mod:fixtable(Alias, Tab, Bool). db_erase(Tab, Key) -> db_erase(val({Tab, storage_type}), Tab, Key). db_erase(ram_copies, Tab, Key) -> ?ets_delete(Tab, Key), ok; db_erase(disc_copies, Tab, Key) -> ?ets_delete(Tab, Key), ok; -db_erase(disc_only_copies, Tab, Key) -> dets:delete(Tab, Key). +db_erase(disc_only_copies, Tab, Key) -> dets:delete(Tab, Key); +db_erase({ext, Alias, Mod}, Tab, Key) -> + Mod:delete(Alias, Tab, Key), + ok. db_match_erase(Tab, '_') -> db_delete_all(val({Tab, storage_type}),Tab); @@ -1143,7 +1246,10 @@ db_match_erase(Tab, Pat) -> db_match_erase(val({Tab, storage_type}), Tab, Pat). db_match_erase(ram_copies, Tab, Pat) -> ?ets_match_delete(Tab, Pat), ok; db_match_erase(disc_copies, Tab, Pat) -> ?ets_match_delete(Tab, Pat), ok; -db_match_erase(disc_only_copies, Tab, Pat) -> dets:match_delete(Tab, Pat). +db_match_erase(disc_only_copies, Tab, Pat) -> dets:match_delete(Tab, Pat); +db_match_erase({ext, Alias, Mod}, Tab, Pat) -> + Mod:match_delete(Alias, Tab, Pat), + ok. db_delete_all(ram_copies, Tab) -> ets:delete_all_objects(Tab); db_delete_all(disc_copies, Tab) -> ets:delete_all_objects(Tab); @@ -1153,31 +1259,41 @@ db_first(Tab) -> db_first(val({Tab, storage_type}), Tab). db_first(ram_copies, Tab) -> ?ets_first(Tab); db_first(disc_copies, Tab) -> ?ets_first(Tab); -db_first(disc_only_copies, Tab) -> dets:first(Tab). +db_first(disc_only_copies, Tab) -> dets:first(Tab); +db_first({ext, Alias, Mod}, Tab) -> + Mod:first(Alias, Tab). db_next_key(Tab, Key) -> db_next_key(val({Tab, storage_type}), Tab, Key). db_next_key(ram_copies, Tab, Key) -> ?ets_next(Tab, Key); db_next_key(disc_copies, Tab, Key) -> ?ets_next(Tab, Key); -db_next_key(disc_only_copies, Tab, Key) -> dets:next(Tab, Key). +db_next_key(disc_only_copies, Tab, Key) -> dets:next(Tab, Key); +db_next_key({ext, Alias, Mod}, Tab, Key) -> + Mod:next(Alias, Tab, Key). db_last(Tab) -> db_last(val({Tab, storage_type}), Tab). db_last(ram_copies, Tab) -> ?ets_last(Tab); db_last(disc_copies, Tab) -> ?ets_last(Tab); -db_last(disc_only_copies, Tab) -> dets:first(Tab). %% Dets don't have order +db_last(disc_only_copies, Tab) -> dets:first(Tab); %% Dets don't have order +db_last({ext, Alias, Mod}, Tab) -> + Mod:last(Alias, Tab). db_prev_key(Tab, Key) -> db_prev_key(val({Tab, storage_type}), Tab, Key). db_prev_key(ram_copies, Tab, Key) -> ?ets_prev(Tab, Key); db_prev_key(disc_copies, Tab, Key) -> ?ets_prev(Tab, Key); -db_prev_key(disc_only_copies, Tab, Key) -> dets:next(Tab, Key). %% Dets don't have order +db_prev_key(disc_only_copies, Tab, Key) -> dets:next(Tab, Key); %% Dets don't have order +db_prev_key({ext, Alias, Mod}, Tab, Key) -> + Mod:prev(Alias, Tab, Key). db_slot(Tab, Pos) -> db_slot(val({Tab, storage_type}), Tab, Pos). db_slot(ram_copies, Tab, Pos) -> ?ets_slot(Tab, Pos); db_slot(disc_copies, Tab, Pos) -> ?ets_slot(Tab, Pos); -db_slot(disc_only_copies, Tab, Pos) -> dets:slot(Tab, Pos). +db_slot(disc_only_copies, Tab, Pos) -> dets:slot(Tab, Pos); +db_slot({ext, Alias, Mod}, Tab, Pos) -> + Mod:slot(Alias, Tab, Pos). db_update_counter(Tab, C, Val) -> db_update_counter(val({Tab, storage_type}), Tab, C, Val). @@ -1186,13 +1302,16 @@ db_update_counter(ram_copies, Tab, C, Val) -> db_update_counter(disc_copies, Tab, C, Val) -> ?ets_update_counter(Tab, C, Val); db_update_counter(disc_only_copies, Tab, C, Val) -> - dets:update_counter(Tab, C, Val). + dets:update_counter(Tab, C, Val); +db_update_counter({ext, Alias, Mod}, Tab, C, Val) -> + Mod:update_counter(Alias, Tab, C, Val). db_erase_tab(Tab) -> db_erase_tab(val({Tab, storage_type}), Tab). db_erase_tab(ram_copies, Tab) -> ?ets_delete_table(Tab); db_erase_tab(disc_copies, Tab) -> ?ets_delete_table(Tab); -db_erase_tab(disc_only_copies, _Tab) -> ignore. +db_erase_tab(disc_only_copies, _Tab) -> ignore; +db_erase_tab({ext, _Alias, _Mod}, _Tab) -> ignore. %% assuming that Tab is a valid ets-table dets_to_ets(Tabname, Tab, File, Type, Rep, Lock) -> diff --git a/lib/mnesia/src/mnesia_loader.erl b/lib/mnesia/src/mnesia_loader.erl index 41fcd76fcb..c710470a2c 100644 --- a/lib/mnesia/src/mnesia_loader.erl +++ b/lib/mnesia/src/mnesia_loader.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2014. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -147,6 +147,19 @@ do_get_disc_copy2(Tab, Reason, Storage, Type) when Storage == disc_only_copies - {error, Error} -> {not_loaded, {"Failed to create dets table", Error}} end + end; + +do_get_disc_copy2(Tab, Reason, Storage = {ext, Alias, Mod}, _Type) -> + Cs = val({Tab, cstruct}), + case mnesia_monitor:unsafe_create_external(Tab, Alias, Mod, Cs) of + ok -> + ok = ext_load_table(Mod, Alias, Tab, Reason), + mnesia_index:init_index(Tab, Storage), + set({Tab, load_node}, node()), + set({Tab, load_reason}, Reason), + {loaded, ok}; + Other -> + {not_loaded, Other} end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -259,7 +272,7 @@ init_receiver(Node, Tab,Storage,Cs,Reason) -> true = lists:member(Node, Active), {SenderPid, TabSize, DetsData} = start_remote_sender(Node,Tab,Storage), - Init = table_init_fun(SenderPid), + Init = table_init_fun(SenderPid, Storage), Args = [self(),Tab,Storage,Cs,SenderPid, TabSize,DetsData,Init], Pid = spawn_link(?MODULE, spawned_receiver, Args), @@ -289,7 +302,12 @@ start_remote_sender(Node,Tab,Storage) -> mnesia_controller:start_remote_sender(Node, Tab, self(), Storage), put(mnesia_table_sender_node, {Tab, Node}), receive - {SenderPid, {first, TabSize}} -> + {SenderPid, {first, _} = Msg} + when is_pid(SenderPid), element(1, Storage) == ext -> + {ext, Alias, Mod} = Storage, + {Sz, Data} = Mod:receiver_first_message(SenderPid, Msg, Alias, Tab), + {SenderPid, Sz, Data}; + {SenderPid, {first, TabSize}} =_M1 -> {SenderPid, TabSize, false}; {SenderPid, {first, TabSize, DetsData}} -> {SenderPid, TabSize, DetsData}; @@ -299,16 +317,18 @@ start_remote_sender(Node,Tab,Storage) -> down(Tab, Storage) end. -table_init_fun(SenderPid) -> +table_init_fun(SenderPid, Storage) -> fun(read) -> Receiver = self(), SenderPid ! {Receiver, more}, - get_data(SenderPid, Receiver) + get_data(SenderPid, Receiver, Storage); + (close) -> + ok end. %% Add_table_copy get's it's own locks. start_receiver(Tab,Storage,Cs,SenderPid,TabSize,DetsData,{dumper,{add_table_copy,_}}) -> - Init = table_init_fun(SenderPid), + Init = table_init_fun(SenderPid, Storage), case do_init_table(Tab,Storage,Cs,SenderPid,TabSize,DetsData,self(), Init) of Err = {error, _} -> SenderPid ! {copier_done, node()}, @@ -322,9 +342,12 @@ spawned_receiver(ReplyTo,Tab,Storage,Cs, SenderPid,TabSize,DetsData, Init) -> Done = do_init_table(Tab,Storage,Cs, SenderPid,TabSize,DetsData, ReplyTo, Init), - ReplyTo ! {self(),Done}, - unlink(ReplyTo), - unlink(whereis(mnesia_controller)), + try + ReplyTo ! {self(),Done}, + unlink(ReplyTo), + unlink(whereis(mnesia_controller)) + catch _:_ -> ok %% avoid error reports when stopping down mnesia + end, exit(normal). wait_on_load_complete(Pid) -> @@ -391,7 +414,15 @@ create_table(Tab, TabSize, Storage, Cs) -> {Storage, Tab}; Else -> Else - end + end; + element(1, Storage) == ext -> + {_, Alias, Mod} = Storage, + case mnesia_monitor:unsafe_create_external(Tab, Alias, Mod, Cs) of + ok -> + {Storage, Tab}; + Else -> + Else + end end. tab_receiver(Node, Tab, Storage, Cs, OrigTabRec) -> @@ -409,21 +440,25 @@ tab_receiver(Node, Tab, Storage, Cs, OrigTabRec) -> tab_receiver(Node, Tab, Storage, Cs, OrigTabRec) end. -make_table_fun(Pid, TabRec) -> +make_table_fun(Pid, TabRec, Storage) -> fun(close) -> ok; + ({read, Msg}) -> + Pid ! {TabRec, Msg}, + get_data(Pid, TabRec, Storage); (read) -> - get_data(Pid, TabRec) + get_data(Pid, TabRec, Storage) end. -get_data(Pid, TabRec) -> +get_data(Pid, TabRec, Storage) -> receive {Pid, {more_z, CompressedRecs}} when is_binary(CompressedRecs) -> - Pid ! {TabRec, more}, - {zlib_uncompress(CompressedRecs), make_table_fun(Pid,TabRec)}; + maybe_reply(Pid, {TabRec, more}, Storage), + {zlib_uncompress(CompressedRecs), + make_table_fun(Pid, TabRec, Storage)}; {Pid, {more, Recs}} -> - Pid ! {TabRec, more}, - {Recs, make_table_fun(Pid,TabRec)}; + maybe_reply(Pid, {TabRec, more}, Storage), + {Recs, make_table_fun(Pid, TabRec, Storage)}; {Pid, no_more} -> end_of_input; {copier_done, Node} -> @@ -431,13 +466,48 @@ get_data(Pid, TabRec) -> Node -> {copier_done, Node}; _ -> - get_data(Pid, TabRec) + get_data(Pid, TabRec, Storage) end; {'EXIT', Pid, Reason} -> handle_exit(Pid, Reason), - get_data(Pid, TabRec) + get_data(Pid, TabRec, Storage) end. +maybe_reply(_, _, {ext, _, _}) -> + ignore; +maybe_reply(Pid, Msg, _) -> + Pid ! Msg. + +ext_init_table(Alias, Mod, Tab, Fun, State, Sender) -> + ok = ext_load_table(Mod, Alias, Tab, {net_load, node(Sender)}), + ext_init_table(read, Alias, Mod, Tab, Fun, State, Sender). + +ext_load_table(Mod, Alias, Tab, Reason) -> + CS = val({Tab, cstruct}), + Mod:load_table(Alias, Tab, Reason, mnesia_schema:cs2list(CS)). + + +ext_init_table(Action, Alias, Mod, Tab, Fun, State, Sender) -> + case Fun(Action) of + {copier_done, Node} -> + verbose("Receiver of table ~p crashed on ~p (more)~n", [Tab, Node]), + down(Tab, {ext,Alias,Mod}); + {Data, NewFun} -> + case Mod:receive_data(Data, Alias, Tab, Sender, State) of + {more, NewState} -> + ext_init_table({read, more}, Alias, Mod, + Tab, NewFun, NewState, Sender); + {{more,Msg}, NewState} -> + ext_init_table({read, Msg}, Alias, Mod, + Tab, NewFun, NewState, Sender) + end; + end_of_input -> + Mod:receive_done(Alias, Tab, Sender, State), + ok = Fun(close) + end. + +init_table(Tab, {ext,Alias,Mod}, Fun, State, Sender) -> + ext_init_table(Alias, Mod, Tab, Fun, State, Sender); init_table(Tab, disc_only_copies, Fun, DetsInfo,Sender) -> ErtsVer = erlang:system_info(version), case DetsInfo of @@ -567,7 +637,10 @@ handle_last({ram_copies, Tab}, _Type, DatBin) -> ok; false -> ok - end. + end; + +handle_last(_Storage, _Type, nobin) -> + ok. down(Tab, Storage) -> case Storage of @@ -578,7 +651,10 @@ down(Tab, Storage) -> disc_only_copies -> TmpFile = mnesia_lib:tab2tmp(Tab), mnesia_lib:dets_sync_close(Tab), - file:delete(TmpFile) + file:delete(TmpFile); + {ext, Alias, Mod} -> + catch Mod:close_table(Alias, Tab), + catch Mod:delete_table(Alias, Tab) end, mnesia_checkpoint:tm_del_copy(Tab, node()), mnesia_controller:sync_del_table_copy_whereabouts(Tab, node()), @@ -603,21 +679,35 @@ db_erase({ram_copies, Tab}, Key) -> db_erase({disc_copies, Tab}, Key) -> true = ?ets_delete(Tab, Key); db_erase({disc_only_copies, Tab}, Key) -> - ok = dets:delete(Tab, Key). + ok = dets:delete(Tab, Key); +db_erase({{ext, Alias, Mod}, Tab}, Key) -> + ok = Mod:delete(Alias, Tab, Key). db_match_erase({ram_copies, Tab} , Pat) -> true = ?ets_match_delete(Tab, Pat); db_match_erase({disc_copies, Tab} , Pat) -> true = ?ets_match_delete(Tab, Pat); db_match_erase({disc_only_copies, Tab}, Pat) -> - ok = dets:match_delete(Tab, Pat). + ok = dets:match_delete(Tab, Pat); +db_match_erase({{ext, Alias, Mod}, Tab}, Pat) -> + % "ets style" is to return true + % "dets style" is to return N | { error, Reason } + % or sometimes ok (?) + % be nice and accept both + case Mod:match_delete(Alias, Tab, Pat) of + N when is_integer (N) -> ok; + true -> ok; + ok -> ok + end. db_put({ram_copies, Tab}, Val) -> true = ?ets_insert(Tab, Val); db_put({disc_copies, Tab}, Val) -> true = ?ets_insert(Tab, Val); db_put({disc_only_copies, Tab}, Val) -> - ok = dets:insert(Tab, Val). + ok = dets:insert(Tab, Val); +db_put({{ext, Alias, Mod}, Tab}, Val) -> + ok = Mod:insert(Alias, Tab, Val). %% This code executes at the remote site where the data is %% executes in a special copier process. @@ -636,47 +726,62 @@ send_table(Pid, Tab, RemoteS) -> unknown -> {error, {no_exists, Tab}}; Storage -> - %% Send first - TabSize = mnesia:table_info(Tab, size), - KeysPerTransfer = calc_nokeys(Storage, Tab), - ChunkData = dets:info(Tab, bchunk_format), - - UseDetsChunk = - Storage == RemoteS andalso - Storage == disc_only_copies andalso - ChunkData /= undefined, - if - UseDetsChunk == true -> - DetsInfo = erlang:system_info(version), - Pid ! {self(), {first, TabSize, {DetsInfo, ChunkData}}}; - true -> - Pid ! {self(), {first, TabSize}} - end, + do_send_table(Pid, Tab, Storage, RemoteS) + end. - %% Debug info - put(mnesia_table_sender, {Tab, node(Pid), Pid}), - {Init, Chunk} = reader_funcs(UseDetsChunk, Tab, Storage, KeysPerTransfer), - - SendIt = fun() -> - NeedLock = need_lock(Tab), - {atomic, ok} = prepare_copy(Pid, Tab, Storage, NeedLock), - send_more(Pid, 1, Chunk, Init(), Tab), - finish_copy(Pid, Tab, Storage, RemoteS, NeedLock) - end, - - try SendIt() of - {_, receiver_died} -> ok; - {atomic, no_more} -> ok - catch - throw:receiver_died -> - cleanup_tab_copier(Pid, Storage, Tab), - ok; - error:Reason -> %% Prepare failed - cleanup_tab_copier(Pid, Storage, Tab), - {error, {tab_copier, Tab, {Reason, erlang:get_stacktrace()}}} - after - unlink(whereis(mnesia_tm)) - end +do_send_table(Pid, Tab, Storage, RemoteS) -> + {Init, Chunk} = + case Storage of + {ext, Alias, Mod} -> + case Mod:sender_init(Alias, Tab, RemoteS, Pid) of + {standard, I, C} -> + Pid ! {self(), {first, Mod:info(Alias, Tab, size)}}, + {I, C}; + {_, _} = Res -> + Res + end; + Storage -> + %% Send first + TabSize = mnesia:table_info(Tab, size), + KeysPerTransfer = calc_nokeys(Storage, Tab), + ChunkData = dets:info(Tab, bchunk_format), + + UseDetsChunk = + Storage == RemoteS andalso + Storage == disc_only_copies andalso + ChunkData /= undefined, + if + UseDetsChunk == true -> + DetsInfo = erlang:system_info(version), + Pid ! {self(), {first, TabSize, {DetsInfo, ChunkData}}}; + true -> + Pid ! {self(), {first, TabSize}} + end, + {_I, _C} = + reader_funcs(UseDetsChunk, Tab, Storage, KeysPerTransfer) + end, + %% Debug info + put(mnesia_table_sender, {Tab, node(Pid), Pid}), + + SendIt = fun() -> + NeedLock = need_lock(Tab), + {atomic, ok} = prepare_copy(Pid, Tab, Storage, NeedLock), + send_more(Pid, 1, Chunk, Init(), Tab, Storage), + finish_copy(Pid, Tab, Storage, RemoteS, NeedLock) + end, + + try SendIt() of + {_, receiver_died} -> ok; + {atomic, no_more} -> ok + catch + throw:receiver_died -> + cleanup_tab_copier(Pid, Storage, Tab), + ok; + error:Reason -> %% Prepare failed + cleanup_tab_copier(Pid, Storage, Tab), + {error, {tab_copier, Tab, {Reason, erlang:get_stacktrace()}}} + after + unlink(whereis(mnesia_tm)) end. prepare_copy(Pid, Tab, Storage, NeedLock) -> @@ -726,20 +831,32 @@ update_where_to_write([H|T], Tab, AddNode) -> [{update_where_to_write, [add, Tab, AddNode], self()}]), update_where_to_write(T, Tab, AddNode). -send_more(Pid, N, Chunk, DataState, Tab) -> +send_more(Pid, N, Chunk, DataState, Tab, Storage) -> receive {NewPid, more} -> case send_packet(N - 1, NewPid, Chunk, DataState) of New when is_integer(New) -> New - 1; NewData -> - send_more(NewPid, ?MAX_NOPACKETS, Chunk, NewData, Tab) + send_more(NewPid, ?MAX_NOPACKETS, Chunk, NewData, + Tab, Storage) + end; + {NewPid, {more, Msg}} when element(1, Storage) == ext -> + {ext, Alias, Mod} = Storage, + {NewChunk, NewState} = + Mod:sender_handle_info(Msg, Alias, Tab, NewPid, DataState), + case send_packet(N - 1, NewPid, NewChunk, NewState) of + New when is_integer(New) -> + New -1; + NewData -> + send_more(NewPid, N, NewChunk, NewData, Tab, + Storage) end; {_NewPid, {old_protocol, Tab}} -> Storage = val({Tab, storage_type}), {Init, NewChunk} = reader_funcs(false, Tab, Storage, calc_nokeys(Storage, Tab)), - send_more(Pid, 1, NewChunk, Init(), Tab); + send_more(Pid, 1, NewChunk, Init(), Tab, Storage); {copier_done, Node} when Node == node(Pid)-> verbose("Receiver of table ~p crashed on ~p (more)~n", [Tab, Node]), @@ -802,9 +919,15 @@ send_packet(_N, _Pid, _Chunk, DataState) -> finish_copy(Pid, Tab, Storage, RemoteS, NeedLock) -> RecNode = node(Pid), DatBin = dat2bin(Tab, Storage, RemoteS), + Node = node(Pid), Trans = fun() -> NeedLock andalso mnesia:read_lock_table(Tab), + %% Check that receiver is still alive + receive {copier_done, Node} -> + throw(receiver_died) + after 0 -> ok + end, A = val({Tab, access_mode}), mnesia_controller:sync_and_block_table_whereabouts(Tab, RecNode, RemoteS, A), cleanup_tab_copier(Pid, Storage, Tab), @@ -813,7 +936,7 @@ finish_copy(Pid, Tab, Storage, RemoteS, NeedLock) -> receive {Pid, no_more} -> % Dont bother about the spurious 'more' message no_more; - {copier_done, Node} when Node == node(Pid)-> + {copier_done, Node} -> verbose("Tab receiver ~p crashed (more): ~p~n", [Tab, Node]), receiver_died end diff --git a/lib/mnesia/src/mnesia_locker.erl b/lib/mnesia/src/mnesia_locker.erl index 5766f22e92..59fd89059f 100644 --- a/lib/mnesia/src/mnesia_locker.erl +++ b/lib/mnesia/src/mnesia_locker.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_log.erl b/lib/mnesia/src/mnesia_log.erl index 36135418c8..9536effd42 100644 --- a/lib/mnesia/src/mnesia_log.erl +++ b/lib/mnesia/src/mnesia_log.erl @@ -224,17 +224,12 @@ sappend(Log, Term) -> ok = disk_log:log(Log, Term). %% Write commit records to the latest_log -log(C) when C#commit.disc_copies == [], - C#commit.disc_only_copies == [], - C#commit.schema_ops == [] -> - ignore; log(C) -> - case mnesia_monitor:use_dir() of + case need_log(C) andalso mnesia_monitor:use_dir() of true -> if is_record(C, commit) -> - C2 = C#commit{ram_copies = [], snmp = []}, - append(latest_log, C2); + append(latest_log, strip_snmp(C)); true -> %% Either a commit record as binary %% or some decision related info @@ -247,17 +242,12 @@ log(C) -> %% Synced -slog(C) when C#commit.disc_copies == [], - C#commit.disc_only_copies == [], - C#commit.schema_ops == [] -> - ignore; slog(C) -> - case mnesia_monitor:use_dir() of + case need_log(C) andalso mnesia_monitor:use_dir() of true -> if is_record(C, commit) -> - C2 = C#commit{ram_copies = [], snmp = []}, - sappend(latest_log, C2); + sappend(latest_log, strip_snmp(C)); true -> %% Either a commit record as binary %% or some decision related info @@ -268,6 +258,13 @@ slog(C) -> ignore end. +need_log(#commit{disc_copies=[], disc_only_copies=[], schema_ops=[], ext=Ext}) -> + lists:keymember(ext_copies, 1, Ext); +need_log(_) -> true. + +strip_snmp(#commit{ext=[]}=CR) -> CR; +strip_snmp(#commit{ext=Ext}=CR) -> + CR#commit{ext=lists:keydelete(snmp, 1, Ext)}. %% Stuff related to the file LOG diff --git a/lib/mnesia/src/mnesia_monitor.erl b/lib/mnesia/src/mnesia_monitor.erl index 081d746257..ab78c9b13e 100644 --- a/lib/mnesia/src/mnesia_monitor.erl +++ b/lib/mnesia/src/mnesia_monitor.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,6 +32,7 @@ init/0, mktab/2, unsafe_mktab/2, + unsafe_create_external/4, mnesia_down/2, needs_protocol_conversion/1, negotiate_protocol/1, @@ -82,9 +83,9 @@ going_down = [], tm_started = false, early_connects = [], connecting, mq = [], remote_node_status = []}). --define(current_protocol_version, {8,2}). +-define(current_protocol_version, {8,3}). --define(previous_protocol_version, {8,1}). +-define(previous_protocol_version, {8,2}). start() -> gen_server:start_link({local, ?MODULE}, ?MODULE, @@ -129,6 +130,8 @@ close_log(Name) -> unsafe_close_log(Name) -> unsafe_call({unsafe_close_log, Name}). +unsafe_create_external(Tab, Alias, Mod, Cs) -> + unsafe_call({unsafe_create_external, Tab, Alias, Mod, Cs}). disconnect(Node) -> cast({disconnect, Node}). @@ -193,7 +196,7 @@ protocol_version() -> %% A sorted list of acceptable protocols the %% preferred protocols are first in the list acceptable_protocol_versions() -> - [protocol_version(), ?previous_protocol_version]. + [protocol_version(), ?previous_protocol_version, {8,1}]. needs_protocol_conversion(Node) -> case {?catch_val({protocol, Node}), protocol_version()} of @@ -405,6 +408,14 @@ handle_call({unsafe_close_log, Name}, _From, State) -> _ = disk_log:close(Name), {reply, ok, State}; +handle_call({unsafe_create_external, Tab, Alias, Mod, Cs}, _From, State) -> + case catch Mod:create_table(Alias, Tab, mnesia_schema:cs2list(Cs)) of + {'EXIT', ExitReason} -> + {reply, {error, ExitReason}, State}; + Reply -> + {reply, Reply, State} + end; + handle_call({negotiate_protocol, Mon, _Version, _Protocols}, _From, State) when State#state.tm_started == false -> State2 = State#state{early_connects = [node(Mon) | State#state.early_connects]}, @@ -658,6 +669,7 @@ get_env(E) -> env() -> [ access_module, + allow_index_on_key, auto_repair, backup_module, debug, @@ -671,19 +683,23 @@ env() -> extra_db_nodes, ignore_fallback_at_startup, fallback_error_function, + fold_chunk_size, max_wait_for_decision, schema_location, core_dir, pid_sort_order, no_table_loaders, dc_dump_limit, - send_compressed + send_compressed, + schema ]. default_env(access_module) -> mnesia; default_env(auto_repair) -> true; +default_env(allow_index_on_key) -> + false; default_env(backup_module) -> mnesia_backup; default_env(debug) -> @@ -709,6 +725,8 @@ default_env(ignore_fallback_at_startup) -> false; default_env(fallback_error_function) -> {mnesia, lkill}; +default_env(fold_chunk_size) -> + 100; default_env(max_wait_for_decision) -> infinity; default_env(schema_location) -> @@ -722,7 +740,9 @@ default_env(no_table_loaders) -> default_env(dc_dump_limit) -> 4; default_env(send_compressed) -> - 0. + 0; +default_env(schema) -> + []. check_type(Env, Val) -> try do_check_type(Env, Val) @@ -730,6 +750,7 @@ check_type(Env, Val) -> end. do_check_type(access_module, A) when is_atom(A) -> A; +do_check_type(allow_index_on_key, B) -> bool(B); do_check_type(auto_repair, B) -> bool(B); do_check_type(backup_module, B) when is_atom(B) -> B; do_check_type(debug, debug) -> debug; @@ -753,6 +774,8 @@ do_check_type(extra_db_nodes, L) when is_list(L) -> (A) when is_atom(A) -> true end, lists:filter(Fun, L); +do_check_type(fold_chunk_size, I) when is_integer(I), I > 0; + I =:= infinity -> I; do_check_type(max_wait_for_decision, infinity) -> infinity; do_check_type(max_wait_for_decision, I) when is_integer(I), I > 0 -> I; do_check_type(schema_location, M) -> media(M); @@ -766,7 +789,8 @@ do_check_type(pid_sort_order, "standard") -> standard; do_check_type(pid_sort_order, _) -> false; do_check_type(no_table_loaders, N) when is_integer(N), N > 0 -> N; do_check_type(dc_dump_limit,N) when is_number(N), N > 0 -> N; -do_check_type(send_compressed, L) when is_integer(L), L >= 0, L =< 9 -> L. +do_check_type(send_compressed, L) when is_integer(L), L >= 0, L =< 9 -> L; +do_check_type(schema, L) when is_list(L) -> L. bool(true) -> true; bool(false) -> false. diff --git a/lib/mnesia/src/mnesia_recover.erl b/lib/mnesia/src/mnesia_recover.erl index 9420e57afa..b204fb282f 100644 --- a/lib/mnesia/src/mnesia_recover.erl +++ b/lib/mnesia/src/mnesia_recover.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2014. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_registry.erl b/lib/mnesia/src/mnesia_registry.erl index abf0b745a1..55ddc3d2fd 100644 --- a/lib/mnesia/src/mnesia_registry.erl +++ b/lib/mnesia/src/mnesia_registry.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_schema.erl b/lib/mnesia/src/mnesia_schema.erl index 782493fb4f..0e4017e4c3 100644 --- a/lib/mnesia/src/mnesia_schema.erl +++ b/lib/mnesia/src/mnesia_schema.erl @@ -29,6 +29,16 @@ -module(mnesia_schema). -export([ + add_backend_type/2, + do_add_backend_type/2, + delete_backend_type/1, + do_delete_backend_type/1, + backend_types/0, + add_index_plugin/3, + do_add_index_plugin/3, + delete_index_plugin/1, + do_delete_index_plugin/1, + index_plugins/0, add_snmp/2, add_table_copy/3, add_table_index/2, @@ -55,14 +65,16 @@ dump_tables/1, ensure_no_schema/1, get_create_list/1, - get_initial_schema/2, + get_initial_schema/3, get_table_properties/1, info/0, info/1, init/1, + init_backends/0, insert_cstruct/3, is_remote_member/1, list2cs/1, + list2cs/2, lock_schema/0, merge_schema/0, merge_schema/1, @@ -110,7 +122,8 @@ do_delete_table/1, do_read_table_property/2, do_delete_table_property/2, - do_write_table_property/2]). + do_write_table_property/2, + do_change_table_copy_type/3]). -include("mnesia.hrl"). -include_lib("kernel/include/file.hrl"). @@ -138,12 +151,30 @@ init(IgnoreFallback) -> verbose("Schema initiated from: ~p~n", [Source]), set({schema, tables}, []), set({schema, local_tables}, []), + do_set_schema(schema), Tabs = set_schema(?ets_first(schema)), lists:foreach(fun(Tab) -> clear_whereabouts(Tab) end, Tabs), set({schema, where_to_read}, node()), set({schema, load_node}, node()), set({schema, load_reason}, initial), - mnesia_controller:add_active_replica(schema, node()). + mnesia_controller:add_active_replica(schema, node()), + init_backends(). + + +init_backends() -> + Backends = lists:foldl(fun({Alias, Mod}, Acc) -> + orddict:append(Mod, Alias, Acc) + end, orddict:new(), get_ext_types()), + [init_backend(Mod, Aliases) || {Mod, Aliases} <- Backends], + ok. + +init_backend(Mod, [_|_] = Aliases) -> + case Mod:init_backend() of + ok -> + Mod:add_aliases(Aliases); + Error -> + mnesia:abort({backend_init_error, Error}) + end. exit_on_error({error, Reason}) -> exit(Reason); @@ -180,6 +211,7 @@ do_set_schema(Tab, Cs) -> set({Tab, ram_copies}, Cs#cstruct.ram_copies), set({Tab, disc_copies}, Cs#cstruct.disc_copies), set({Tab, disc_only_copies}, Cs#cstruct.disc_only_copies), + set({Tab, external_copies}, Cs#cstruct.external_copies), set({Tab, load_order}, Cs#cstruct.load_order), set({Tab, access_mode}, Cs#cstruct.access_mode), set({Tab, majority}, Cs#cstruct.majority), @@ -195,15 +227,21 @@ do_set_schema(Tab, Cs) -> set({Tab, arity}, Arity), RecName = Cs#cstruct.record_name, set({Tab, record_name}, RecName), - set({Tab, record_validation}, {RecName, Arity, Type}), set({Tab, wild_pattern}, wild(RecName, Arity)), - set({Tab, index}, Cs#cstruct.index), + set({Tab, index}, [P || {P,_} <- Cs#cstruct.index]), + case Cs#cstruct.index of + [] -> + set({Tab, index_info}, mnesia_index:index_info(Type, [])); + _ -> + ignore + end, %% create actual index tabs later set({Tab, cookie}, Cs#cstruct.cookie), set({Tab, version}, Cs#cstruct.version), set({Tab, cstruct}, Cs), Storage = mnesia_lib:schema_cs_to_storage_type(node(), Cs), set({Tab, storage_type}, Storage), + set_record_validation(Tab, Storage, RecName, Arity, Type), mnesia_lib:add({schema, tables}, Tab), Ns = mnesia_lib:cs_to_nodes(Cs), case lists:member(node(), Ns) of @@ -213,7 +251,24 @@ do_set_schema(Tab, Cs) -> mnesia_lib:add({schema, local_tables}, Tab); false -> ignore - end. + end, + set_ext_types(Tab, get_ext_types(), Cs#cstruct.external_copies). + +set_record_validation(Tab, {ext,Alias,Mod}, RecName, Arity, Type) -> + set({Tab, record_validation}, {RecName, Arity, Type, Alias, Mod}); +set_record_validation(Tab, _, RecName, Arity, Type) -> + set({Tab, record_validation}, {RecName, Arity, Type}). + +set_ext_types(Tab, ExtTypes, ExtCopies) -> + lists:foreach( + fun({Type, _} = Key) -> + Nodes = case lists:keyfind(Key, 1, ExtCopies) of + {_, Ns} -> Ns; + false -> [] + end, + set({Tab, Type}, Nodes) + end, ExtTypes). + wild(RecName, Arity) -> Wp0 = list_to_tuple(lists:duplicate(Arity, '_')), @@ -525,9 +580,14 @@ do_read_disc_schema(Fname, Keep) -> Res. get_initial_schema(SchemaStorage, Nodes) -> + get_initial_schema(SchemaStorage, Nodes, []). + +get_initial_schema(SchemaStorage, Nodes, Properties) -> % + UserProps = initial_schema_properties(Properties), Cs = #cstruct{name = schema, record_name = schema, - attributes = [table, cstruct]}, + attributes = [table, cstruct], + user_properties = UserProps}, Cs2 = case SchemaStorage of ram_copies -> Cs#cstruct{ram_copies = Nodes}; @@ -535,6 +595,35 @@ get_initial_schema(SchemaStorage, Nodes) -> end, cs2list(Cs2). +initial_schema_properties(Props0) -> + DefaultProps = remove_duplicates(mnesia_monitor:get_env(schema)), + Props = lists:foldl( + fun({K,V}, Acc) -> + lists:keystore(K, 1, Acc, {K,V}) + end, DefaultProps, remove_duplicates(Props0)), + initial_schema_properties_(Props). + +initial_schema_properties_([{backend_types, Types}|Props]) -> + lists:foreach(fun({Name, Module}) -> + verify_backend_type(Name, Module) + end, Types), + [{mnesia_backend_types, Types}|initial_schema_properties_(Props)]; +initial_schema_properties_([{index_plugins, Plugins}|Props]) -> + lists:foreach(fun({Name, Module, Function}) -> + verify_index_plugin(Name, Module, Function) + end, Plugins), + [{mnesia_index_plugins, Plugins}|initial_schema_properties_(Props)]; +initial_schema_properties_([P|_Props]) -> + mnesia:abort({bad_schema_property, P}); +initial_schema_properties_([]) -> + []. + +remove_duplicates([{K,_} = H|T]) -> + [H | remove_duplicates([X || {K1,_} = X <- T, + K1 =/= K])]; +remove_duplicates([]) -> + []. + read_cstructs_from_disc() -> %% Assumptions: %% - local schema lock in global @@ -551,8 +640,9 @@ read_cstructs_from_disc() -> {type, set}], case dets:open_file(make_ref(), Args) of {ok, Tab} -> + ExtTypes = get_ext_types_disc(), Fun = fun({_, _, List}) -> - {continue, list2cs(List)} + {continue, list2cs(List, ExtTypes)} end, Cstructs = dets:traverse(Tab, Fun), dets:close(Tab), @@ -632,7 +722,7 @@ do_insert_schema_ops(_Store, []) -> api_list2cs(List) when is_list(List) -> Name = pick(unknown, name, List, must), - Keys = check_keys(Name, List, record_info(fields, cstruct)), + Keys = check_keys(Name, List), check_duplicates(Name, Keys), list2cs(List); api_list2cs(Other) -> @@ -647,7 +737,15 @@ cs2list(Cs) when is_record(Cs, cstruct) -> cs2list(CreateList) when is_list(CreateList) -> CreateList; -%% since 4.6 +cs2list(Cs) when element(1, Cs) == cstruct, tuple_size(Cs) == 20 -> + Tags = [name,type, + ram_copies,disc_copies,disc_only_copies,external_copies, + load_order,access_mode,majority,index,snmp,local_content, + record_name,attributes, + user_properties,frag_properties,storage_properties, + cookie,version], + rec2list(Tags, Tags, 2, Cs); +%% since vsn-4.6 (protocol 8.2 or older) cs2list(Cs) when element(1, Cs) == cstruct, tuple_size(Cs) == 19 -> Tags = [name,type,ram_copies,disc_copies,disc_only_copies, load_order,access_mode,majority,index,snmp,local_content, @@ -657,8 +755,38 @@ cs2list(Cs) when element(1, Cs) == cstruct, tuple_size(Cs) == 19 -> rec2list(Tags, Tags, 2, Cs). cs2list(false, Cs) -> - cs2list(Cs). + cs2list(Cs); +cs2list({8,3}, Cs) -> + cs2list(Cs); +cs2list({8,Minor}, Cs) when Minor =:= 2; Minor =:= 1 -> + Orig = record_info(fields, cstruct), + Tags = [name,type,ram_copies,disc_copies,disc_only_copies, + load_order,access_mode,majority,index,snmp,local_content, + record_name,attributes, + user_properties,frag_properties,storage_properties, + cookie,version], + CsList = rec2list(Tags, Orig, 2, Cs), + case proplists:get_value(index, CsList, []) of + [] -> CsList; + NewFormat -> + OldFormat = [Pos || {Pos, _Pref} <- NewFormat], + lists:keyreplace(index, 1, CsList, {index, OldFormat}) + end. +rec2list([index | Tags], [index|Orig], Pos, Rec) -> + Val = element(Pos, Rec), + [{index, lists:map( + fun({_, _Type}=P) -> P; + (P) when is_integer(P); is_atom(P) -> {P, ordered} + end, Val)} | rec2list(Tags, Orig, Pos + 1, Rec)]; +rec2list([external_copies | Tags], Orig0, Pos, Rec) -> + Orig = case Orig0 of + [external_copies|Rest] -> Rest; + _ -> Orig0 + end, + Val = element(Pos, Rec), + [{Alias, Ns} || {{Alias,_}, Ns} <- Val] + ++ rec2list(Tags, Orig, Pos+1, Rec); rec2list([Tag | Tags], [Tag | Orig], Pos, Rec) -> Val = element(Pos, Rec), [{Tag, Val} | rec2list(Tags, Orig, Pos + 1, Rec)]; @@ -667,17 +795,33 @@ rec2list([], _, _Pos, _Rec) -> rec2list(Tags, [_|Orig], Pos, Rec) -> rec2list(Tags, Orig, Pos+1, Rec). -normalize_cs(Cstructs, _Node) -> - Cstructs. +normalize_cs(Cstructs, Node) -> + %% backward-compatibility hack; normalize before returning + case need_old_cstructs([Node]) of + false -> + Cstructs; + Version -> + %% some other format + [convert_cs(Version, Cs) || Cs <- Cstructs] + end. + +convert_cs(Version, Cs) -> + Fields = [Value || {_, Value} <- cs2list(Version, Cs)], + list_to_tuple([cstruct|Fields]). + +list2cs(List) -> + list2cs(List, get_ext_types()). -list2cs(List) when is_list(List) -> +list2cs(List, ExtTypes) when is_list(List) -> Name = pick(unknown, name, List, must), Type = pick(Name, type, List, set), Rc0 = pick(Name, ram_copies, List, []), Dc = pick(Name, disc_copies, List, []), Doc = pick(Name, disc_only_copies, List, []), - Rc = case {Rc0, Dc, Doc} of - {[], [], []} -> [node()]; + + Ext = pick_external_copies(List, ExtTypes), + Rc = case {Rc0, Dc, Doc, Ext} of + {[], [], [], []} -> [node()]; _ -> Rc0 end, LC = pick(Name, local_content, List, false), @@ -695,8 +839,6 @@ list2cs(List) when is_list(List) -> Ix = pick(Name, index, List, []), verify({alt, [nil, list]}, mnesia_lib:etype(Ix), {bad_type, Name, {index, [Ix]}}), - Ix2 = [attr_to_pos(I, Attrs) || I <- Ix], - Frag = pick(Name, frag_properties, List, []), verify({alt, [nil, list]}, mnesia_lib:etype(Frag), {badarg, Name, {frag_properties, Frag}}), @@ -723,24 +865,48 @@ list2cs(List) when is_list(List) -> DetsOpts = proplists:get_value(dets, BEProps, []), is_list(DetsOpts) orelse mnesia:abort({badarg, Name, {dets, DetsOpts}}), [CheckProp(Prop, BadDetsOpts) || Prop <- DetsOpts], - #cstruct{name = Name, - ram_copies = Rc, - disc_copies = Dc, - disc_only_copies = Doc, - type = Type, - index = Ix2, - snmp = Snmp, - load_order = LoadOrder, - access_mode = AccessMode, - majority = Majority, - local_content = LC, - record_name = RecName, - attributes = Attrs, - user_properties = lists:sort(UserProps), - frag_properties = lists:sort(Frag), - storage_properties = lists:sort(BEProps), - cookie = Cookie, - version = Version}. + + case lists:keymember(mnesia, 1, application:which_applications()) of + true -> + Keys = check_keys(Name, List), + check_duplicates(Name, Keys); + false -> + %% check_keys/2 cannot be executed when mnesia is not + %% running, due to it not being possible to read what ext + %% backends are loaded. + %%% this doesn't work - disabled for now: + %%%Keys = check_keys(Name, List, record_info(fields, cstruct)), + %%%check_duplicates(Name, Keys) + ignore + end, + + Cs0 = #cstruct{name = Name, + ram_copies = Rc, + disc_copies = Dc, + disc_only_copies = Doc, + external_copies = Ext, + type = Type, + index = Ix, + snmp = Snmp, + load_order = LoadOrder, + access_mode = AccessMode, + majority = Majority, + local_content = LC, + record_name = RecName, + attributes = Attrs, + user_properties = lists:sort(UserProps), + frag_properties = lists:sort(Frag), + storage_properties = lists:sort(BEProps), + cookie = Cookie, + version = Version}, + case Ix of + [] -> Cs0; + [_|_] -> + Ix2 = expand_index_attrs(Cs0), + Cs0#cstruct{index = Ix2} + end; +list2cs(Other, _ExtTypes) -> + mnesia:abort({badarg, Other}). pick(Tab, Key, List, Default) -> case lists:keysearch(Key, 1, List) of @@ -754,6 +920,79 @@ pick(Tab, Key, List, Default) -> mnesia:abort({bad_type, Tab, BadArg}) end. +pick_external_copies(_List, []) -> + []; +pick_external_copies(List, ExtTypes) -> + lists:foldr( + fun({K, Val}, Acc) -> + case lists:keyfind(K, 1, ExtTypes) of + false -> + Acc; + {_, Mod} -> + [{{K,Mod}, Val}|Acc] + end + end, [], List). + +expand_storage_type(S) when S==ram_copies; + S==disc_copies; + S==disc_only_copies -> + S; +expand_storage_type(S) -> + case lists:keyfind(S, 1, get_ext_types()) of + false -> + mnesia:abort({bad_type, {storage_type, S}}); + {Alias, Mod} -> + {ext, Alias, Mod} + end. + +get_ext_types() -> + get_schema_user_property(mnesia_backend_types). + +get_index_plugins() -> + get_schema_user_property(mnesia_index_plugins). + +get_schema_user_property(Key) -> + Tab = schema, + %% Must work reliably both within transactions and outside of transactions + Res = case get(mnesia_activity_state) of + undefined -> + dirty_read_table_property(Tab, Key); + _ -> + do_read_table_property(Tab, Key) + end, + case Res of + undefined -> + []; + {_, Types} -> + Types + end. + +get_ext_types_disc() -> + try get_ext_types_disc_() + catch + error:_ ->[] + end. + +get_ext_types_disc_() -> + case mnesia_schema:remote_read_schema() of + {ok, _, Prop} -> + K1 = user_properties, + case lists:keyfind(K1, 1, Prop) of + {K1, UserProp} -> + K2 = mnesia_backend_types, + case lists:keyfind(K2, 1, UserProp) of + {K2, Types} -> + Types; + _ -> + [] + end; + _ -> + [] + end; + _ -> + [] + end. + %% Convert attribute name to integer if neccessary attr_tab_to_pos(_Tab, Pos) when is_integer(Pos) -> Pos; @@ -761,6 +1000,7 @@ attr_tab_to_pos(Tab, Attr) -> attr_to_pos(Attr, val({Tab, attributes})). %% Convert attribute name to integer if neccessary +attr_to_pos({_} = P, _) -> P; attr_to_pos(Pos, _Attrs) when is_integer(Pos) -> Pos; attr_to_pos(Attr, Attrs) when is_atom(Attr) -> @@ -775,8 +1015,18 @@ attr_to_pos(Attr, [_ | Attrs], Pos) -> attr_to_pos(Attr, _, _) -> mnesia:abort({bad_type, Attr}). +check_keys(Tab, Attrs) -> + Types = [T || {T,_} <- get_ext_types()], + check_keys(Tab, Attrs, Types ++ record_info(fields, cstruct)). + check_keys(Tab, [{Key, _Val} | Tail], Items) -> - case lists:member(Key, Items) of + Key1 = if + is_tuple(Key) -> + element(1, Key); + true -> + Key + end, + case lists:member(Key1, Items) of true -> [Key | check_keys(Tab, Tail, Items)]; false -> mnesia:abort({badarg, Tab, Key}) end; @@ -800,7 +1050,92 @@ has_duplicates([]) -> false. %% This is the only place where we check the validity of data -verify_cstruct(Cs) when is_record(Cs, cstruct) -> + +verify_cstruct(#cstruct{} = Cs) -> + assert_correct_cstruct(Cs), + Cs1 = verify_external_copies( + Cs#cstruct{index = expand_index_attrs(Cs)}), + assert_correct_cstruct(Cs1), + Cs1. + +expand_index_attrs(#cstruct{index = Ix, attributes = Attrs, + name = Tab} = Cs) -> + Prefered = prefered_index_types(Cs), + expand_index_attrs(Ix, Tab, Attrs, Prefered). + +expand_index_attrs(Ix, Tab, Attrs, Prefered) -> + lists:map(fun(P) when is_integer(P); is_atom(P) -> + {attr_to_pos(P, Attrs), Prefered}; + ({A} = P) when is_atom(A) -> + {P, Prefered}; + ({P, Type}) -> + {attr_to_pos(P, Attrs), Type}; + (_Other) -> + mnesia:abort({bad_type, Tab, {index, Ix}}) + end, Ix). + +prefered_index_types(#cstruct{external_copies = Ext}) -> + ExtTypes = [mnesia_lib:semantics(S, index_types) || + {S,Ns} <- Ext, Ns =/= []], + case intersect_types(ExtTypes) of + [] -> ordered; + [Pref|_] -> Pref + end. + +intersect_types([]) -> + []; +intersect_types([S1, S2|Rest]) -> + intersect_types([S1 -- (S1 -- S2)|Rest]); +intersect_types([S]) -> + S. + +verify_external_copies(#cstruct{external_copies = []} = Cs) -> + Cs; +verify_external_copies(#cstruct{name = Tab, external_copies = EC} = Cs) -> + Bad = {bad_type, Tab, {external_copies, EC}}, + AllECNodes = lists:concat([Ns || {_, Ns} <- EC, + is_list(Ns)]), + verify(true, length(lists:usort(AllECNodes)) == length(AllECNodes), Bad), + CsL = cs2list(Cs), + CsL1 = lists:foldl( + fun({{Alias, Mod}, Ns} = _X, CsLx) -> + BadTab = fun(Why) -> + {Why, Tab, {{ext, Alias, Mod},Ns}} + end, + verify(atom, mnesia_lib:etype(Mod), BadTab), + verify(true, fun() -> + lists:all(fun is_atom/1, Ns) + end, BadTab), + check_semantics(Mod, Alias, BadTab, Cs), + try Mod:check_definition(Alias, Tab, Ns, CsLx) of + ok -> + CsLx; + {ok, CsLx1} -> + CsLx1; + {error, Reason} -> + mnesia:abort(BadTab(Reason)) + catch + error:E -> + mnesia:abort(BadTab(E)) + end; + (_, CsLx) -> + CsLx + end, CsL, EC), + list2cs(CsL1). + +check_semantics(Mod, Alias, BadTab, #cstruct{type = Type}) -> + Ext = {ext, Alias, Mod}, + case lists:member(mnesia_lib:semantics(Ext, storage), [ram_copies, disc_copies, + disc_only_copies]) of + false -> mnesia:abort(BadTab(invalid_storage)); + true -> ok + end, + case lists:member(Type, mnesia_lib:semantics(Ext, types)) of + false -> mnesia:abort(BadTab(bad_type)); + true -> ok + end. + +assert_correct_cstruct(Cs) when is_record(Cs, cstruct) -> verify_nodes(Cs), Tab = Cs#cstruct.name, @@ -841,22 +1176,30 @@ verify_cstruct(Cs) when is_record(Cs, cstruct) -> Attrs), Index = Cs#cstruct.index, + verify({alt, [nil, list]}, mnesia_lib:etype(Index), {bad_type, Tab, {index, Index}}), + IxPlugins = get_index_plugins(), + AllowIndexOnKey = check_if_allow_index_on_key(), IxFun = - fun(Pos) -> - verify(true, fun() -> - if - is_integer(Pos), - Pos > 2, - Pos =< Arity -> - true; - true -> false - end - end, - {bad_type, Tab, {index, [Pos]}}) - end, + fun(Pos) -> + verify( + true, fun() -> + I = index_pos(Pos), + case Pos of + {_, T} -> + (T==bag orelse T==ordered) + andalso good_ix_pos( + I, AllowIndexOnKey, + Arity, IxPlugins); + _ -> + good_ix_pos(Pos, AllowIndexOnKey, + Arity, IxPlugins) + end + end, + {bad_type, Tab, {index, [Pos]}}) + end, lists:foreach(IxFun, Index), LC = Cs#cstruct.local_content, @@ -880,7 +1223,9 @@ verify_cstruct(Cs) when is_record(Cs, cstruct) -> {badarg, Tab, {snmp, Snmp}}), CheckProp = fun(Prop) when is_tuple(Prop), size(Prop) >= 1 -> ok; - (Prop) -> mnesia:abort({bad_type, Tab, {user_properties, [Prop]}}) + (Prop) -> + mnesia:abort({bad_type, Tab, + {user_properties, [Prop]}}) end, lists:foreach(CheckProp, Cs#cstruct.user_properties), @@ -900,17 +1245,45 @@ verify_cstruct(Cs) when is_record(Cs, cstruct) -> mnesia:abort({bad_type, Tab, {version, Version}}) end. +good_ix_pos({_} = P, _, _, Plugins) -> + lists:keymember(P, 1, Plugins); +good_ix_pos(I, true, Arity, _) when is_integer(I) -> + I >= 0 andalso I =< Arity; +good_ix_pos(I, false, Arity, _) when is_integer(I) -> + I > 2 andalso I =< Arity; +good_ix_pos(_, _, _, _) -> + false. + + +check_if_allow_index_on_key() -> + case mnesia_monitor:get_env(allow_index_on_key) of + true -> + true; + _ -> + false + end. + verify_nodes(Cs) -> Tab = Cs#cstruct.name, Ram = Cs#cstruct.ram_copies, Disc = Cs#cstruct.disc_copies, DiscOnly = Cs#cstruct.disc_only_copies, + Ext = lists:append([Ns || {_,Ns} <- Cs#cstruct.external_copies]), LoadOrder = Cs#cstruct.load_order, verify({alt, [nil, list]}, mnesia_lib:etype(Ram), {bad_type, Tab, {ram_copies, Ram}}), verify({alt, [nil, list]}, mnesia_lib:etype(Disc), {bad_type, Tab, {disc_copies, Disc}}), + lists:foreach( + fun({BE, Ns}) -> + verify({alt, [nil, list]}, mnesia_lib:etype(Ns), + {bad_type, Tab, {BE, Ns}}), + lists:foreach(fun(N) -> + verify(atom, mnesia_lib:etype(N), + {bad_type, Tab, {BE, Ns}}) + end, Ns) + end, Cs#cstruct.external_copies), case Tab of schema -> verify([], DiscOnly, {bad_type, Tab, {disc_only_copies, DiscOnly}}); @@ -922,12 +1295,15 @@ verify_nodes(Cs) -> verify(integer, mnesia_lib:etype(LoadOrder), {bad_type, Tab, {load_order, LoadOrder}}), - Nodes = Ram ++ Disc ++ DiscOnly, + Nodes = Ram ++ Disc ++ DiscOnly ++ Ext, verify(list, mnesia_lib:etype(Nodes), {combine_error, Tab, - [{ram_copies, []}, {disc_copies, []}, {disc_only_copies, []}]}), + [{ram_copies, []}, {disc_copies, []}, + {disc_only_copies, []}, {external_copies, []}]}), verify(false, has_duplicates(Nodes), {combine_error, Tab, Nodes}), - AtomCheck = fun(N) -> verify(atom, mnesia_lib:etype(N), {bad_type, Tab, N}) end, + AtomCheck = fun(N) -> + verify(atom, mnesia_lib:etype(N), {bad_type, Tab, N}) + end, lists:foreach(AtomCheck, Nodes). verify(Expected, Fun, Error) when is_function(Fun) -> @@ -1010,28 +1386,194 @@ check_active([{badrpc, Reason} | _Replies], Expl, Tab) -> check_active([], _Expl, _Tab) -> ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Function for definining an external backend type + +add_backend_type(Name, Module) -> + case schema_transaction(fun() -> do_add_backend_type(Name, Module) end) of + {atomic, NeedsInit} -> + case NeedsInit of + true -> + Module:init_backend(); + false -> + ignore + end, + Module:add_aliases([Name]), + {atomic, ok}; + Other -> + Other + end. + +do_add_backend_type(Name, Module) -> + verify_backend_type(Name, Module), + Types = case do_read_table_property(schema, mnesia_backend_types) of + undefined -> + []; + {_, Ts} -> + case lists:keymember(Name, 1, Ts) of + true -> + mnesia:abort({backend_type_already_exists, Name}); + false -> + Ts + end + end, + ModuleRegistered = lists:keymember(Module, 2, Types), + do_write_table_property(schema, {mnesia_backend_types, + [{Name, Module}|Types]}), + ModuleRegistered. + +delete_backend_type(Name) -> + schema_transaction(fun() -> do_delete_backend_type(Name) end). + +do_delete_backend_type(Name) -> + case do_read_table_property(schema, mnesia_backend_types) of + undefined -> + []; + {_, Ts} -> + case lists:keyfind(Name, 1, Ts) of + {_, Mod} -> + case using_backend_type(Name, Mod) of + [_|_] = Tabs -> + mnesia:abort({backend_in_use, {Name, Tabs}}); + [] -> + do_write_table_property( + schema, {mnesia_backend_types, + lists:keydelete(Name, 1, Ts)}) + end; + false -> + mnesia:abort({no_such_backend, Name}) + end + end. + +using_backend_type(Name, Mod) -> + Ext = ets:select(mnesia_gvar, + [{ {{'$1',external_copies},'$2'}, [], [{{'$1','$2'}}] }]), + Entry = {Name, Mod}, + [T || {T,C} <- Ext, + lists:keymember(Entry, 1, C)]. + +verify_backend_type(Name, Module) -> + case legal_backend_name(Name) of + false -> + mnesia:abort({bad_type, {backend_type,Name,Module}}); + true -> + ok + end, + ExpectedExports = mnesia_backend_type:behaviour_info(callbacks), + Exports = try Module:module_info(exports) + catch + error:_ -> + mnesia:abort({undef_backend, Module}) + end, + case ExpectedExports -- Exports of + [] -> + ok; + _Other -> + io:fwrite(user, "Missing backend_type exports: ~p~n", [_Other]), + mnesia:abort({bad_type, {backend_type,Name,Module}}) + end. + +legal_backend_name(Name) -> + is_atom(Name) andalso + (not lists:member(Name, record_info(fields, cstruct))). + +%% Used e.g. by mnesia:system_info(backend_types). +backend_types() -> + [ram_copies, disc_copies, disc_only_copies | + [T || {T,_} <- get_ext_types()]]. + +add_index_plugin(Name, Module, Function) -> + schema_transaction( + fun() -> do_add_index_plugin(Name, Module, Function) end). + +do_add_index_plugin(Name, Module, Function) -> + verify_index_plugin(Name, Module, Function), + Plugins = case do_read_table_property(schema, mnesia_index_plugins) of + undefined -> + []; + {_, Ps} -> + case lists:keymember(Name, 1, Ps) of + true -> + mnesia:abort({index_plugin_already_exists, Name}); + false -> + Ps + end + end, + do_write_table_property(schema, {mnesia_index_plugins, + [{Name, Module, Function}|Plugins]}). + +delete_index_plugin(P) -> + schema_transaction( + fun() -> do_delete_index_plugin(P) end). + +do_delete_index_plugin({A} = P) when is_atom(A) -> + Plugins = get_index_plugins(), + case lists:keyfind(P, 1, Plugins) of + false -> + mnesia:abort({no_exists, {index_plugin, P}}); + _Found -> + case ets:select(mnesia_gvar, + [{ {{'$1',{index,{P,'_'}}},'_'},[],['$1']}, + { {{'$1',{index,P}},'_'},[],['$1']}], 1) of + {[_], _} -> + mnesia:abort({plugin_in_use, P}); + '$end_of_table' -> + do_write_table_property( + schema, {mnesia_index_plugins, + lists:keydelete(P, 1, Plugins)}) + end + end. + +verify_index_plugin({A} = Name, Module, Function) + when is_atom(A), is_atom(Module), is_atom(Function) -> + case code:ensure_loaded(Module) of + {error, nofile} -> + mnesia:abort({bad_type, {index_plugin,Name,Module,Function}}); + {module,_} -> + %% Index plugins are called as Module:Function(Tab, Pos, Obj) + case erlang:function_exported(Module, Function, 3) of + true -> + ok; + false -> + mnesia:abort( + {bad_type, {index_plugin,Name,Module,Function}}) + end + end; +verify_index_plugin(Name, Module, Function) -> + mnesia:abort({bad_type, {index_plugin,Name,Module,Function}}). + + +%% Used e.g. by mnesia:system_info(backend_types). +index_plugins() -> + get_index_plugins(). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Here's the real interface function to create a table -create_table(TabDef) -> - schema_transaction(fun() -> do_multi_create_table(TabDef) end). +create_table([_|_] = TabDef) -> + schema_transaction(fun() -> do_multi_create_table(TabDef) end); +create_table(Arg) -> {aborted, {badarg, Arg}}. %% And the corresponding do routines .... do_multi_create_table(TabDef) -> get_tid_ts_and_lock(schema, write), ensure_writable(schema), + do_create_table(TabDef), + ok. + +do_create_table(TabDef) when is_list(TabDef) -> Cs = api_list2cs(TabDef), case Cs#cstruct.frag_properties of [] -> - do_create_table(Cs); + do_create_table_1(Cs); _Props -> CsList = mnesia_frag:expand_cstruct(Cs), - lists:foreach(fun do_create_table/1, CsList) - end, - ok. + lists:foreach(fun do_create_table_1/1, CsList) + end. -do_create_table(Cs) -> +do_create_table_1(Cs) -> {_Mod, _Tid, Ts} = get_tid_ts_and_lock(schema, none), Store = Ts#tidstore.store, do_insert_schema_ops(Store, make_create_table(Cs)). @@ -1041,14 +1583,9 @@ make_create_table(Cs) -> verify(false, check_if_exists(Tab), {already_exists, Tab}), unsafe_make_create_table(Cs). -% unsafe_do_create_table(Cs) -> -% {_Mod, Tid, Ts} = get_tid_ts_and_lock(schema, none), -% Store = Ts#tidstore.store, -% do_insert_schema_ops(Store, unsafe_make_create_table(Cs)). - -unsafe_make_create_table(Cs) -> +unsafe_make_create_table(Cs0) -> {_Mod, Tid, Ts} = get_tid_ts_and_lock(schema, none), - verify_cstruct(Cs), + Cs = verify_cstruct(Cs0), Tab = Cs#cstruct.name, %% Check that we have all disc replica nodes running @@ -1196,8 +1733,7 @@ make_add_table_copy(Tab, Node, Storage) -> Cs = incr_version(val({Tab, cstruct})), Ns = mnesia_lib:cs_to_nodes(Cs), verify(false, lists:member(Node, Ns), {already_exists, Tab, Node}), - Cs2 = new_cs(Cs, Node, Storage, add), - verify_cstruct(Cs2), + Cs2 = verify_cstruct(new_cs(Cs, Node, Storage, add)), %% Check storage and if node is running IsRunning = lists:member(Node, val({current, db_nodes})), @@ -1245,14 +1781,14 @@ make_del_table_copy(Tab, Node) -> _ when Tab == schema -> %% ensure_active(Cs2), ensure_not_active(Tab, Node), - verify_cstruct(Cs2), + Cs3 = verify_cstruct(Cs2), Ops = remove_node_from_tabs(val({schema, tables}), Node), - [{op, del_table_copy, ram_copies, Node, vsn_cs2list(Cs2)} | Ops]; + [{op, del_table_copy, ram_copies, Node, vsn_cs2list(Cs3)} | Ops]; _ -> ensure_active(Cs), - verify_cstruct(Cs2), - get_tid_ts_and_lock(Tab, write), - [{op, del_table_copy, Storage, Node, vsn_cs2list(Cs2)}] + Cs3 = verify_cstruct(Cs2), + get_tid_ts_and_lock(Tab, write), + [{op, del_table_copy, Storage, Node, vsn_cs2list(Cs3)}] end. remove_node_from_tabs([], _Node) -> @@ -1278,9 +1814,9 @@ remove_node_from_tabs([Tab|Rest], Node) -> [{op, delete_table, vsn_cs2list(Cs)} | remove_node_from_tabs(Rest, Node)]; _Ns -> - verify_cstruct(Cs2), + Cs3 = verify_cstruct(Cs2), get_tid_ts_and_lock(Tab, write), - [{op, del_table_copy, ram_copies, Node, vsn_cs2list(Cs2)}| + [{op, del_table_copy, ram_copies, Node, vsn_cs2list(Cs3)}| remove_node_from_tabs(Rest, Node)] end end. @@ -1298,8 +1834,34 @@ new_cs(Cs, Node, disc_copies, del) -> new_cs(Cs, Node, disc_only_copies, del) -> Cs#cstruct{disc_only_copies = lists:delete(Node , Cs#cstruct.disc_only_copies)}; -new_cs(Cs, _Node, Storage, _Op) -> - mnesia:abort({badarg, Cs#cstruct.name, Storage}). +new_cs(#cstruct{external_copies = ExtCps} = Cs, Node, Storage0, Op) -> + Storage = case Storage0 of + {ext, Alias, _} -> Alias; + Alias -> Alias + end, + ExtTypes = get_ext_types(), + case lists:keyfind(Storage, 1, ExtTypes) of + false -> + mnesia:abort({badarg, Cs#cstruct.name, Storage}); + {_, Mod} -> + Key = {Storage, Mod}, + case {lists:keymember(Key, 1, ExtCps), Op} of + {false, del} -> + Cs; + {false, add} -> + Cs#cstruct{external_copies = [{Key, [Node]}|ExtCps]}; + {true, _} -> + F = fun({K, Ns}) when K == Key -> + case Op of + del -> {K, lists:delete(Node, Ns)}; + add -> {K, opt_add(Node, Ns)} + end; + (X) -> + X + end, + Cs#cstruct{external_copies = lists:map(F, ExtCps)} + end + end. opt_add(N, L) -> [N | lists:delete(N, L)]. @@ -1335,8 +1897,7 @@ make_move_table(Tab, FromNode, ToNode) -> verify(true, lists:member(ToNode, Running), {not_active, schema, ToNode}), Cs2 = new_cs(Cs, ToNode, Storage, add), - Cs3 = new_cs(Cs2, FromNode, Storage, del), - verify_cstruct(Cs3), + Cs3 = verify_cstruct(new_cs(Cs2, FromNode, Storage, del)), [{op, add_table_copy, Storage, ToNode, vsn_cs2list(Cs2)}, {op, sync_trans}, {op, del_table_copy, Storage, FromNode, vsn_cs2list(Cs3)}]. @@ -1363,9 +1924,11 @@ make_change_table_copy_type(Tab, Node, ToS) -> Cs = incr_version(val({Tab, cstruct})), FromS = mnesia_lib:storage_type_at_node(Node, Tab), - case compare_storage_type(false, FromS, ToS) of + ToSExp = expand_storage_type(ToS), + + case compare_storage_type(false, FromS, ToSExp) of {same, _} -> - mnesia:abort({already_exists, Tab, Node, ToS}); + mnesia:abort({already_exists, Tab, Node, ToSExp}); {diff, _} -> ignore; incompatible -> @@ -1373,10 +1936,8 @@ make_change_table_copy_type(Tab, Node, ToS) -> end, Cs2 = new_cs(Cs, Node, FromS, del), - Cs3 = new_cs(Cs2, Node, ToS, add), - verify_cstruct(Cs3), - - [{op, change_table_copy_type, Node, FromS, ToS, vsn_cs2list(Cs3)}]. + Cs3 = verify_cstruct(new_cs(Cs2, Node, ToS, add)), + [{op, change_table_copy_type, Node, FromS, ToSExp, vsn_cs2list(Cs3)}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% change index functions .... @@ -1398,11 +1959,12 @@ make_add_table_index(Tab, Pos) -> Cs = incr_version(val({Tab, cstruct})), ensure_active(Cs), Ix = Cs#cstruct.index, - verify(false, lists:member(Pos, Ix), {already_exists, Tab, Pos}), + verify(false, lists:keymember(index_pos(Pos), 1, Ix), + {already_exists, Tab, Pos}), Ix2 = lists:sort([Pos | Ix]), - Cs2 = Cs#cstruct{index = Ix2}, - verify_cstruct(Cs2), - [{op, add_index, Pos, vsn_cs2list(Cs2)}]. + Cs2 = verify_cstruct(Cs#cstruct{index = Ix2}), + NewPosInfo = lists:keyfind(Pos, 1, Cs2#cstruct.index), + [{op, add_index, NewPosInfo, vsn_cs2list(Cs2)}]. del_table_index(Tab, Pos) -> schema_transaction(fun() -> do_del_table_index(Tab, Pos) end). @@ -1420,9 +1982,8 @@ make_del_table_index(Tab, Pos) -> Cs = incr_version(val({Tab, cstruct})), ensure_active(Cs), Ix = Cs#cstruct.index, - verify(true, lists:member(Pos, Ix), {no_exists, Tab, Pos}), - Cs2 = Cs#cstruct{index = lists:delete(Pos, Ix)}, - verify_cstruct(Cs2), + verify(true, lists:keymember(Pos, 1, Ix), {no_exists, Tab, Pos}), + Cs2 = verify_cstruct(Cs#cstruct{index = lists:keydelete(Pos, 1, Ix)}), [{op, del_index, Pos, vsn_cs2list(Cs2)}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1444,8 +2005,7 @@ make_add_snmp(Tab, Ustruct) -> verify([], Cs#cstruct.snmp, {already_exists, Tab, snmp}), Error = {badarg, Tab, snmp, Ustruct}, verify(true, mnesia_snmp_hook:check_ustruct(Ustruct), Error), - Cs2 = Cs#cstruct{snmp = Ustruct}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{snmp = Ustruct}), [{op, add_snmp, Ustruct, vsn_cs2list(Cs2)}]. del_snmp(Tab) -> @@ -1462,8 +2022,7 @@ make_del_snmp(Tab) -> ensure_writable(schema), Cs = incr_version(val({Tab, cstruct})), ensure_active(Cs), - Cs2 = Cs#cstruct{snmp = []}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{snmp = []}), [{op, del_snmp, vsn_cs2list(Cs2)}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1492,19 +2051,21 @@ make_transform(Tab, Fun, NewAttrs, NewRecName) -> Cs = incr_version(val({Tab, cstruct})), ensure_active(Cs), ensure_writable(Tab), - case mnesia_lib:val({Tab, index}) of + case Cs#cstruct.index of [] -> - Cs2 = Cs#cstruct{attributes = NewAttrs, record_name = NewRecName}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct( + Cs#cstruct{attributes = NewAttrs, + record_name = NewRecName}), [{op, transform, Fun, vsn_cs2list(Cs2)}]; PosList -> - DelIdx = fun(Pos, Ncs) -> + DelIdx = fun({Pos,_}, Ncs) -> Ix = Ncs#cstruct.index, - Ncs1 = Ncs#cstruct{index = lists:delete(Pos, Ix)}, + Ix2 = lists:keydelete(Pos, 1, Ix), + Ncs1 = Ncs#cstruct{index = Ix2}, Op = {op, del_index, Pos, vsn_cs2list(Ncs1)}, {Op, Ncs1} end, - AddIdx = fun(Pos, Ncs) -> + AddIdx = fun({_,_} = Pos, Ncs) -> Ix = Ncs#cstruct.index, Ix2 = lists:sort([Pos | Ix]), Ncs1 = Ncs#cstruct{index = Ix2}, @@ -1514,10 +2075,16 @@ make_transform(Tab, Fun, NewAttrs, NewRecName) -> {DelOps, Cs1} = lists:mapfoldl(DelIdx, Cs, PosList), Cs2 = Cs1#cstruct{attributes = NewAttrs, record_name = NewRecName}, {AddOps, Cs3} = lists:mapfoldl(AddIdx, Cs2, PosList), - verify_cstruct(Cs3), - lists:flatten([DelOps, {op, transform, Fun, vsn_cs2list(Cs2)}, AddOps]) + _ = verify_cstruct(Cs3), % just a sanity check + lists:flatten([DelOps, {op, transform, Fun, vsn_cs2list(Cs2)}, + AddOps]) end. +index_pos({Pos,_}) -> Pos; +index_pos(Pos) when is_integer(Pos) -> Pos; +index_pos({P} = Pos) when is_atom(P) -> Pos. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% @@ -1537,8 +2104,7 @@ make_change_table_access_mode(Tab, Mode) -> ensure_active(Cs), OldMode = Cs#cstruct.access_mode, verify(false, OldMode == Mode, {already_exists, Tab, Mode}), - Cs2 = Cs#cstruct{access_mode = Mode}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{access_mode = Mode}), [{op, change_table_access_mode, vsn_cs2list(Cs2), OldMode, Mode}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1558,8 +2124,7 @@ make_change_table_load_order(Tab, LoadOrder) -> Cs = incr_version(val({Tab, cstruct})), ensure_active(Cs), OldLoadOrder = Cs#cstruct.load_order, - Cs2 = Cs#cstruct{load_order = LoadOrder}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{load_order = LoadOrder}), [{op, change_table_load_order, vsn_cs2list(Cs2), OldLoadOrder, LoadOrder}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1605,6 +2170,7 @@ write_table_property(Tab, Prop) when is_tuple(Prop), size(Prop) >= 1 -> schema_transaction(fun() -> do_write_table_property(Tab, Prop) end); write_table_property(Tab, Prop) -> {aborted, {bad_type, Tab, Prop}}. + do_write_table_property(Tab, Prop) -> TidTs = get_tid_ts_and_lock(schema, write), {_, _, Ts} = TidTs, @@ -1636,8 +2202,7 @@ make_write_table_properties(Tab, [Prop | Props], Cs) -> PropKey = element(1, Prop), DelProps = lists:keydelete(PropKey, 1, OldProps), MergedProps = lists:merge(DelProps, [Prop]), - Cs2 = Cs#cstruct{user_properties = MergedProps}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{user_properties = MergedProps}), [{op, write_property, vsn_cs2list(Cs2), Prop} | make_write_table_properties(Tab, Props, Cs2)]; make_write_table_properties(_Tab, [], _Cs) -> @@ -1682,22 +2247,38 @@ do_read_table_property(Tab, Key) -> {_, _, Ts} = TidTs, Store = Ts#tidstore.store, Props = ets:foldl( - fun({op, create_table, [{name, T}|Opts]}, _Acc) - when T==Tab -> + fun({op, announce_im_running,_,Opts,_,_}, _Acc) when Tab==schema -> + find_props(Opts); + ({op, create_table, [{name, T}|Opts]}, _Acc) + when T==Tab -> find_props(Opts); ({op, Op, [{name,T}|Opts], _Prop}, _Acc) - when T==Tab, Op==write_property; Op==delete_property -> + when T==Tab, Op==write_property; + T==Tab, Op==delete_property -> find_props(Opts); ({op, delete_table, [{name,T}|_]}, _Acc) when T==Tab -> []; (_Other, Acc) -> Acc - end, [], Store), - case lists:keysearch(Key, 1, Props) of - {value, Property} -> - Property; - false -> + end, undefined, Store), + case Props of + undefined -> + get_tid_ts_and_lock(Tab, read), + dirty_read_table_property(Tab, Key); + _ when is_list(Props) -> + case lists:keyfind(Key, 1, Props) of + false -> + undefined; + Other -> + Other + end + end. + +dirty_read_table_property(Tab, Key) -> + try ets:lookup_element(mnesia_gvar, {Tab,user_property,Key}, 2) + catch + error:_ -> undefined end. @@ -1757,8 +2338,7 @@ make_delete_table_properties(Tab, PropKeys) -> make_delete_table_properties(Tab, [PropKey | PropKeys], Cs) -> OldProps = Cs#cstruct.user_properties, Props = lists:keydelete(PropKey, 1, OldProps), - Cs2 = Cs#cstruct{user_properties = Props}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{user_properties = Props}), [{op, delete_property, vsn_cs2list(Cs2), PropKey} | make_delete_table_properties(Tab, PropKeys, Cs2)]; make_delete_table_properties(_Tab, [], _Cs) -> @@ -1899,6 +2479,11 @@ prepare_op(Tid, {op, create_table, TabDef}, _WaitFor) -> create_disc_only_table(Tab,Cs), insert_cstruct(Tid, Cs, false), {true, optional}; + {ext, Alias, Mod} -> + mnesia_lib:set({Tab, create_table},true), + create_external_table(Alias, Tab, Mod, Cs), + insert_cstruct(Tid, Cs, false), + {true, optional}; unknown -> %% No replica on this node mnesia_lib:set({Tab, create_table},true), insert_cstruct(Tid, Cs, false), @@ -1967,6 +2552,12 @@ prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) NotActive = mnesia_lib:not_active_here(Tab), + if Tab =/= schema -> + check_if_disc_required(FromS, ToS); + true -> + ok + end, + if NotActive == true -> mnesia:abort({not_active, Tab, node()}); @@ -2006,6 +2597,15 @@ prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) mnesia:abort({combine_error, Tab, ToS}) end; + element(1,FromS) == ext; element(1,ToS) == ext -> + if ToS == ram_copies -> + create_ram_table(Tab, Cs); + true -> + ok + end, + mnesia_dumper:dump_to_logfile(FromS, Tab), + mnesia_checkpoint:tm_change_table_copy_type(Tab, FromS, ToS); + FromS == ram_copies -> case mnesia_monitor:use_dir() of true -> @@ -2021,7 +2621,9 @@ prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) disc_only_copies -> mnesia_dumper:raw_named_dump_table(Tab, dmp) end, - mnesia_checkpoint:tm_change_table_copy_type(Tab, FromS, ToS) + mnesia_checkpoint:tm_change_table_copy_type(Tab, + FromS, + ToS) end; false -> mnesia:abort({has_no_disc, node()}) @@ -2029,6 +2631,7 @@ prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) FromS == disc_copies, ToS == disc_only_copies -> mnesia_dumper:raw_named_dump_table(Tab, dmp); + FromS == disc_only_copies -> Type = Cs#cstruct.type, create_ram_table(Tab, Cs), @@ -2040,6 +2643,7 @@ prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) Err = "Failed to copy disc data to ram", mnesia:abort({system_limit, Tab, {Err,Reason}}) end; + true -> ignore end, @@ -2103,6 +2707,8 @@ prepare_op(_Tid, {op, transform, Fun, TabDef}, _WaitFor) -> {true, Objs, mandatory} catch _:Reason -> mnesia_lib:db_fixtable(Storage, Tab, false), + mnesia_lib:important("Transform function failed: '~p' in '~p'", + [Reason, erlang:get_stacktrace()]), exit({"Bad transform function", Tab, Fun, node(), Reason}) end end; @@ -2119,6 +2725,22 @@ prepare_op(_Tid, {op, merge_schema, TabDef}, _WaitFor) -> prepare_op(_Tid, _Op, _WaitFor) -> {true, optional}. +check_if_disc_required(FromS, ToS) -> + FromSem = mnesia_lib:semantics(FromS, storage), + ToSem = mnesia_lib:semantics(ToS, storage), + case {FromSem, ToSem} of + {ram_copies, _} when ToSem == disc_copies; + ToSem == disc_only_copies -> + case mnesia_monitor:use_dir() of + true -> + ok; + false -> + mnesia:abort({has_no_disc, node()}) + end; + _ -> + ok + end. + create_ram_table(Tab, #cstruct{type=Type, storage_properties=Props}) -> EtsOpts = proplists:get_value(ets, Props, []), Args = [{keypos, 2}, public, named_table, Type | EtsOpts], @@ -2160,6 +2782,14 @@ create_disc_only_table(Tab, #cstruct{type=Type, storage_properties=Props}) -> mnesia:abort({system_limit, Tab, {Err,Reason}}) end. +create_external_table(Alias, Tab, Mod, Cs) -> + case mnesia_monitor:unsafe_create_external(Tab, Alias, Mod, Cs) of + ok -> + ok; + {error,Reason} -> + Err = "Failed to create external table", + mnesia:abort({system_limit, Tab, {Err,Reason}}) + end. receive_sync([], Pids) -> Pids; @@ -2294,7 +2924,10 @@ undo_prepare_op(Tid, {op, create_table, TabDef}) -> mnesia_monitor:unsafe_close_dets(Tab), Dat = mnesia_lib:tab2dat(Tab), %% disc_delete_table(Tab, Storage), - file:delete(Dat) + file:delete(Dat); + {ext, Alias, Mod} -> + Mod:close_table(Alias, Tab), + Mod:delete_table(Alias, Tab) end; undo_prepare_op(Tid, {op, add_table_copy, Storage, Node, TabDef}) -> @@ -2394,6 +3027,8 @@ ram_delete_table(Tab, Storage) -> case Storage of unknown -> ignore; + {ext, _, _} -> + ignore; disc_only_copies -> ignore; _Else -> @@ -2455,13 +3090,27 @@ purge_known_files([File | Tail], KeepFiles, Dir, Suffixes) -> ignore; true -> AbsFile = filename:join([Dir, File]), - file:delete(AbsFile) - end + delete_recursive(AbsFile) + end end, purge_known_files(Tail, KeepFiles, Dir, Suffixes); purge_known_files([], _KeepFiles, _Dir, _Suffixes) -> ok. +%% Removes a directory or file recursively +delete_recursive(Path) -> + case filelib:is_dir(Path) of + true -> + {ok, Names} = file:list_dir(Path), + lists:foreach(fun(Name) -> + delete_recursive(filename:join(Path, Name)) + end, + Names), + file:del_dir(Path); + false -> + file:delete(Path) + end. + has_known_suffix(_File, _Suffixes, true) -> true; has_known_suffix(File, [Suffix | Tail], false) -> @@ -2469,11 +3118,33 @@ has_known_suffix(File, [Suffix | Tail], false) -> has_known_suffix(_File, [], Bool) -> Bool. -known_suffixes() -> real_suffixes() ++ tmp_suffixes(). +known_suffixes() -> known_suffixes(get_ext_types_disc()). + +known_suffixes(Ext) -> real_suffixes(Ext) ++ tmp_suffixes(Ext). -real_suffixes() -> [".DAT", ".LOG", ".BUP", ".DCL", ".DCD"]. +real_suffixes(Ext) -> [".DAT", ".LOG", ".BUP", ".DCL", ".DCD"] ++ ext_real_suffixes(Ext). -tmp_suffixes() -> [".TMP", ".BUPTMP", ".RET", ".DMP"]. +tmp_suffixes() -> tmp_suffixes(get_ext_types_disc()). + +tmp_suffixes(Ext) -> [".TMP", ".BUPTMP", ".RET", ".DMP", "."] ++ ext_tmp_suffixes(Ext). + +ext_real_suffixes(Ext) -> + try lists:foldl(fun(Mod, Acc) -> Acc++Mod:real_suffixes() end, [], + [M || {_,M} <- Ext]) + catch + error:E -> + verbose("Cant find real ext suffixes (~p)~n", [E]), + [] + end. + +ext_tmp_suffixes(Ext) -> + try lists:foldl(fun(Mod, Acc) -> Acc++Mod:tmp_suffixes() end, [], + [M || {_,M} <- Ext]) + catch + error:E -> + verbose("Cant find tmp ext suffixes (~p)~n", [E]), + [] + end. info() -> Tabs = lists:sort(val({schema, tables})), @@ -2591,12 +3262,12 @@ do_restore(R, BupSchema) -> arrange_restore(R, Fun, Recs) -> R2 = R#r{insert_op = Fun, recs = Recs}, - case mnesia_bup:iterate(R#r.module, fun restore_items/4, R#r.opaque, R2) of + case mnesia_bup:iterate(R#r.module, fun restore_items/5, R#r.opaque, R2) of {ok, R3} -> R3#r.recs; {error, Reason} -> mnesia:abort(Reason) end. -restore_items([Rec | Recs], Header, Schema, R) -> +restore_items([Rec | Recs], Header, Schema, Ext, R) -> Tab = element(1, Rec), case lists:keysearch(Tab, 1, R#r.tables) of {value, {Tab, Where0, Snmp, RecName}} -> @@ -2609,13 +3280,13 @@ restore_items([Rec | Recs], Header, Schema, R) -> {Rest, NRecs} = restore_tab_items([Rec | Recs], Tab, RecName, Where, Snmp, R#r.recs, R#r.insert_op), - restore_items(Rest, Header, Schema, R#r{recs = NRecs}); + restore_items(Rest, Header, Schema, Ext, R#r{recs = NRecs}); false -> Rest = skip_tab_items(Recs, Tab), - restore_items(Rest, Header, Schema, R) + restore_items(Rest, Header, Schema, Ext, R) end; -restore_items([], _Header, _Schema, R) -> +restore_items([], _Header, _Schema, _Ext, R) -> R. restore_func(Tab, R) -> @@ -2629,8 +3300,14 @@ restore_func(Tab, R) -> where_to_commit(Tab, CsList) -> Ram = [{N, ram_copies} || N <- pick(Tab, ram_copies, CsList, [])], Disc = [{N, disc_copies} || N <- pick(Tab, disc_copies, CsList, [])], - DiscO = [{N, disc_only_copies} || N <- pick(Tab, disc_only_copies, CsList, [])], - Ram ++ Disc ++ DiscO. + DiscO = [{N, disc_only_copies} || + N <- pick(Tab, disc_only_copies, CsList, [])], + ExtNodes = [{Alias, Mod, pick(Tab, Alias, CsList, [])} || + {Alias, Mod} <- get_ext_types()], + Ext = lists:foldl(fun({Alias, Mod, Ns}, Acc) -> + [{N, {ext, Alias, Mod}} || N <- Ns] ++ Acc + end, [], ExtNodes), + Ram ++ Disc ++ DiscO ++ Ext. %% Changes of the Meta info of schema itself is not allowed restore_schema([{schema, schema, _List} | Schema], R) -> @@ -2709,7 +3386,13 @@ make_dump_tables([schema | _Tabs]) -> make_dump_tables([Tab | Tabs]) -> get_tid_ts_and_lock(Tab, read), TabDef = get_create_list(Tab), - DiscResident = val({Tab, disc_copies}) ++ val({Tab, disc_only_copies}), + DiscResident = + val({Tab, disc_copies}) ++ + val({Tab, disc_only_copies}) ++ + lists:concat([Ns || {{A,M},Ns} <- val({Tab, external_copies}), + lists:member( + mnesia_lib:semantics({ext,A,M},storage), + [disc_copies, disc_only_copies])]), verify([], DiscResident, {"Only allowed on ram_copies", Tab, DiscResident}), [{op, dump_table, unknown, TabDef} | make_dump_tables(Tabs)]; @@ -2721,7 +3404,9 @@ merge_schema() -> schema_transaction(fun() -> do_merge_schema([]) end). merge_schema(UserFun) -> - schema_transaction(fun() -> UserFun(fun(Arg) -> do_merge_schema(Arg) end) end). + schema_transaction(fun() -> + UserFun(fun(Arg) -> do_merge_schema(Arg) end) + end). do_merge_schema(LockTabs0) -> {_Mod, Tid, Ts} = get_tid_ts_and_lock(schema, write), @@ -2795,11 +3480,23 @@ do_merge_schema(LockTabs0) -> end. fetch_cstructs(Node) -> - rpc:call(Node, mnesia_controller, get_remote_cstructs, []). + Convert = mnesia_monitor:needs_protocol_conversion(Node), + case rpc:call(Node, mnesia_controller, get_remote_cstructs, []) of + {cstructs, Cs0, RemoteRunning1} when Convert -> + {cstructs, [list2cs(cs2list(Cs)) || Cs <- Cs0], RemoteRunning1}; + Result -> + Result + end. -need_old_cstructs() -> false. +need_old_cstructs() -> + need_old_cstructs(val({schema, where_to_write})). -need_old_cstructs(_Nodes) -> false. +need_old_cstructs(Nodes) -> + Filter = fun(Node) -> mnesia_monitor:needs_protocol_conversion(Node) end, + case lists:filter(Filter, Nodes) of + [] -> false; + Ns -> lists:min([element(1, ?catch_val({protocol, Node})) || Node <- Ns]) + end. tab_to_nodes(Tab) when is_atom(Tab) -> Cs = val({Tab, cstruct}), @@ -2952,8 +3649,8 @@ do_make_merge_schema(Node, NeedsConv, RemoteCs = #cstruct{}) -> %% invariants must be enforced in order to allow merge of cstructs. %% %% Returns a new cstruct or issues a fatal error -merge_cstructs(Cs, RemoteCs, Force) -> - verify_cstruct(Cs), +merge_cstructs(Cs0, RemoteCs, Force) -> + Cs = verify_cstruct(Cs0), try do_merge_cstructs(Cs, RemoteCs, Force) of MergedCs when is_record(MergedCs, cstruct) -> MergedCs @@ -2963,15 +3660,15 @@ merge_cstructs(Cs, RemoteCs, Force) -> error:Reason -> exit(Reason) end. -do_merge_cstructs(Cs, RemoteCs, Force) -> - verify_cstruct(RemoteCs), +do_merge_cstructs(Cs, RemoteCs0, Force) -> + RemoteCs = verify_cstruct(RemoteCs0), Ns = mnesia_lib:uniq(mnesia_lib:cs_to_nodes(Cs) ++ mnesia_lib:cs_to_nodes(RemoteCs)), {AnythingNew, MergedCs} = merge_storage_type(Ns, false, Cs, RemoteCs, Force), - MergedCs2 = merge_versions(AnythingNew, MergedCs, RemoteCs, Force), - verify_cstruct(MergedCs2), - MergedCs2. + verify_cstruct( + merge_versions(AnythingNew, MergedCs, RemoteCs, Force)). + merge_storage_type([N | Ns], AnythingNew, Cs, RemoteCs, Force) -> Local = mnesia_lib:cs_to_storage_type(N, Cs), @@ -3115,4 +3812,3 @@ unannounce_im_running([N | Ns]) -> unannounce_im_running(Ns); unannounce_im_running([]) -> ok. - diff --git a/lib/mnesia/src/mnesia_snmp_hook.erl b/lib/mnesia/src/mnesia_snmp_hook.erl index 1599ff1e65..58a20c6051 100644 --- a/lib/mnesia/src/mnesia_snmp_hook.erl +++ b/lib/mnesia/src/mnesia_snmp_hook.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_snmp_sup.erl b/lib/mnesia/src/mnesia_snmp_sup.erl index 862eaaec77..ed579d01c5 100644 --- a/lib/mnesia/src/mnesia_snmp_sup.erl +++ b/lib/mnesia/src/mnesia_snmp_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_sp.erl b/lib/mnesia/src/mnesia_sp.erl index 71c1e7dd0c..d65c659690 100644 --- a/lib/mnesia/src/mnesia_sp.erl +++ b/lib/mnesia/src/mnesia_sp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_subscr.erl b/lib/mnesia/src/mnesia_subscr.erl index 246d0e8697..c2748f5bae 100644 --- a/lib/mnesia/src/mnesia_subscr.erl +++ b/lib/mnesia/src/mnesia_subscr.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_sup.erl b/lib/mnesia/src/mnesia_sup.erl index 3a9360aa6d..4aece81308 100644 --- a/lib/mnesia/src/mnesia_sup.erl +++ b/lib/mnesia/src/mnesia_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -60,9 +60,10 @@ init() -> Flags = {one_for_all, 0, 3600}, % Should be rest_for_one policy Event = event_procs(), + Ext = ext_procs(), Kernel = kernel_procs(), - {ok, {Flags, Event ++ Kernel}}. + {ok, {Flags, Event ++ Ext ++ Kernel}}. event_procs() -> KillAfter = timer:seconds(30), @@ -75,6 +76,11 @@ kernel_procs() -> KA = infinity, [{K, {K, start, []}, permanent, KA, supervisor, [K, supervisor]}]. +ext_procs() -> + K = mnesia_ext_sup, + KA = infinity, + [{K, {K, start, []}, permanent, KA, supervisor, [K, supervisor]}]. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% event handler diff --git a/lib/mnesia/src/mnesia_text.erl b/lib/mnesia/src/mnesia_text.erl index 34655b0318..21adca813a 100644 --- a/lib/mnesia/src/mnesia_text.erl +++ b/lib/mnesia/src/mnesia_text.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/src/mnesia_tm.erl b/lib/mnesia/src/mnesia_tm.erl index 1d3eb87036..305bf14bcf 100644 --- a/lib/mnesia/src/mnesia_tm.erl +++ b/lib/mnesia/src/mnesia_tm.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,7 +42,8 @@ put_activity_id/2, block_tab/1, unblock_tab/1, - fixtable/3 + fixtable/3, + new_cr_format/1 ]). %% sys callback functions @@ -79,6 +80,7 @@ start() -> init(Parent) -> register(?MODULE, self()), process_flag(trap_exit, true), + process_flag(message_queue_data, off_heap), %% Initialize the schema IgnoreFallback = mnesia_monitor:get_env(ignore_fallback_at_startup), @@ -206,10 +208,10 @@ doit_loop(#state{coordinators=Coordinators,participants=Participants,supervisor= {_From, {async_dirty, Tid, Commit, Tab}} -> case lists:member(Tab, State#state.blocked_tabs) of false -> - do_async_dirty(Tid, Commit, Tab), + do_async_dirty(Tid, new_cr_format(Commit), Tab), doit_loop(State); true -> - Item = {async_dirty, Tid, Commit, Tab}, + Item = {async_dirty, Tid, new_cr_format(Commit), Tab}, State2 = State#state{dirty_queue = [Item | State#state.dirty_queue]}, doit_loop(State2) end; @@ -217,10 +219,10 @@ doit_loop(#state{coordinators=Coordinators,participants=Participants,supervisor= {From, {sync_dirty, Tid, Commit, Tab}} -> case lists:member(Tab, State#state.blocked_tabs) of false -> - do_sync_dirty(From, Tid, Commit, Tab), + do_sync_dirty(From, Tid, new_cr_format(Commit), Tab), doit_loop(State); true -> - Item = {sync_dirty, From, Tid, Commit, Tab}, + Item = {sync_dirty, From, Tid, new_cr_format(Commit), Tab}, State2 = State#state{dirty_queue = [Item | State#state.dirty_queue]}, doit_loop(State2) end; @@ -241,10 +243,11 @@ doit_loop(#state{coordinators=Coordinators,participants=Participants,supervisor= reply(From, {error, {system_limit, Msg, Reason}}, State) end; - {From, {ask_commit, Protocol, Tid, Commit, DiscNs, RamNs}} -> + {From, {ask_commit, Protocol, Tid, Commit0, DiscNs, RamNs}} -> ?eval_debug_fun({?MODULE, doit_ask_commit}, [{tid, Tid}, {prot, Protocol}]), mnesia_checkpoint:tm_enter_pending(Tid, DiscNs, RamNs), + Commit = new_cr_format(Commit0), Pid = case Protocol of asym_trans when node(Tid#tid.pid) /= node() -> @@ -948,7 +951,7 @@ return_abort(Fun, Args, Reason) -> if Level == 1 -> mnesia_locker:async_release_tid(Nodes, Tid), - ?MODULE ! {delete_transaction, Tid}, + ?SAFE(?MODULE ! {delete_transaction, Tid}), erase(mnesia_activity_state), flush_downs(), ?SAFE(unlink(whereis(?MODULE))), @@ -1137,14 +1140,14 @@ arrange(Tid, Store, Type) -> reverse([]) -> []; reverse([H=#commit{ram_copies=Ram, disc_copies=DC, - disc_only_copies=DOC,snmp = Snmp} + disc_only_copies=DOC, ext=Ext} |R]) -> [ H#commit{ - ram_copies = lists:reverse(Ram), - disc_copies = lists:reverse(DC), - disc_only_copies = lists:reverse(DOC), - snmp = lists:reverse(Snmp) + ram_copies = lists:reverse(Ram), + disc_copies = lists:reverse(DC), + disc_only_copies = lists:reverse(DOC), + ext = [{Type, lists:reverse(E)} || {Type,E} <- Ext] } | reverse(R)]. @@ -1311,8 +1314,13 @@ pick_node({dirty,_}, Node, [], Done) -> pick_node(_Tid, Node, [], _Done) -> mnesia:abort({bad_commit, {missing_lock, Node}}). -prepare_node(Node, Storage, [Item | Items], Rec, Kind) when Kind == snmp -> - Rec2 = Rec#commit{snmp = [Item | Rec#commit.snmp]}, +prepare_node(Node, Storage, [Item | Items], #commit{ext=Ext0}=Rec, Kind) when Kind == snmp -> + Rec2 = case lists:keytake(snmp, 1, Ext0) of + false -> + Rec#commit{ext = [{snmp,[Item]}|Ext0]}; + {_, {snmp,Snmp},Ext} -> + Rec#commit{ext = [{snmp,[Item|Snmp]}|Ext]} + end, prepare_node(Node, Storage, Items, Rec2, Kind); prepare_node(Node, Storage, [Item | Items], Rec, Kind) when Kind /= schema -> Rec2 = @@ -1323,7 +1331,15 @@ prepare_node(Node, Storage, [Item | Items], Rec, Kind) when Kind /= schema -> Rec#commit{disc_copies = [Item | Rec#commit.disc_copies]}; disc_only_copies -> Rec#commit{disc_only_copies = - [Item | Rec#commit.disc_only_copies]} + [Item | Rec#commit.disc_only_copies]}; + {ext, Alias, Mod} -> + Ext0 = Rec#commit.ext, + case lists:keytake(ext_copies, 1, Ext0) of + false -> + Rec#commit{ext = [{ext_copies, [{{ext,Alias,Mod}, Item}]}|Ext0]}; + {_,{_,EC},Ext} -> + Rec#commit{ext = [{ext_copies, [{{ext,Alias,Mod}, Item}|EC]}|Ext]} + end end, prepare_node(Node, Storage, Items, Rec2, Kind); prepare_node(_Node, _Storage, Items, Rec, Kind) @@ -1750,16 +1766,30 @@ do_commit(Tid, Bin) when is_binary(Bin) -> do_commit(Tid, binary_to_term(Bin)); do_commit(Tid, C) -> do_commit(Tid, C, optional). + do_commit(Tid, Bin, DumperMode) when is_binary(Bin) -> do_commit(Tid, binary_to_term(Bin), DumperMode); do_commit(Tid, C, DumperMode) -> mnesia_dumper:update(Tid, C#commit.schema_ops, DumperMode), - R = do_snmp(Tid, C#commit.snmp), + R = do_snmp(Tid, proplists:get_value(snmp, C#commit.ext, [])), R2 = do_update(Tid, ram_copies, C#commit.ram_copies, R), R3 = do_update(Tid, disc_copies, C#commit.disc_copies, R2), R4 = do_update(Tid, disc_only_copies, C#commit.disc_only_copies, R3), + R5 = do_update_ext(Tid, C#commit.ext, R4), mnesia_subscr:report_activity(Tid), - R4. + R5. + +%% This could/should be optimized +do_update_ext(_Tid, [], OldRes) -> OldRes; +do_update_ext(Tid, Ext, OldRes) -> + case lists:keyfind(ext_copies, 1, Ext) of + false -> OldRes; + {_, Ops} -> + Do = fun({{ext, _,_} = Storage, Op}, R) -> + do_update(Tid, Storage, [Op], R) + end, + lists:foldl(Do, OldRes, Ops) + end. %% Update the items do_update(Tid, Storage, [Op | Ops], OldRes) -> @@ -1782,12 +1812,12 @@ do_update(_Tid, _Storage, [], Res) -> Res. do_update_op(Tid, Storage, {{Tab, K}, Obj, write}) -> - commit_write(?catch_val({Tab, commit_work}), Tid, + commit_write(?catch_val({Tab, commit_work}), Tid, Storage, Tab, K, Obj, undefined), mnesia_lib:db_put(Storage, Tab, Obj); do_update_op(Tid, Storage, {{Tab, K}, Val, delete}) -> - commit_delete(?catch_val({Tab, commit_work}), Tid, Tab, K, Val, undefined), + commit_delete(?catch_val({Tab, commit_work}), Tid, Storage, Tab, K, Val, undefined), mnesia_lib:db_erase(Storage, Tab, K); do_update_op(Tid, Storage, {{Tab, K}, {RecName, Incr}, update_counter}) -> @@ -1805,86 +1835,84 @@ do_update_op(Tid, Storage, {{Tab, K}, {RecName, Incr}, update_counter}) -> mnesia_lib:db_put(Storage, Tab, Zero), {Zero, []} end, - commit_update(?catch_val({Tab, commit_work}), Tid, Tab, + commit_update(?catch_val({Tab, commit_work}), Tid, Storage, Tab, K, NewObj, OldObjs), element(3, NewObj); do_update_op(Tid, Storage, {{Tab, Key}, Obj, delete_object}) -> commit_del_object(?catch_val({Tab, commit_work}), - Tid, Tab, Key, Obj, undefined), + Tid, Storage, Tab, Key, Obj), mnesia_lib:db_match_erase(Storage, Tab, Obj); do_update_op(Tid, Storage, {{Tab, Key}, Obj, clear_table}) -> - commit_clear(?catch_val({Tab, commit_work}), Tid, Tab, Key, Obj), + commit_clear(?catch_val({Tab, commit_work}), Tid, Storage, Tab, Key, Obj), mnesia_lib:db_match_erase(Storage, Tab, Obj). -commit_write([], _, _, _, _, _) -> ok; -commit_write([{checkpoints, CpList}|R], Tid, Tab, K, Obj, Old) -> +commit_write([], _, _, _, _, _, _) -> ok; +commit_write([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj, Old) -> mnesia_checkpoint:tm_retain(Tid, Tab, K, write, CpList), - commit_write(R, Tid, Tab, K, Obj, Old); -commit_write([H|R], Tid, Tab, K, Obj, Old) + commit_write(R, Tid, Storage, Tab, K, Obj, Old); +commit_write([H|R], Tid, Storage, Tab, K, Obj, Old) when element(1, H) == subscribers -> mnesia_subscr:report_table_event(H, Tab, Tid, Obj, write, Old), - commit_write(R, Tid, Tab, K, Obj, Old); -commit_write([H|R], Tid, Tab, K, Obj, Old) + commit_write(R, Tid, Storage, Tab, K, Obj, Old); +commit_write([H|R], Tid, Storage, Tab, K, Obj, Old) when element(1, H) == index -> - mnesia_index:add_index(H, Tab, K, Obj, Old), - commit_write(R, Tid, Tab, K, Obj, Old). + mnesia_index:add_index(H, Storage, Tab, K, Obj, Old), + commit_write(R, Tid, Storage, Tab, K, Obj, Old). -commit_update([], _, _, _, _, _) -> ok; -commit_update([{checkpoints, CpList}|R], Tid, Tab, K, Obj, _) -> +commit_update([], _, _, _, _, _, _) -> ok; +commit_update([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj, _) -> Old = mnesia_checkpoint:tm_retain(Tid, Tab, K, write, CpList), - commit_update(R, Tid, Tab, K, Obj, Old); -commit_update([H|R], Tid, Tab, K, Obj, Old) + commit_update(R, Tid, Storage, Tab, K, Obj, Old); +commit_update([H|R], Tid, Storage, Tab, K, Obj, Old) when element(1, H) == subscribers -> mnesia_subscr:report_table_event(H, Tab, Tid, Obj, write, Old), - commit_update(R, Tid, Tab, K, Obj, Old); -commit_update([H|R], Tid, Tab, K, Obj, Old) + commit_update(R, Tid, Storage, Tab, K, Obj, Old); +commit_update([H|R], Tid,Storage, Tab, K, Obj, Old) when element(1, H) == index -> - mnesia_index:add_index(H, Tab, K, Obj, Old), - commit_update(R, Tid, Tab, K, Obj, Old). + mnesia_index:add_index(H, Storage, Tab, K, Obj, Old), + commit_update(R, Tid, Storage, Tab, K, Obj, Old). -commit_delete([], _, _, _, _, _) -> ok; -commit_delete([{checkpoints, CpList}|R], Tid, Tab, K, Obj, _) -> +commit_delete([], _, _, _, _, _, _) -> ok; +commit_delete([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj, _) -> Old = mnesia_checkpoint:tm_retain(Tid, Tab, K, delete, CpList), - commit_delete(R, Tid, Tab, K, Obj, Old); -commit_delete([H|R], Tid, Tab, K, Obj, Old) + commit_delete(R, Tid, Storage, Tab, K, Obj, Old); +commit_delete([H|R], Tid, Storage, Tab, K, Obj, Old) when element(1, H) == subscribers -> mnesia_subscr:report_table_event(H, Tab, Tid, Obj, delete, Old), - commit_delete(R, Tid, Tab, K, Obj, Old); -commit_delete([H|R], Tid, Tab, K, Obj, Old) + commit_delete(R, Tid, Storage, Tab, K, Obj, Old); +commit_delete([H|R], Tid, Storage, Tab, K, Obj, Old) when element(1, H) == index -> - mnesia_index:delete_index(H, Tab, K), - commit_delete(R, Tid, Tab, K, Obj, Old). + mnesia_index:delete_index(H, Storage, Tab, K), + commit_delete(R, Tid, Storage, Tab, K, Obj, Old). commit_del_object([], _, _, _, _, _) -> ok; -commit_del_object([{checkpoints, CpList}|R], Tid, Tab, K, Obj, _) -> - Old = mnesia_checkpoint:tm_retain(Tid, Tab, K, delete_object, CpList), - commit_del_object(R, Tid, Tab, K, Obj, Old); -commit_del_object([H|R], Tid, Tab, K, Obj, Old) - when element(1, H) == subscribers -> - mnesia_subscr:report_table_event(H, Tab, Tid, Obj, delete_object, Old), - commit_del_object(R, Tid, Tab, K, Obj, Old); -commit_del_object([H|R], Tid, Tab, K, Obj, Old) - when element(1, H) == index -> - mnesia_index:del_object_index(H, Tab, K, Obj, Old), - commit_del_object(R, Tid, Tab, K, Obj, Old). - -commit_clear([], _, _, _, _) -> ok; -commit_clear([{checkpoints, CpList}|R], Tid, Tab, K, Obj) -> +commit_del_object([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj) -> + mnesia_checkpoint:tm_retain(Tid, Tab, K, delete_object, CpList), + commit_del_object(R, Tid, Storage, Tab, K, Obj); +commit_del_object([H|R], Tid, Storage, Tab, K, Obj) when element(1, H) == subscribers -> + mnesia_subscr:report_table_event(H, Tab, Tid, Obj, delete_object), + commit_del_object(R, Tid, Storage, Tab, K, Obj); +commit_del_object([H|R], Tid, Storage, Tab, K, Obj) when element(1, H) == index -> + mnesia_index:del_object_index(H, Storage, Tab, K, Obj), + commit_del_object(R, Tid, Storage, Tab, K, Obj). + +commit_clear([], _, _, _, _, _) -> ok; +commit_clear([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj) -> mnesia_checkpoint:tm_retain(Tid, Tab, K, clear_table, CpList), - commit_clear(R, Tid, Tab, K, Obj); -commit_clear([H|R], Tid, Tab, K, Obj) + commit_clear(R, Tid, Storage, Tab, K, Obj); +commit_clear([H|R], Tid, Storage, Tab, K, Obj) when element(1, H) == subscribers -> mnesia_subscr:report_table_event(H, Tab, Tid, Obj, clear_table, undefined), - commit_clear(R, Tid, Tab, K, Obj); -commit_clear([H|R], Tid, Tab, K, Obj) + commit_clear(R, Tid, Storage, Tab, K, Obj); +commit_clear([H|R], Tid, Storage, Tab, K, Obj) when element(1, H) == index -> mnesia_index:clear_index(H, Tab, K, Obj), - commit_clear(R, Tid, Tab, K, Obj). + commit_clear(R, Tid, Storage, Tab, K, Obj). do_snmp(_, []) -> ok; -do_snmp(Tid, [Head | Tail]) -> +do_snmp(Tid, [Head|Tail]) -> try mnesia_snmp_hook:update(Head) catch _:Reason -> %% This should only happen when we recently have @@ -1896,31 +1924,34 @@ do_snmp(Tid, [Head | Tail]) -> end, do_snmp(Tid, Tail). -commit_nodes([C | Tail], AccD, AccR) - when C#commit.disc_copies == [], - C#commit.disc_only_copies == [], - C#commit.schema_ops == [] -> - commit_nodes(Tail, AccD, [C#commit.node | AccR]); commit_nodes([C | Tail], AccD, AccR) -> - commit_nodes(Tail, [C#commit.node | AccD], AccR); + case C of + #commit{disc_copies=[], disc_only_copies=[], schema_ops=[], ext=Ext} -> + case lists:keyfind(ext_copies, 1, Ext) of + false -> commit_nodes(Tail, AccD, [C#commit.node | AccR]); + _ -> commit_nodes(Tail, [C#commit.node | AccD], AccR) + end; + _ -> + commit_nodes(Tail, [C#commit.node | AccD], AccR) + end; commit_nodes([], AccD, AccR) -> {AccD, AccR}. commit_decision(D, [C | Tail], AccD, AccR) -> N = C#commit.node, {D2, Tail2} = - case C#commit.schema_ops of - [] when C#commit.disc_copies == [], - C#commit.disc_only_copies == [] -> - commit_decision(D, Tail, AccD, [N | AccR]); - [] -> + case C of + #commit{disc_copies=[], disc_only_copies=[], schema_ops=[], ext=Ext} -> + case lists:keyfind(ext_copies, 1, Ext) of + false -> commit_decision(D, Tail, AccD, [N | AccR]); + _ -> commit_decision(D, Tail, [N | AccD], AccR) + end; + #commit{schema_ops=[]} -> commit_decision(D, Tail, [N | AccD], AccR); - Ops -> + #commit{schema_ops=Ops} -> case ram_only_ops(N, Ops) of - true -> - commit_decision(D, Tail, AccD, [N | AccR]); - false -> - commit_decision(D, Tail, [N | AccD], AccR) + true -> commit_decision(D, Tail, AccD, [N | AccR]); + false -> commit_decision(D, Tail, [N | AccD], AccR) end end, {D2, [C#commit{decision = D2} | Tail2]}; @@ -1948,7 +1979,7 @@ sync_send_dirty(Tid, [Head | Tail], Tab, WaitFor) -> Res = do_dirty(Tid, Head), {WF, Res}; true -> - {?MODULE, Node} ! {self(), {sync_dirty, Tid, Head, Tab}}, + {?MODULE, Node} ! {self(), {sync_dirty, Tid, ext_format(Head), Tab}}, sync_send_dirty(Tid, Tail, Tab, [Node | WaitFor]) end; sync_send_dirty(_Tid, [], _Tab, WaitFor) -> @@ -1967,11 +1998,11 @@ async_send_dirty(Tid, [Head | Tail], Tab, ReadNode, WaitFor, Res) -> NewRes = do_dirty(Tid, Head), async_send_dirty(Tid, Tail, Tab, ReadNode, WaitFor, NewRes); ReadNode == Node -> - {?MODULE, Node} ! {self(), {sync_dirty, Tid, Head, Tab}}, + {?MODULE, Node} ! {self(), {sync_dirty, Tid, ext_format(Head), Tab}}, NewRes = {'EXIT', {aborted, {node_not_running, Node}}}, async_send_dirty(Tid, Tail, Tab, ReadNode, [Node | WaitFor], NewRes); true -> - {?MODULE, Node} ! {self(), {async_dirty, Tid, Head, Tab}}, + {?MODULE, Node} ! {self(), {async_dirty, Tid, ext_format(Head), Tab}}, async_send_dirty(Tid, Tail, Tab, ReadNode, WaitFor, Res) end; async_send_dirty(_Tid, [], _Tab, _ReadNode, WaitFor, Res) -> @@ -2028,23 +2059,29 @@ ask_commit(Protocol, Tid, [Head | Tail], DiscNs, RamNs, WaitFor, Local) -> Node == node() -> ask_commit(Protocol, Tid, Tail, DiscNs, RamNs, WaitFor, Head); true -> - Bin = opt_term_to_binary(Protocol, Head, DiscNs++RamNs), - Msg = {ask_commit, Protocol, Tid, Bin, DiscNs, RamNs}, + CR = ext_format(Head), + Msg = {ask_commit, Protocol, Tid, CR, DiscNs, RamNs}, {?MODULE, Node} ! {self(), Msg}, ask_commit(Protocol, Tid, Tail, DiscNs, RamNs, [Node | WaitFor], Local) end; ask_commit(_Protocol, _Tid, [], _DiscNs, _RamNs, WaitFor, Local) -> {WaitFor, Local}. -%% This used to test protocol conversion between mnesia-nodes -%% but it is really dependent on the emulator version on the -%% two nodes (if funs are sent which they are in transform table op). -%% to be safe we let erts do the translation (many times maybe and thus -%% slower but it works. -% opt_term_to_binary(asym_trans, Head, Nodes) -> -% opt_term_to_binary(Nodes, Head); -opt_term_to_binary(_Protocol, Head, _Nodes) -> - Head. +ext_format(#commit{ext=[]}=CR) -> CR; +ext_format(#commit{node=Node, ext=Ext}=CR) -> + case mnesia_monitor:needs_protocol_conversion(Node) of + true -> + case lists:keyfind(snmp, 1, Ext) of + false -> CR#commit{ext=[]}; + {snmp, List} -> CR#commit{ext=List} + end; + false -> CR + end. + +new_cr_format(#commit{ext=[]}=Cr) -> Cr; +new_cr_format(#commit{ext=[{_,_}|_]}=Cr) -> Cr; +new_cr_format(#commit{ext=Snmp}=Cr) -> + Cr#commit{ext=[{snmp,Snmp}]}. rec_all([Node | Tail], Tid, Res, Pids) -> receive diff --git a/lib/mnesia/test/Makefile b/lib/mnesia/test/Makefile index 5799e34e13..5b61b1af65 100644 --- a/lib/mnesia/test/Makefile +++ b/lib/mnesia/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2012. All Rights Reserved. +# Copyright Ericsson AB 1996-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -52,7 +52,8 @@ MODULES= \ mnesia_schema_recovery_test \ mnesia_measure_test \ mnesia_cost \ - mnesia_dbn_meters + mnesia_dbn_meters \ + ext_test DocExamplesDir := ../doc/src/ diff --git a/lib/mnesia/test/ext_test.erl b/lib/mnesia/test/ext_test.erl new file mode 100644 index 0000000000..3d1cc40c09 --- /dev/null +++ b/lib/mnesia/test/ext_test.erl @@ -0,0 +1,237 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ext_test). + +%% Initializations +-export([init_backend/0, add_aliases/1, remove_aliases/1, + check_definition/4, semantics/2]). + +-export([ + create_table/3, load_table/4, + delete_table/2, close_table/2, sync_close_table/2, + + sender_init/4, + receiver_first_message/4, receive_data/5, receive_done/4, + + index_is_consistent/3, is_index_consistent/2, + + real_suffixes/0, tmp_suffixes/0, + + info/3, + fixtable/3, + validate_key/6, validate_record/6, + + first/2, last/2, next/3, prev/3, slot/3, + + insert/3, update_counter/4, + lookup/3, + delete/3, match_delete/3, + select/1, select/3, select/4, repair_continuation/2 + ]). + +-ifdef(DEBUG). +-define(DBG(DATA), io:format("~p:~p: ~p~n",[?MODULE, ?LINE, DATA])). +-define(DBG(FORMAT, ARGS), io:format("~p:~p: " ++ FORMAT,[?MODULE, ?LINE] ++ ARGS)). +-else. +-define(DBG(DATA), ok). +-define(DBG(FORMAT, ARGS), ok). +-endif. + +%% types() -> +%% [{fs_copies, ?MODULE}, +%% {raw_fs_copies, ?MODULE}]. + +semantics(ext_ets, storage) -> ram_copies; +semantics(ext_ets, types ) -> [set, ordered_set, bag]; +semantics(ext_ets, index_types) -> [ordered]; +semantics(_Alias, _) -> + undefined. + +%% valid_op(_, _) -> +%% true. + +init_backend() -> + ?DBG(init_backend), + ok. + +add_aliases(_As) -> + ?DBG(_As), + ok. + +remove_aliases(_) -> + ok. + + +%% Table operations + +check_definition(ext_ets, _Tab, _Nodes, _Props) -> + ?DBG("~p ~p ~p~n", [_Tab, _Nodes, _Props]), + ok. + +create_table(ext_ets, Tab, Props) when is_atom(Tab) -> + Tid = ets:new(Tab, [public, proplists:get_value(type, Props, set), {keypos, 2}]), + ?DBG("~p Create: ~p(~p) ~p~n", [self(), Tab, Tid, Props]), + mnesia_lib:set({?MODULE, Tab}, Tid), + ok; +create_table(_, Tag={Tab, index, {_Where, Type0}}, _Opts) -> + Type = case Type0 of + ordered -> ordered_set; + _ -> Type0 + end, + Tid = ets:new(Tab, [public, Type]), + ?DBG("~p(~p) ~p~n", [Tab, Tid, Tag]), + mnesia_lib:set({?MODULE, Tag}, Tid), + ok; +create_table(_, Tag={_Tab, retainer, ChkPName}, _Opts) -> + Tid = ets:new(ChkPName, [set, public, {keypos, 2}]), + ?DBG("~p(~p) ~p~n", [_Tab, Tid, Tag]), + mnesia_lib:set({?MODULE, Tag}, Tid), + ok. + +delete_table(ext_ets, Tab) -> + try + ets:delete(mnesia_lib:val({?MODULE,Tab})), + mnesia_lib:unset({?MODULE,Tab}), + ok + catch _:_ -> + ?DBG({double_delete, Tab}), + ok + end. + +load_table(ext_ets, _Tab, init_index, _Cs) -> ok; +load_table(ext_ets, _Tab, _LoadReason, _Cs) -> + ?DBG("Load ~p ~p~n", [_Tab, _LoadReason]), + ok. +%% mnesia_monitor:unsafe_create_external(Tab, ext_ets, ?MODULE, Cs). + +sender_init(Alias, Tab, _RemoteStorage, _Pid) -> + KeysPerTransfer = 100, + {standard, + fun() -> mnesia_lib:db_init_chunk({ext,Alias,?MODULE}, Tab, KeysPerTransfer) end, + fun(Cont) -> mnesia_lib:db_chunk({ext,Alias,?MODULE}, Cont) end}. + +receiver_first_message(Sender, {first, Size}, _Alias, Tab) -> + ?DBG({first,Size}), + {Size, {Tab, Sender}}. + +receive_data(Data, ext_ets, Name, _Sender, {Name, Tab, _Sender}=State) -> + ?DBG({Data,State}), + true = ets:insert(Tab, Data), + {more, State}; +receive_data(Data, Alias, Tab, Sender, {Name, Sender}) -> + receive_data(Data, Alias, Tab, Sender, {Name, mnesia_lib:val({?MODULE,Tab}), Sender}). + +receive_done(_Alias, _Tab, _Sender, _State) -> + ?DBG({done,_State}), + ok. + +close_table(Alias, Tab) -> sync_close_table(Alias, Tab). + +sync_close_table(ext_ets, _Tab) -> + ?DBG(_Tab). + +fixtable(ext_ets, Tab, Bool) -> + ?DBG({Tab,Bool}), + ets:safe_fixtable(mnesia_lib:val({?MODULE,Tab}), Bool). + +info(ext_ets, Tab, Type) -> + ?DBG({Tab,Type}), + Tid = mnesia_lib:val({?MODULE,Tab}), + try ets:info(Tid, Type) of + Val -> Val + catch _:_ -> + undefined + end. + +real_suffixes() -> + [".dat"]. + +tmp_suffixes() -> + []. + +%% Index + +index_is_consistent(_Alias, _Ix, _Bool) -> ok. % Ignore for now +is_index_consistent(_Alias, _Ix) -> false. % Always rebuild + +%% Record operations + +validate_record(_Alias, _Tab, RecName, Arity, Type, _Obj) -> + {RecName, Arity, Type}. + +validate_key(_Alias, _Tab, RecName, Arity, Type, _Key) -> + {RecName, Arity, Type}. + +insert(ext_ets, Tab, Obj) -> + ?DBG({Tab,Obj}), + try + ets:insert(mnesia_lib:val({?MODULE,Tab}), Obj), + ok + catch _:Reason -> + io:format("CRASH ~p ~p~n",[Reason, mnesia_lib:val({?MODULE,Tab})]) + end. + +lookup(ext_ets, Tab, Key) -> + ets:lookup(mnesia_lib:val({?MODULE,Tab}), Key). + +delete(ext_ets, Tab, Key) -> + ets:delete(mnesia_lib:val({?MODULE,Tab}), Key). + +match_delete(ext_ets, Tab, Pat) -> + ets:match_delete(mnesia_lib:val({?MODULE,Tab}), Pat). + +first(ext_ets, Tab) -> + ets:first(mnesia_lib:val({?MODULE,Tab})). + +last(Alias, Tab) -> first(Alias, Tab). + +next(ext_ets, Tab, Key) -> + ets:next(mnesia_lib:val({?MODULE,Tab}), Key). + +prev(Alias, Tab, Key) -> + next(Alias, Tab, Key). + +slot(ext_ets, Tab, Pos) -> + ets:slot(mnesia_lib:val({?MODULE,Tab}), Pos). + +update_counter(ext_ets, Tab, C, Val) -> + ets:update_counter(mnesia_lib:val({?MODULE,Tab}), C, Val). + +select('$end_of_table' = End) -> End; +select({ext_ets, C}) -> ets:select(C). + +select(Alias, Tab, Ms) -> + Res = select(Alias, Tab, Ms, 100000), + select_1(Res). + +select_1('$end_of_table') -> []; +select_1({Acc, C}) -> + case ets:select(C) of + '$end_of_table' -> Acc; + {New, Cont} -> + select_1({New ++ Acc, Cont}) + end. + +select(ext_ets, Tab, Ms, Limit) when is_integer(Limit); Limit =:= infinity -> + ets:select(mnesia_lib:val({?MODULE,Tab}), Ms, Limit). + +repair_continuation({Alias, Cont}, Ms) -> + {Alias, ets:repair_continuation(Cont, Ms)}. diff --git a/lib/mnesia/test/mnesia_SUITE.erl b/lib/mnesia/test/mnesia_SUITE.erl index 070f51e6a8..3ec4847c5d 100644 --- a/lib/mnesia/test/mnesia_SUITE.erl +++ b/lib/mnesia/test/mnesia_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_atomicity_test.erl b/lib/mnesia/test/mnesia_atomicity_test.erl index e3e0eaaf75..cc32ba3826 100644 --- a/lib/mnesia/test/mnesia_atomicity_test.erl +++ b/lib/mnesia/test/mnesia_atomicity_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2011. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -700,11 +700,19 @@ start_restart_check(RestartOp, ReplicaNeed, Config) -> %% mnesia shall be killed at that node, where A is reading %% the information from - kill_where_to_read(TabName, N1, [N2, N3]), + Read = kill_where_to_read(TabName, N1, [N2, N3]), %% wait some time to let mnesia go down and spread those news around %% fun A shall be able to finish its job before being restarted - wait(500), + Wait = fun(Loop) -> + wait(300), + sys:get_status(mnesia_monitor), + case lists:member(Read, mnesia_lib:val({current, db_nodes})) of + true -> Loop(Loop); + false -> ok + end + end, + Wait(Wait), A ! go_ahead, %% the sticky write doesnt work on remote nodes !!! @@ -772,10 +780,12 @@ kill_where_to_read(TabName, N1, Nodes) -> Read = rpc:call(N1,mnesia,table_info, [TabName, where_to_read]), case lists:member(Read, Nodes) of true -> - mnesia_test_lib:kill_mnesia([Read]); + mnesia_test_lib:kill_mnesia([Read]), + Read; false -> ?error("Fault while killing Mnesia: ~p~n", [Read]), - mnesia_test_lib:kill_mnesia(Nodes) + mnesia_test_lib:kill_mnesia(Nodes), + Read end. sync_tid_release() -> diff --git a/lib/mnesia/test/mnesia_bench_SUITE.erl b/lib/mnesia/test/mnesia_bench_SUITE.erl index 962b57ae6f..7c86db383d 100644 --- a/lib/mnesia/test/mnesia_bench_SUITE.erl +++ b/lib/mnesia/test/mnesia_bench_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012. All Rights Reserved. +%% Copyright Ericsson AB 2012-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_config_backup.erl b/lib/mnesia/test/mnesia_config_backup.erl index a95f305b43..239101ca92 100644 --- a/lib/mnesia/test/mnesia_config_backup.erl +++ b/lib/mnesia/test/mnesia_config_backup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_config_event.erl b/lib/mnesia/test/mnesia_config_event.erl index 6a35e14363..4299b14f36 100644 --- a/lib/mnesia/test/mnesia_config_event.erl +++ b/lib/mnesia/test/mnesia_config_event.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_config_test.erl b/lib/mnesia/test/mnesia_config_test.erl index 089fbc06dc..45da909264 100644 --- a/lib/mnesia/test/mnesia_config_test.erl +++ b/lib/mnesia/test/mnesia_config_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2014. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -693,9 +693,9 @@ event_module(Config) when is_list(Config) -> end, ?match({[ok, ok], []}, rpc:multicall(Nodes, mnesia, start, [Def])), - receive after 1000 -> ok end, + receive after 2000 -> ok end, mnesia_event ! {get_log, self()}, - DebugLog1 = receive + DebugLog1 = receive {log, L1} -> L1 after 10000 -> [timeout] end, @@ -706,9 +706,9 @@ event_module(Config) when is_list(Config) -> ?match({[ok], []}, rpc:multicall([N2], mnesia, start, [])), - receive after 1000 -> ok end, + receive after 2000 -> ok end, mnesia_event ! {get_log, self()}, - DebugLog = receive + DebugLog = receive {log, L} -> L after 10000 -> [timeout] end, @@ -1100,8 +1100,8 @@ dynamic_basic(Config) when is_list(Config) -> ?match(ok, mnesia:dirty_write({tab1, 1, 1})), ?match(ok, mnesia:dirty_write({tab2, 1, 1})), - - ?match(ok, rpc:call(N2, mnesia, start, [[{extra_db_nodes, [N1]}]])), + + ?match(ok, rpc:call(N2, mnesia, start, [[{extra_db_nodes, [N1]}, {schema, ?BACKEND}]])), ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [[tab1,tab2],5000])), io:format("Here ~p ~n",[?LINE]), check_storage(N2, N1, [N3]), @@ -1112,7 +1112,7 @@ dynamic_basic(Config) when is_list(Config) -> ?match(ok, mnesia:delete_schema([N3])), io:format("T1 ~p ~n",[rpc:call(N3,?MODULE,c_nodes,[])]), - ?match(ok, rpc:call(N3, mnesia, start, [])), + ?match(ok, rpc:call(N3, mnesia, start, [[{schema, ?BACKEND}]])), io:format("T2 ~p ~n",[rpc:call(N3,?MODULE,c_nodes,[])]), timer:sleep(2000), io:format("T3 ~p ~n",[rpc:call(N3,?MODULE,c_nodes,[])]), @@ -1127,7 +1127,7 @@ dynamic_basic(Config) when is_list(Config) -> ?match([], mnesia_test_lib:kill_mnesia([N3])), ?match(ok, mnesia:delete_schema([N3])), - ?match(ok, rpc:call(N3, mnesia, start, [])), + ?match(ok, rpc:call(N3, mnesia, start, [[{schema, ?BACKEND}]])), ?match({ok, [N3]}, sort(?rpc_connect(N1, [N3]))), ?match(ok, rpc:call(N3, mnesia, wait_for_tables, [[tab1,tab2],5000])), io:format("Here ~p ~n",[?LINE]), @@ -1143,7 +1143,7 @@ dynamic_basic(Config) when is_list(Config) -> % mnesia should come up now. ?match({atomic, ok}, mnesia:add_table_copy(tab1, N2, ram_copies)), - ?match(ok, rpc:call(N2, mnesia, start, [])), + ?match(ok, rpc:call(N2, mnesia, start, [[{schema, ?BACKEND}]])), ?match({ok, _}, sort(?rpc_connect(N2, [N3]))), ?match(SNs, sort(rpc:call(N1, mnesia, system_info, [running_db_nodes]))), @@ -1162,7 +1162,7 @@ dynamic_basic(Config) when is_list(Config) -> ?match([N3,N1], sort(rpc:call(N1, mnesia, system_info, [running_db_nodes]))), ?match([N3,N1], sort(rpc:call(N3, mnesia, system_info, [running_db_nodes]))), - ?match(ok, rpc:call(N2, mnesia, start, [])), + ?match(ok, rpc:call(N2, mnesia, start, [[{schema, ?BACKEND}]])), ?match({ok, _}, sort(?rpc_connect(N3, [N2]))), ?match(SNs, sort(rpc:call(N1, mnesia, system_info, [running_db_nodes]))), @@ -1192,7 +1192,7 @@ dynamic_ext(Config) when is_list(Config) -> mnesia_test_lib:kill_mnesia([N2]), ?match(ok, mnesia:delete_schema([N2])), - ?match(ok, rpc:call(N2, mnesia, start, [[{extra_db_nodes, [N1]}]])), + ?match(ok, rpc:call(N2, mnesia, start, [[{extra_db_nodes, [N1]}, {schema, ?BACKEND}]])), ?match(SNs, sort(rpc:call(N1, mnesia, system_info, [running_db_nodes]))), ?match(SNs, sort(rpc:call(N2, mnesia, system_info, [running_db_nodes]))), @@ -1213,7 +1213,7 @@ dynamic_ext(Config) when is_list(Config) -> ?match(ok, mnesia:dirty_write({tab3, 42, T})), ?match(stopped, rpc:call(N2, mnesia, stop, [])), - ?match(ok, rpc:call(N2, mnesia, start, [])), + ?match(ok, rpc:call(N2, mnesia, start, [[{schema, ?BACKEND}]])), ?match(SNs, sort(rpc:call(N2, mnesia, system_info, [running_db_nodes]))), ?match(ok, mnesia:wait_for_tables([tab0,tab1,tab2,tab3], 10000)), ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [[tab1,tab2,tab3], 100])), @@ -1226,7 +1226,7 @@ dynamic_ext(Config) when is_list(Config) -> ?match(stopped, rpc:call(N1, mnesia, stop, [])), - ?match(ok, rpc:call(N2, mnesia, start, [[{extra_db_nodes,[N1,N2]}]])), + ?match(ok, rpc:call(N2, mnesia, start, [[{extra_db_nodes,[N1,N2]}, {schema, ?BACKEND}]])), ?match({timeout,[tab0]}, rpc:call(N2, mnesia, wait_for_tables, [[tab0], 500])), ?match(ok, rpc:call(N1, mnesia, start, [[{extra_db_nodes, [N1,N2]}]])), @@ -1238,7 +1238,7 @@ dynamic_ext(Config) when is_list(Config) -> ?match(stopped, rpc:call(N1, mnesia, stop, [])), mnesia_test_lib:kill_mnesia([N2]), ?match(ok, mnesia:delete_schema([N2])), - ?match(ok, rpc:call(N1, mnesia, start, [[{extra_db_nodes, [N1,N2]}]])), + ?match(ok, rpc:call(N1, mnesia, start, [[{extra_db_nodes, [N1,N2]}, {schema, ?BACKEND}]])), ?match({timeout,[tab0]}, rpc:call(N1, mnesia, wait_for_tables, [[tab0], 500])), ?match(ok, rpc:call(N2, mnesia, start, [[{extra_db_nodes,[N1,N2]}]])), diff --git a/lib/mnesia/test/mnesia_consistency_test.erl b/lib/mnesia/test/mnesia_consistency_test.erl index 087aef86c9..2fe1bd34e6 100644 --- a/lib/mnesia/test/mnesia_consistency_test.erl +++ b/lib/mnesia/test/mnesia_consistency_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -565,6 +565,7 @@ consistency_after_fallback_3_disc_only(Config) when is_list(Config) -> consistency_after_fallback(disc_only_copies, 3, Config). consistency_after_fallback(ReplicaType, NodeConfig, Config) -> + put(mnesia_test_verbose, true), %%?verbose("Starting consistency_after_fallback2 at ~p~n", [self()]), Delay = 5, Nodes = ?acquire_nodes(NodeConfig, [{tc_timeout, timer:minutes(10)} | Config]), @@ -594,10 +595,11 @@ consistency_after_fallback(ReplicaType, NodeConfig, Config) -> ?match(ok, mnesia_tpcb:verify_tabs()), %% Stop and then start mnesia and check table consistency - %%?verbose("Restarting Mnesia~n", []), + ?verbose("Kill Mnesia~n", []), mnesia_test_lib:kill_mnesia(Nodes), + ?verbose("Start Mnesia~n", []), mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller,history]), - + ?verbose("Verify tabs~n", []), ?match(ok, mnesia_tpcb:verify_tabs()), if ReplicaType == ram_copies -> @@ -663,10 +665,10 @@ consistency_after_restore(ReplicaType, Op, Config) -> [lists:foreach(fun(E) -> ok = mnesia:dirty_write({Tab, E, 2}) end, NList) || Tab <- Tabs], - Pids1 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carA, Op]), ok} || _ <- lists:seq(1, 5)], - Pids2 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carB, Op]), ok} || _ <- lists:seq(1, 5)], - Pids3 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carC, Op]), ok} || _ <- lists:seq(1, 5)], - Pids4 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carD, Op]), ok} || _ <- lists:seq(1, 5)], + Pids1 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carA, Op]), carA} || _ <- lists:seq(1, 5)], + Pids2 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carB, Op]), carB} || _ <- lists:seq(1, 5)], + Pids3 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carC, Op]), carC} || _ <- lists:seq(1, 5)], + Pids4 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carD, Op]), carD} || _ <- lists:seq(1, 5)], AllPids = Pids1 ++ Pids2 ++ Pids3 ++ Pids4, @@ -676,19 +678,38 @@ consistency_after_restore(ReplicaType, Op, Config) -> Else -> Else end end, - + timer:sleep(timer:seconds(Delay)), %% Let changers grab locks ?verbose("Doing restore~n", []), ?match(Tabs, Restore(File, [{default_op, Op}])), - timer:sleep(timer:seconds(Delay)), %% Let em die + Collect = fun(Msg, Acc) -> + receive Msg -> Acc + after 10000 -> [Msg|Acc] + end + end, - ?match_multi_receive(AllPids), + Failed1 = lists:foldl(Collect, [], AllPids), + Failed = lists:foldl(Collect, [], Failed1), + + case Failed of + [] -> ok; + _ -> + ?match([], Failed), + io:format("TIME: ~p sec~n", [erlang:system_time(seconds) band 16#FF]), + Dbg = fun({_, Pid, Tab}) -> + io:format("Tab ~p: ~p~n",[Tab, process_info(Pid, current_stacktrace)]), + [io:format(" ~p~n", [Rec]) || Rec <- mnesia:dirty_match_object({Tab, '_', '_'})] + end, + [Dbg(Msg) || Msg <- Failed], + io:format(" Held: ~p~n", [mnesia_locker:get_held_locks()]), + io:format("Queue: ~p~n", [mnesia_locker:get_lock_queue()]) + end, - case ?match(ok, restore_verify_tabs(Tabs)) of - {success, ok} -> + case ?match(ok, restore_verify_tabs(Tabs)) of + {success, ok} -> file:delete(File); - _ -> + _ -> {T, M, S} = time(), File2 = ?flat_format("consistency_error~w~w~w.BUP", [T, M, S]), file:rename(File, File2) @@ -696,19 +717,22 @@ consistency_after_restore(ReplicaType, Op, Config) -> ?verify_mnesia(Nodes, []). change_tab(Father, Tab, Test) -> - Key = random:uniform(20), + Key = rand:uniform(20), Update = fun() -> + Time = erlang:system_time(seconds) band 16#FF, + case put(time, Time) of + Time -> ok; + _ -> io:format("~p ~p ~p sec~n", [self(), Tab, Time]) + end, case mnesia:read({Tab, Key}) of - [{Tab, Key, 1}] -> - quit; - [{Tab, Key, _N}] -> - mnesia:write({Tab, Key, 3}) + [{Tab, Key, 1}] -> quit; + [{Tab, Key, _N}] -> mnesia:write({Tab, Key, 3}) end end, case mnesia:transaction(Update) of {atomic, quit} -> - exit(ok); - {aborted, {no_exists, Tab}} when Test == recreate_tables ->%% I'll allow this + exit(Tab); + {aborted, {no_exists, Tab}} when Test == recreate_tables -> %% I'll allow this change_tab(Father, Tab, Test); {atomic, ok} -> change_tab(Father, Tab, Test) diff --git a/lib/mnesia/test/mnesia_cost.erl b/lib/mnesia/test/mnesia_cost.erl index 714dbaef27..a3fc8dfe20 100644 --- a/lib/mnesia/test/mnesia_cost.erl +++ b/lib/mnesia/test/mnesia_cost.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_dbn_meters.erl b/lib/mnesia/test/mnesia_dbn_meters.erl index 5c3ea08a1d..407d7c65b8 100644 --- a/lib/mnesia/test/mnesia_dbn_meters.erl +++ b/lib/mnesia/test/mnesia_dbn_meters.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_dirty_access_test.erl b/lib/mnesia/test/mnesia_dirty_access_test.erl index 0d57e5a1b1..6d970ac990 100644 --- a/lib/mnesia/test/mnesia_dirty_access_test.erl +++ b/lib/mnesia/test/mnesia_dirty_access_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -40,43 +40,40 @@ all() -> groups() -> [{dirty_write, [], - [dirty_write_ram, dirty_write_disc, - dirty_write_disc_only]}, + [dirty_write_ram, dirty_write_disc, dirty_write_disc_only, + dirty_write_xets]}, {dirty_read, [], - [dirty_read_ram, dirty_read_disc, - dirty_read_disc_only]}, + [dirty_read_ram, dirty_read_disc, dirty_read_disc_only, dirty_read_xets]}, {dirty_update_counter, [], [dirty_update_counter_ram, dirty_update_counter_disc, - dirty_update_counter_disc_only]}, + dirty_update_counter_disc_only, dirty_update_counter_xets]}, {dirty_delete, [], [dirty_delete_ram, dirty_delete_disc, - dirty_delete_disc_only]}, + dirty_delete_disc_only, dirty_delete_xets]}, {dirty_delete_object, [], [dirty_delete_object_ram, dirty_delete_object_disc, - dirty_delete_object_disc_only]}, + dirty_delete_object_disc_only, dirty_delete_object_xets]}, {dirty_match_object, [], [dirty_match_object_ram, dirty_match_object_disc, - dirty_match_object_disc_only]}, + dirty_match_object_disc_only, dirty_match_object_xets]}, {dirty_index, [], [{group, dirty_index_match_object}, {group, dirty_index_read}, {group, dirty_index_update}]}, {dirty_index_match_object, [], - [dirty_index_match_object_ram, - dirty_index_match_object_disc, - dirty_index_match_object_disc_only]}, + [dirty_index_match_object_ram, dirty_index_match_object_disc, + dirty_index_match_object_disc_only, dirty_index_match_object_xets]}, {dirty_index_read, [], [dirty_index_read_ram, dirty_index_read_disc, - dirty_index_read_disc_only]}, + dirty_index_read_disc_only, dirty_index_read_xets]}, {dirty_index_update, [], - [dirty_index_update_set_ram, - dirty_index_update_set_disc, - dirty_index_update_set_disc_only, + [dirty_index_update_set_ram, dirty_index_update_set_disc, + dirty_index_update_set_disc_only, dirty_index_update_set_xets, dirty_index_update_bag_ram, dirty_index_update_bag_disc, - dirty_index_update_bag_disc_only]}, + dirty_index_update_bag_disc_only, dirty_index_update_bag_xets]}, {dirty_iter, [], - [dirty_iter_ram, dirty_iter_disc, - dirty_iter_disc_only]}, + [dirty_iter_ram, dirty_iter_disc, dirty_iter_disc_only, + dirty_iter_xets]}, {admin_tests, [], [del_table_copy_1, del_table_copy_2, del_table_copy_3, add_table_copy_1, add_table_copy_2, add_table_copy_3, @@ -106,6 +103,9 @@ dirty_write_disc_only(suite) -> []; dirty_write_disc_only(Config) when is_list(Config) -> dirty_write(Config, disc_only_copies). +dirty_write_xets(Config) when is_list(Config) -> + dirty_write(Config, ext_ets). + dirty_write(Config, Storage) -> [Node1] = Nodes = ?acquire_nodes(1, Config), Tab = dirty_write, @@ -137,6 +137,9 @@ dirty_read_disc_only(suite) -> []; dirty_read_disc_only(Config) when is_list(Config) -> dirty_read(Config, disc_only_copies). +dirty_read_xets(Config) when is_list(Config) -> + dirty_read(Config, ext_ets). + dirty_read(Config, Storage) -> [Node1] = Nodes = ?acquire_nodes(1, Config), Tab = dirty_read, @@ -180,6 +183,9 @@ dirty_update_counter_disc_only(suite) -> []; dirty_update_counter_disc_only(Config) when is_list(Config) -> dirty_update_counter(Config, disc_only_copies). +dirty_update_counter_xets(Config) when is_list(Config) -> + dirty_update_counter(Config, ext_ets). + dirty_update_counter(Config, Storage) -> [Node1] = Nodes = ?acquire_nodes(1, Config), Tab = dirty_update_counter, @@ -222,6 +228,9 @@ dirty_delete_disc_only(suite) -> []; dirty_delete_disc_only(Config) when is_list(Config) -> dirty_delete(Config, disc_only_copies). +dirty_delete_xets(Config) when is_list(Config) -> + dirty_delete(Config, ext_ets). + dirty_delete(Config, Storage) -> [Node1] = Nodes = ?acquire_nodes(1, Config), Tab = dirty_delete, @@ -259,6 +268,9 @@ dirty_delete_object_disc_only(suite) -> []; dirty_delete_object_disc_only(Config) when is_list(Config) -> dirty_delete_object(Config, disc_only_copies). +dirty_delete_object_xets(Config) when is_list(Config) -> + dirty_delete_object(Config, ext_ets). + dirty_delete_object(Config, Storage) -> [Node1] = Nodes = ?acquire_nodes(1, Config), Tab = dirty_delete_object, @@ -302,6 +314,9 @@ dirty_match_object_disc_only(suite) -> []; dirty_match_object_disc_only(Config) when is_list(Config) -> dirty_match_object(Config, disc_only_copies). +dirty_match_object_xets(Config) when is_list(Config) -> + dirty_match_object(Config, ext_ets). + dirty_match_object(Config, Storage) -> [Node1] = Nodes = ?acquire_nodes(1, Config), Tab = dirty_match, @@ -326,7 +341,6 @@ dirty_match_object(Config, Storage) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Dirty read matching records by using an index - dirty_index_match_object_ram(suite) -> []; dirty_index_match_object_ram(Config) when is_list(Config) -> dirty_index_match_object(Config, ram_copies). @@ -339,6 +353,9 @@ dirty_index_match_object_disc_only(suite) -> []; dirty_index_match_object_disc_only(Config) when is_list(Config) -> dirty_index_match_object(Config, disc_only_copies). +dirty_index_match_object_xets(Config) when is_list(Config) -> + dirty_index_match_object(Config, ext_ets). + dirty_index_match_object(Config, Storage) -> [Node1] = Nodes = ?acquire_nodes(1, Config), Tab = dirty_index_match_object, @@ -376,6 +393,9 @@ dirty_index_read_disc_only(suite) -> []; dirty_index_read_disc_only(Config) when is_list(Config) -> dirty_index_read(Config, disc_only_copies). +dirty_index_read_xets(Config) when is_list(Config) -> + dirty_index_read(Config, ext_ets). + dirty_index_read(Config, Storage) -> [Node1] = Nodes = ?acquire_nodes(1, Config), Tab = dirty_index_read, @@ -437,78 +457,85 @@ dirty_index_update_set_disc_only(suite) -> []; dirty_index_update_set_disc_only(Config) when is_list(Config) -> dirty_index_update_set(Config, disc_only_copies). +dirty_index_update_set_xets(Config) when is_list(Config) -> + dirty_index_update_set(Config, ext_ets). + dirty_index_update_set(Config, Storage) -> - [Node1] = Nodes = ?acquire_nodes(1, Config), - Tab = index_test, - ValPos = v1, + [Node1] = Nodes = ?acquire_nodes(1, Config), + Tab = index_test, + ValPos = v1, ValPos2 = v3, Def = [{attributes, [k, v1, v2, v3]}, {Storage, [Node1]}, - {index, [ValPos]}], - ?match({atomic, ok}, mnesia:create_table(Tab, Def)), - + {index, [ValPos]}], + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + Pat1 = {Tab, '$1', 2, '$2', '$3'}, - Pat2 = {Tab, '$1', '$2', '$3', '$4'}, - - Rec1 = {Tab, 1, 2, 3, 4}, + Pat2 = {Tab, '$1', '$2', '$3', '$4'}, + Pat3 = {Tab, '_', '_', '_', {4, 14}}, + + Rec1 = {Tab, 1, 2, 3, {4, 14}}, Rec2 = {Tab, 2, 2, 13, 14}, - Rec3 = {Tab, 1, 12, 13, 14}, - Rec4 = {Tab, 4, 2, 13, 14}, - + Rec3 = {Tab, 1, 12, 13, 14}, + Rec4 = {Tab, 4, 2, 13, 14}, + ?match([], mnesia:dirty_index_read(Tab, 2, ValPos)), ?match(ok, mnesia:dirty_write(Rec1)), ?match([Rec1], mnesia:dirty_index_read(Tab, 2, ValPos)), - + ?match(ok, mnesia:dirty_write(Rec2)), R1 = mnesia:dirty_index_read(Tab, 2, ValPos), ?match([Rec1, Rec2], lists:sort(R1)), - + ?match(ok, mnesia:dirty_write(Rec3)), R2 = mnesia:dirty_index_read(Tab, 2, ValPos), ?match([Rec2], lists:sort(R2)), ?match([Rec2], mnesia:dirty_index_match_object(Pat1, ValPos)), - - {atomic, R3} = mnesia:transaction(fun() -> mnesia:match_object(Pat2) end), + + {atomic, R3} = mnesia:transaction(fun() -> mnesia:match_object(Pat2) end), ?match([Rec3, Rec2], lists:sort(R3)), - + ?match(ok, mnesia:dirty_write(Rec4)), R4 = mnesia:dirty_index_read(Tab, 2, ValPos), ?match([Rec2, Rec4], lists:sort(R4)), - + ?match(ok, mnesia:dirty_delete({Tab, 4})), ?match([Rec2], mnesia:dirty_index_read(Tab, 2, ValPos)), - + ?match({atomic, ok}, mnesia:del_table_index(Tab, ValPos)), ?match({atomic, ok}, mnesia:transaction(fun() -> mnesia:write(Rec4) end)), ?match({atomic, ok}, mnesia:add_table_index(Tab, ValPos)), ?match({atomic, ok}, mnesia:add_table_index(Tab, ValPos2)), - + R5 = mnesia:dirty_match_object(Pat2), ?match([Rec3, Rec2, Rec4], lists:sort(R5)), - + R6 = mnesia:dirty_index_read(Tab, 2, ValPos), - ?match([Rec2, Rec4], lists:sort(R6)), - ?match([], mnesia:dirty_index_read(Tab, 4, ValPos2)), + ?match([Rec2, Rec4], lists:sort(R6)), + ?match([], mnesia:dirty_index_read(Tab, {4,14}, ValPos2)), R7 = mnesia:dirty_index_read(Tab, 14, ValPos2), ?match([Rec3, Rec2, Rec4], lists:sort(R7)), ?match({atomic, ok}, mnesia:transaction(fun() -> mnesia:write(Rec1) end)), R8 = mnesia:dirty_index_read(Tab, 2, ValPos), ?match([Rec1, Rec2, Rec4], lists:sort(R8)), - ?match([Rec1], mnesia:dirty_index_read(Tab, 4, ValPos2)), + ?match([Rec1], mnesia:dirty_index_read(Tab, {4,14}, ValPos2)), + ?match([Rec1], mnesia:dirty_match_object(Pat3)), + ?match([Rec1], mnesia:dirty_index_match_object(Pat3, ValPos2)), + R9 = mnesia:dirty_index_read(Tab, 14, ValPos2), ?match([Rec2, Rec4], lists:sort(R9)), ?match({atomic, ok}, mnesia:transaction(fun() -> mnesia:delete_object(Rec2) end)), R10 = mnesia:dirty_index_read(Tab, 2, ValPos), ?match([Rec1, Rec4], lists:sort(R10)), - ?match([Rec1], mnesia:dirty_index_read(Tab, 4, ValPos2)), + ?match([Rec1], mnesia:dirty_index_read(Tab, {4,14}, ValPos2)), ?match([Rec4], mnesia:dirty_index_read(Tab, 14, ValPos2)), ?match(ok, mnesia:dirty_delete({Tab, 4})), R11 = mnesia:dirty_index_read(Tab, 2, ValPos), ?match([Rec1], lists:sort(R11)), - ?match([Rec1], mnesia:dirty_index_read(Tab, 4, ValPos2)), + ?match([Rec1], mnesia:dirty_index_read(Tab, {4,14}, ValPos2)), ?match([], mnesia:dirty_index_read(Tab, 14, ValPos2)), ?verify_mnesia(Nodes, []). @@ -525,6 +552,9 @@ dirty_index_update_bag_disc_only(suite) -> []; dirty_index_update_bag_disc_only(Config)when is_list(Config) -> dirty_index_update_bag(Config, disc_only_copies). +dirty_index_update_bag_xets(Config) when is_list(Config) -> + dirty_index_update_bag(Config, ext_ets). + dirty_index_update_bag(Config, Storage) -> [Node1] = Nodes = ?acquire_nodes(1, Config), Tab = index_test, @@ -631,7 +661,6 @@ dirty_index_update_bag(Config, Storage) -> %% Dirty iteration %% dirty_slot, dirty_first, dirty_next - dirty_iter_ram(suite) -> []; dirty_iter_ram(Config) when is_list(Config) -> dirty_iter(Config, ram_copies). @@ -644,6 +673,9 @@ dirty_iter_disc_only(suite) -> []; dirty_iter_disc_only(Config) when is_list(Config) -> dirty_iter(Config, disc_only_copies). +dirty_iter_xets(Config) when is_list(Config) -> + dirty_iter(Config, ext_ets). + dirty_iter(Config, Storage) -> [Node1] = Nodes = ?acquire_nodes(1, Config), Tab = dirty_iter, diff --git a/lib/mnesia/test/mnesia_durability_test.erl b/lib/mnesia/test/mnesia_durability_test.erl index 4c9438df3e..97bc84a2d8 100644 --- a/lib/mnesia/test/mnesia_durability_test.erl +++ b/lib/mnesia/test/mnesia_durability_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2014. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_evil_backup.erl b/lib/mnesia/test/mnesia_evil_backup.erl index e605fa7926..044cf501fd 100644 --- a/lib/mnesia/test/mnesia_evil_backup.erl +++ b/lib/mnesia/test/mnesia_evil_backup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2011. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -325,7 +325,7 @@ restore(Config, Op) -> end, ?match(ok, file:delete(File1)), ?match(ok, file:delete(File2)), - ?match([], Check() -- Before), + ?match([], Check() -- (Before ++ [{ok, latest_log}, {ok, previous_log}])), ?verify_mnesia(Nodes, []). @@ -723,18 +723,18 @@ bup_records(File, Mod) -> exit(Reason) end. -sops_with_checkpoint(doc) -> +sops_with_checkpoint(doc) -> ["Test schema operations during a checkpoint"]; sops_with_checkpoint(suite) -> []; sops_with_checkpoint(Config) when is_list(Config) -> - Ns = ?acquire_nodes(2, Config), - + Ns = [N1,N2] = ?acquire_nodes(2, Config), + ?match({ok, cp1, Ns}, mnesia:activate_checkpoint([{name, cp1},{max,mnesia:system_info(tables)}])), - Tab = tab, + Tab = tab, ?match({atomic, ok}, mnesia:create_table(Tab, [{disc_copies,Ns}])), OldRecs = [{Tab, K, -K} || K <- lists:seq(1, 5)], [mnesia:dirty_write(R) || R <- OldRecs], - + ?match({ok, cp2, Ns}, mnesia:activate_checkpoint([{name, cp2},{max,mnesia:system_info(tables)}])), File1 = "cp1_delete_me.BUP", ?match(ok, mnesia:dirty_write({Tab,6,-6})), @@ -742,16 +742,16 @@ sops_with_checkpoint(Config) when is_list(Config) -> ?match(ok, mnesia:dirty_write({Tab,7,-7})), File2 = "cp2_delete_me.BUP", ?match(ok, mnesia:backup_checkpoint(cp2, File2)), - + ?match(ok, mnesia:deactivate_checkpoint(cp1)), ?match(ok, mnesia:backup_checkpoint(cp2, File1)), ?match(ok, mnesia:dirty_write({Tab,8,-8})), - + ?match({atomic,ok}, mnesia:delete_table(Tab)), ?match({error,_}, mnesia:backup_checkpoint(cp2, File2)), ?match({'EXIT',_}, mnesia:dirty_write({Tab,9,-9})), - ?match({atomic,_}, mnesia:restore(File1, [{default_op, recreate_tables}])), + ?match({atomic,_}, mnesia:restore(File1, [{default_op, recreate_tables}])), Test = fun(N) when N > 5 -> ?error("To many records in backup ~p ~n", [N]); (N) -> case mnesia:dirty_read(Tab,N) of [{Tab,N,B}] when -B =:= N -> ok; @@ -759,8 +759,29 @@ sops_with_checkpoint(Config) when is_list(Config) -> end end, [Test(N) || N <- mnesia:dirty_all_keys(Tab)], - ?match({aborted,enoent}, mnesia:restore(File2, [{default_op, recreate_tables}])), - + ?match({aborted,enoent}, mnesia:restore(File2, [{default_op, recreate_tables}])), + + %% Mnesia crashes when deleting a table during backup + ?match([], mnesia_test_lib:stop_mnesia([N2])), + Tab2 = ram, + ?match({atomic, ok}, mnesia:create_table(Tab2, [{ram_copies,[N1]}])), + ?match({ok, cp3, _}, mnesia:activate_checkpoint([{name, cp3}, + {ram_overrides_dump,true}, + {min,[Tab2]}])), + Write = fun Loop (N) -> + case N > 0 of + true -> + mnesia:dirty_write({Tab2, N+100, N+100}), + Loop(N-1); + false -> + ok + end + end, + ok = Write(100000), + spawn_link(fun() -> ?match({atomic, ok},mnesia:delete_table(Tab2)) end), + + %% We don't check result here, depends on timing of above call + mnesia:backup_checkpoint(cp3, File2), file:delete(File1), file:delete(File2), - ?verify_mnesia(Ns, []). + ?verify_mnesia([N1], [N2]). diff --git a/lib/mnesia/test/mnesia_evil_coverage_test.erl b/lib/mnesia/test/mnesia_evil_coverage_test.erl index 68b521c939..6e34040bc4 100644 --- a/lib/mnesia/test/mnesia_evil_coverage_test.erl +++ b/lib/mnesia/test/mnesia_evil_coverage_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -72,7 +72,9 @@ groups() -> {record_name_dirty_access, [], [record_name_dirty_access_ram, record_name_dirty_access_disc, - record_name_dirty_access_disc_only]}]. + record_name_dirty_access_disc_only, + record_name_dirty_access_xets + ]}]. init_per_group(_GroupName, Config) -> Config. @@ -112,6 +114,7 @@ system_info(Config) when is_list(Config) -> ?match(I when is_integer(I), mnesia:system_info(transaction_log_writes)), ?match(I when is_integer(I), mnesia:system_info(send_compressed)), ?match(L when is_list(L), mnesia:system_info(all)), + ?match(L when is_list(L), mnesia:system_info(backend_types)), ?match({'EXIT', {aborted, Reason }} when element(1, Reason) == badarg , mnesia:system_info(ali_baba)), ?verify_mnesia(Nodes, []). @@ -132,11 +135,11 @@ table_info(Config) when is_list(Config) -> Schema = case mnesia_test_lib:diskless(Config) of true -> [{type, Type}, {attributes, Attrs}, {index, [ValPos]}, - {ram_copies, Nodes}]; + {ram_copies, [Node1, Node2]}, {ext_ets, [Node3]}]; false -> [{type, Type}, {attributes, Attrs}, {index, [ValPos]}, - {disc_only_copies, [Node1]}, {ram_copies, [Node2]}, - {disc_copies, [Node3]}] + {disc_only_copies, [Node1]}, {ram_copies, [Node2]}, + {ext_ets, [Node3]}] end, ?match({atomic, ok}, mnesia:create_table(Tab, Schema)), @@ -144,28 +147,22 @@ table_info(Config) when is_list(Config) -> Keys = lists:seq(1, Size), Records = [{Tab, A, 7} || A <- Keys], lists:foreach(fun(Rec) -> ?match(ok, mnesia:dirty_write(Rec)) end, Records), - ?match(Mem when is_integer(Mem), mnesia:table_info(Tab, memory)), - ?match(Size, mnesia:table_info(Tab, size)), - ?match(Type, mnesia:table_info(Tab, type)), case mnesia_test_lib:diskless(Config) of true -> ?match(Nodes, mnesia:table_info(Tab, ram_copies)); false -> - ?match([Node3], mnesia:table_info(Tab, mnesia_test_lib:storage_type(disc_copies, Config))), + ?match([Node3], mnesia:table_info(Tab, ext_ets)), ?match([Node2], mnesia:table_info(Tab, ram_copies)), ?match([Node1], mnesia:table_info(Tab, mnesia_test_lib:storage_type(disc_only_copies, Config))) end, Read = [Node1, Node2, Node3], - ?match(true, lists:member(mnesia:table_info(Tab, where_to_read), Read)), Write = ?sort([Node1, Node2, Node3]), - ?match(Write, ?sort(mnesia:table_info(Tab, where_to_write))), - ?match([ValPos], mnesia:table_info(Tab, index)), - ?match(Arity, mnesia:table_info(Tab, arity)), - ?match(Attrs, mnesia:table_info(Tab, attributes)), - ?match({Tab, '_', '_'}, mnesia:table_info(Tab, wild_pattern)), - ?match({atomic, Attrs}, mnesia:transaction(fun() -> - mnesia:table_info(Tab, attributes) end)), + + {[ok,ok,ok], []} = rpc:multicall(Nodes, ?MODULE, info_check, + [Tab, Read, Write, Size, Type, ValPos, Arity, Attrs]), + + ?match({atomic, Attrs}, mnesia:transaction(fun() -> mnesia:table_info(Tab, attributes) end)), ?match(L when is_list(L), mnesia:table_info(Tab, all)), @@ -179,6 +176,17 @@ table_info(Config) when is_list(Config) -> ?match(0, mnesia:table_info(tab_info, size)), ?verify_mnesia([Node1, Node3], [Node2]). +info_check(Tab, Read, Write, Size, Type, ValPos, Arity, Attrs) -> + ?match(true, lists:member(mnesia:table_info(Tab, where_to_read), Read)), + ?match(Write, ?sort(mnesia:table_info(Tab, where_to_write))), + ?match(Mem when is_integer(Mem), mnesia:table_info(Tab, memory)), + ?match(Size, mnesia:table_info(Tab, size)), + ?match(Type, mnesia:table_info(Tab, type)), + ?match([ValPos], mnesia:table_info(Tab, index)), + ?match(Arity, mnesia:table_info(Tab, arity)), + ?match(Attrs, mnesia:table_info(Tab, attributes)), + ?match({Tab, '_', '_'}, mnesia:table_info(Tab, wild_pattern)), + ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Check the error descriptions @@ -216,11 +224,12 @@ db_node_lifecycle(Config) when is_list(Config) -> [Node1, Node2, Node3] = AllNodes = ?acquire_nodes(3, Config), Tab = db_node_lifecycle, - Who = fun(T) -> + Who = fun(T) -> L1 = mnesia:table_info(T, ram_copies), L2 = mnesia:table_info(T, disc_copies), L3 = mnesia:table_info(T, disc_only_copies), - L1 ++ L2 ++ L3 + L4 = mnesia:table_info(T, ext_ets), + L1 ++ L2 ++ L3 ++ L4 end, SNs = ?sort(AllNodes), @@ -235,6 +244,7 @@ db_node_lifecycle(Config) when is_list(Config) -> ?match({error, _}, mnesia:create_schema([foo@bar])), ?match(ok, mnesia:start()), ?match(false, mnesia:system_info(use_dir)), + ?match([ram_copies, disc_copies, disc_only_copies], mnesia:system_info(backend_types)), ?match({atomic, ok}, mnesia:create_table(Tab, [])), ?match({aborted, {has_no_disc, Node1}}, mnesia:dump_tables([Tab])), ?match({aborted, {has_no_disc, Node1}}, mnesia:change_table_copy_type(Tab, node(), disc_copies)), @@ -242,7 +252,7 @@ db_node_lifecycle(Config) when is_list(Config) -> ?match(stopped, mnesia:stop()), - ?match(ok, mnesia:create_schema(AllNodes)), + ?match(ok, mnesia:create_schema(AllNodes, ?BACKEND)), ?match([], mnesia_test_lib:start_mnesia(AllNodes)), ?match([SNs, SNs, SNs], @@ -271,12 +281,15 @@ db_node_lifecycle(Config) when is_list(Config) -> Tab3 = not_local, Tab4 = local, Tab5 = remote, + Tab6 = ext1, Tabs = [Schema, [{name, Tab2}, {disc_copies, AllNodes}], [{name, Tab3}, {ram_copies, [Node2, Node3]}], [{name, Tab4}, {disc_only_copies, [Node1]}], - [{name, Tab5}, {disc_only_copies, [Node2]}]], + [{name, Tab5}, {disc_only_copies, [Node2]}], + [{name, Tab6}, {ext_ets, [Node1, Node2]}] + ], [?match({atomic, ok}, mnesia:create_table(T)) || T <- Tabs ], @@ -287,31 +300,28 @@ db_node_lifecycle(Config) when is_list(Config) -> ?match({aborted, {node_not_running, Node1}}, mnesia:del_table_copy(schema, Node2)), - ?match([], mnesia_test_lib:start_mnesia([Node1],[Tab2,Tab4])), + ?match([], mnesia_test_lib:start_mnesia([Node1],[Tab2,Tab4,Tab6])), ?match([], mnesia_test_lib:stop_mnesia([Node2])), - ?match({atomic, ok}, - mnesia:del_table_copy(schema, Node2)), + ?match({atomic, ok}, mnesia:del_table_copy(schema, Node2)), - %% Check + %% Check RemNodes = AllNodes -- [Node2], - ?match(RemNodes, mnesia:system_info(db_nodes)), + ?match(RemNodes, mnesia:system_info(db_nodes)), ?match([Node1], Who(Tab)), ?match(RemNodes, Who(Tab2)), ?match([Node3], Who(Tab3)), ?match([Node1], Who(Tab4)), ?match({'EXIT', {aborted, {no_exists, _, _}}}, Who(Tab5)), + ?match([Node1], Who(Tab6)), - ?match({atomic, ok}, - mnesia:change_table_copy_type(Tab2, Node3, ram_copies)), + ?match({atomic, ok}, mnesia:change_table_copy_type(Tab2, Node3, ram_copies)), - ?match({atomic, ok}, - mnesia:change_table_copy_type(schema, Node3, ram_copies)), + ?match({atomic, ok}, mnesia:change_table_copy_type(schema, Node3, ram_copies)), ?match([], mnesia_test_lib:stop_mnesia([Node3])), - ?match({atomic, ok}, - mnesia:del_table_copy(schema, Node3)), - ?match([Node1], mnesia:system_info(db_nodes)), + ?match({atomic, ok}, mnesia:del_table_copy(schema, Node3)), + ?match([Node1], mnesia:system_info(db_nodes)), ?match([Node1], Who(Tab)), ?match([Node1], Who(Tab2)), ?match({'EXIT', {aborted, {no_exists, _, _}}}, Who(Tab3)), @@ -363,7 +373,8 @@ start_and_stop(Config) when is_list(Config) -> checkpoint(suite) -> []; checkpoint(Config) when is_list(Config) -> checkpoint(2, Config), - checkpoint(3, Config). + checkpoint(3, Config), + ok. checkpoint(NodeConfig, Config) -> [Node1 | _] = TabNodes = ?acquire_nodes(NodeConfig, Config), @@ -381,7 +392,7 @@ checkpoint(NodeConfig, Config) -> CreateTab(Type, 3, [lists:last(TabNodes)])] ++ Acc end, - Types = [ram_copies, disc_copies, disc_only_copies], + Types = [ram_copies, disc_copies, disc_only_copies, ext_ets], Tabs = lists:foldl(CreateTabs, [], Types), Recs = ?sort([{T, N, N} || T <- Tabs, N <- lists:seq(1, 10)]), lists:foreach(fun(R) -> ?match(ok, mnesia:dirty_write(R)) end, Recs), @@ -430,25 +441,29 @@ checkpoint(NodeConfig, Config) -> replica_location(suite) -> []; replica_location(Config) when is_list(Config) -> [Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config), - Tab = replica_location, %% Create three replicas - Schema = [{name, Tab}, {disc_only_copies, [Node1]}, - {ram_copies, [Node2]}, {disc_copies, [Node3]}], - ?match({atomic, ok}, mnesia:create_table(Schema)), - ?match([], ?vrl(Tab, [Node1], [Node2], [Node3], Nodes)), + Check = fun(Tab, Schema) -> + ?match({atomic, ok}, mnesia:create_table([{name, Tab}|Schema])), + ?match([], ?vrl(Tab, [Node1], [Node2], [Node3], Nodes)), - %% Delete one replica - ?match({atomic, ok}, mnesia:del_table_copy(Tab, Node2)), - ?match([], ?vrl(Tab, [Node1], [], [Node3], Nodes)), + %% Delete one replica + ?match({atomic, ok}, mnesia:del_table_copy(Tab, Node2)), + ?match([], ?vrl(Tab, [Node1], [], [Node3], Nodes)), - %% Move one replica - ?match({atomic, ok}, mnesia:move_table_copy(Tab, Node1, Node2)), - ?match([], ?vrl(Tab, [Node2], [], [Node3], Nodes)), + %% Move one replica + ?match({atomic, ok}, mnesia:move_table_copy(Tab, Node1, Node2)), + ?match([], ?vrl(Tab, [Node2], [], [Node3], Nodes)), + + %% Change replica type + ?match({atomic, ok}, mnesia:change_table_copy_type(Tab, Node2, ram_copies)), + ?match([], ?vrl(Tab, [], [Node2], [Node3], Nodes)) + end, + Check(replica_location, [{disc_only_copies, [Node1]}, + {ram_copies, [Node2]}, {disc_copies, [Node3]}]), - %% Change replica type - ?match({atomic, ok}, mnesia:change_table_copy_type(Tab, Node2, ram_copies)), - ?match([], ?vrl(Tab, [], [Node2], [Node3], Nodes)), + Check(ext_location, [{disc_only_copies, [Node1]}, + {ext_ets, [Node2]}, {disc_copies, [Node3]}]), ?verify_mnesia(Nodes, []). @@ -720,7 +735,7 @@ replica_management(Config) when is_list(Config) -> %% ?match({atomic, ok}, mnesia:create_table([{name, Tab}, {attributes, Attrs}, - {ram_copies, [Node1, Node3]}])), + {ram_copies, [Node1]}, {ext_ets, [Node3]}])), [?match(ok, mnesia:dirty_write({Tab, K, K + 2})) || K <-lists:seq(1, 10)], ?match([], ?vrl(Tab, [], [Node1, Node3], [], Nodes)), %% R - - @@ -757,7 +772,7 @@ replica_management(Config) when is_list(Config) -> ?match([], ?vrl(Tab, [Node2], [], [Node1], Nodes)), ?match([0,10,10], ?SS(rpc:multicall(Nodes, mnesia, table_info, [Tab, size]))), %% D DO - - ?match({atomic, ok}, mnesia:add_table_copy(Tab, Node3, ram_copies)), + ?match({atomic, ok}, mnesia:add_table_copy(Tab, Node3, ext_ets)), ?match([], ?vrl(Tab, [Node2], [Node3], [Node1], Nodes)), ?match([10,10,10], ?SS(rpc:multicall(Nodes, mnesia, table_info, [Tab, size]))), %% D DO R @@ -784,7 +799,7 @@ replica_management(Config) when is_list(Config) -> ?match([10,10,10], ?SS(rpc:multicall(Nodes, mnesia, table_info, [Tab, size]))), %% D DO D0 - ?match({atomic, ok}, mnesia:change_table_copy_type(Tab, Node3, ram_copies)), + ?match({atomic, ok}, mnesia:change_table_copy_type(Tab, Node3, ext_ets)), ?match([], ?vrl(Tab, [Node2], [Node3], [Node1], Nodes)), ?match([10,10,10], ?SS(rpc:multicall(Nodes, mnesia, table_info, [Tab, size]))), %% D DO R @@ -841,18 +856,31 @@ replica_management(Config) when is_list(Config) -> ?match([], ?vrl(Tab, [Node3], [], [Node2], Nodes)), ?match([0,10,10], ?SS(rpc:multicall(Nodes, mnesia, table_info, [Tab, size]))), %% - D DO + ?match({atomic, ok}, mnesia:change_table_copy_type(Tab, Node3, ext_ets)), + ?match([], ?vrl(Tab, [], [Node3], [Node2], Nodes)), + ?match([0,10,10], ?SS(rpc:multicall(Nodes, mnesia, table_info, [Tab, size]))), + %% - D ER + ?match({atomic, ok}, mnesia:move_table_copy(Tab, Node3, Node1)), + ?match([], ?vrl(Tab, [], [Node1], [Node2], Nodes)), + ?match([0,10,10], ?SS(rpc:multicall(Nodes, mnesia, table_info, [Tab, size]))), + %% ER D - + ?match({aborted, _}, mnesia:move_table_copy(Tab, Node1, Node2)), + ?match({aborted, _}, mnesia:move_table_copy(Tab, Node3, Node2)), + ?match({atomic, ok}, mnesia:move_table_copy(Tab, Node1, Node3)), + %% - D ER ?match([], mnesia_test_lib:stop_mnesia([Node3])), ?match({atomic,ok}, mnesia:transaction(fun() -> mnesia:write({Tab, 43, sync_me}) end)), - ?match([], ?vrl(Tab, [Node3], [], [Node2],Nodes -- [Node3])), - %% - D DO + ?match([], ?vrl(Tab, [], [Node3], [Node2],Nodes -- [Node3])), + %% - D ER ?match({aborted,Reason56} when element(1, Reason56) == not_active, mnesia:move_table_copy(Tab, Node3, Node1)), - ?match([], ?vrl(Tab, [Node3], [], [Node2],Nodes -- [Node3])), - %% DO D - + ?match([], ?vrl(Tab, [], [Node3], [Node2],Nodes -- [Node3])), + %% - D ER ?match([], mnesia_test_lib:start_mnesia([Node3])), - ?match([], ?vrl(Tab, [Node3], [], [Node2], Nodes)), - %% DO D - + ?match([], ?vrl(Tab, [], [Node3], [Node2], Nodes)), + %% - D ER + ?match([{Tab,43,sync_me}], mnesia:dirty_read({Tab,43})), %% %% Transformer @@ -990,7 +1018,7 @@ local_content(Config) when is_list(Config) -> ?match([], mnesia_test_lib:stop_mnesia([Node3])), %% Added for OTP-44306 - ?match(ok, rpc:call(Node3, mnesia, start, [])), + ?match(ok, rpc:call(Node3, mnesia, start, [[{schema, ?BACKEND}]])), ?match({ok, _}, mnesia:change_config(extra_db_nodes, [Node3])), mnesia_test_lib:sync_tables([Node3], [Tab1]), @@ -1427,7 +1455,7 @@ unsupp_user_props(Config) when is_list(Config) -> table_info, [silly1, user_properties])), ?match([{prop,propval2}], rpc:call(Node1, mnesia, table_info, [silly2, user_properties])), - ?match([{prop,propval3}], rpc:call(Node1, mnesia, + ?match([_,{prop,propval3}], rpc:call(Node1, mnesia, table_info, [schema, user_properties])), F2 = fun() -> @@ -2262,6 +2290,10 @@ record_name_dirty_access_disc_only(suite) -> record_name_dirty_access_disc_only(Config) when is_list(Config) -> record_name_dirty_access(disc_only_copies, Config). +record_name_dirty_access_xets(Config) when is_list(Config) -> + record_name_dirty_access(ext_ets, Config). + + record_name_dirty_access(Storage, Config) -> [Node1, _Node2] = Nodes = ?acquire_nodes(2, Config), diff --git a/lib/mnesia/test/mnesia_examples_test.erl b/lib/mnesia/test/mnesia_examples_test.erl index 889ec50520..808e62d9c2 100644 --- a/lib/mnesia/test/mnesia_examples_test.erl +++ b/lib/mnesia/test/mnesia_examples_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_frag_test.erl b/lib/mnesia/test/mnesia_frag_test.erl index c6430240c8..9f2102beb2 100644 --- a/lib/mnesia/test/mnesia_frag_test.erl +++ b/lib/mnesia/test/mnesia_frag_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2010. All Rights Reserved. +%% Copyright Ericsson AB 1999-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_inconsistent_database_test.erl b/lib/mnesia/test/mnesia_inconsistent_database_test.erl index c6ca3c6853..3fd2c6ad5b 100644 --- a/lib/mnesia/test/mnesia_inconsistent_database_test.erl +++ b/lib/mnesia/test/mnesia_inconsistent_database_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_install_test.erl b/lib/mnesia/test/mnesia_install_test.erl index e0145f3a8b..103f85b3d6 100644 --- a/lib/mnesia/test/mnesia_install_test.erl +++ b/lib/mnesia/test/mnesia_install_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_isolation_test.erl b/lib/mnesia/test/mnesia_isolation_test.erl index 4dd0c01d05..63940ec05c 100644 --- a/lib/mnesia/test/mnesia_isolation_test.erl +++ b/lib/mnesia/test/mnesia_isolation_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_majority_test.erl b/lib/mnesia/test/mnesia_majority_test.erl index f2e7a09e5e..9811de6ae7 100644 --- a/lib/mnesia/test/mnesia_majority_test.erl +++ b/lib/mnesia/test/mnesia_majority_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_measure_test.erl b/lib/mnesia/test/mnesia_measure_test.erl index b184c5efdf..ad71fafecb 100644 --- a/lib/mnesia/test/mnesia_measure_test.erl +++ b/lib/mnesia/test/mnesia_measure_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_meter.erl b/lib/mnesia/test/mnesia_meter.erl index 73f35a4a46..c278dfde93 100644 --- a/lib/mnesia/test/mnesia_meter.erl +++ b/lib/mnesia/test/mnesia_meter.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_nice_coverage_test.erl b/lib/mnesia/test/mnesia_nice_coverage_test.erl index 3e00cc7325..ffbe36e48d 100644 --- a/lib/mnesia/test/mnesia_nice_coverage_test.erl +++ b/lib/mnesia/test/mnesia_nice_coverage_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_qlc_test.erl b/lib/mnesia/test/mnesia_qlc_test.erl index 90fb8540c2..5067e86521 100644 --- a/lib/mnesia/test/mnesia_qlc_test.erl +++ b/lib/mnesia/test/mnesia_qlc_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2011. All Rights Reserved. +%% Copyright Ericsson AB 2004-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_recovery_test.erl b/lib/mnesia/test/mnesia_recovery_test.erl index e47ed9c6c0..130b87346f 100644 --- a/lib/mnesia/test/mnesia_recovery_test.erl +++ b/lib/mnesia/test/mnesia_recovery_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -504,12 +504,21 @@ with_checkpoint(Config, Type) when is_list(Config) -> ?match(ok, mnesia:deactivate_checkpoint(sune)), ?match([], check_chkp(Nodes)), + Wait = fun(Loop) -> + timer:sleep(300), + sys:get_status(mnesia_monitor), + case lists:member(Kill, mnesia_lib:val({current, db_nodes})) of + true -> Loop(Loop); + false -> ok + end + end, + case Kill of Node1 -> ignore; Node2 -> mnesia_test_lib:kill_mnesia([Kill]), - timer:sleep(500), %% Just to help debugging + Wait(Wait), ?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune}, {max, mnesia:system_info(tables)}, {ram_overrides_dump, true}])), @@ -576,7 +585,7 @@ delete_during_start(Config) when is_list(Config) -> mnesia_test_lib:kill_mnesia([N2,N3]), %% timer:sleep(500), ?match({[ok,ok],[]}, rpc:multicall([N2,N3], mnesia,start, - [[{extra_db_nodes,[N1]}]])), + [[{extra_db_nodes,[N1]}, {schema, ?BACKEND}]])), [Tab1,Tab2,Tab3|_] = Tabs, ?match({atomic, ok}, mnesia:delete_table(Tab1)), ?match({atomic, ok}, mnesia:delete_table(Tab2)), @@ -1542,7 +1551,7 @@ disc_less(Config) when is_list(Config) -> ?match(ok, rpc:call(Node2, mnesia, start, [])), timer:sleep(500), - ?match(ok, rpc:call(Node3, mnesia, start, [[{extra_db_nodes, [Node1, Node2]}]])), + ?match(ok, rpc:call(Node3, mnesia, start, [[{extra_db_nodes, [Node1, Node2]}, {schema, ?BACKEND}]])), ?match(ok, rpc:call(Node3, mnesia, wait_for_tables, [[Tab1, Tab2, Tab3], 20000])), ?match(ok, rpc:call(Node1, mnesia, wait_for_tables, [[Tab1, Tab2, Tab3], 20000])), diff --git a/lib/mnesia/test/mnesia_registry_test.erl b/lib/mnesia/test/mnesia_registry_test.erl index 9d9190d70d..3df37a2c8c 100644 --- a/lib/mnesia/test/mnesia_registry_test.erl +++ b/lib/mnesia/test/mnesia_registry_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_schema_recovery_test.erl b/lib/mnesia/test/mnesia_schema_recovery_test.erl index 05c5639d60..ca2dd74b34 100644 --- a/lib/mnesia/test/mnesia_schema_recovery_test.erl +++ b/lib/mnesia/test/mnesia_schema_recovery_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2014. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_test_lib.erl b/lib/mnesia/test/mnesia_test_lib.erl index 9d3b277e07..0fabdc7929 100644 --- a/lib/mnesia/test/mnesia_test_lib.erl +++ b/lib/mnesia/test/mnesia_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -142,6 +142,9 @@ %% included for test server compatibility %% assume that all test cases only takes Config as sole argument init_per_testcase(_Func, Config) -> + Env = application:get_all_env(mnesia), + [application:unset_env(mnesia, Key, [{timeout, infinity}]) || + {Key, _} <- Env, Key /= included_applications], global:register_name(mnesia_global_logger, group_leader()), Config. @@ -260,6 +263,7 @@ slave_start_link(Host, Name, Retries) -> Path = code:get_path(), ok = rpc:call(NewNode, file, set_cwd, [Cwd]), true = rpc:call(NewNode, code, set_path, [Path]), + ok = rpc:call(NewNode, error_logger, tty, [false]), spawn_link(NewNode, ?MODULE, slave_sup, []), rpc:multicall([node() | nodes()], global, sync, []), {ok, NewNode}; @@ -668,11 +672,13 @@ do_prepare([delete_schema | Actions], Selected, All, Config, File, Line) -> end, do_prepare(Actions, Selected, All, Config, File, Line); do_prepare([create_schema | Actions], Selected, All, Config, File, Line) -> + Ext = ?BACKEND, case diskless(Config) of true -> + rpc:multicall(Selected, application, set_env, [mnesia, schema, Ext]), skip; _Else -> - case mnesia:create_schema(Selected) of + case mnesia:create_schema(Selected, Ext) of ok -> ignore; BadNodes -> @@ -975,7 +981,6 @@ reload_appls([Appl | Appls], Selected) -> {Ok2temp, Empty} = rpc:multicall(Selected, application, unload, [Appl]), Conv = fun({error,{not_loaded,mnesia}}) -> ok; (Else) -> Else end, Ok2 = {lists:map(Conv, Ok2temp), Empty}, - Ok3 = rpc:multicall(Selected, application, load, [Appl]), if Ok /= Ok2 -> @@ -1040,7 +1045,8 @@ verify_replica_location(Tab, DiscOnly0, Ram0, Disc0, AliveNodes0) -> S1 = ?match(AliveNodes, lists:sort(mnesia:system_info(running_db_nodes))), S2 = ?match(DiscOnly, lists:sort(mnesia:table_info(Tab, disc_only_copies))), - S3 = ?match(Ram, lists:sort(mnesia:table_info(Tab, ram_copies))), + S3 = ?match(Ram, lists:sort(mnesia:table_info(Tab, ram_copies) ++ + mnesia:table_info(Tab, ext_ets))), S4 = ?match(Disc, lists:sort(mnesia:table_info(Tab, disc_copies))), S5 = ?match(Write, lists:sort(mnesia:table_info(Tab, where_to_write))), S6 = case lists:member(This, Read) of diff --git a/lib/mnesia/test/mnesia_test_lib.hrl b/lib/mnesia/test/mnesia_test_lib.hrl index dc3b5b9af8..ba7eb10ea2 100644 --- a/lib/mnesia/test/mnesia_test_lib.hrl +++ b/lib/mnesia/test/mnesia_test_lib.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -150,3 +150,5 @@ -define(verify_mnesia(Ups, Downs), mnesia_test_lib:verify_mnesia(Ups, Downs, ?FILE, ?LINE)). + +-define(BACKEND, [{backend_types, [{ext_ets, ext_test},{ext_dets, ext_test}]}]). diff --git a/lib/mnesia/test/mnesia_tpcb.erl b/lib/mnesia/test/mnesia_tpcb.erl index c6eda1c448..fb39ee321d 100644 --- a/lib/mnesia/test/mnesia_tpcb.erl +++ b/lib/mnesia/test/mnesia_tpcb.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/mnesia/test/mnesia_trans_access_test.erl b/lib/mnesia/test/mnesia_trans_access_test.erl index a371d58e75..4ed73ea859 100644 --- a/lib/mnesia/test/mnesia_trans_access_test.erl +++ b/lib/mnesia/test/mnesia_trans_access_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -307,6 +307,7 @@ select14(Config) when is_list(Config) -> %% Some Helpers Trans = fun(Fun) -> mnesia:transaction(Fun) end, + Dirty = fun(Fun) -> mnesia:async_dirty(Fun) end, LoopHelp = fun('$end_of_table',_) -> []; ({Recs,Cont},Fun) -> Sel = mnesia:select(Cont), @@ -334,8 +335,13 @@ select14(Config) when is_list(Config) -> ?match({atomic, [OneRec]}, Trans(fun() -> Loop(Tab, OnePat) end)), ?match({atomic, All}, Trans(fun() -> Loop(Tab, AllPat) end)), - {atomic,{_, Cont}} = Trans(fun() -> mnesia:select(Tab, OnePat, 1, read) end), - ?match({aborted, wrong_transaction}, Trans(fun() -> mnesia:select(Cont) end)), + {atomic,{_, ContOne}} = Trans(fun() -> mnesia:select(Tab, OnePat, 1, read) end), + ?match({aborted, wrong_transaction}, Trans(fun() -> mnesia:select(ContOne) end)), + ?match('$end_of_table', Dirty(fun() -> mnesia:select(ContOne) end)), + + {atomic,{_, ContAll}} = Trans(fun() -> mnesia:select(Tab, AllPat, 1, read) end), + ?match({aborted, wrong_transaction}, Trans(fun() -> mnesia:select(ContAll) end)), + ?match({[_], _}, Dirty(fun() -> mnesia:select(ContAll) end)), ?match({aborted, _}, Trans(fun() -> mnesia:select(Tab, {match, '$1', 2},1,read) end)), ?match({aborted, _}, Trans(fun() -> mnesia:select(Tab, [{'_', [], '$1'}],1,read) end)), @@ -931,20 +937,20 @@ index_update_bag(Config)when is_list(Config) -> [IPos] = mnesia_lib:val({Tab,index}), ITab = mnesia_lib:val({index_test,{index, IPos}}), io:format("~n Index ~p @ ~p => ~p ~n~n",[IPos,ITab, ets:tab2list(ITab)]), - ?match([{2,1},{2,2},{12,1}], lists:keysort(1,ets:tab2list(ITab))), + %?match([{2,1},{2,2},{12,1}], lists:keysort(1,ets:tab2list(ITab))), ?match({atomic, ok}, mnesia:transaction(fun() -> mnesia:write(Rec5) end)), {atomic, R60} = mnesia:transaction(fun() -> mnesia:index_read(Tab, 2, ValPos) end), ?match([Rec1,Rec5,Rec2], lists:sort(R60)), - ?match([{2,1},{2,2},{12,1}], lists:keysort(1,ets:tab2list(ITab))), + %?match([{2,1},{2,2},{12,1}], lists:keysort(1,ets:tab2list(ITab))), ?match({atomic, ok}, mnesia:transaction(fun() -> mnesia:delete_object(Rec3) end)), {atomic, R61} = mnesia:transaction(fun() -> mnesia:index_read(Tab, 2, ValPos) end), ?match([Rec1,Rec5,Rec2], lists:sort(R61)), {atomic, R62} = mnesia:transaction(fun() -> mnesia:index_read(Tab,12, ValPos) end), ?match([], lists:sort(R62)), - ?match([{2,1},{2,2}], lists:keysort(1,ets:tab2list(ITab))), + %% ?match([{2,1},{2,2}], lists:keysort(1,ets:tab2list(ITab))), %% reset for rest of testcase ?match({atomic, ok}, mnesia:transaction(fun() -> mnesia:write(Rec3) end)), @@ -1142,8 +1148,9 @@ create_live_table_index(Config, Storage) -> ?match([{atomic,ok}|_], [Create(N) || N <- lists:seq(1,50)]), ?match([], mnesia_test_lib:stop_mnesia([N2,N3])), - ?match(ok, rpc:call(N2, mnesia, start, [[{extra_db_nodes,[N1]}]])), - ?match(ok, rpc:call(N3, mnesia, start, [[{extra_db_nodes,[N1]}]])), + Ext = [{schema, ?BACKEND}], + ?match(ok, rpc:call(N2, mnesia, start, [[{extra_db_nodes,[N1]}|Ext]])), + ?match(ok, rpc:call(N3, mnesia, start, [[{extra_db_nodes,[N1]}|Ext]])), ?match({atomic, ok}, mnesia:add_table_index(Tab, ValPos)), diff --git a/lib/mnesia/test/mt.erl b/lib/mnesia/test/mt.erl index 1256dbb80a..5a981bf539 100644 --- a/lib/mnesia/test/mt.erl +++ b/lib/mnesia/test/mt.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2011. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -80,6 +80,8 @@ resolve(Suite0) when is_atom(Suite0) -> {Suite, Case} -> {Suite, is_group(Suite,Case)} end; +resolve({Suite0, {group, Case}}) -> + resolve({Suite0, Case}); resolve({Suite0, Case}) when is_atom(Suite0), is_atom(Case) -> case alias(Suite0) of Suite when is_atom(Suite) -> diff --git a/lib/mnesia/vsn.mk b/lib/mnesia/vsn.mk index 194bc439a0..e272a469bb 100644 --- a/lib/mnesia/vsn.mk +++ b/lib/mnesia/vsn.mk @@ -1 +1 @@ -MNESIA_VSN = 4.13.4 +MNESIA_VSN = 4.14.3 |