Merge branch 'maint-20' into maint

* maint-20:
  Updated OTP version
  Update release notes
  Update version numbers
  crypto: Fix crash in compute_key(ecdh, ...) on badarg
  Relax add_table_copy restriction
  Fixed #Ref ordering bug
  Test #Ref ordering in lists and ets
  Do NOT disc_load from ram_copies when master_node is set
  ssl: Make sure that a correct cipher suite is selected
  ssl: Correct handling of empty server SNI extension

5 files changed, 194 insertions, 24 deletions

diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml
index 7134e3d1e4..b4816f9fa4 100644
--- a/lib/mnesia/doc/src/notes.xml
+++ b/lib/mnesia/doc/src/notes.xml
@@ -54,6 +54,32 @@
 
 </section>
 
+<section><title>Mnesia 4.15.3.1</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    When master node is set do not force a load from
+	    ram_copies replica when there are no available
+	    disc_copies, since that would load an empty table. Wait
+	    until a disk replica is available or until user
+	    explicitly force_loads the table.</p>
+          <p>
+	    Own Id: OTP-15221 Aux Id: ERIERL-217 </p>
+        </item>
+        <item>
+          <p>
+	    Allow to add replicas even if all other replicas are down
+	    when the other replicase are not stored on disk.</p>
+          <p>
+	    Own Id: OTP-15226 Aux Id: ERIERL-221 </p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
 <section><title>Mnesia 4.15.3</title>
 
     <section><title>Fixed Bugs and Malfunctions</title>
diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl
index f81ba783f2..882de0d613 100644
--- a/lib/mnesia/src/mnesia_controller.erl
+++ b/lib/mnesia/src/mnesia_controller.erl
@@ -772,22 +772,6 @@ handle_call({unannounce_add_table_copy, [Tab, Node], From}, ReplyTo, State) ->
 	    noreply(State#state{early_msgs = [{call, Msg, undefined} | Msgs]})
     end;
 
-handle_call({net_load, Tab, Cs}, From, State) ->
-    State2 =
-	case State#state.schema_is_merged of
-	    true ->
-		Worker = #net_load{table = Tab,
-				   opt_reply_to = From,
-				   reason = {dumper,{add_table_copy, unknown}},
-				   cstruct = Cs
-				  },
-		add_worker(Worker, State);
-	    false ->
-		reply(From, {not_loaded, schema_not_merged}),
-		State
-	end,
-    noreply(State2);
-
 handle_call(Msg, From, State) when State#state.schema_is_merged /= true ->
     %% Buffer early messages
     Msgs = State#state.early_msgs,
@@ -1457,7 +1441,8 @@ orphan_tables([Tab | Tabs], Node, Ns, Local, Remote) ->
 				    L = [Tab | Local],
 				    orphan_tables(Tabs, Node, Ns, L, Remote);
 				Masters ->
-				    R = [{Tab, Masters} | Remote],
+                                    %% Do not disc_load table from RamCopyHolders
+				    R = [{Tab, Masters -- RamCopyHolders} | Remote],
 				    orphan_tables(Tabs, Node, Ns, Local, R)
 			    end;
 			_ ->
@@ -2162,6 +2147,15 @@ load_table_fun(#net_load{cstruct=Cs, table=Tab, reason=Reason, opt_reply_to=Repl
 		       {dumper,{add_table_copy,_}} -> true;
 		       _ -> false
 		   end,
+
+    OnlyRamCopies = case Cs of
+                        #cstruct{disc_copies = DC,
+                                 disc_only_copies = DOC,
+                                 external_copies = Ext} ->
+                            [] =:= (DC ++ (DOC ++ Ext)) -- [node()];
+                        _ ->
+                            false
+                    end,
     if
 	ReadNode == node() ->
 	    %% Already loaded locally
@@ -2173,6 +2167,8 @@ load_table_fun(#net_load{cstruct=Cs, table=Tab, reason=Reason, opt_reply_to=Repl
 	    end;
 	AccessMode == read_only, not AddTableCopy ->
 	    fun() -> disc_load_table(Tab, Reason, ReplyTo) end;
+        Active =:= [], AddTableCopy, OnlyRamCopies ->
+            fun() -> disc_load_table(Tab, Reason, ReplyTo) end;
 	true ->
 	    fun() ->
 		    %% Either we cannot read the table yet
diff --git a/lib/mnesia/src/mnesia_loader.erl b/lib/mnesia/src/mnesia_loader.erl
index ebe924a86e..2cdae0c906 100644
--- a/lib/mnesia/src/mnesia_loader.erl
+++ b/lib/mnesia/src/mnesia_loader.erl
@@ -67,7 +67,7 @@ do_get_disc_copy2(Tab, Reason, Storage, Type) when Storage == disc_copies ->
     EtsOpts = proplists:get_value(ets, StorageProps, []),
     Args = [{keypos, 2}, public, named_table, Type | EtsOpts],
     case Reason of
-	{dumper, _} -> %% Resources already allocated
+	{dumper, DR} when is_atom(DR) -> %% Resources already allocated
 	    ignore;
 	_ ->
 	    mnesia_monitor:mktab(Tab, Args),
@@ -91,8 +91,8 @@ do_get_disc_copy2(Tab, Reason, Storage, Type) when Storage == ram_copies ->
     EtsOpts = proplists:get_value(ets, StorageProps, []),
     Args = [{keypos, 2}, public, named_table, Type | EtsOpts],
     case Reason of
-	{dumper, _} -> %% Resources allready allocated
-	    ignore;
+	{dumper, DR} when is_atom(DR) ->
+            ignore; %% Resources already allocated
 	_ ->
 	    mnesia_monitor:mktab(Tab, Args),
 	    Fname = mnesia_lib:tab2dcd(Tab),
@@ -131,7 +131,7 @@ do_get_disc_copy2(Tab, Reason, Storage, Type) when Storage == disc_only_copies -
 	    {repair, mnesia_monitor:get_env(auto_repair)} 
 	    | DetsOpts],
     case Reason of
-	{dumper, _} ->
+	{dumper, DR} when is_atom(DR) ->
 	    mnesia_index:init_index(Tab, Storage),
 	    snmpify(Tab, Storage),
 	    set({Tab, load_node}, node()),
diff --git a/lib/mnesia/test/mnesia_durability_test.erl b/lib/mnesia/test/mnesia_durability_test.erl
index 62199d8b9a..ccbfdc9738 100644
--- a/lib/mnesia/test/mnesia_durability_test.erl
+++ b/lib/mnesia/test/mnesia_durability_test.erl
@@ -46,6 +46,7 @@
          master_nodes/1, starting_master_nodes/1,
          master_on_non_local_tables/1,
          remote_force_load_with_local_master_node/1,
+         master_node_with_ram_copy_2/1, master_node_with_ram_copy_3/1,
          dump_ram_copies/1, dump_disc_copies/1, dump_disc_only/1]).
 
 -include("mnesia_test_lib.hrl").
@@ -91,7 +92,8 @@ groups() ->
      {load_tables_with_master_tables, [],
       [master_nodes, starting_master_nodes,
        master_on_non_local_tables,
-       remote_force_load_with_local_master_node]},
+       remote_force_load_with_local_master_node,
+       master_node_with_ram_copy_2, master_node_with_ram_copy_3]},
      {durability_of_dump_tables, [],
       [dump_ram_copies, dump_disc_copies, dump_disc_only]}].
 
@@ -1165,6 +1167,107 @@ remote_force_load_with_local_master_node(Config) when is_list(Config) ->
 
     ?verify_mnesia(Nodes, []).
 
+master_node_with_ram_copy_2(Config) when is_list(Config) ->
+    [A, B] = Nodes = ?acquire_nodes(2, Config),
+    Tab = ?FUNCTION_NAME,
+    ?match({atomic,ok}, mnesia:create_table(Tab, [{disc_copies, [A]}, {ram_copies, [B]}])),
+    ?match({atomic,ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
+
+    %% Test that we don't load from ram_copies
+    ?match(stopped, rpc:call(A, mnesia, stop, [])),
+    ?match(stopped, rpc:call(B, mnesia, stop, [])),
+    ?match(ok, rpc:call(B, mnesia, start, [])),
+    ?match({timeout, [Tab]}, rpc:call(B, mnesia, wait_for_tables, [[Tab], 1000])),
+    ?match(ok, rpc:call(A, mnesia, start, [])),
+    ?match(ok, rpc:call(B, mnesia, wait_for_tables, [[Tab], 3000])),
+    ?match([{Tab, 1, init}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
+    ?match([{Tab, 1, init}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
+
+    %% Test that master_nodes set to ram_copy node require force_load
+    ?match(ok, rpc:call(A, mnesia, set_master_nodes, [[B]])),
+    ?match(stopped, rpc:call(A, mnesia, stop, [])),
+    ?match(stopped, rpc:call(B, mnesia, stop, [])),
+    ?match(ok, rpc:call(B, mnesia, start, [])),
+    ?match({timeout, [Tab]}, rpc:call(B, mnesia, wait_for_tables, [[Tab], 1000])),
+    ?match(ok, rpc:call(A, mnesia, start, [])),
+    ?match({timeout, [Tab]}, rpc:call(B, mnesia, wait_for_tables, [[Tab], 1000])),
+
+    ?match(yes, rpc:call(A, mnesia, force_load_table, [Tab])),
+    ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 1000])),
+    ?match(ok, rpc:call(B, mnesia, wait_for_tables, [[Tab], 1000])),
+    ?match([{Tab, 1, init}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
+    ?match([{Tab, 1, init}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
+
+    ?verify_mnesia(Nodes, []).
+
+
+master_node_with_ram_copy_3(Config) when is_list(Config) ->
+    [A, B, C] = Nodes = ?acquire_nodes(3, Config),
+    Tab = ?FUNCTION_NAME,
+    ?match({atomic,ok}, mnesia:create_table(Tab, [{disc_copies, [A,C]}, {ram_copies, [B]}])),
+    ?match({atomic,ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
+
+    %% Test that we don't load from ram_copies
+    ?match(stopped, rpc:call(A, mnesia, stop, [])),
+    ?match(stopped, rpc:call(C, mnesia, stop, [])),
+    ?match(stopped, rpc:call(B, mnesia, stop, [])),
+    ?match(ok, rpc:call(B, mnesia, start, [])),
+    ?match({timeout, [Tab]}, rpc:call(B, mnesia, wait_for_tables, [[Tab], 1000])),
+    ?match(ok, rpc:call(A, mnesia, start, [])),
+    ?match(ok, rpc:call(C, mnesia, start, [])),
+    ?match(ok, rpc:call(B, mnesia, wait_for_tables, [[Tab], 3000])),
+    ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 3000])),
+    ?match([{Tab, 1, init}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
+    ?match([{Tab, 1, init}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
+    ?match([{Tab, 1, init}], rpc:call(C, mnesia, dirty_read, [{Tab, 1}])),
+
+    %% Test that master_nodes set to ram_copy node will wait until loaded
+    ?match(ok, rpc:call(A, mnesia, set_master_nodes, [[B]])),
+    ?match(stopped, rpc:call(A, mnesia, stop, [])),
+    ?match({atomic,ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, update})])),
+    ?match(stopped, rpc:call(C, mnesia, stop, [])),
+    ?match({atomic,ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, ram_copies})])),
+    ?match(stopped, rpc:call(B, mnesia, stop, [])),
+    ?match(ok, rpc:call(B, mnesia, start, [])),
+    ?match({timeout, [Tab]}, rpc:call(B, mnesia, wait_for_tables, [[Tab], 500])),
+    ?match(ok, rpc:call(A, mnesia, start, [])),
+    ?match({timeout, [Tab]}, rpc:call(A, mnesia, wait_for_tables, [[Tab], 500])),
+    ?match(ok, rpc:call(C, mnesia, start, [])),
+    ?match(ok, rpc:call(B, mnesia, wait_for_tables, [[Tab], 3000])),
+    ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 3000])),
+    ?match([{Tab, 1, update}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
+    ?match([{Tab, 1, update}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
+    ?match([{Tab, 1, update}], rpc:call(C, mnesia, dirty_read, [{Tab, 1}])),
+
+    %% Test that master_nodes set to ram_copy node requires force load
+    ?match({atomic,ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
+    ?match(ok, rpc:call(A, mnesia, set_master_nodes, [[B]])),
+    ?match(ok, rpc:call(C, mnesia, set_master_nodes, [[B]])),
+
+    ?match(stopped, rpc:call(A, mnesia, stop, [])),
+    ?match({atomic,ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, update})])),
+    ?match(stopped, rpc:call(C, mnesia, stop, [])),
+    ?match({atomic,ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, ram_copies})])),
+    ?match(stopped, rpc:call(B, mnesia, stop, [])),
+    ?match(ok, rpc:call(B, mnesia, start, [])),
+    ?match({timeout, [Tab]}, rpc:call(B, mnesia, wait_for_tables, [[Tab], 500])),
+    ?match(ok, rpc:call(A, mnesia, start, [])),
+    ?match({timeout, [Tab]}, rpc:call(A, mnesia, wait_for_tables, [[Tab], 500])),
+    ?match(ok, rpc:call(C, mnesia, start, [])),
+    ?match({timeout, [Tab]}, rpc:call(A, mnesia, wait_for_tables, [[Tab], 500])),
+    ?match({timeout, [Tab]}, rpc:call(B, mnesia, wait_for_tables, [[Tab], 500])),
+    ?match({timeout, [Tab]}, rpc:call(B, mnesia, wait_for_tables, [[Tab], 500])),
+    ?match(yes, rpc:call(C, mnesia, force_load_table, [Tab])),
+
+    ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 3000])),
+    ?match(ok, rpc:call(B, mnesia, wait_for_tables, [[Tab], 3000])),
+    ?match(ok, rpc:call(C, mnesia, wait_for_tables, [[Tab], 3000])),
+    ?match([{Tab, 1, update}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
+    ?match([{Tab, 1, update}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
+    ?match([{Tab, 1, update}], rpc:call(C, mnesia, dirty_read, [{Tab, 1}])),
+
+    ?verify_mnesia(Nodes, []).
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 
@@ -1415,7 +1518,7 @@ do_disc_durability(Config,CopyType) ->
 		  [{Tab_bag, 22, a_2222}], [{Tab_bag, 33, a_3333}],
 		  [{Tab_set, counter, 10}]]),
 
-    timer:sleep(1000), %% Debugging strange msgs..
+    timer:sleep(500), %% Debugging strange msgs..
     ?log("Flushed ~p ~n", [mnesia_test_lib:flush()]),
     ?verify_mnesia(Nodes, []).
 
diff --git a/lib/mnesia/test/mnesia_evil_coverage_test.erl b/lib/mnesia/test/mnesia_evil_coverage_test.erl
index eb1f987cf0..77236940d4 100644
--- a/lib/mnesia/test/mnesia_evil_coverage_test.erl
+++ b/lib/mnesia/test/mnesia_evil_coverage_test.erl
@@ -31,6 +31,7 @@
          db_node_lifecycle/1, evil_delete_db_node/1, start_and_stop/1,
          checkpoint/1, table_lifecycle/1, storage_options/1,
          add_copy_conflict/1, add_copy_when_going_down/1,
+         add_copy_with_down/1,
          replica_management/1, clear_table_during_load/1,
          schema_availability/1, local_content/1,
          replica_location/1, user_properties/1, unsupp_user_props/1,
@@ -65,7 +66,8 @@ all() ->
      db_node_lifecycle, evil_delete_db_node, start_and_stop,
      checkpoint, table_lifecycle, storage_options, 
      add_copy_conflict,
-     add_copy_when_going_down, replica_management, clear_table_during_load,
+     add_copy_when_going_down, add_copy_with_down, replica_management,
+     clear_table_during_load,
      schema_availability, local_content,
      {group, table_access_modifications}, replica_location,
      {group, table_sync}, user_properties, unsupp_user_props,
@@ -732,6 +734,49 @@ add_copy_when_going_down(Config) ->
     ?match_receive({test,{aborted,_}}),
     ?verify_mnesia([Node2], []).
 
+add_copy_with_down(suite) -> [];
+add_copy_with_down(Config) ->
+    %% Allow add_table_copy() with ram_copies even all other replicas are down
+    Nodes = [Node1, Node2, Node3] = ?acquire_nodes(3, Config),
+    ?match({atomic, ok}, mnesia:create_table(a, [{ram_copies, [Node3]}, {disc_copies, [Node2]}])),
+    stopped = rpc:call(Node2, mnesia, stop, []),
+    stopped = rpc:call(Node3, mnesia, stop, []),
+    ?match({aborted, _}, mnesia:add_table_copy(a, Node1, ram_copies)),
+    ?match({aborted, _}, mnesia:del_table_copy(a, Node2)),
+    ok = rpc:call(Node3, mnesia, start, []),
+    ?match({aborted, _}, mnesia:add_table_copy(a, Node1, ram_copies)),
+    ?match([], mnesia_test_lib:start_mnesia([Node2], [a])),
+    ?match({atomic, ok}, mnesia:change_table_copy_type(a, Node2, ram_copies)),
+    stopped = rpc:call(Node2, mnesia, stop, []),
+    stopped = rpc:call(Node3, mnesia, stop, []),
+    ?match({atomic, ok}, mnesia:add_table_copy(a, Node1, ram_copies)),
+    ?match(ok, mnesia:dirty_write({a,1,1})),
+    ?match([], mnesia_test_lib:start_mnesia([Node2,Node3], [a])),
+    ?match([{a,1,1}], rpc:call(Node1, mnesia, dirty_read, [{a,1}])),
+    ?match([{a,1,1}], rpc:call(Node2, mnesia, dirty_read, [{a,1}])),
+    ?match([{a,1,1}], rpc:call(Node3, mnesia, dirty_read, [{a,1}])),
+
+    ?match({atomic, ok}, mnesia:del_table_copy(a, Node1)),
+    stopped = rpc:call(Node2, mnesia, stop, []),
+    stopped = rpc:call(Node3, mnesia, stop, []),
+    ?match({atomic, ok}, mnesia:add_table_copy(a, Node1, disc_copies)),
+    ?match(ok, mnesia:dirty_write({a,1,1})),
+    ?match([], mnesia_test_lib:start_mnesia([Node2,Node3], [a])),
+    ?match([{a,1,1}], rpc:call(Node1, mnesia, dirty_read, [{a,1}])),
+    ?match([{a,1,1}], rpc:call(Node2, mnesia, dirty_read, [{a,1}])),
+    ?match([{a,1,1}], rpc:call(Node3, mnesia, dirty_read, [{a,1}])),
+
+    ?match({atomic, ok}, mnesia:del_table_copy(a, Node1)),
+    stopped = rpc:call(Node2, mnesia, stop, []),
+    stopped = rpc:call(Node3, mnesia, stop, []),
+    ?match({atomic, ok}, mnesia:add_table_copy(a, Node1, disc_only_copies)),
+    ?match(ok, mnesia:dirty_write({a,1,1})),
+    ?match([], mnesia_test_lib:start_mnesia([Node2,Node3], [a])),
+    ?match([{a,1,1}], rpc:call(Node1, mnesia, dirty_read, [{a,1}])),
+    ?match([{a,1,1}], rpc:call(Node2, mnesia, dirty_read, [{a,1}])),
+    ?match([{a,1,1}], rpc:call(Node3, mnesia, dirty_read, [{a,1}])),
+
+    ?verify_mnesia(Nodes, []).
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% Add, drop and move replicas, change storage types