%% %% %CopyrightBegin% %% %% Copyright Ericsson AB 2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at %% %% http://www.apache.org/licenses/LICENSE-2.0 %% %% Unless required by applicable law or agreed to in writing, software %% distributed under the License is distributed on an "AS IS" BASIS, %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. %% %% %CopyrightEnd% %% -module(persistent_term_SUITE). -include_lib("common_test/include/ct.hrl"). -export([all/0,suite/0,init_per_suite/1,end_per_suite/1, basic/1,purging/1,sharing/1,get_trapping/1, info/1,info_trapping/1,killed_while_trapping/1, off_heap_values/1,keys/1,collisions/1, init_restart/1, put_erase_trapping/1, killed_while_trapping_put/1, killed_while_trapping_erase/1]). %% -export([test_init_restart_cmd/1]). suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,10}}]. all() -> [basic,purging,sharing,get_trapping,info,info_trapping, killed_while_trapping,off_heap_values,keys,collisions, init_restart, put_erase_trapping, killed_while_trapping_put, killed_while_trapping_erase]. init_per_suite(Config) -> %% Put a term in the dict so that we know that the testcases handle %% stray terms left by stdlib or other test suites. persistent_term:put(init_per_suite, {?MODULE}), Config. end_per_suite(Config) -> persistent_term:erase(init_per_suite), Config. basic(_Config) -> Chk = chk(), N = 777, Seq = lists:seq(1, N), par(2, N, Seq, Chk), seq(3, Seq, Chk), seq(3, Seq, Chk), %Same values. _ = [begin Key = {?MODULE,{key,I}}, true = persistent_term:erase(Key), false = persistent_term:erase(Key), {'EXIT',{badarg,_}} = (catch persistent_term:get(Key)), {not_present,Key} = persistent_term:get(Key, {not_present,Key}) end || I <- Seq], [] = [P || {{?MODULE,_},_}=P <- pget(Chk)], chk(Chk). par(C, N, Seq, Chk) -> _ = [spawn_link(fun() -> ok = persistent_term:put({?MODULE,{key,I}}, {value,C*I}) end) || I <- Seq], Result = wait(N, Chk), _ = [begin Double = C*I, {{?MODULE,{key,I}},{value,Double}} = Res end || {I,Res} <- lists:zip(Seq, Result)], ok. seq(C, Seq, Chk) -> _ = [ok = persistent_term:put({?MODULE,{key,I}}, {value,C*I}) || I <- Seq], All = pget(Chk), All = [P || {{?MODULE,_},_}=P <- All], All = [{Key,persistent_term:get(Key)} || {Key,_} <- All], Result = lists:sort(All), _ = [begin Double = C*I, {{?MODULE,{key,I}},{value,Double}} = Res end || {I,Res} <- lists:zip(Seq, Result)], ok. wait(N, Chk) -> All = [P || {{?MODULE,_},_}=P <- pget(Chk)], case length(All) of N -> All = [{Key,persistent_term:get(Key)} || {Key,_} <- All], lists:sort(All); _ -> receive after 10 -> ok end, wait(N, Chk) end. %% Make sure that terms that have been erased are copied into all %% processes that still hold a pointer to them. purging(_Config) -> Chk = chk(), do_purging(fun(K) -> persistent_term:put(K, {?MODULE,new}) end, replaced), do_purging(fun persistent_term:erase/1, erased), chk(Chk). do_purging(Eraser, Type) -> Parent = self(), Key = {?MODULE,?FUNCTION_NAME}, ok = persistent_term:put(Key, {term,[<<"abc",0:777/unit:8>>]}), Ps0 = [spawn_monitor(fun() -> purging_tester(Parent, Key) end) || _ <- lists:seq(1, 50)], Ps = maps:from_list(Ps0), purging_recv(gotten, Ps), Eraser(Key), _ = [P ! {Parent,Type} || P <- maps:keys(Ps)], purging_wait(Ps). purging_recv(Tag, Ps) when map_size(Ps) > 0 -> receive {Pid,Tag} -> true = is_map_key(Pid, Ps), purging_recv(Tag, maps:remove(Pid, Ps)) end; purging_recv(_, _) -> ok. purging_wait(Ps) when map_size(Ps) > 0 -> receive {'DOWN',Ref,process,Pid,Reason} -> normal = Reason, Ref = map_get(Pid, Ps), purging_wait(maps:remove(Pid, Ps)) end; purging_wait(_) -> ok. purging_tester(Parent, Key) -> Term = persistent_term:get(Key), purging_check_term(Term), 0 = erts_debug:size_shared(Term), Parent ! {self(),gotten}, receive {Parent,erased} -> {'EXIT',{badarg,_}} = (catch persistent_term:get(Key)), purging_tester_1(Term); {Parent,replaced} -> {?MODULE,new} = persistent_term:get(Key), purging_tester_1(Term) end. %% Wait for the term to be copied into this process. purging_tester_1(Term) -> purging_check_term(Term), receive after 1 -> ok end, case erts_debug:size_shared(Term) of 0 -> purging_tester_1(Term); Size -> %% The term has been copied into this process. purging_check_term(Term), Size = erts_debug:size(Term) end. purging_check_term({term,[<<"abc",0:777/unit:8>>]}) -> ok. %% Test that sharing is preserved when storing terms. sharing(_Config) -> Chk = chk(), Depth = 10, Size = 2*Depth, Shared = lists:foldl(fun(_, A) -> [A|A] end, [], lists:seq(1, Depth)), Size = erts_debug:size(Shared), Key = {?MODULE,?FUNCTION_NAME}, ok = persistent_term:put(Key, Shared), SharedStored = persistent_term:get(Key), Size = erts_debug:size(SharedStored), 0 = erts_debug:size_shared(SharedStored), {Pid,Ref} = spawn_monitor(fun() -> Term = persistent_term:get(Key), Size = erts_debug:size(Term), 0 = erts_debug:size_shared(Term), true = Term =:= SharedStored end), receive {'DOWN',Ref,process,Pid,normal} -> true = persistent_term:erase(Key), Size = erts_debug:size(SharedStored), chk(Chk) end. %% Test trapping of persistent_term:get/0. get_trapping(_Config) -> Chk = chk(), %% Assume that the get/0 traps after 4000 iterations %% in a non-debug emulator. N = case test_server:timetrap_scale_factor() of 1 -> 10000; _ -> 1000 end, spawn_link(fun() -> get_trapping_create(N) end), All = do_get_trapping(N, [], Chk), N = get_trapping_check_result(lists:sort(All), 1), erlang:garbage_collect(), get_trapping_erase(N), chk(Chk). do_get_trapping(N, Prev, Chk) -> case pget(Chk) of Prev when length(Prev) >= N -> All = [P || {{?MODULE,{get_trapping,_}},_}=P <- Prev], case length(All) of N -> All; _ -> do_get_trapping(N, Prev, Chk) end; New -> receive after 1 -> ok end, do_get_trapping(N, New, Chk) end. get_trapping_create(0) -> ok; get_trapping_create(N) -> ok = persistent_term:put({?MODULE,{get_trapping,N}}, N), get_trapping_create(N-1). get_trapping_check_result([{{?MODULE,{get_trapping,N}},N}|T], N) -> get_trapping_check_result(T, N+1); get_trapping_check_result([], N) -> N-1. get_trapping_erase(0) -> ok; get_trapping_erase(N) -> true = persistent_term:erase({?MODULE,{get_trapping,N}}), get_trapping_erase(N-1). %% Test retrieving information about persistent terms. info(_Config) -> Chk = chk(), %% White box test of info/0. N = 100, try Overhead = info_literal_area_overhead(), io:format("Overhead = ~p\n", [Overhead]), info_wb(N, Overhead, info_info()) after _ = [_ = persistent_term:erase({?MODULE,I}) || I <- lists:seq(1, N)] end, chk(Chk). %% White box test of persistent_term:info/0. We take into account %% that there might already exist persistent terms (created by the %% OTP standard libraries), but we assume that they are not %% changed during the execution of this test case. info_wb(0, _, _) -> ok; info_wb(N, Overhead, {BaseCount,BaseMemory}) -> Key = {?MODULE,N}, Value = lists:seq(1, N), ok = persistent_term:put(Key, Value), %% Calculate the extra memory needed for this term. WordSize = erlang:system_info(wordsize), ExtraMemory = Overhead + 2 * N * WordSize, %% Call persistent_term:info/0. {Count,Memory} = info_info(), %% There should be one more persistent term. Count = BaseCount + 1, %% Verify that the amount of memory is correct. case BaseMemory + ExtraMemory of Memory -> %% Exactly right. The size of the hash table was not changed. ok; Expected -> %% The size of the hash table has been doubled to avoid filling %% the table to more than 50 percent. The previous number %% of entries must have been exactly half the size of the %% hash table. The expected number of extra words added by %% the resizing will be twice that number. ExtraWords = BaseCount * 2, true = ExtraWords * WordSize =:= (Memory - Expected) end, info_wb(N-1, Overhead, {Count,Memory}). info_info() -> #{count:=Count,memory:=Memory} = persistent_term:info(), true = is_integer(Count) andalso Count >= 0, true = is_integer(Memory) andalso Memory >= 0, {Count,Memory}. %% Calculate the number of extra bytes needed for storing each term in %% the literal, assuming that the key is a tuple of size 2 with %% immediate elements. The calculated number is the size of the %% ErtsLiteralArea struct excluding the storage for the literal term %% itself. info_literal_area_overhead() -> Key1 = {?MODULE,1}, Key2 = {?MODULE,2}, #{memory:=Mem0} = persistent_term:info(), ok = persistent_term:put(Key1, literal), #{memory:=Mem1} = persistent_term:info(), ok = persistent_term:put(Key2, literal), #{memory:=Mem2} = persistent_term:info(), true = persistent_term:erase(Key1), true = persistent_term:erase(Key2), %% The size of the hash table may have doubled when inserting %% one of the keys. To avoiding counting the change in the hash %% table size, take the smaller size increase. min(Mem2-Mem1, Mem1-Mem0). %% Test trapping of persistent_term:info/0. info_trapping(_Config) -> Chk = chk(), %% Assume that the info/0 traps after 4000 iterations %% in a non-debug emulator. N = case test_server:timetrap_scale_factor() of 1 -> 10000; _ -> 1000 end, spawn_link(fun() -> info_trapping_create(N) end), All = do_info_trapping(N, 0, Chk), N = info_trapping_check_result(lists:sort(All), 1), erlang:garbage_collect(), info_trapping_erase(N), chk(Chk). do_info_trapping(N, PrevMem, Chk) -> case info_info() of {M,Mem} when M >= N -> true = Mem >= PrevMem, All = [P || {{?MODULE,{info_trapping,_}},_}=P <- pget(Chk)], case length(All) of N -> All; _ -> do_info_trapping(N, PrevMem, Chk) end; {_,Mem} -> true = Mem >= PrevMem, receive after 1 -> ok end, do_info_trapping(N, Mem, Chk) end. info_trapping_create(0) -> ok; info_trapping_create(N) -> ok = persistent_term:put({?MODULE,{info_trapping,N}}, N), info_trapping_create(N-1). info_trapping_check_result([{{?MODULE,{info_trapping,N}},N}|T], N) -> info_trapping_check_result(T, N+1); info_trapping_check_result([], N) -> N-1. info_trapping_erase(0) -> ok; info_trapping_erase(N) -> true = persistent_term:erase({?MODULE,{info_trapping,N}}), info_trapping_erase(N-1). %% Test that hash tables are deallocated if a process running %% persistent_term:get/0 is killed. killed_while_trapping(_Config) -> Chk = chk(), N = case test_server:timetrap_scale_factor() of 1 -> 20000; _ -> 2000 end, kwt_put(N), kwt_spawn(10), kwt_erase(N), chk(Chk). kwt_put(0) -> ok; kwt_put(N) -> ok = persistent_term:put({?MODULE,{kwt,N}}, N), kwt_put(N-1). kwt_spawn(0) -> ok; kwt_spawn(N) -> Pids = [spawn(fun kwt_getter/0) || _ <- lists:seq(1, 20)], erlang:yield(), _ = [exit(Pid, kill) || Pid <- Pids], kwt_spawn(N-1). kwt_getter() -> _ = persistent_term:get(), kwt_getter(). kwt_erase(0) -> ok; kwt_erase(N) -> true = persistent_term:erase({?MODULE,{kwt,N}}), kwt_erase(N-1). %% Test storing off heap values (such as ref-counted binaries). off_heap_values(_Config) -> Chk = chk(), Key = {?MODULE,?FUNCTION_NAME}, Val = {a,list_to_binary(lists:seq(0, 255)),make_ref(),fun() -> ok end}, ok = persistent_term:put(Key, Val), FetchedVal = persistent_term:get(Key), Val = FetchedVal, true = persistent_term:erase(Key), off_heap_values_wait(FetchedVal, Val), chk(Chk). off_heap_values_wait(FetchedVal, Val) -> case erts_debug:size_shared(FetchedVal) of 0 -> Val = FetchedVal, ok; _ -> erlang:yield(), off_heap_values_wait(FetchedVal, Val) end. %% Test some more data types as keys. Use the module name as a key %% to minimize the risk of collision with any key used %% by the OTP libraries. keys(_Config) -> Chk = chk(), do_key(?MODULE), do_key([?MODULE]), do_key(?MODULE_STRING), do_key(list_to_binary(?MODULE_STRING)), chk(Chk). do_key(Key) -> Val = term_to_binary(Key), ok = persistent_term:put(Key, Val), StoredVal = persistent_term:get(Key), Val = StoredVal, true = persistent_term:erase(Key). %% Create persistent terms with keys that are known to collide. %% Delete them in random order, making sure that all others %% terms can still be found. collisions(_Config) -> Chk = chk(), %% Create persistent terms with random keys. Keys = lists:flatten(colliding_keys()), Kvs = [{K,rand:uniform(1000)} || K <- Keys], _ = [ok = persistent_term:put(K, V) || {K,V} <- Kvs], _ = [V = persistent_term:get(K) || {K,V} <- Kvs], %% Now delete the persistent terms in random order. collisions_delete(lists:keysort(2, Kvs), Chk), chk(Chk). collisions_delete([{Key,Val}|Kvs], Chk) -> Val = persistent_term:get(Key), true = persistent_term:erase(Key), true = lists:sort(pget(Chk)) =:= lists:sort(Kvs), _ = [V = persistent_term:get(K) || {K,V} <- Kvs], collisions_delete(Kvs, Chk); collisions_delete([], _) -> ok. colliding_keys() -> %% Collisions found by Jesper L. Andersen for breaking maps. L = [[764492191,2361333849], [49527266765044,90940896816021,20062927283041,267080852079651], [249858369443708,206247021789428,20287304470696,25847120931175], [10645228898670,224705626119556,267405565521452,258214397180678], [264783762221048,166955943492306,98802957003141,102012488332476], [69425677456944,177142907243411,137138950917722,228865047699598], [116031213307147,29203342183358,37406949328742,255198080174323], [200358182338308,235207156008390,120922906095920,116215987197289], [58728890318426,68877471005069,176496507286088,221041411345780], [91094120814795,50665258299931,256093108116737,19777509566621], [74646746200247,98350487270564,154448261001199,39881047281135], [23408943649483,164410325820923,248161749770122,274558342231648], [169531547115055,213630535746863,235098262267796,200508473898303], [235098564415817,85039146398174,51721575960328,173069189684390], [176136386396069,155368359051606,147817099696487,265419485459634], [137542881551462,40028925519736,70525669519846,63445773516557], [173854695142814,114282444507812,149945832627054,99605565798831], [177686773562184,127158716984798,132495543008547], [227073396444896,139667311071766,158915951283562], [26212438434289,94902985796531,198145776057315], [266279278943923,58550737262493,74297973216378], [32373606512065,131854353044428,184642643042326], [34335377662439,85341895822066,273492717750246]], %% Verify that the keys still collide (this will fail if the %% internal hash function has been changed). erts_debug:set_internal_state(available_internal_state, true), try case erlang:system_info(wordsize) of 8 -> verify_colliding_keys(L); 4 -> %% Not guaranteed to collide on a 32-bit system. ok end after erts_debug:set_internal_state(available_internal_state, false) end, L. verify_colliding_keys([[K|Ks]|Gs]) -> Hash = internal_hash(K), [Hash] = lists:usort([internal_hash(Key) || Key <- Ks]), verify_colliding_keys(Gs); verify_colliding_keys([]) -> ok. internal_hash(Term) -> erts_debug:get_internal_state({internal_hash,Term}). %% Test that all persistent terms are erased by init:restart/0. init_restart(_Config) -> File = "command_file", ok = file:write_file(File, term_to_binary(restart)), {ok,[[Erl]]} = init:get_argument(progname), ModPath = filename:dirname(code:which(?MODULE)), Cmd = Erl ++ " -pa " ++ ModPath ++ " -noshell " "-run " ++ ?MODULE_STRING ++ " test_init_restart_cmd " ++ File, io:format("~s\n", [Cmd]), Expected = "12ok", case os:cmd(Cmd) of Expected -> ok; Actual -> io:format("Expected: ~s", [Expected]), io:format("Actual: ~s\n", [Actual]), ct:fail(unexpected_output) end. test_init_restart_cmd([File]) -> try do_test_init_restart_cmd(File) catch C:R -> io:format("\n~p ~p\n", [C,R]), halt() end, receive _ -> ok end. do_test_init_restart_cmd(File) -> {ok,Bin} = file:read_file(File), Seq = lists:seq(1, 50), case binary_to_term(Bin) of restart -> _ = [persistent_term:put({?MODULE,I}, {value,I}) || I <- Seq], ok = file:write_file(File, term_to_binary(was_restarted)), io:put_chars("1"), init:restart(), receive _ -> ok end; was_restarted -> io:put_chars("2"), ok = file:delete(File), _ = [begin Key = {?MODULE,I}, {'EXIT',{badarg,_}} = (catch persistent_term:get(Key)) end || I <- Seq], io:put_chars("ok"), init:stop() end. %% Check that there is the same number of persistents terms before %% and after each test case. chk() -> {persistent_term:info(), persistent_term:get()}. chk({Info, _Initial} = Chk) -> Info = persistent_term:info(), Key = {?MODULE,?FUNCTION_NAME}, ok = persistent_term:put(Key, {term,Info}), Term = persistent_term:get(Key), true = persistent_term:erase(Key), chk_not_stuck(Term), [persistent_term:erase(K) || {K, _} <- pget(Chk)], ok. chk_not_stuck(Term) -> %% Hash tables to be deleted are put onto a queue. %% Make sure that the queue isn't stuck by a table with %% a non-zero ref count. case erts_debug:size_shared(Term) of 0 -> erlang:yield(), chk_not_stuck(Term); _ -> ok end. pget({_, Initial}) -> persistent_term:get() -- Initial. killed_while_trapping_put(_Config) -> erts_debug:set_internal_state(available_internal_state, true), repeat( fun() -> NrOfPutsInChild = 10000, do_puts(2500, my_value), Pid = spawn(fun() -> do_puts(NrOfPutsInChild, my_value2) end), timer:sleep(1), erlang:exit(Pid, kill), do_erases(NrOfPutsInChild) end, 10), erts_debug:set_internal_state(available_internal_state, false). killed_while_trapping_erase(_Config) -> erts_debug:set_internal_state(available_internal_state, true), repeat( fun() -> NrOfErases = 2500, do_puts(NrOfErases, my_value), Pid = spawn(fun() -> do_erases(NrOfErases) end), timer:sleep(1), erlang:exit(Pid, kill), do_erases(NrOfErases) end, 10), erts_debug:set_internal_state(available_internal_state, false). put_erase_trapping(_Config) -> NrOfItems = 5000, erts_debug:set_internal_state(available_internal_state, true), do_puts(NrOfItems, first), do_puts(NrOfItems, second), do_erases(NrOfItems), erts_debug:set_internal_state(available_internal_state, false). do_puts(0, _) -> ok; do_puts(NrOfPuts, ValuePrefix) -> Key = {?MODULE, NrOfPuts}, Value = {ValuePrefix, NrOfPuts}, erts_debug:set_internal_state(reds_left, rand:uniform(250)), persistent_term:put(Key, Value), Value = persistent_term:get(Key), do_puts(NrOfPuts - 1, ValuePrefix). do_erases(0) -> ok; do_erases(NrOfErases) -> Key = {?MODULE,NrOfErases}, erts_debug:set_internal_state(reds_left, rand:uniform(500)), persistent_term:erase(Key), not_found = persistent_term:get(Key, not_found), do_erases(NrOfErases - 1). repeat(_Fun, 0) -> ok; repeat(Fun, N) -> Fun(), repeat(Fun, N-1).