diff options
Diffstat (limited to 'lib/stdlib/test/ets_SUITE.erl')
| -rw-r--r-- | lib/stdlib/test/ets_SUITE.erl | 1636 |
1 files changed, 1181 insertions, 455 deletions
diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index aa2e352c29..2c0692855f 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -67,6 +67,7 @@ meta_newdel_unnamed/1, meta_newdel_named/1]). -export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1, smp_select_replace/1, otp_8166/1, otp_8732/1, delete_unfix_race/1]). +-export([throughput_benchmark/0, test_throughput_benchmark/1]). -export([exit_large_table_owner/1, exit_many_large_table_owner/1, exit_many_tables_owner/1, @@ -144,7 +145,8 @@ all() -> massive_ets_all, take, whereis_table, - delete_unfix_race]. + delete_unfix_race, + test_throughput_benchmark]. groups() -> [{new, [], @@ -743,7 +745,7 @@ select_bound_chunk(_Config) -> repeat_for_opts(fun select_bound_chunk_do/1, [all_types]). select_bound_chunk_do(Opts) -> - T = ets:new(x, Opts), + T = ets_new(x, Opts), ets:insert(T, [{key, 1}]), {[{key, 1}], '$end_of_table'} = ets:select(T, [{{key,1},[],['$_']}], 100000), ok. @@ -789,7 +791,7 @@ check_badarg({'EXIT', {badarg, [{M,F,A,_} | _]}}, M, F, Args) -> %% Test ets:delete_all_objects/1. t_delete_all_objects(Config) when is_list(Config) -> EtsMem = etsmem(), - repeat_for_opts(fun t_delete_all_objects_do/1), + repeat_for_opts_all_set_table_types(fun t_delete_all_objects_do/1), verify_etsmem(EtsMem). get_kept_objects(T) -> @@ -809,7 +811,10 @@ t_delete_all_objects_do(Opts) -> true = ets:delete_all_objects(T), '$end_of_table' = ets:next(T,O), 0 = ets:info(T,size), - 4000 = get_kept_objects(T), + case ets:info(T,type) of + ordered_set -> ok; + _ -> 4000 = get_kept_objects(T) + end, ets:safe_fixtable(T,false), 0 = ets:info(T,size), 0 = get_kept_objects(T), @@ -820,7 +825,7 @@ t_delete_all_objects_do(Opts) -> ets:delete(T), %% Test delete_all_objects is atomic - T2 = ets:new(t_delete_all_objects, [public | Opts]), + T2 = ets_new(t_delete_all_objects, [public | Opts]), Self = self(), Inserters = [spawn_link(fun() -> inserter(T2, 100*1000, 1, Self) end) || _ <- [1,2,3,4]], [receive {Ipid, running} -> ok end || Ipid <- Inserters], @@ -2372,17 +2377,29 @@ write_concurrency(Config) when is_list(Config) -> Yes6 = ets_new(foo,[duplicate_bag,protected,{write_concurrency,true}]), No3 = ets_new(foo,[duplicate_bag,private,{write_concurrency,true}]), - No4 = ets_new(foo,[ordered_set,public,{write_concurrency,true}]), - No5 = ets_new(foo,[ordered_set,protected,{write_concurrency,true}]), - No6 = ets_new(foo,[ordered_set,private,{write_concurrency,true}]), - - No7 = ets_new(foo,[public,{write_concurrency,false}]), - No8 = ets_new(foo,[protected,{write_concurrency,false}]), + Yes7 = ets_new(foo,[ordered_set,public,{write_concurrency,true}]), + Yes8 = ets_new(foo,[ordered_set,protected,{write_concurrency,true}]), + Yes9 = ets_new(foo,[ordered_set,{write_concurrency,true}]), + Yes10 = ets_new(foo,[{write_concurrency,true},ordered_set,public]), + Yes11 = ets_new(foo,[{write_concurrency,true},ordered_set,protected]), + Yes12 = ets_new(foo,[set,{write_concurrency,false}, + {write_concurrency,true},ordered_set,public]), + Yes13 = ets_new(foo,[private,public,set,{write_concurrency,false}, + {write_concurrency,true},ordered_set]), + No4 = ets_new(foo,[ordered_set,private,{write_concurrency,true}]), + No5 = ets_new(foo,[ordered_set,public,{write_concurrency,false}]), + No6 = ets_new(foo,[ordered_set,protected,{write_concurrency,false}]), + No7 = ets_new(foo,[ordered_set,private,{write_concurrency,false}]), + + No8 = ets_new(foo,[public,{write_concurrency,false}]), + No9 = ets_new(foo,[protected,{write_concurrency,false}]), YesMem = ets:info(Yes1,memory), NoHashMem = ets:info(No1,memory), + YesTreeMem = ets:info(Yes7,memory), NoTreeMem = ets:info(No4,memory), - io:format("YesMem=~p NoHashMem=~p NoTreeMem=~p\n",[YesMem,NoHashMem,NoTreeMem]), + io:format("YesMem=~p NoHashMem=~p NoTreeMem=~p YesTreeMem=~p\n",[YesMem,NoHashMem, + NoTreeMem,YesTreeMem]), YesMem = ets:info(Yes2,memory), YesMem = ets:info(Yes3,memory), @@ -2391,13 +2408,24 @@ write_concurrency(Config) when is_list(Config) -> YesMem = ets:info(Yes6,memory), NoHashMem = ets:info(No2,memory), NoHashMem = ets:info(No3,memory), + YesTreeMem = ets:info(Yes7,memory), + YesTreeMem = ets:info(Yes8,memory), + YesTreeMem = ets:info(Yes9,memory), + YesTreeMem = ets:info(Yes10,memory), + YesTreeMem = ets:info(Yes11,memory), + YesTreeMem = ets:info(Yes12,memory), + YesTreeMem = ets:info(Yes13,memory), + NoTreeMem = ets:info(No4,memory), NoTreeMem = ets:info(No5,memory), NoTreeMem = ets:info(No6,memory), - NoHashMem = ets:info(No7,memory), + NoTreeMem = ets:info(No7,memory), NoHashMem = ets:info(No8,memory), + NoHashMem = ets:info(No9,memory), true = YesMem > NoHashMem, true = YesMem > NoTreeMem, + true = YesMem > YesTreeMem, + true = YesTreeMem < NoTreeMem, {'EXIT',{badarg,_}} = (catch ets_new(foo,[public,{write_concurrency,foo}])), {'EXIT',{badarg,_}} = (catch ets_new(foo,[public,{write_concurrency}])), @@ -2405,8 +2433,8 @@ write_concurrency(Config) when is_list(Config) -> {'EXIT',{badarg,_}} = (catch ets_new(foo,[public,write_concurrency])), lists:foreach(fun(T) -> ets:delete(T) end, - [Yes1,Yes2,Yes3,Yes4,Yes5,Yes6, - No1,No2,No3,No4,No5,No6,No7,No8]), + [Yes1,Yes2,Yes3,Yes4,Yes5,Yes6,Yes7,Yes8,Yes9,Yes10,Yes11,Yes12,Yes13, + No1,No2,No3,No4,No5,No6,No7,No8,No9]), verify_etsmem(EtsMem), ok. @@ -3051,24 +3079,26 @@ pick_all_backwards(T) -> %% Small test case for both set and bag type ets tables. setbag(Config) when is_list(Config) -> EtsMem = etsmem(), - Set = ets_new(set,[set]), - Bag = ets_new(bag,[bag]), - Key = {foo,bar}, - - %% insert some value - ets:insert(Set,{Key,val1}), - ets:insert(Bag,{Key,val1}), - - %% insert new value for same key again - ets:insert(Set,{Key,val2}), - ets:insert(Bag,{Key,val2}), - - %% check - [{Key,val2}] = ets:lookup(Set,Key), - [{Key,val1},{Key,val2}] = ets:lookup(Bag,Key), - - true = ets:delete(Set), - true = ets:delete(Bag), + lists:foreach(fun(SetType) -> + Set = ets_new(SetType,[SetType]), + Bag = ets_new(bag,[bag]), + Key = {foo,bar}, + + %% insert some value + ets:insert(Set,{Key,val1}), + ets:insert(Bag,{Key,val1}), + + %% insert new value for same key again + ets:insert(Set,{Key,val2}), + ets:insert(Bag,{Key,val2}), + + %% check + [{Key,val2}] = ets:lookup(Set,Key), + [{Key,val1},{Key,val2}] = ets:lookup(Bag,Key), + + true = ets:delete(Set), + true = ets:delete(Bag) + end, [set, cat_ord_set,stim_cat_ord_set,ordered_set]), verify_etsmem(EtsMem). %% Test case to check proper return values for illegal ets_new() calls. @@ -3101,11 +3131,13 @@ named(Config) when is_list(Config) -> %% Test case to check if specified keypos works. keypos2(Config) when is_list(Config) -> EtsMem = etsmem(), - Tab = make_table(foo, - [set,{keypos,2}], - [{val,key}, {val2,key}]), - [{val2,key}] = ets:lookup(Tab,key), - true = ets:delete(Tab), + lists:foreach(fun(SetType) -> + Tab = make_table(foo, + [SetType,{keypos,2}], + [{val,key}, {val2,key}]), + [{val2,key}] = ets:lookup(Tab,key), + true = ets:delete(Tab) + end, [set, cat_ord_set,stim_cat_ord_set,ordered_set]), verify_etsmem(EtsMem). %% Privacy check. Check that a named(public/private/protected) table @@ -3197,7 +3229,7 @@ rotate_tuple(Tuple, N) -> %% Check lookup in an empty table and lookup of a non-existing key. empty(Config) when is_list(Config) -> - repeat_for_opts(fun empty_do/1). + repeat_for_opts_all_table_types(fun empty_do/1). empty_do(Opts) -> EtsMem = etsmem(), @@ -3210,7 +3242,7 @@ empty_do(Opts) -> %% Check proper return values for illegal insert operations. badinsert(Config) when is_list(Config) -> - repeat_for_opts(fun badinsert_do/1). + repeat_for_opts_all_table_types(fun badinsert_do/1). badinsert_do(Opts) -> EtsMem = etsmem(), @@ -3234,7 +3266,7 @@ badinsert_do(Opts) -> time_lookup(Config) when is_list(Config) -> %% just for timing, really EtsMem = etsmem(), - Values = repeat_for_opts(fun time_lookup_do/1), + Values = repeat_for_opts_all_table_types(fun time_lookup_do/1), verify_etsmem(EtsMem), {comment,lists:flatten(io_lib:format( "~p ets lookups/s",[Values]))}. @@ -3431,7 +3463,7 @@ delete_tab_do(Opts) -> %% Check that ets:delete/1 works and that other processes can run. delete_large_tab(Config) when is_list(Config) -> - ct:timetrap({minutes,30}), %% valgrind needs a lot + ct:timetrap({minutes,60}), %% valgrind needs a lot Data = [{erlang:phash2(I, 16#ffffff),I} || I <- lists:seq(1, 200000)], EtsMem = etsmem(), repeat_for_opts(fun(Opts) -> delete_large_tab_do(Opts,Data) end), @@ -3440,10 +3472,18 @@ delete_large_tab(Config) when is_list(Config) -> delete_large_tab_do(Opts,Data) -> delete_large_tab_1(foo_hash, Opts, Data, false), delete_large_tab_1(foo_tree, [ordered_set | Opts], Data, false), - delete_large_tab_1(foo_hash, Opts, Data, true). + delete_large_tab_1(foo_tree, [stim_cat_ord_set | Opts], Data, false), + delete_large_tab_1(foo_hash_fix, Opts, Data, true). delete_large_tab_1(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + delete_large_tab_2(Name, Flags, Data, Fix) + end. + +delete_large_tab_2(Name, Flags, Data, Fix) -> Tab = ets_new(Name, Flags), ets:insert(Tab, Data), @@ -3511,9 +3551,17 @@ delete_large_named_table(Config) when is_list(Config) -> delete_large_named_table_do(Opts,Data) -> delete_large_named_table_1(foo_hash, [named_table | Opts], Data, false), delete_large_named_table_1(foo_tree, [ordered_set,named_table | Opts], Data, false), + delete_large_named_table_1(foo_tree, [stim_cat_ord_set,named_table | Opts], Data, false), delete_large_named_table_1(foo_hash, [named_table | Opts], Data, true). delete_large_named_table_1(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + delete_large_named_table_2(Name, Flags, Data, Fix) + end. + +delete_large_named_table_2(Name, Flags, Data, Fix) -> Tab = ets_new(Name, Flags), ets:insert(Tab, Data), @@ -3548,16 +3596,27 @@ evil_delete_do(Opts,Data) -> verify_etsmem(EtsMem), evil_delete_owner(foo_tree, [ordered_set | Opts], Data, false), verify_etsmem(EtsMem), + evil_delete_owner(foo_catree, [stim_cat_ord_set | Opts], Data, false), + verify_etsmem(EtsMem), TabA = evil_delete_not_owner(foo_hash, Opts, Data, false), verify_etsmem(EtsMem), TabB = evil_delete_not_owner(foo_hash, Opts, Data, true), verify_etsmem(EtsMem), TabC = evil_delete_not_owner(foo_tree, [ordered_set | Opts], Data, false), verify_etsmem(EtsMem), + TabD = evil_delete_not_owner(foo_catree, [stim_cat_ord_set | Opts], Data, false), + verify_etsmem(EtsMem), lists:foreach(fun(T) -> undefined = ets:info(T) end, - [TabA,TabB,TabC]). + [TabA,TabB,TabC,TabD]). evil_delete_not_owner(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + evil_delete_not_owner_1(Name, Flags, Data, Fix) + end. + +evil_delete_not_owner_1(Name, Flags, Data, Fix) -> io:format("Not owner: ~p, fix = ~p", [Name,Fix]), Tab = ets_new(Name, [public|Flags]), ets:insert(Tab, Data), @@ -3583,6 +3642,13 @@ evil_delete_not_owner(Name, Flags, Data, Fix) -> Tab. evil_delete_owner(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + evil_delete_owner_1(Name, Flags, Data, Fix) + end. + +evil_delete_owner_1(Name, Flags, Data, Fix) -> Fun = fun() -> Tab = ets_new(Name, [public|Flags]), ets:insert(Tab, Data), @@ -3770,7 +3836,7 @@ verify_rescheduling_exit(Config, ForEachData, Flags, Fix, NOTabs, NOProcs) -> %% Make sure that slots for ets tables are cleared properly. table_leak(Config) when is_list(Config) -> - repeat_for_opts(fun(Opts) -> table_leak_1(Opts,20000) end). + repeat_for_opts_all_non_stim_table_types(fun(Opts) -> table_leak_1(Opts,20000) end). table_leak_1(_,0) -> ok; table_leak_1(Opts,N) -> @@ -3833,7 +3899,7 @@ match_delete3_do(Opts) -> %% Test ets:first/1 & ets:next/2. firstnext(Config) when is_list(Config) -> - repeat_for_opts(fun firstnext_do/1). + repeat_for_opts_all_set_table_types(fun firstnext_do/1). firstnext_do(Opts) -> EtsMem = etsmem(), @@ -3855,15 +3921,20 @@ firstnext_collect(Tab,Key,List) -> %% Tests ets:first/1 & ets:next/2. firstnext_concurrent(Config) when is_list(Config) -> - register(master, self()), - ets_init(?MODULE, 20), - [dynamic_go() || _ <- lists:seq(1, 2)], - receive - after 5000 -> ok - end. + lists:foreach( + fun(TableType) -> + register(master, self()), + TableName = list_to_atom(atom_to_list(?MODULE) ++ atom_to_list(TableType)), + ets_init(TableName, 20, TableType), + [dynamic_go(TableName) || _ <- lists:seq(1, 2)], + receive + after 5000 -> ok + end, + unregister(master) + end, repeat_for_opts_atom2list(ord_set_types)). -ets_init(Tab, N) -> - ets_new(Tab, [named_table,public,ordered_set]), +ets_init(Tab, N, TableType) -> + ets_new(Tab, [named_table,public,TableType]), cycle(Tab, lists:seq(1,N+1)). cycle(_Tab, [H|T]) when H > length(T)-> ok; @@ -3871,9 +3942,9 @@ cycle(Tab, L) -> ets:insert(Tab,list_to_tuple(L)), cycle(Tab, tl(L)++[hd(L)]). -dynamic_go() -> my_spawn_link(fun dynamic_init/0). +dynamic_go(TableName) -> my_spawn_link(fun() -> dynamic_init(TableName) end). -dynamic_init() -> [dyn_lookup(?MODULE) || _ <- lists:seq(1, 10)]. +dynamic_init(TableName) -> [dyn_lookup(TableName) || _ <- lists:seq(1, 10)]. dyn_lookup(T) -> dyn_lookup(T, ets:first(T)). @@ -3891,7 +3962,7 @@ dyn_lookup(T, K) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% slot(Config) when is_list(Config) -> - repeat_for_opts(fun slot_do/1). + repeat_for_opts_all_set_table_types(fun slot_do/1). slot_do(Opts) -> EtsMem = etsmem(), @@ -3916,7 +3987,7 @@ slot_loop(Tab,SlotNo,EltsSoFar) -> match1(Config) when is_list(Config) -> - repeat_for_opts(fun match1_do/1). + repeat_for_opts_all_set_table_types(fun match1_do/1). match1_do(Opts) -> EtsMem = etsmem(), @@ -3979,7 +4050,7 @@ match2_do(Opts) -> %% Some ets:match_object tests. match_object(Config) when is_list(Config) -> - repeat_for_opts(fun match_object_do/1). + repeat_for_opts_all_set_table_types(fun match_object_do/1). match_object_do(Opts) -> EtsMem = etsmem(), @@ -4079,7 +4150,7 @@ match_object_do(Opts) -> %% Tests that db_match_object does not generate a `badarg' when %% resuming a search with no previous matches. match_object2(Config) when is_list(Config) -> - repeat_for_opts(fun match_object2_do/1). + repeat_for_opts_all_table_types(fun match_object2_do/1). match_object2_do(Opts) -> EtsMem = etsmem(), @@ -4104,18 +4175,21 @@ match_object2_do(Opts) -> %% OTP-3319. Test tab2list. tab2list(Config) when is_list(Config) -> - EtsMem = etsmem(), - Tab = make_table(foo, - [ordered_set], - [{a,b}, {c,b}, {b,b}, {a,c}]), - [{a,c},{b,b},{c,b}] = ets:tab2list(Tab), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_all_ord_set_table_types( + fun(Opts) -> + EtsMem = etsmem(), + Tab = make_table(foo, + Opts, + [{a,b}, {c,b}, {b,b}, {a,c}]), + [{a,c},{b,b},{c,b}] = ets:tab2list(Tab), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end). %% Simple general small test. If this fails, ets is in really bad %% shape. misc1(Config) when is_list(Config) -> - repeat_for_opts(fun misc1_do/1). + repeat_for_opts_all_table_types(fun misc1_do/1). misc1_do(Opts) -> EtsMem = etsmem(), @@ -4133,7 +4207,7 @@ misc1_do(Opts) -> %% Check the safe_fixtable function. safe_fixtable(Config) when is_list(Config) -> - repeat_for_opts(fun safe_fixtable_do/1). + repeat_for_opts_all_table_types(fun safe_fixtable_do/1). safe_fixtable_do(Opts) -> EtsMem = etsmem(), @@ -4191,10 +4265,42 @@ safe_fixtable_do(Opts) -> %% Tests ets:info result for required tuples. info(Config) when is_list(Config) -> - repeat_for_opts(fun info_do/1). + repeat_for_opts_all_table_types(fun info_do/1). info_do(Opts) -> EtsMem = etsmem(), + TableType = lists:foldl( + fun(Item, Curr) -> + case Item of + set -> set; + ordered_set -> ordered_set; + cat_ord_set -> ordered_set; + stim_cat_ord_set -> ordered_set; + bag -> bag; + duplicate_bag -> duplicate_bag; + _ -> Curr + end + end, set, Opts), + PublicOrCurr = + fun(Curr) -> + case lists:member({write_concurrency, false}, Opts) or + lists:member(private, Opts) or + lists:member(protected, Opts) of + true -> Curr; + false -> public + end + end, + Protection = lists:foldl( + fun(Item, Curr) -> + case Item of + public -> public; + protected -> protected; + private -> private; + cat_ord_set -> PublicOrCurr(Curr); %% Special items + stim_cat_ord_set -> PublicOrCurr(Curr); + _ -> Curr + end + end, protected, Opts), MeMyselfI=self(), ThisNode=node(), Tab = ets_new(foobar, [{keypos, 2} | Opts]), @@ -4208,9 +4314,9 @@ info_do(Opts) -> {value, {size, 0}} = lists:keysearch(size, 1, Res), {value, {node, ThisNode}} = lists:keysearch(node, 1, Res), {value, {named_table, false}} = lists:keysearch(named_table, 1, Res), - {value, {type, set}} = lists:keysearch(type, 1, Res), + {value, {type, TableType}} = lists:keysearch(type, 1, Res), {value, {keypos, 2}} = lists:keysearch(keypos, 1, Res), - {value, {protection, protected}} = + {value, {protection, Protection}} = lists:keysearch(protection, 1, Res), {value, {id, Tab}} = lists:keysearch(id, 1, Res), true = ets:delete(Tab), @@ -4255,20 +4361,29 @@ dups_do(Opts) -> %% Test the ets:tab2file function on an empty ets table. tab2file(Config) when is_list(Config) -> FName = filename:join([proplists:get_value(priv_dir, Config),"tab2file_case"]), - tab2file_do(FName, []), - tab2file_do(FName, [{sync,true}]), - tab2file_do(FName, [{sync,false}]), - {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [{sync,yes}])), - {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [sync])), + tab2file_do(FName, [], set), + tab2file_do(FName, [], ordered_set), + tab2file_do(FName, [], cat_ord_set), + tab2file_do(FName, [], stim_cat_ord_set), + tab2file_do(FName, [{sync,true}], set), + tab2file_do(FName, [{sync,false}], set), + {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [{sync,yes}], set)), + {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [sync], set)), ok. -tab2file_do(FName, Opts) -> +tab2file_do(FName, Opts, TableType) -> %% Write an empty ets table to a file, read back and check properties. - Tab = ets_new(ets_SUITE_foo_tab, [named_table, set, public, + Tab = ets_new(ets_SUITE_foo_tab, [named_table, TableType, public, {keypos, 2}, compressed, {write_concurrency,true}, {read_concurrency,true}]), + ActualTableType = + case TableType of + cat_ord_set -> ordered_set; + stim_cat_ord_set -> ordered_set; + _ -> TableType + end, catch file:delete(FName), Res = ets:tab2file(Tab, FName, Opts), true = ets:delete(Tab), @@ -4279,7 +4394,7 @@ tab2file_do(FName, Opts) -> public = ets:info(Tab2, protection), true = ets:info(Tab2, named_table), 2 = ets:info(Tab2, keypos), - set = ets:info(Tab2, type), + ActualTableType = ets:info(Tab2, type), true = ets:info(Tab2, compressed), Smp = erlang:system_info(smp_support), Smp = ets:info(Tab2, read_concurrency), @@ -4292,7 +4407,7 @@ tab2file_do(FName, Opts) -> tab2file2(Config) when is_list(Config) -> repeat_for_opts(fun(Opts) -> tab2file2_do(Opts, Config) - end, [[set,bag],compressed]). + end, [[stim_cat_ord_set,cat_ord_set,set,bag],compressed]). tab2file2_do(Opts, Config) -> EtsMem = etsmem(), @@ -4353,7 +4468,7 @@ fill_tab2(Tab, Val, Num) -> %% Test verification of tables with object count extended_info. tabfile_ext1(Config) when is_list(Config) -> - repeat_for_opts(fun(Opts) -> tabfile_ext1_do(Opts, Config) end). + repeat_for_opts_all_set_table_types(fun(Opts) -> tabfile_ext1_do(Opts, Config) end). tabfile_ext1_do(Opts,Config) -> FName = filename:join([proplists:get_value(priv_dir, Config),"nisse.dat"]), @@ -4391,7 +4506,7 @@ tabfile_ext1_do(Opts,Config) -> %% Test verification of tables with md5sum extended_info. tabfile_ext2(Config) when is_list(Config) -> - repeat_for_opts(fun(Opts) -> tabfile_ext2_do(Opts,Config) end). + repeat_for_opts_all_set_table_types(fun(Opts) -> tabfile_ext2_do(Opts,Config) end). tabfile_ext2_do(Opts,Config) -> FName = filename:join([proplists:get_value(priv_dir, Config),"olle.dat"]), @@ -4428,71 +4543,77 @@ tabfile_ext2_do(Opts,Config) -> %% Test verification of (named) tables without extended info. tabfile_ext3(Config) when is_list(Config) -> - FName = filename:join([proplists:get_value(priv_dir, Config),"namn.dat"]), - FName2 = filename:join([proplists:get_value(priv_dir, Config),"ncountflip.dat"]), - L = lists:seq(1,10), - Name = make_ref(), - ?MODULE = ets_new(?MODULE,[named_table]), - [ets:insert(?MODULE,{X,integer_to_list(X)}) || X <- L], - ets:tab2file(?MODULE,FName), - {error,cannot_create_table} = ets:file2tab(FName), - true = ets:delete(?MODULE), - {ok,?MODULE} = ets:file2tab(FName), - true = ets:delete(?MODULE), - disk_log:open([{name,Name},{file,FName}]), - {_,[H2|T2]} = disk_log:chunk(Name,start), - disk_log:close(Name), - NewT2=lists:keydelete(8,1,T2), - file:delete(FName2), - disk_log:open([{name,Name},{file,FName2},{mode,read_write}]), - disk_log:log_terms(Name,[H2|NewT2]), - disk_log:close(Name), - 9 = length(ets:tab2list(element(2,ets:file2tab(FName2)))), - true = ets:delete(?MODULE), - {error,invalid_object_count} = ets:file2tab(FName2,[{verify,true}]), - {'EXIT',_} = (catch ets:delete(?MODULE)), - {ok,_} = ets:tabfile_info(FName2), - {ok,_} = ets:tabfile_info(FName), - file:delete(FName), - file:delete(FName2), + repeat_for_all_set_table_types( + fun(Opts) -> + FName = filename:join([proplists:get_value(priv_dir, Config),"namn.dat"]), + FName2 = filename:join([proplists:get_value(priv_dir, Config),"ncountflip.dat"]), + L = lists:seq(1,10), + Name = make_ref(), + ?MODULE = ets_new(?MODULE,[named_table|Opts]), + [ets:insert(?MODULE,{X,integer_to_list(X)}) || X <- L], + ets:tab2file(?MODULE,FName), + {error,cannot_create_table} = ets:file2tab(FName), + true = ets:delete(?MODULE), + {ok,?MODULE} = ets:file2tab(FName), + true = ets:delete(?MODULE), + disk_log:open([{name,Name},{file,FName}]), + {_,[H2|T2]} = disk_log:chunk(Name,start), + disk_log:close(Name), + NewT2=lists:keydelete(8,1,T2), + file:delete(FName2), + disk_log:open([{name,Name},{file,FName2},{mode,read_write}]), + disk_log:log_terms(Name,[H2|NewT2]), + disk_log:close(Name), + 9 = length(ets:tab2list(element(2,ets:file2tab(FName2)))), + true = ets:delete(?MODULE), + {error,invalid_object_count} = ets:file2tab(FName2,[{verify,true}]), + {'EXIT',_} = (catch ets:delete(?MODULE)), + {ok,_} = ets:tabfile_info(FName2), + {ok,_} = ets:tabfile_info(FName), + file:delete(FName), + file:delete(FName2) + end), ok. %% Tests verification of large table with md5 sum. tabfile_ext4(Config) when is_list(Config) -> - FName = filename:join([proplists:get_value(priv_dir, Config),"bauta.dat"]), - LL = lists:seq(1,10000), - TL = ets_new(x,[]), - Name2 = make_ref(), - [ets:insert(TL,{X,integer_to_list(X)}) || X <- LL], - ok = ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), - {ok, Name2} = disk_log:open([{name, Name2}, {file, FName}, - {mode, read_only}]), - {C,[_|_]} = disk_log:chunk(Name2,start), - {_,[_|_]} = disk_log:chunk(Name2,C), - disk_log:close(Name2), - true = lists:sort(ets:tab2list(TL)) =:= - lists:sort(ets:tab2list(element(2,ets:file2tab(FName)))), - Res = [begin - {ok,FD} = file:open(FName,[binary,read,write]), - {ok, Bin} = file:pread(FD,0,1000), - <<B1:N/binary,Ch:8,B2/binary>> = Bin, - Ch2 = (Ch + 1) rem 255, - Bin2 = <<B1/binary,Ch2:8,B2/binary>>, - ok = file:pwrite(FD,0,Bin2), - ok = file:close(FD), - X = case ets:file2tab(FName) of - {ok,TL2} -> - true = lists:sort(ets:tab2list(TL)) =/= - lists:sort(ets:tab2list(TL2)); - _ -> - totally_broken - end, - {error,Y} = ets:file2tab(FName,[{verify,true}]), - ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), - {X,Y} - end || N <- lists:seq(500,600)], - io:format("~p~n",[Res]), - file:delete(FName), + repeat_for_all_set_table_types( + fun(Opts) -> + FName = filename:join([proplists:get_value(priv_dir, Config),"bauta.dat"]), + LL = lists:seq(1,10000), + TL = ets_new(x,Opts), + Name2 = make_ref(), + [ets:insert(TL,{X,integer_to_list(X)}) || X <- LL], + ok = ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), + {ok, Name2} = disk_log:open([{name, Name2}, {file, FName}, + {mode, read_only}]), + {C,[_|_]} = disk_log:chunk(Name2,start), + {_,[_|_]} = disk_log:chunk(Name2,C), + disk_log:close(Name2), + true = lists:sort(ets:tab2list(TL)) =:= + lists:sort(ets:tab2list(element(2,ets:file2tab(FName)))), + Res = [begin + {ok,FD} = file:open(FName,[binary,read,write]), + {ok, Bin} = file:pread(FD,0,1000), + <<B1:N/binary,Ch:8,B2/binary>> = Bin, + Ch2 = (Ch + 1) rem 255, + Bin2 = <<B1/binary,Ch2:8,B2/binary>>, + ok = file:pwrite(FD,0,Bin2), + ok = file:close(FD), + X = case ets:file2tab(FName) of + {ok,TL2} -> + true = lists:sort(ets:tab2list(TL)) =/= + lists:sort(ets:tab2list(TL2)); + _ -> + totally_broken + end, + {error,Y} = ets:file2tab(FName,[{verify,true}]), + ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), + {X,Y} + end || N <- lists:seq(500,600)], + io:format("~p~n",[Res]), + file:delete(FName) + end), ok. %% Test that no disk_log is left open when file has been corrupted. @@ -4556,11 +4677,11 @@ make_sub_binary(List, Num) when is_list(List) -> %% Perform multiple lookups for every key in a large table. heavy_lookup(Config) when is_list(Config) -> - repeat_for_opts(fun heavy_lookup_do/1). + repeat_for_opts_all_set_table_types(fun heavy_lookup_do/1). heavy_lookup_do(Opts) -> EtsMem = etsmem(), - Tab = ets_new(foobar_table, [set, protected, {keypos, 2} | Opts]), + Tab = ets_new(foobar_table, [{keypos, 2} | Opts]), ok = fill_tab2(Tab, 0, 7000), _ = [do_lookup(Tab, 6999) || _ <- lists:seq(1, 50)], true = ets:delete(Tab), @@ -4579,11 +4700,11 @@ do_lookup(Tab, N) -> %% Perform multiple lookups for every element in a large table. heavy_lookup_element(Config) when is_list(Config) -> - repeat_for_opts(fun heavy_lookup_element_do/1). + repeat_for_opts_all_set_table_types(fun heavy_lookup_element_do/1). heavy_lookup_element_do(Opts) -> EtsMem = etsmem(), - Tab = ets_new(foobar_table, [set, protected, {keypos, 2} | Opts]), + Tab = ets_new(foobar_table, [{keypos, 2} | Opts]), ok = fill_tab2(Tab, 0, 7000), %% lookup ALL elements 50 times Laps = 50 div syrup_factor(), @@ -4606,14 +4727,14 @@ do_lookup_element(Tab, N, M) -> heavy_concurrent(Config) when is_list(Config) -> - ct:timetrap({minutes,30}), %% valgrind needs a lot of time - repeat_for_opts(fun do_heavy_concurrent/1). + ct:timetrap({minutes,120}), %% valgrind needs a lot of time + repeat_for_opts_all_set_table_types(fun do_heavy_concurrent/1). do_heavy_concurrent(Opts) -> Size = 10000, Laps = 10000 div syrup_factor(), EtsMem = etsmem(), - Tab = ets_new(blupp, [set, public, {keypos, 2} | Opts]), + Tab = ets_new(blupp, [public, {keypos, 2} | Opts]), ok = fill_tab2(Tab, 0, Size), Procs = lists:map( fun (N) -> @@ -4647,48 +4768,68 @@ do_heavy_concurrent_proc(Tab, N, Offs) -> fold_empty(Config) when is_list(Config) -> - EtsMem = etsmem(), - Tab = make_table(a, [], []), - [] = ets:foldl(fun(_X) -> exit(hej) end, [], Tab), - [] = ets:foldr(fun(_X) -> exit(hej) end, [], Tab), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_opts_all_set_table_types( + fun(Opts) -> + EtsMem = etsmem(), + Tab = make_table(a, Opts, []), + [] = ets:foldl(fun(_X) -> exit(hej) end, [], Tab), + [] = ets:foldr(fun(_X) -> exit(hej) end, [], Tab), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end), + ok. foldl(Config) when is_list(Config) -> - EtsMem = etsmem(), - L = [{a,1}, {c,3}, {b,2}], - LS = lists:sort(L), - Tab = make_table(a, [bag], L), - LS = lists:sort(ets:foldl(fun(E,A) -> [E|A] end, [], Tab)), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_opts_all_table_types( + fun(Opts) -> + EtsMem = etsmem(), + L = [{a,1}, {c,3}, {b,2}], + LS = lists:sort(L), + Tab = make_table(a, Opts, L), + LS = lists:sort(ets:foldl(fun(E,A) -> [E|A] end, [], Tab)), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end), + ok. foldr(Config) when is_list(Config) -> - EtsMem = etsmem(), - L = [{a,1}, {c,3}, {b,2}], - LS = lists:sort(L), - Tab = make_table(a, [bag], L), - LS = lists:sort(ets:foldr(fun(E,A) -> [E|A] end, [], Tab)), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_opts_all_table_types( + fun(Opts) -> + EtsMem = etsmem(), + L = [{a,1}, {c,3}, {b,2}], + LS = lists:sort(L), + Tab = make_table(a, Opts, L), + LS = lists:sort(ets:foldr(fun(E,A) -> [E|A] end, [], Tab)), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end), + ok. foldl_ordered(Config) when is_list(Config) -> - EtsMem = etsmem(), - L = [{a,1}, {c,3}, {b,2}], - LS = lists:sort(L), - Tab = make_table(a, [ordered_set], L), - LS = lists:reverse(ets:foldl(fun(E,A) -> [E|A] end, [], Tab)), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_opts_all_ord_set_table_types( + fun(Opts) -> + EtsMem = etsmem(), + L = [{a,1}, {c,3}, {b,2}], + LS = lists:sort(L), + Tab = make_table(a, Opts, L), + LS = lists:reverse(ets:foldl(fun(E,A) -> [E|A] end, [], Tab)), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end), + ok. foldr_ordered(Config) when is_list(Config) -> - EtsMem = etsmem(), - L = [{a,1}, {c,3}, {b,2}], - LS = lists:sort(L), - Tab = make_table(a, [ordered_set], L), - LS = ets:foldr(fun(E,A) -> [E|A] end, [], Tab), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_opts_all_ord_set_table_types( + fun(Opts) -> + EtsMem = etsmem(), + L = [{a,1}, {c,3}, {b,2}], + LS = lists:sort(L), + Tab = make_table(a, Opts, L), + LS = ets:foldr(fun(E,A) -> [E|A] end, [], Tab), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end), + ok. %% Test ets:member BIF. member(Config) when is_list(Config) -> @@ -5087,27 +5228,30 @@ gen_dets_filename(Config,N) -> "testdets_" ++ integer_to_list(N) ++ ".dets"). otp_6842_select_1000(Config) when is_list(Config) -> - Tab = ets_new(xxx,[ordered_set]), - [ets:insert(Tab,{X,X}) || X <- lists:seq(1,10000)], - AllTrue = lists:duplicate(10,true), - AllTrue = - [ length( - element(1, - ets:select(Tab,[{'_',[],['$_']}],X*1000))) =:= - X*1000 || X <- lists:seq(1,10) ], - Sequences = [[1000,1000,1000,1000,1000,1000,1000,1000,1000,1000], - [2000,2000,2000,2000,2000], - [3000,3000,3000,1000], - [4000,4000,2000], - [5000,5000], - [6000,4000], - [7000,3000], - [8000,2000], - [9000,1000], - [10000]], - AllTrue = [ check_seq(Tab, ets:select(Tab,[{'_',[],['$_']}],hd(L)),L) || - L <- Sequences ], - ets:delete(Tab), + repeat_for_opts_all_ord_set_table_types( + fun(Opts) -> + Tab = ets_new(xxx,Opts), + [ets:insert(Tab,{X,X}) || X <- lists:seq(1,10000)], + AllTrue = lists:duplicate(10,true), + AllTrue = + [ length( + element(1, + ets:select(Tab,[{'_',[],['$_']}],X*1000))) =:= + X*1000 || X <- lists:seq(1,10) ], + Sequences = [[1000,1000,1000,1000,1000,1000,1000,1000,1000,1000], + [2000,2000,2000,2000,2000], + [3000,3000,3000,1000], + [4000,4000,2000], + [5000,5000], + [6000,4000], + [7000,3000], + [8000,2000], + [9000,1000], + [10000]], + AllTrue = [ check_seq(Tab, ets:select(Tab,[{'_',[],['$_']}],hd(L)),L) || + L <- Sequences ], + ets:delete(Tab) + end), ok. check_seq(_,'$end_of_table',[]) -> @@ -5119,17 +5263,21 @@ check_seq(A,B,C) -> false. otp_6338(Config) when is_list(Config) -> - L = binary_to_term(<<131,108,0,0,0,2,104,2,108,0,0,0,2,103,100,0,19,112,112, - 98,49,95,98,115,49,50,64,98,108,97,100,101,95,48,95,53, - 0,0,33,50,0,0,0,4,1,98,0,0,23,226,106,100,0,4,101,120, - 105,116,104,2,108,0,0,0,2,104,2,100,0,3,115,98,109,100, - 0,19,112,112,98,50,95,98,115,49,50,64,98,108,97,100, - 101,95,48,95,56,98,0,0,18,231,106,100,0,4,114,101,99, - 118,106>>), - T = ets_new(xxx,[ordered_set]), - lists:foreach(fun(X) -> ets:insert(T,X) end,L), - [[4839,recv]] = ets:match(T,{[{sbm,ppb2_bs12@blade_0_8},'$1'],'$2'}), - ets:delete(T). + repeat_for_opts_all_ord_set_table_types( + fun(Opts) -> + L = binary_to_term(<<131,108,0,0,0,2,104,2,108,0,0,0,2,103,100,0,19,112,112, + 98,49,95,98,115,49,50,64,98,108,97,100,101,95,48,95,53, + 0,0,33,50,0,0,0,4,1,98,0,0,23,226,106,100,0,4,101,120, + 105,116,104,2,108,0,0,0,2,104,2,100,0,3,115,98,109,100, + 0,19,112,112,98,50,95,98,115,49,50,64,98,108,97,100, + 101,95,48,95,56,98,0,0,18,231,106,100,0,4,114,101,99, + 118,106>>), + T = ets_new(xxx,Opts), + lists:foreach(fun(X) -> ets:insert(T,X) end,L), + [[4839,recv]] = ets:match(T,{[{sbm,ppb2_bs12@blade_0_8},'$1'],'$2'}), + ets:delete(T) + end), + ok. %% Elements could come in the wrong order in a bag if a rehash occurred. otp_5340(Config) when is_list(Config) -> @@ -5199,7 +5347,7 @@ otp_7665_act(Tab,Min,Max,DelNr) -> %% Whitebox testing of meta name table hashing. meta_wb(Config) when is_list(Config) -> EtsMem = etsmem(), - repeat_for_opts(fun meta_wb_do/1), + repeat_for_opts_all_non_stim_table_types(fun meta_wb_do/1), verify_etsmem(EtsMem). @@ -5268,13 +5416,16 @@ colliding_names(Name) -> %% OTP_6913: Grow and shrink. grow_shrink(Config) when is_list(Config) -> - EtsMem = etsmem(), - - Set = ets_new(a, [set]), - grow_shrink_0(0, 3071, 3000, 5000, Set), - ets:delete(Set), - - verify_etsmem(EtsMem). + repeat_for_all_set_table_types( + fun(Opts) -> + EtsMem = etsmem(), + + Set = ets_new(a, Opts), + grow_shrink_0(0, 3071, 3000, 5000, Set), + ets:delete(Set), + + verify_etsmem(EtsMem) + end). grow_shrink_0(N, _, _, Max, _) when N >= Max -> ok; @@ -5304,7 +5455,8 @@ grow_pseudo_deleted(Config) when is_list(Config) -> grow_pseudo_deleted_do() -> lists:foreach(fun(Type) -> grow_pseudo_deleted_do(Type) end, - [set,bag,duplicate_bag]). + [set,cat_ord_set,stim_cat_ord_set, + ordered_set,bag,duplicate_bag]). grow_pseudo_deleted_do(Type) -> process_flag(scheduler,1), @@ -5319,7 +5471,12 @@ grow_pseudo_deleted_do(Type) -> [true]}]), Left = Mult*(Mod-1), Left = ets:info(T,size), - Mult = get_kept_objects(T), + case Type of + cat_ord_set -> ok; + stim_cat_ord_set -> ok; + ordered_set -> ok; + _ -> Mult = get_kept_objects(T) + end, filltabstr(T,Mult), my_spawn_opt( fun() -> @@ -5357,7 +5514,8 @@ shrink_pseudo_deleted(Config) when is_list(Config) -> shrink_pseudo_deleted_do() -> lists:foreach(fun(Type) -> shrink_pseudo_deleted_do(Type) end, - [set,bag,duplicate_bag]). + [set,cat_ord_set,stim_cat_ord_set, + ordered_set,bag,duplicate_bag]). shrink_pseudo_deleted_do(Type) -> process_flag(scheduler,1), @@ -5371,7 +5529,12 @@ shrink_pseudo_deleted_do(Type) -> [{'>', '$1', Half}], [true]}]), Half = ets:info(T,size), - Half = get_kept_objects(T), + case Type of + cat_ord_set -> ok; + stim_cat_ord_set -> ok; + ordered_set -> ok; + _ -> Half = get_kept_objects(T) + end, my_spawn_opt( fun()-> true = ets:info(T,fixed), Self ! start, @@ -5471,10 +5634,14 @@ meta_newdel_named(Config) when is_list(Config) -> %% Concurrent insert's on same table. smp_insert(Config) when is_list(Config) -> - ets_new(smp_insert,[named_table,public,{write_concurrency,true}]), + repeat_for_opts(fun smp_insert_do/1, + [[set,ordered_set,stim_cat_ord_set]]). + +smp_insert_do(Opts) -> + ets_new(smp_insert,[named_table,public,{write_concurrency,true}|Opts]), InitF = fun(_) -> ok end, ExecF = fun(_) -> true = ets:insert(smp_insert,{rand:uniform(10000)}) - end, + end, FiniF = fun(_) -> ok end, run_smp_workers(InitF,ExecF,FiniF,100000), verify_table_load(smp_insert), @@ -5482,33 +5649,41 @@ smp_insert(Config) when is_list(Config) -> %% Concurrent deletes on same fixated table. smp_fixed_delete(Config) when is_list(Config) -> - only_if_smp(fun()->smp_fixed_delete_do() end). - -smp_fixed_delete_do() -> - T = ets_new(foo,[public,{write_concurrency,true}]), - %%Mem = ets:info(T,memory), - NumOfObjs = 100000, - filltabint(T,NumOfObjs), - ets:safe_fixtable(T,true), - Buckets = num_of_buckets(T), - InitF = fun([ProcN,NumOfProcs|_]) -> {ProcN,NumOfProcs} end, - ExecF = fun({Key,_}) when Key > NumOfObjs -> - [end_of_work]; - ({Key,Increment}) -> - true = ets:delete(T,Key), - {Key+Increment,Increment} - end, - FiniF = fun(_) -> ok end, - run_sched_workers(InitF,ExecF,FiniF,NumOfObjs), - 0 = ets:info(T,size), - true = ets:info(T,fixed), - Buckets = num_of_buckets(T), - NumOfObjs = get_kept_objects(T), - ets:safe_fixtable(T,false), - %% Will fail as unfix does not shrink the table: - %%Mem = ets:info(T,memory), - %%verify_table_load(T), - ets:delete(T). + only_if_smp(fun()-> + repeat_for_opts(fun smp_fixed_delete_do/1, + [[set,ordered_set,stim_cat_ord_set]]) + end). + +smp_fixed_delete_do(Opts) -> + begin + T = ets_new(foo,[public,{write_concurrency,true}|Opts]), + %%Mem = ets:info(T,memory), + NumOfObjs = 100000, + filltabint(T,NumOfObjs), + ets:safe_fixtable(T,true), + Buckets = num_of_buckets(T), + InitF = fun([ProcN,NumOfProcs|_]) -> {ProcN,NumOfProcs} end, + ExecF = fun({Key,_}) when Key > NumOfObjs -> + [end_of_work]; + ({Key,Increment}) -> + true = ets:delete(T,Key), + {Key+Increment,Increment} + end, + FiniF = fun(_) -> ok end, + run_sched_workers(InitF,ExecF,FiniF,NumOfObjs), + 0 = ets:info(T,size), + true = ets:info(T,fixed), + Buckets = num_of_buckets(T), + case ets:info(T,type) of + set -> NumOfObjs = get_kept_objects(T); + _ -> ok + end, + ets:safe_fixtable(T,false), + %% Will fail as unfix does not shrink the table: + %%Mem = ets:info(T,memory), + %%verify_table_load(T), + ets:delete(T) + end. %% ERL-720 %% Provoke race between ets:delete and table unfix (by select_count) @@ -5551,7 +5726,12 @@ delete_unfix_race(Config) when is_list(Config) -> verify_etsmem(EtsMem). num_of_buckets(T) -> - element(1,ets:info(T,stats)). + case ets:info(T,type) of + set -> element(1,ets:info(T,stats)); + bag -> element(1,ets:info(T,stats)); + duplicate_bag -> element(1,ets:info(T,stats)); + _ -> ok + end. %% Fixate hash table while other process is busy doing unfix. smp_unfix_fix(Config) when is_list(Config) -> @@ -5716,98 +5896,112 @@ otp_8166_zombie_creator(T,Deleted) -> verify_table_load(T) -> - Stats = ets:info(T,stats), - {Buckets,AvgLen,StdDev,ExpSD,_MinLen,_MaxLen,_} = Stats, - ok = if - AvgLen > 1.2 -> - io:format("Table overloaded: Stats=~p\n~p\n", - [Stats, ets:info(T)]), - false; - - Buckets>256, AvgLen < 0.47 -> - io:format("Table underloaded: Stats=~p\n~p\n", - [Stats, ets:info(T)]), - false; - - StdDev > ExpSD*2 -> - io:format("Too large standard deviation (poor hashing?)," - " stats=~p\n~p\n",[Stats, ets:info(T)]), - false; - - true -> - io:format("Stats = ~p\n",[Stats]), - ok - end. + case ets:info(T,type) of + ordered_set -> ok; + _ -> + Stats = ets:info(T,stats), + {Buckets,AvgLen,StdDev,ExpSD,_MinLen,_MaxLen,_} = Stats, + ok = if + AvgLen > 1.2 -> + io:format("Table overloaded: Stats=~p\n~p\n", + [Stats, ets:info(T)]), + false; + + Buckets>256, AvgLen < 0.47 -> + io:format("Table underloaded: Stats=~p\n~p\n", + [Stats, ets:info(T)]), + false; + + StdDev > ExpSD*2 -> + io:format("Too large standard deviation (poor hashing?)," + " stats=~p\n~p\n",[Stats, ets:info(T)]), + false; + + true -> + io:format("Stats = ~p\n",[Stats]), + ok + end + end. %% ets:select on a tree with NIL key object. otp_8732(Config) when is_list(Config) -> - Tab = ets_new(noname,[ordered_set]), - filltabstr(Tab,999), - ets:insert(Tab,{[],"nasty NIL object"}), - [] = ets:match(Tab,{'_',nomatch}), %% Will hang if bug not fixed + repeat_for_all_ord_set_table_types( + fun(Opts) -> + Tab = ets_new(noname,Opts), + filltabstr(Tab,999), + ets:insert(Tab,{[],"nasty NIL object"}), + [] = ets:match(Tab,{'_',nomatch}) %% Will hang if bug not fixed + end), ok. %% Run concurrent select_delete (and inserts) on same table. smp_select_delete(Config) when is_list(Config) -> - T = ets_new(smp_select_delete,[named_table,public,{write_concurrency,true}]), - Mod = 17, - Zeros = erlang:make_tuple(Mod,0), - InitF = fun(_) -> Zeros end, - ExecF = fun(Diffs0) -> - case rand:uniform(20) of - 1 -> - Mod = 17, - Eq = rand:uniform(Mod) - 1, - Deleted = ets:select_delete(T, - [{{'_', '$1'}, - [{'=:=', {'rem', '$1', Mod}, Eq}], - [true]}]), - Diffs1 = setelement(Eq+1, Diffs0, - element(Eq+1,Diffs0) - Deleted), - Diffs1; - _ -> - Key = rand:uniform(10000), - Eq = Key rem Mod, - case ets:insert_new(T,{Key,Key}) of - true -> - Diffs1 = setelement(Eq+1, Diffs0, - element(Eq+1,Diffs0)+1), - Diffs1; - false -> Diffs0 - end - end - end, - FiniF = fun(Result) -> Result end, - Results = run_sched_workers(InitF,ExecF,FiniF,20000), - TotCnts = lists:foldl(fun(Diffs, Sum) -> add_lists(Sum,tuple_to_list(Diffs)) end, - lists:duplicate(Mod, 0), Results), - io:format("TotCnts = ~p\n",[TotCnts]), - LeftInTab = lists:foldl(fun(N,Sum) -> Sum+N end, - 0, TotCnts), - io:format("LeftInTab = ~p\n",[LeftInTab]), - LeftInTab = ets:info(T,size), - lists:foldl(fun(Cnt,Eq) -> - WasCnt = ets:select_count(T, - [{{'_', '$1'}, - [{'=:=', {'rem', '$1', Mod}, Eq}], - [true]}]), - io:format("~p: ~p =?= ~p\n",[Eq,Cnt,WasCnt]), - Cnt = WasCnt, - Eq+1 - end, - 0, TotCnts), - %% May fail as select_delete does not shrink table (enough) - %%verify_table_load(T), - LeftInTab = ets:select_delete(T, [{{'$1','$1'}, [], [true]}]), - 0 = ets:info(T,size), - false = ets:info(T,fixed), - ets:delete(T). + repeat_for_opts(fun smp_select_delete_do/1, + [[set,ordered_set,stim_cat_ord_set], read_concurrency, compressed]). + +smp_select_delete_do(Opts) -> + begin % indentation + T = ets_new(smp_select_delete,[named_table,public,{write_concurrency,true}|Opts]), + Mod = 17, + Zeros = erlang:make_tuple(Mod,0), + InitF = fun(_) -> Zeros end, + ExecF = fun(Diffs0) -> + case rand:uniform(20) of + 1 -> + Mod = 17, + Eq = rand:uniform(Mod) - 1, + Deleted = ets:select_delete(T, + [{{'_', '$1'}, + [{'=:=', {'rem', '$1', Mod}, Eq}], + [true]}]), + Diffs1 = setelement(Eq+1, Diffs0, + element(Eq+1,Diffs0) - Deleted), + Diffs1; + _ -> + Key = rand:uniform(10000), + Eq = Key rem Mod, + case ets:insert_new(T,{Key,Key}) of + true -> + Diffs1 = setelement(Eq+1, Diffs0, + element(Eq+1,Diffs0)+1), + Diffs1; + false -> Diffs0 + end + end + end, + FiniF = fun(Result) -> Result end, + Results = run_sched_workers(InitF,ExecF,FiniF,20000), + TotCnts = lists:foldl(fun(Diffs, Sum) -> add_lists(Sum,tuple_to_list(Diffs)) end, + lists:duplicate(Mod, 0), Results), + io:format("TotCnts = ~p\n",[TotCnts]), + LeftInTab = lists:foldl(fun(N,Sum) -> Sum+N end, + 0, TotCnts), + io:format("LeftInTab = ~p\n",[LeftInTab]), + LeftInTab = ets:info(T,size), + lists:foldl(fun(Cnt,Eq) -> + WasCnt = ets:select_count(T, + [{{'_', '$1'}, + [{'=:=', {'rem', '$1', Mod}, Eq}], + [true]}]), + io:format("~p: ~p =?= ~p\n",[Eq,Cnt,WasCnt]), + Cnt = WasCnt, + Eq+1 + end, + 0, TotCnts), + %% May fail as select_delete does not shrink table (enough) + %%verify_table_load(T), + LeftInTab = ets:select_delete(T, [{{'$1','$1'}, [], [true]}]), + 0 = ets:info(T,size), + false = ets:info(T,fixed), + ets:delete(T) + end, % indentation + ok. smp_select_replace(Config) when is_list(Config) -> repeat_for_opts(fun smp_select_replace_do/1, - [[set,ordered_set,duplicate_bag]]). + [[set,ordered_set,stim_cat_ord_set,duplicate_bag]]). smp_select_replace_do(Opts) -> T = ets_new(smp_select_replace, @@ -5847,7 +6041,8 @@ smp_select_replace_do(Opts) -> %% Test different types. types(Config) when is_list(Config) -> init_externals(), - repeat_for_opts(fun types_do/1, [[set,ordered_set],compressed]). + repeat_for_opts(fun types_do/1, [repeat_for_opts_atom2list(set_types), + compressed]). types_do(Opts) -> EtsMem = etsmem(), @@ -5874,7 +6069,7 @@ types_do(Opts) -> %% OTP-9932: Memory overwrite when inserting large integers in compressed bag. %% Will crash with segv on 64-bit opt if not fixed. otp_9932(Config) when is_list(Config) -> - T = ets:new(xxx, [bag, compressed]), + T = ets_new(xxx, [bag, compressed]), Fun = fun(N) -> Key = {1316110174588445 bsl N,1316110174588583 bsl N}, S = {Key, Key}, @@ -5890,48 +6085,56 @@ otp_9932(Config) when is_list(Config) -> %% vm-deadlock caused by race between ets:delete and others on %% write_concurrency table. otp_9423(Config) when is_list(Config) -> - InitF = fun(_) -> {0,0} end, - ExecF = fun({S,F}) -> - receive - stop -> - io:format("~p got stop\n", [self()]), - [end_of_work | {"Succeded=",S,"Failed=",F}] - after 0 -> - %%io:format("~p (~p) doing lookup\n", [self(), {S,F}]), - try ets:lookup(otp_9423, key) of - [] -> {S+1,F} - catch - error:badarg -> {S,F+1} - end - end - end, - FiniF = fun(R) -> R end, - case run_smp_workers(InitF, ExecF, FiniF, infinite, 1) of - Pids when is_list(Pids) -> - %%[P ! start || P <- Pids], - repeat(fun() -> ets:new(otp_9423, [named_table, public, {write_concurrency,true}]), - ets:delete(otp_9423) - end, 10000), - [P ! stop || P <- Pids], - wait_pids(Pids), - ok; + repeat_for_all_non_stim_set_table_types( + fun(Opts) -> + InitF = fun(_) -> {0,0} end, + ExecF = fun({S,F}) -> + receive + stop -> + io:format("~p got stop\n", [self()]), + [end_of_work | {"Succeded=",S,"Failed=",F}] + after 0 -> + %%io:format("~p (~p) doing lookup\n", [self(), {S,F}]), + try ets:lookup(otp_9423, key) of + [] -> {S+1,F} + catch + error:badarg -> {S,F+1} + end + end + end, + FiniF = fun(R) -> R end, + case run_smp_workers(InitF, ExecF, FiniF, infinite, 1) of + Pids when is_list(Pids) -> + %%[P ! start || P <- Pids], + repeat(fun() -> ets_new(otp_9423, [named_table, public, + {write_concurrency,true}|Opts]), + ets:delete(otp_9423) + end, 10000), + [P ! stop || P <- Pids], + wait_pids(Pids), + ok; + + Skipped -> Skipped + end + end). - Skipped -> Skipped - end. %% Corrupted binary in compressed table otp_10182(Config) when is_list(Config) -> - Bin = <<"aHR0cDovL2hvb3RzdWl0ZS5jb20vYy9wcm8tYWRyb2xsLWFi">>, - Key = {test, Bin}, - Value = base64:decode(Bin), - In = {Key,Value}, - Db = ets:new(undefined, [set, protected, {read_concurrency, true}, compressed]), - ets:insert(Db, In), - [Out] = ets:lookup(Db, Key), - io:format("In : ~p\nOut: ~p\n", [In,Out]), - ets:delete(Db), - In = Out. + repeat_for_opts_all_table_types( + fun(Opts) -> + Bin = <<"aHR0cDovL2hvb3RzdWl0ZS5jb20vYy9wcm8tYWRyb2xsLWFi">>, + Key = {test, Bin}, + Value = base64:decode(Bin), + In = {Key,Value}, + Db = ets_new(undefined, Opts), + ets:insert(Db, In), + [Out] = ets:lookup(Db, Key), + io:format("In : ~p\nOut: ~p\n", [In,Out]), + ets:delete(Db), + In = Out + end). %% Test that ets:all include/exclude tables that we know are created/deleted ets_all(Config) when is_list(Config) -> @@ -6022,19 +6225,23 @@ take(Config) when is_list(Config) -> ets:insert(T1, {{'not',<<"immediate">>},ok}), [{{'not',<<"immediate">>},ok}] = ets:take(T1, {'not',<<"immediate">>}), %% Same with ordered tables. - T2 = ets_new(b, [ordered_set]), - [] = ets:take(T2, foo), - ets:insert(T2, {foo,bar}), - [] = ets:take(T2, bar), - [{foo,bar}] = ets:take(T2, foo), - [] = ets:tab2list(T2), - ets:insert(T2, {{'not',<<"immediate">>},ok}), - [{{'not',<<"immediate">>},ok}] = ets:take(T2, {'not',<<"immediate">>}), - %% Arithmetically-equal keys. - ets:insert(T2, [{1.0,float},{2,integer}]), - [{1.0,float}] = ets:take(T2, 1), - [{2,integer}] = ets:take(T2, 2.0), - [] = ets:tab2list(T2), + repeat_for_all_ord_set_table_types( + fun(Opts) -> + T2 = ets_new(b, Opts), + [] = ets:take(T2, foo), + ets:insert(T2, {foo,bar}), + [] = ets:take(T2, bar), + [{foo,bar}] = ets:take(T2, foo), + [] = ets:tab2list(T2), + ets:insert(T2, {{'not',<<"immediate">>},ok}), + [{{'not',<<"immediate">>},ok}] = ets:take(T2, {'not',<<"immediate">>}), + %% Arithmetically-equal keys. + ets:insert(T2, [{1.0,float},{2,integer}]), + [{1.0,float}] = ets:take(T2, 1), + [{2,integer}] = ets:take(T2, 2.0), + [] = ets:tab2list(T2), + ets:delete(T2) + end), %% Same with bag. T3 = ets_new(c, [bag]), ets:insert(T3, [{1,1},{1,2},{3,3}]), @@ -6042,7 +6249,6 @@ take(Config) when is_list(Config) -> [{3,3}] = ets:take(T3, 3), [] = ets:tab2list(T3), ets:delete(T1), - ets:delete(T2), ets:delete(T3), ok. @@ -6077,9 +6283,371 @@ whereis_table(Config) when is_list(Config) -> ok. -%% -%% Utility functions: -%% + +%% The following work functions are used by +%% throughput_benchmark/4. They are declared on the top level beacuse +%% declaring them as function local funs cause a scalability issue. +get_op([{_,O}], _RandNum) -> + O; +get_op([{Prob,O}|Rest], RandNum) -> + case RandNum < Prob of + true -> O; + false -> get_op(Rest, RandNum) + end. +do_op(Table, ProbHelpTab, Range, Operations) -> + RandNum = rand:uniform(), + Op = get_op(ProbHelpTab, RandNum), + #{ Op := TheOp} = Operations, + TheOp(Table, Range). +do_work(WorksDoneSoFar, Table, ProbHelpTab, Range, Operations) -> + receive + stop -> WorksDoneSoFar + after + 0 -> do_op(Table, ProbHelpTab, Range, Operations), + do_work(WorksDoneSoFar + 1, Table, ProbHelpTab, Range, Operations) + end. + +prefill_table(T, KeyRange, Num) -> + Seed = rand:uniform(KeyRange), + %%io:format("prefill_table: Seed = ~p\n", [Seed]), + RState = unique_rand_start(KeyRange, Seed), + prefill_table_loop(T, RState, Num), + Num = ets:info(T, size). + +prefill_table_loop(_, _, 0) -> + ok; +prefill_table_loop(T, RS0, N) -> + {Key, RS1} = unique_rand_next(RS0), + ets:insert(T, {Key}), + prefill_table_loop(T, RS1, N-1). + +throughput_benchmark() -> + throughput_benchmark(false, not_set, not_set). + +throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> + NrOfSchedulers = erlang:system_info(schedulers), + %% Definitions of operations that are supported by the benchmark + NextSeqOp = + fun (T, KeyRange, SeqSize) -> + Start = rand:uniform(KeyRange), + Last = + lists:foldl( + fun(_, Prev) -> + case Prev of + '$end_of_table'-> ok; + _ -> + try ets:next(T, Prev) of + Normal -> Normal + catch + error:badarg -> + % sets (not ordered_sets) cannot handle when the argument + % to next is not in the set + rand:uniform(KeyRange) + end + end + end, + Start, + lists:seq(1, SeqSize)), + case Last =:= -1 of + true -> io:format("Will never be printed"); + false -> ok + end + end, + PartialSelectOp = + fun (T, KeyRange, SeqSize) -> + Start = rand:uniform(KeyRange), + Last = Start + SeqSize, + case -1 =:= ets:select_count(T, + ets:fun2ms(fun({X}) when X > Start andalso X =< Last -> true end)) of + true -> io:format("Will never be printed"); + false -> ok + end + + end, + %% Mapping benchmark operation names to their corresponding functions that do them + Operations = + #{insert => + fun(T,KeyRange) -> + Num = rand:uniform(KeyRange), + ets:insert(T, {Num}) + end, + delete => + fun(T,KeyRange) -> + Num = rand:uniform(KeyRange), + ets:delete(T, Num) + end, + lookup => + fun(T,KeyRange) -> + Num = rand:uniform(KeyRange), + ets:lookup(T, Num) + end, + nextseq10 => + fun(T,KeyRange) -> NextSeqOp(T,KeyRange,10) end, + nextseq100 => + fun(T,KeyRange) -> NextSeqOp(T,KeyRange,100) end, + nextseq1000 => + fun(T,KeyRange) -> NextSeqOp(T,KeyRange,1000) end, + selectAll => + fun(T,_KeyRange) -> + case -1 =:= ets:select_count(T, ets:fun2ms(fun(X) -> true end)) of + true -> io:format("Will never be printed"); + false -> ok + end + end, + partial_select1000 => + fun(T,KeyRange) -> PartialSelectOp(T,KeyRange,1000) end + }, + %% Helper functions + CalculateThreadCounts = fun Calculate([Count|Rest]) -> + case Count > NrOfSchedulers of + true -> lists:reverse(Rest); + false -> Calculate([Count*2,Count|Rest]) + end + end, + CalculateOpsProbHelpTab = + fun Calculate([{_, OpName}], _) -> + [{1.0, OpName}]; + Calculate([{OpPropability, OpName}|Res], Current) -> + NewCurrent = Current + OpPropability, + [{NewCurrent, OpName}| Calculate(Res, NewCurrent)] + end, + RenderScenario = + fun R([], StringSoFar) -> + StringSoFar; + R([{Fraction, Operation}], StringSoFar) -> + io_lib:format("~s ~f% ~p",[StringSoFar, Fraction * 100.0, Operation]); + R([{Fraction, Operation}|Rest], StringSoFar) -> + R(Rest, + io_lib:format("~s ~f% ~p, ",[StringSoFar, Fraction * 100.0, Operation])) + end, + SafeFixTableIfRequired = + fun(Table, Scenario, On) -> + case set =:= ets:info(Table, type) of + true -> + HasNotRequiringOp = + lists:search( + fun({_,nextseq10}) -> true; + ({_,nextseq100}) -> true; + ({_,nextseq1000}) -> true; + (_) -> false + end, Scenario), + case HasNotRequiringOp of + false -> ok; + _ -> ets:safe_fixtable(Table, On) + end; + false -> ok + end + end, + %% Function that runs a benchmark instance and returns the number + %% of operations that were performed + RunBenchmark = + fun(NrOfProcs, TableConfig, Scenario, + Range, Duration, RecoverTime) -> + ProbHelpTab = CalculateOpsProbHelpTab(Scenario, 0), + Table = ets:new(t, TableConfig), + prefill_table(Table, Range, Range div 2), + SafeFixTableIfRequired(Table, Scenario, true), + ParentPid = self(), + ChildPids = + lists:map( + fun(_N) -> + spawn(fun() -> + receive start -> ok end, + WorksDone = + do_work(0, Table, ProbHelpTab, Range, Operations), + ParentPid ! WorksDone + end) + end, lists:seq(1, NrOfProcs)), + lists:foreach(fun(Pid) -> Pid ! start end, ChildPids), + timer:sleep(Duration), + lists:foreach(fun(Pid) -> Pid ! stop end, ChildPids), + TotalWorksDone = lists:foldl( + fun(_, Sum) -> + receive + Count -> Sum + Count + end + end, 0, ChildPids), + SafeFixTableIfRequired(Table, Scenario, false), + ets:delete(Table), + timer:sleep(RecoverTime), + TotalWorksDone + end, + %% + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %%%% Benchmark Configuration %%%%%%%%%%%%%%%%%%%%%%%% + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %% + %% Change the following variables to configure the benchmark runs + ThreadCounts = + case TestMode of + true -> [1, NrOfSchedulers]; + false -> CalculateThreadCounts([1]) + end, + KeyRanges = % Sizes of the key ranges + case TestMode of + true -> [50000]; + false -> [1000000] + end, + Duration = + case BenchmarkRunMs of % Duration of a benchmark run in milliseconds + not_set -> 30000; + _ -> BenchmarkRunMs + end, + TimeMsToSleepAfterEachBenchmarkRun = + case RecoverTimeMs of + not_set -> 1000; + _ -> RecoverTimeMs + end, + TableTypes = % The table types that will be benchmarked + [ + [ordered_set, public], + [ordered_set, public, {write_concurrency, true}], + [ordered_set, public, {read_concurrency, true}], + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public], + [set, public, {write_concurrency, true}], + [set, public, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + Scenarios = % Benchmark scenarios (the fractions should add up to approximately 1.0) + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {1.0, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq10} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq1000} + ], + [ + {1.0, nextseq1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, partial_select1000} + ] + ], + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %%%% End of Benchmark Configuration %%%%%%%%%%%%%%%% + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %% Prepare for memory check + EtsMem = case TestMode of + true -> etsmem(); + false -> ok + end, + %% Run the benchmark + io:format("# Each instance of the benchmark runs for ~w seconds:~n", [Duration/1000]), + io:format("# The result of a benchmark instance is presented as a number representing~n"), + io:format("# the number of operations performed per second:~n~n~n"), + io:format("# To plot graphs for the results below:~n"), + io:format("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n"), + io:format("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n"), + io:format("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n"), + io:format("# step 1 and press the Render button~n~n"), + io:format("#BENCHMARK STARTED$~n"), + %% The following loop runs all benchmark scenarios and prints the results (i.e, operations/second) + lists:foreach( + fun(KeyRange) -> + lists:foreach( + fun(Scenario) -> + io:format("Scenario: ~s | Key Range Size: ~w$~n", + [RenderScenario(Scenario, ""), + KeyRange]), + lists:foreach( + fun(ThreadCount) -> + io:format("; ~w",[ThreadCount]) + end, + ThreadCounts), + io:format("$~n",[]), + lists:foreach( + fun(TableType) -> + io:format("~w ",[TableType]), + lists:foreach( + fun(ThreadCount) -> + Result = RunBenchmark(ThreadCount, + TableType, + Scenario, + KeyRange, + Duration, + TimeMsToSleepAfterEachBenchmarkRun), + io:format("; ~f",[Result/(Duration/1000.0)]) + end, + ThreadCounts), + io:format("$~n",[]) + end, + TableTypes) + end, + Scenarios) + end, + KeyRanges), + io:format("~n#BENCHMARK ENDED$~n~n"), + case TestMode of + true -> verify_etsmem(EtsMem); + false -> ok + end. + +test_throughput_benchmark(Config) when is_list(Config) -> + throughput_benchmark(true, 100, 0). + add_lists(L1,L2) -> add_lists(L1,L2,[]). @@ -6140,8 +6708,11 @@ wait_pids(Pids, Acc) -> {Pid,Result} -> true = lists:member(Pid,Pids), Others = lists:delete(Pid,Pids), - io:format("wait_pid got ~p from ~p, still waiting for ~p\n",[Result,Pid,Others]), + %%io:format("wait_pid got ~p from ~p\n",[Result,Pid]), wait_pids(Others,[Result | Acc]) + after 60*1000 -> + io:format("Still waiting for workers ~p\n",[Pids]), + wait_pids(Pids, Acc) end. @@ -6165,48 +6736,25 @@ wait_for_memory_deallocations() -> wait_for_memory_deallocations() end. - etsmem() -> - wait_for_memory_deallocations(), - - AllTabs = lists:map(fun(T) -> {T,ets:info(T,name),ets:info(T,size), - ets:info(T,memory),ets:info(T,type)} - end, ets:all()), - - EtsAllocInfo = erlang:system_info({allocator,ets_alloc}), - ErlangMemoryEts = try erlang:memory(ets) catch error:notsup -> notsup end, - - Mem = - {ErlangMemoryEts, - case EtsAllocInfo of - false -> undefined; - MemInfo -> - CS = lists:foldl( - fun ({instance, _, L}, Acc) -> - {value,{mbcs,MBCS}} = lists:keysearch(mbcs, 1, L), - {value,{sbcs,SBCS}} = lists:keysearch(sbcs, 1, L), - NewAcc = [MBCS, SBCS | Acc], - case lists:keysearch(mbcs_pool, 1, L) of - {value,{mbcs_pool, MBCS_POOL}} -> - [MBCS_POOL|NewAcc]; - _ -> NewAcc - end - end, - [], - MemInfo), - lists:foldl( - fun(L, {Bl0,BlSz0}) -> - {value,BlTup} = lists:keysearch(blocks, 1, L), - blocks = element(1, BlTup), - Bl = element(2, BlTup), - {value,BlSzTup} = lists:keysearch(blocks_size, 1, L), - blocks_size = element(1, BlSzTup), - BlSz = element(2, BlSzTup), - {Bl0+Bl,BlSz0+BlSz} - end, {0,0}, CS) - end}, - {Mem,AllTabs}. + % The following is done twice to avoid an inconsistent memory + % "snapshot" (see verify_etsmem/2). + lists:foldl( + fun(_,_) -> + wait_for_memory_deallocations(), + + AllTabs = lists:map(fun(T) -> {T,ets:info(T,name),ets:info(T,size), + ets:info(T,memory),ets:info(T,type)} + end, ets:all()), + EtsAllocSize = erts_debug:alloc_blocks_size(ets_alloc), + ErlangMemoryEts = try erlang:memory(ets) catch error:notsup -> notsup end, + + Mem = {ErlangMemoryEts, EtsAllocSize}, + {Mem, AllTabs} + end, + not_used, + lists:seq(1,2)). verify_etsmem(MI) -> wait_for_test_procs(), @@ -6227,15 +6775,15 @@ verify_etsmem({MemInfo,AllTabs}, Try) -> end; {MemInfo2, AllTabs2} -> - io:format("Expected: ~p", [MemInfo]), - io:format("Actual: ~p", [MemInfo2]), - io:format("Changed tables before: ~p\n",[AllTabs -- AllTabs2]), - io:format("Changed tables after: ~p\n", [AllTabs2 -- AllTabs]), + io:format("#Expected: ~p", [MemInfo]), + io:format("#Actual: ~p", [MemInfo2]), + io:format("#Changed tables before: ~p\n",[AllTabs -- AllTabs2]), + io:format("#Changed tables after: ~p\n", [AllTabs2 -- AllTabs]), case Try < 2 of true -> - io:format("\nThis discrepancy could be caused by an " + io:format("\n#This discrepancy could be caused by an " "inconsistent memory \"snapshot\"" - "\nTry again...\n", []), + "\n#Try again...\n", []), verify_etsmem({MemInfo, AllTabs}, Try+1); false -> ct:fail("Failed memory check") @@ -6709,22 +7257,49 @@ make_unaligned_sub_binary(List) -> repeat_for_opts(F) -> repeat_for_opts(F, [write_concurrency, read_concurrency, compressed]). +repeat_for_opts_all_table_types(F) -> + repeat_for_opts(F, [all_types, write_concurrency, read_concurrency, compressed]). + +repeat_for_opts_all_non_stim_table_types(F) -> + repeat_for_opts(F, [all_non_stim_types, write_concurrency, read_concurrency, compressed]). + +repeat_for_opts_all_set_table_types(F) -> + repeat_for_opts(F, [set_types, write_concurrency, read_concurrency, compressed]). + +repeat_for_all_set_table_types(F) -> + repeat_for_opts(F, [set_types]). + +repeat_for_all_ord_set_table_types(F) -> + repeat_for_opts(F, [ord_set_types]). + +repeat_for_all_non_stim_set_table_types(F) -> + repeat_for_opts(F, [all_non_stim_set_types]). + +repeat_for_opts_all_ord_set_table_types(F) -> + repeat_for_opts(F, [ord_set_types, write_concurrency, read_concurrency, compressed]). + repeat_for_opts(F, OptGenList) when is_function(F, 1) -> repeat_for_opts(F, OptGenList, []). repeat_for_opts(F, [], Acc) -> lists:foldl(fun(Opts, RV_Acc) -> OptList = lists:filter(fun(E) -> E =/= void end, Opts), - io:format("Calling with options ~p\n",[OptList]), - RV = F(OptList), - case RV_Acc of - {comment,_} -> RV_Acc; - _ -> case RV of - {comment,_} -> RV; - _ -> [RV | RV_Acc] - end - end - end, [], Acc); + case is_redundant_opts_combo(OptList) of + true -> + %%io:format("Ignoring redundant options ~p\n",[OptList]), + ok; + false -> + io:format("Calling with options ~p\n",[OptList]), + RV = F(OptList), + case RV_Acc of + {comment,_} -> RV_Acc; + _ -> case RV of + {comment,_} -> RV; + _ -> [RV | RV_Acc] + end + end + end + end, [], Acc); repeat_for_opts(F, [OptList | Tail], []) when is_list(OptList) -> repeat_for_opts(F, Tail, [[Opt] || Opt <- OptList]); repeat_for_opts(F, [OptList | Tail], AccList) when is_list(OptList) -> @@ -6732,14 +7307,119 @@ repeat_for_opts(F, [OptList | Tail], AccList) when is_list(OptList) -> repeat_for_opts(F, [Atom | Tail], AccList) when is_atom(Atom) -> repeat_for_opts(F, [repeat_for_opts_atom2list(Atom) | Tail ], AccList). -repeat_for_opts_atom2list(all_types) -> [set,ordered_set,bag,duplicate_bag]; +repeat_for_opts_atom2list(set_types) -> [set,ordered_set,stim_cat_ord_set,cat_ord_set]; +repeat_for_opts_atom2list(ord_set_types) -> [ordered_set,stim_cat_ord_set,cat_ord_set]; +repeat_for_opts_atom2list(all_types) -> [set,ordered_set,stim_cat_ord_set,cat_ord_set,bag,duplicate_bag]; +repeat_for_opts_atom2list(all_non_stim_types) -> [set,ordered_set,cat_ord_set,bag,duplicate_bag]; +repeat_for_opts_atom2list(all_non_stim_set_types) -> [set,ordered_set,cat_ord_set]; repeat_for_opts_atom2list(write_concurrency) -> [{write_concurrency,false},{write_concurrency,true}]; repeat_for_opts_atom2list(read_concurrency) -> [{read_concurrency,false},{read_concurrency,true}]; repeat_for_opts_atom2list(compressed) -> [compressed,void]. +is_redundant_opts_combo(Opts) -> + (lists:member(stim_cat_ord_set, Opts) orelse + lists:member(cat_ord_set, Opts)) + andalso + (lists:member({write_concurrency, false}, Opts) orelse + lists:member(private, Opts) orelse + lists:member(protected, Opts)). + ets_new(Name, Opts) -> - %%ets:new(Name, [compressed | Opts]). - ets:new(Name, Opts). + ReplaceStimOrdSetHelper = + fun (MOpts) -> + lists:map(fun (I) -> + case I of + stim_cat_ord_set -> ordered_set; + cat_ord_set -> ordered_set; + _ -> I + end + end, MOpts) + end, + EtsNewHelper = + fun (MOpts) -> + UseOpts = ReplaceStimOrdSetHelper(MOpts), + case get(ets_new_opts) of + UseOpts -> + silence; %% suppress identical table opts spam + _ -> + put(ets_new_opts, UseOpts), + io:format("ets:new(~p, ~p)~n", [Name, UseOpts]) + end, + ets:new(Name, UseOpts) + end, + case (lists:member(stim_cat_ord_set, Opts) or + lists:member(cat_ord_set, Opts)) andalso + (not lists:member({write_concurrency, false}, Opts)) andalso + (not lists:member(private, Opts)) andalso + (not lists:member(protected, Opts)) of + true -> + NewOpts1 = + case lists:member({write_concurrency, true}, Opts) of + true -> Opts; + false -> [{write_concurrency, true}|Opts] + end, + NewOpts2 = + case lists:member(public, NewOpts1) of + true -> NewOpts1; + false -> [public|NewOpts1] + end, + T = EtsNewHelper(NewOpts2), + case lists:member(stim_cat_ord_set, Opts) of + true -> stimulate_contention(T); + false -> ok + end, + T; + false -> + EtsNewHelper(Opts) + end. + +% This function do the following to the input ETS table: +% 1. Perform a number of concurrent insert operations +% 2. Remove all inserted items +% +% The purpose of this function is to stimulate fine grained locking in +% tables of types ordered_set with the write_concurrency options +% turned on. Such tables are implemented as CA trees* and thus +% activates fine grained locking only when lock contention is +% detected. +% +% A Contention Adapting Approach to Concurrent Ordered Sets +% Journal of Parallel and Distributed Computing, 2018 +% Kjell Winblad and Konstantinos Sagonas +% https://doi.org/10.1016/j.jpdc.2017.11.007 +stimulate_contention(T) -> + NrOfSchedulers = erlang:system_info(schedulers), + ParentPid = self(), + KeyRange = 100000, + ChildPids = + lists:map(fun(_N) -> + spawn(fun() -> + receive start -> ok end, + stimulate_contention_do_inserts(T, KeyRange, 0), + ParentPid ! done + end) + end, lists:seq(1, NrOfSchedulers)), + lists:foreach(fun(Pid) -> Pid ! start end, ChildPids), + timer:sleep(100), + lists:foreach(fun(Pid) -> Pid ! stop end, ChildPids), + lists:foreach(fun(_P) -> receive done -> ok end end, ChildPids), + lists:foreach(fun(N) -> ets:delete(T, N) end, lists:seq(0, KeyRange)). + + + +stimulate_contention_do_inserts(T, KeyRange, 0) -> + OpsBetweenStopCheck = 10000, + receive + stop -> ok + after + 0 -> stimulate_contention_do_inserts(T, KeyRange, OpsBetweenStopCheck) + end; +stimulate_contention_do_inserts(T, KeyRange, OpsToNextStopCheck) -> + R = trunc(KeyRange * rand:uniform()), + ets:insert(T,{R,R,R}), + stimulate_contention_do_inserts(T, KeyRange, OpsToNextStopCheck - 1). + + do_tc(Do, Report) -> T1 = erlang:monotonic_time(), @@ -6753,3 +7433,49 @@ syrup_factor() -> valgrind -> 20; _ -> 1 end. + + +%% +%% This is a pseudo random number generator for UNIQUE integers. +%% All integers between 1 and Max will be generated before it repeat itself. +%% It's a variant of this one using quadratic residues by Jeff Preshing: +%% http://preshing.com/20121224/how-to-generate-a-sequence-of-unique-random-integers/ +%% +unique_rand_start(Max, Seed) -> + L = lists:dropwhile(fun(P) -> P < Max end, + primes_3mod4()), + [P | _] = case L of + [] -> + error("Random range too large"); + _ -> + L + end, + 3 = P rem 4, + {0, {Max, P, Seed}}. + +unique_rand_next({N, {Max, P, Seed}=Const}) -> + case dquad(P, N, Seed) + 1 of + RND when RND > Max -> % Too large, skip + unique_rand_next({N+1, Const}); + RND -> + {RND, {N+1, Const}} + end. + +%% A one-to-one relation between all integers 0 =< X < Prime +%% if Prime rem 4 == 3. +quad(Prime, X) -> + Rem = X*X rem Prime, + case 2*X < Prime of + true -> + Rem; + false -> + Prime - Rem + end. + +dquad(Prime, X, Seed) -> + quad(Prime, (quad(Prime, X) + Seed) rem Prime). + +%% Primes where P rem 4 == 3. +primes_3mod4() -> + [103, 211, 503, 1019, 2003, 5003, 10007, 20011, 50023, + 100003, 200003, 500083, 1000003, 2000003]. |