%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 1996-2016. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%
-module(dets_SUITE).
%-define(debug, true).
-ifdef(debug).
-define(format(S, A), io:format(S, A)).
-define(config(X,Y), foo).
-define(t, test_server).
-define(privdir(_), "./dets_SUITE_priv").
-define(datadir(_), "./dets_SUITE_data").
-else.
-include_lib("common_test/include/ct.hrl").
-define(format(S, A), ok).
-define(privdir(Conf), ?config(priv_dir, Conf)).
-define(datadir(Conf), ?config(data_dir, Conf)).
-endif.
-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
init_per_group/2,end_per_group/2,
newly_started/1, basic_v8/1, basic_v9/1,
open_v8/1, open_v9/1, sets_v8/1, sets_v9/1, bags_v8/1,
bags_v9/1, duplicate_bags_v8/1, duplicate_bags_v9/1,
access_v8/1, access_v9/1, dirty_mark/1, dirty_mark2/1,
bag_next_v8/1, bag_next_v9/1, oldbugs_v8/1, oldbugs_v9/1,
unsafe_assumptions/1, truncated_segment_array_v8/1,
truncated_segment_array_v9/1, open_file_v8/1, open_file_v9/1,
init_table_v8/1, init_table_v9/1, repair_v8/1, repair_v9/1,
hash_v8b_v8c/1, phash/1, fold_v8/1, fold_v9/1, fixtable_v8/1,
fixtable_v9/1, match_v8/1, match_v9/1, select_v8/1,
select_v9/1, update_counter/1, badarg/1, cache_sets_v8/1,
cache_sets_v9/1, cache_bags_v8/1, cache_bags_v9/1,
cache_duplicate_bags_v8/1, cache_duplicate_bags_v9/1,
otp_4208/1, otp_4989/1, many_clients/1, otp_4906/1, otp_5402/1,
simultaneous_open/1, insert_new/1, repair_continuation/1,
otp_5487/1, otp_6206/1, otp_6359/1, otp_4738/1, otp_7146/1,
otp_8070/1, otp_8856/1, otp_8898/1, otp_8899/1, otp_8903/1,
otp_8923/1, otp_9282/1, otp_11245/1, otp_11709/1, otp_13229/1,
otp_13260/1]).
-export([dets_dirty_loop/0]).
-export([histogram/1, sum_histogram/1, ave_histogram/1]).
-export([init_per_testcase/2, end_per_testcase/2]).
%% Internal export.
-export([client/2]).
-import(lists,
[append/1, delete/2, duplicate/2, filter/2, foreach/2, keysearch/3,
last/1, map/2, member/2, reverse/1, seq/2, sort/1, usort/1]).
-include_lib("kernel/include/file.hrl").
-define(DETS_SERVER, dets).
%% HEADSZ taken from dets_v8.erl and dets_v9.erl.
-define(HEADSZ_v8, 40).
-define(HEADSZ_v9, (56+28*4+16)).
-define(NO_KEYS_POS_v9, 36).
-define(CLOSED_PROPERLY_POS, 8).
-define(NOT_PROPERLY_CLOSED,0).
-define(CLOSED_PROPERLY,1).
init_per_testcase(_Case, Config) ->
Config.
end_per_testcase(_Case, _Config) ->
ok.
suite() ->
[{ct_hooks,[ts_install_cth]},
{timetrap,{minutes,15}}].
all() ->
[
basic_v8, basic_v9, open_v8, open_v9, sets_v8, sets_v9,
bags_v8, bags_v9, duplicate_bags_v8, duplicate_bags_v9,
newly_started, open_file_v8, open_file_v9,
init_table_v8, init_table_v9, repair_v8, repair_v9,
access_v8, access_v9, oldbugs_v8, oldbugs_v9,
unsafe_assumptions, truncated_segment_array_v8,
truncated_segment_array_v9, dirty_mark, dirty_mark2,
bag_next_v8, bag_next_v9, hash_v8b_v8c, phash, fold_v8,
fold_v9, fixtable_v8, fixtable_v9, match_v8, match_v9,
select_v8, select_v9, update_counter, badarg,
cache_sets_v8, cache_sets_v9, cache_bags_v8,
cache_bags_v9, cache_duplicate_bags_v8,
cache_duplicate_bags_v9, otp_4208, otp_4989,
many_clients, otp_4906, otp_5402, simultaneous_open,
insert_new, repair_continuation, otp_5487, otp_6206,
otp_6359, otp_4738, otp_7146, otp_8070, otp_8856, otp_8898,
otp_8899, otp_8903, otp_8923, otp_9282, otp_11245, otp_11709,
otp_13229, otp_13260
].
groups() ->
[].
init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
ok.
init_per_group(_GroupName, Config) ->
Config.
end_per_group(_GroupName, Config) ->
Config.
%% OTP-3621
newly_started(Config) when is_list(Config) ->
true = is_alive(),
{ok, Node} = test_server:start_node(slave1, slave, []),
[] = rpc:call(Node, dets, all, []),
test_server:stop_node(Node),
ok.
%% Basic test case.
basic_v8(Config) when is_list(Config) ->
basic(Config, 8).
%% Basic test case.
basic_v9(Config) when is_list(Config) ->
basic(Config, 9).
basic(Config, Version) ->
Tab = dets_basic_test,
FName = filename(Tab, Config),
P0 = pps(),
{ok, _} = dets:open_file(Tab,[{file, FName},{version,Version}]),
ok = dets:insert(Tab,{mazda,japan}),
ok = dets:insert(Tab,{toyota,japan}),
ok = dets:insert(Tab,{suzuki,japan}),
ok = dets:insert(Tab,{honda,japan}),
ok = dets:insert(Tab,{renault,france}),
ok = dets:insert(Tab,{citroen,france}),
ok = dets:insert(Tab,{opel,germany}),
ok = dets:insert(Tab,{saab,sweden}),
ok = dets:insert(Tab,{volvo,sweden}),
[{opel,germany}] = dets:lookup(Tab,opel),
Japs = dets:traverse(Tab, fun(Obj) ->
case Obj of
{_, japan} -> {continue, Obj};
_ -> continue
end
end),
4 = length(Japs),
ok = dets:close(Tab),
file:delete(FName),
check_pps(P0),
ok.
open_v8(Config) when is_list(Config) ->
open(Config, 8).
open_v9(Config) when is_list(Config) ->
open(Config, 9).
open(Config, Version) ->
%% Running this test twice means that the Dets server is restarted
%% twice. dets_sup specifies a maximum of 4 restarts in an hour.
%% If this becomes a problem, one should consider running this
%% test on a slave node.
{Sets, Bags, Dups} = args(Config),
All = Sets ++ Bags ++ Dups,
delete_files(All),
Data = make_data(1),
P0 = pps(),
Tabs = open_files(1, All, Version),
initialize(Tabs, Data),
check(Tabs, Data),
foreach(fun(Tab) -> ok = dets:close(Tab) end, Tabs),
%% Now reopen the files
?format("Reopening closed files \n", []),
Tabs = open_files(1, All, Version),
?format("Checking contents of reopened files \n", []),
check(Tabs, Data),
%% crash the dets server
?format("Crashing dets server \n", []),
process_flag(trap_exit, true),
Procs = [whereis(?DETS_SERVER) | [dets:info(Tab, pid) || Tab <- Tabs]],
foreach(fun(Pid) -> exit(Pid, kill) end, Procs),
timer:sleep(100),
c:flush(), %% flush all the EXIT sigs
timer:sleep(200),
%% Now reopen the files again
?format("Reopening crashed files \n", []),
open_files(1, All, Version),
?format("Checking contents of repaired files \n", []),
check(Tabs, Data),
close_all(Tabs),
delete_files(All),
{Ports0, Procs0} = P0,
Test = fun() ->
P1 = pps(),
{Ports1, Procs1} = P1,
show("Old port", Ports0 -- Ports1),
show("New port", Ports1 -- Ports0),
show("Old procs", Procs0 -- Procs1),
show("New procs", Procs1 -- Procs0),
io:format("Remaining Dets-pids (should be nil): ~p~n",
[find_dets_pids()]),
true = Ports1 =:= Ports0,
%% The dets_server process has been restarted:
[_] = Procs0 -- Procs1,
[_] = Procs1 -- Procs0,
ok
end,
case catch Test() of
ok -> ok;
_ ->
timer:sleep(500),
ok = Test()
end.
check(Tabs, Data) ->
foreach(fun(Tab) ->
Kp = dets:info(Tab, keypos),
?format("checking ~p~n", [Tab]),
foreach(fun(Item) ->
case dets:lookup(Tab, k(Kp,Item)) of
[Item] -> ok;
_Other -> bad(Tab,Item)
end
end, Data)
end, Tabs),
ok.
k(Kp, Obj) -> element(Kp, Obj).
bad(_Tab, _Item) ->
?format("Can't find item ~p in ~p ~n", [_Item, _Tab]),
exit(badtab).
%% Perform traversal and match testing on set type dets tables.
sets_v8(Config) when is_list(Config) ->
sets(Config, 8).
%% Perform traversal and match testing on set type dets tables.
sets_v9(Config) when is_list(Config) ->
sets(Config, 9).
sets(Config, Version) ->
{Sets, _, _} = args(Config),
Data = make_data(1),
delete_files(Sets),
P0 = pps(),
Tabs = open_files(1, Sets, Version),
Bigger = [{17,q,w,w}, {48,q,w,w,w,w,w,w}], % 48 requires a bigger buddy
initialize(Tabs, Data++Bigger++Data), % overwrite
Len = length(Data),
foreach(fun(Tab) -> trav_test(Data, Len, Tab) end, Tabs),
size_test(Len, Tabs),
no_keys_test(Tabs),
foreach(fun(Tab) -> del_test(Tab) end, Tabs),
initialize(Tabs, Data),
foreach(fun(Tab) -> del_obj_test(Tab) end, Tabs),
initialize(Tabs, Data),
foreach(fun(Tab) ->
Len = dets:info(Tab, size) end,
Tabs),
foreach(fun(Tab) -> match_test(Data, Tab) end, Tabs),
foreach(fun(Tab) -> match_del_test(Tab) end, Tabs),
close_all(Tabs),
delete_files(Sets),
check_pps(P0),
ok.
%% Perform traversal and match testing on bag type dets tables.
bags_v8(Config) when is_list(Config) ->
bags(Config, 8).
%% Perform traversal and match testing on bag type dets tables.
bags_v9(Config) when is_list(Config) ->
bags(Config, 9).
bags(Config, Version) ->
{_, Bags, _} = args(Config),
Data = make_data(1, bag), %% gives twice as many objects
delete_files(Bags),
P0 = pps(),
Tabs = open_files(1, Bags, Version),
initialize(Tabs, Data++Data),
Len = length(Data),
foreach(fun(Tab) -> trav_test(Data, Len, Tab) end, Tabs),
size_test(Len, Tabs),
no_keys_test(Tabs),
foreach(fun(Tab) -> del_test(Tab) end, Tabs),
initialize(Tabs, Data),
foreach(fun(Tab) -> del_obj_test(Tab) end, Tabs),
initialize(Tabs, Data),
foreach(fun(Tab) ->
Len = dets:info(Tab, size) end,
Tabs),
foreach(fun(Tab) -> match_test(Data, Tab) end, Tabs),
foreach(fun(Tab) -> match_del_test(Tab) end, Tabs),
close_all(Tabs),
delete_files(Bags),
check_pps(P0),
ok.
%% Perform traversal and match testing on duplicate_bag type dets tables.
duplicate_bags_v8(Config) when is_list(Config) ->
duplicate_bags(Config, 8).
%% Perform traversal and match testing on duplicate_bag type dets tables.
duplicate_bags_v9(Config) when is_list(Config) ->
duplicate_bags(Config, 9).
duplicate_bags(Config, Version) when is_list(Config) ->
{_, _, Dups} = args(Config),
Data = make_data(1, duplicate_bag), %% gives twice as many objects
delete_files(Dups),
P0 = pps(),
Tabs = open_files(1, Dups, Version),
initialize(Tabs, Data),
Len = length(Data),
foreach(fun(Tab) -> trav_test(Data, Len, Tab) end, Tabs),
size_test(Len, Tabs),
no_keys_test(Tabs),
foreach(fun(Tab) -> del_test(Tab) end, Tabs),
initialize(Tabs, Data),
foreach(fun(Tab) -> del_obj_test(Tab) end, Tabs),
initialize(Tabs, Data),
foreach(fun(Tab) ->
Len = dets:info(Tab, size) end,
Tabs),
foreach(fun(Tab) -> match_test(Data, Tab) end, Tabs),
foreach(fun(Tab) -> match_del_test(Tab) end, Tabs),
close_all(Tabs),
delete_files(Dups),
check_pps(P0),
ok.
access_v8(Config) when is_list(Config) ->
access(Config, 8).
access_v9(Config) when is_list(Config) ->
access(Config, 9).
access(Config, Version) ->
Args_acc = [[{ram_file, true}, {access, read}],
[{access, read}]],
Args = [[{ram_file, true}],
[]],
{Args_acc_1, _, _} = zip_filename(Args_acc, [], [], Config),
delete_files(Args_acc_1),
{Args_1, _, _} = zip_filename(Args, [], [], Config),
P0 = pps(),
{error, {file_error,_,enoent}} = dets:open_file('1', hd(Args_acc_1)),
Tabs = open_files(1, Args_1, Version),
close_all(Tabs),
Tabs = open_files(1, Args_acc_1, Version),
foreach(fun(Tab) ->
{error, {access_mode,_}} = dets:insert(Tab, {1,2}),
[] = dets:lookup(Tab, 11),
'$end_of_table' = dets:first(Tab),
{error, {access_mode,_}} = dets:delete(Tab, 22)
end, Tabs),
close_all(Tabs),
delete_files(Args_acc_1),
check_pps(P0),
ok.
%% Test that the table is not marked dirty if not written.
dirty_mark(Config) when is_list(Config) ->
true = is_alive(),
Tab = dets_dirty_mark_test,
FName = filename(Tab, Config),
P0 = pps(),
dets:open_file(Tab,[{file, FName}]),
dets:insert(Tab,{mazda,japan}),
dets:insert(Tab,{toyota,japan}),
dets:insert(Tab,{suzuki,japan}),
dets:insert(Tab,{honda,japan}),
dets:insert(Tab,{renault,france}),
dets:insert(Tab,{citroen,france}),
dets:insert(Tab,{opel,germany}),
dets:insert(Tab,{saab,sweden}),
dets:insert(Tab,{volvo,sweden}),
[{opel,germany}] = dets:lookup(Tab,opel),
ok = dets:close(Tab),
Call = fun(P,A) ->
P ! {self(), A},
receive
{P, Ans} ->
Ans
after 5000 ->
exit(other_process_dead)
end
end,
{ok, Node} = test_server:start_node(dets_dirty_mark,
slave,
[{linked, false},
{args, "-pa " ++
filename:dirname
(code:which(?MODULE))}]),
ok = ensure_node(20, Node),
%% io:format("~p~n",[rpc:call(Node, code, get_path, [])]),
%% io:format("~p~n",[rpc:call(Node, file, get_cwd, [])]),
%% io:format("~p~n",[Config]),
Pid = rpc:call(Node,erlang, spawn,
[?MODULE, dets_dirty_loop, []]),
{ok, Tab} = Call(Pid, [open, Tab, [{file, FName}]]),
[{opel,germany}] = Call(Pid, [read,Tab,opel]),
test_server:stop_node(Node),
{ok, Tab} = dets:open_file(Tab,[{file, FName},
{repair,false}]),
ok = dets:close(Tab),
file:delete(FName),
check_pps(P0),
ok.
%% Test that the table is flushed when auto_save is in effect.
dirty_mark2(Config) when is_list(Config) ->
true = is_alive(),
Tab = dets_dirty_mark2_test,
FName = filename(Tab, Config),
P0 = pps(),
dets:open_file(Tab,[{file, FName}]),
dets:insert(Tab,{toyota,japan}),
dets:insert(Tab,{suzuki,japan}),
dets:insert(Tab,{honda,japan}),
dets:insert(Tab,{renault,france}),
dets:insert(Tab,{citroen,france}),
dets:insert(Tab,{opel,germany}),
dets:insert(Tab,{saab,sweden}),
dets:insert(Tab,{volvo,sweden}),
[{opel,germany}] = dets:lookup(Tab,opel),
ok = dets:close(Tab),
Call = fun(P,A) ->
P ! {self(), A},
receive
{P, Ans} ->
Ans
after 5000 ->
exit(other_process_dead)
end
end,
{ok, Node} = test_server:start_node(dets_dirty_mark2,
slave,
[{linked, false},
{args, "-pa " ++
filename:dirname
(code:which(?MODULE))}]),
ok = ensure_node(20, Node),
Pid = rpc:call(Node,erlang, spawn,
[?MODULE, dets_dirty_loop, []]),
{ok, Tab} = Call(Pid, [open, Tab, [{file, FName},{auto_save,1000}]]),
ok = Call(Pid, [write,Tab,{mazda,japan}]),
timer:sleep(2100),
%% Read something, just to give auto save time to finish.
[{opel,germany}] = Call(Pid, [read,Tab,opel]),
test_server:stop_node(Node),
{ok, Tab} = dets:open_file(Tab, [{file, FName}, {repair,false}]),
ok = dets:close(Tab),
file:delete(FName),
check_pps(P0),
ok.
dets_dirty_loop() ->
receive
{From, [open, Name, Args]} ->
Ret = dets:open_file(Name, Args),
From ! {self(), Ret},
dets_dirty_loop();
{From, [read, Name, Key]} ->
Ret = dets:lookup(Name, Key),
From ! {self(), Ret},
dets_dirty_loop();
{From, [write, Name, Value]} ->
Ret = dets:insert(Name, Value),
From ! {self(), Ret},
dets_dirty_loop();
{From, [close, Name]} ->
Ret = dets:close(Name),
From ! {self(), Ret},
dets_dirty_loop()
end.
%% Check that bags and next work as expected.
bag_next_v8(Config) when is_list(Config) ->
bag_next(Config, 8).
%% Check that bags and next work as expected.
bag_next_v9(Config) when is_list(Config) ->
Tab = dets_bag_next_test,
FName = filename(Tab, Config),
%% first and next crash upon error
dets:open_file(Tab,[{file, FName}, {type, bag},{version,9}]),
ok = dets:insert(Tab, [{1,1},{2,2},{3,3},{4,4}]),
FirstKey = dets:first(Tab),
NextKey = dets:next(Tab, FirstKey),
[FirstObj | _] = dets:lookup(Tab, FirstKey),
[NextObj | _] = dets:lookup(Tab, NextKey),
{ok, FirstPos} = dets:where(Tab, FirstObj),
{ok, NextPos} = dets:where(Tab, NextObj),
crash(FName, NextPos+12),
{'EXIT',BadObject1} = (catch dets:next(Tab, FirstKey)),
bad_object(BadObject1, FName),
crash(FName, FirstPos+12),
{'EXIT',BadObject2} = (catch dets:first(Tab)),
bad_object(BadObject2, FName),
dets:close(Tab),
file:delete(FName),
bag_next(Config, 9).
bag_next(Config, Version) ->
Tab = dets_bag_next_test,
FName = filename(Tab, Config),
P0 = pps(),
dets:open_file(Tab,[{file, FName}, {type, bag},{version,Version}]),
dets:insert(Tab,{698,hopp}),
dets:insert(Tab,{186,hopp}),
dets:insert(Tab,{hej,hopp}),
dets:insert(Tab,{186,plopp}),
Loop = fun(N, Last, Self) ->
case N of
0 ->
exit({unterminated_first_next_sequence, N, Last});
_ ->
case Last of
'$end_of_table' ->
ok;
_ ->
Self(N-1, dets:next(Tab,Last), Self)
end
end
end,
ok = Loop(4,dets:first(Tab),Loop),
dets:close(Tab),
file:delete(FName),
check_pps(P0),
ok.
oldbugs_v8(Config) when is_list(Config) ->
oldbugs(Config, 8).
oldbugs_v9(Config) when is_list(Config) ->
oldbugs(Config, 9).
oldbugs(Config, Version) ->
FName = filename(dets_suite_oldbugs_test, Config),
P0 = pps(),
{ok, ob} = dets:open_file(ob, [{version, Version},
{type, bag}, {file, FName}]),
ok = dets:insert(ob, {1, 2}),
ok = dets:insert(ob, {1,3}),
ok = dets:insert(ob, {1, 2}),
2 = dets:info(ob, size), %% assertion
ok = dets:close(ob),
file:delete(FName),
check_pps(P0),
ok.
%% Test that shrinking an object and then expanding it works.
unsafe_assumptions(Config) when is_list(Config) ->
FName = filename(dets_suite_unsafe_assumptions_test, Config),
file:delete(FName),
P0 = pps(),
{ok, a} = dets:open_file(a, [{version,8},{file, FName}]),
O0 = {2,false},
O1 = {1, false},
O2 = {1, true},
O3 = {1, duplicate(20,false)},
O4 = {1, duplicate(25,false)}, % same 2-log as O3
ok = dets:insert(a, O1),
ok = dets:insert(a, O0),
true = [O1,O0] =:= sort(get_all_objects(a)),
true = [O1,O0] =:= sort(get_all_objects_fast(a)),
ok = dets:insert(a, O2),
true = [O2,O0] =:= sort(get_all_objects(a)),
true = [O2,O0] =:= sort(get_all_objects_fast(a)),
ok = dets:insert(a, O3),
true = [O3,O0] =:= sort(get_all_objects(a)),
true = [O3,O0] =:= sort(get_all_objects_fast(a)),
ok = dets:insert(a, O4),
true = [O4,O0] =:= sort(get_all_objects(a)),
true = [O4,O0] =:= sort(get_all_objects_fast(a)),
ok = dets:close(a),
file:delete(FName),
check_pps(P0),
ok.
%% Test that a file where the segment array has been truncated
%% is possible to repair.
truncated_segment_array_v8(Config) when is_list(Config) ->
trunc_seg_array(Config, 8).
%% Test that a file where the segment array has been truncated
%% is possible to repair.
truncated_segment_array_v9(Config) when is_list(Config) ->
trunc_seg_array(Config, 9).
trunc_seg_array(Config, V) ->
TabRef = dets_suite_truncated_segment_array_test,
Fname = filename(TabRef, Config),
%% Create file that needs to be repaired
file:delete(Fname),
P0 = pps(),
{ok, TabRef} = dets:open_file(TabRef, [{file, Fname},{version,V}]),
ok = dets:close(TabRef),
%% Truncate the file
HeadSize = headsz(V),
truncate(Fname, HeadSize + 10),
%% Open the truncated file
io:format("Expect repair:~n"),
{ok, TabRef} = dets:open_file(TabRef,
[{file, Fname}, {repair, true}]),
ok = dets:close(TabRef),
file:delete(Fname),
check_pps(P0),
ok.
%% Test open_file/1.
open_file_v8(Config) when is_list(Config) ->
open_1(Config, 8).
%% Test open_file/1.
open_file_v9(Config) when is_list(Config) ->
T = open_v9,
Fname = filename(T, Config),
{ok, _} = dets:open_file(T, [{file,Fname},{version,9}]),
9 = dets:info(T, version),
true = [self()] =:= dets:info(T, users),
{ok, _} = dets:open_file(T, [{file,Fname},{version,9}]),
{error,incompatible_arguments} =
dets:open_file(T, [{file,Fname},{version,8}]),
true = [self(),self()] =:= dets:info(T, users),
ok = dets:close(T),
true = [self()] =:= dets:info(T, users),
ok = dets:close(T),
undefined = ets:info(T, users),
file:delete(Fname),
open_1(Config, 9).
open_1(Config, V) ->
TabRef = open_file_1_test,
Fname = filename(TabRef, Config),
file:delete(Fname),
P0 = pps(),
{error,{file_error,Fname,enoent}} = dets:open_file(Fname),
ok = file:write_file(Fname, duplicate(100,65)),
{error,{not_a_dets_file,Fname}} = dets:open_file(Fname),
file:delete(Fname),
HeadSize = headsz(V),
{ok, TabRef} = dets:open_file(TabRef, [{file, Fname},{version,V}]),
ok = dets:close(TabRef),
truncate(Fname, HeadSize + 10),
true = dets:is_dets_file(Fname),
io:format("Expect repair:~n"),
{ok, Ref} = dets:open_file(Fname), % repairing
ok = dets:close(Ref),
file:delete(Fname),
%% truncated file header, invalid type
{ok, TabRef} = dets:open_file(TabRef, [{file,Fname},{version,V}]),
ok = ins(TabRef, 3000),
ok = dets:close(TabRef),
TypePos = 12,
crash(Fname, TypePos),
{error, {invalid_type_code,Fname}} = dets:open_file(Fname),
truncate(Fname, HeadSize - 10),
{error,{not_a_dets_file,Fname}} = dets:open_file(Fname),
{error,{not_a_dets_file,Fname}} =
dets:open_file(TabRef, [{file,Fname},{version,V}]),
file:delete(Fname),
{error,{file_error,{foo,bar},_}} = dets:is_dets_file({foo,bar}),
check_pps(P0),
ok.
%% Test initialize_table/2 and from_ets/2.
init_table_v8(Config) when is_list(Config) ->
init_table(Config, 8).
%% Test initialize_table/2 and from_ets/2.
init_table_v9(Config) when is_list(Config) ->
%% Objects are returned in "time order".
T = init_table_v9,
Fname = filename(T, Config),
file:delete(Fname),
L = [{1,a},{2,b},{1,c},{2,c},{1,c},{2,a},{1,b}],
Input = init([L]),
{ok, _} = dets:open_file(T, [{file,Fname},{version,9},
{type,duplicate_bag}]),
ok = dets:init_table(T, Input),
[{1,a},{1,c},{1,c},{1,b}] = dets:lookup(T, 1),
[{2,b},{2,c},{2,a}] = dets:lookup(T, 2),
ok = dets:close(T),
file:delete(Fname),
init_table(Config, 9),
fast_init_table(Config).
init_table(Config, V) ->
TabRef = init_table_test,
Fname = filename(TabRef, Config),
file:delete(Fname),
P0 = pps(),
Args = [{file,Fname},{version,V},{auto_save,120000}],
{ok, _} = dets:open_file(TabRef, Args),
{'EXIT', _} =
(catch dets:init_table(TabRef, fun(foo) -> bar end)),
dets:close(TabRef),
{ok, _} = dets:open_file(TabRef, Args),
{'EXIT', _} = (catch dets:init_table(TabRef, fun() -> foo end)),
dets:close(TabRef),
{ok, _} = dets:open_file(TabRef, Args),
{'EXIT', {badarg, _}} = (catch dets:init_table(TabRef, nofun)),
{'EXIT', {badarg, _}} =
(catch dets:init_table(TabRef, fun(_X) -> end_of_input end,
[{foo,bar}])),
dets:close(TabRef),
{ok, _} = dets:open_file(TabRef, Args),
away = (catch dets:init_table(TabRef, fun(_) -> throw(away) end)),
dets:close(TabRef),
{ok, _} = dets:open_file(TabRef, Args),
{error, {init_fun, fopp}} =
dets:init_table(TabRef, fun(read) -> fopp end),
dets:close(TabRef),
{ok, _} = dets:open_file(TabRef, Args),
dets:safe_fixtable(TabRef, true),
{error, {fixed_table, TabRef}} = dets:init_table(TabRef, init([])),
dets:safe_fixtable(TabRef, false),
ET = ets:new(foo,[]),
ok = dets:from_ets(TabRef, ET),
[] = get_all_objects(TabRef),
[] = get_all_objects_fast(TabRef),
true = ets:insert(ET, {1,a}),
true = ets:insert(ET, {2,b}),
ok = dets:from_ets(TabRef, ET),
[{1,a},{2,b}] = sort(get_all_objects(TabRef)),
[{1,a},{2,b}] = sort(get_all_objects_fast(TabRef)),
true = ets:delete(ET),
120000 = dets:info(TabRef, auto_save),
ok = dets:close(TabRef),
{ok, _} = dets:open_file(TabRef, [{access,read} | Args]),
{error, {access_mode, Fname}} = dets:init_table(TabRef, init([])),
ok = dets:close(TabRef),
{ok, _} = dets:open_file(TabRef, Args),
{error, invalid_objects_list} =
(catch dets:init_table(TabRef, init([[{1,2},bad,{3,4}]]))),
_ = dets:close(TabRef),
file:delete(Fname),
L1 = [[{1,a},{2,b}],[],[{3,c}],[{4,d}],[]],
bulk_init(L1, set, 4, Config, V),
L2 = [[{1,a},{2,b}],[],[{2,q},{3,c}],[{4,d}],[{4,e},{2,q}]],
bulk_init(L2, set, 4, Config, V),
bulk_init(L2, bag, 6, Config, V),
bulk_init(L2, duplicate_bag, 7, Config, V),
bulk_init(L1, set, 4, 512, Config, V),
bulk_init([], set, 0, 10000, Config, V),
file:delete(Fname),
%% Initiate a file that contains a lot of objects.
{ok, _} = dets:open_file(TabRef, [{min_no_slots,10000} | Args]),
ok = ins(TabRef, 6000),
Fun = init_fun(0, 10000),
ok = dets:init_table(TabRef, Fun,{format,term}),
All = sort(get_all_objects(TabRef)),
FAll = get_all_objects_fast(TabRef),
true = All =:= sort(FAll),
true = length(All) =:= 10000,
ok = dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, [{min_no_slots,4000} | Args]),
ok = ins(TabRef, 6000),
FileSize1 = dets:info(TabRef, file_size),
Fun2 = init_fun(0, 4000),
ok = dets:init_table(TabRef, Fun2),
FileSize2 = dets:info(TabRef, file_size),
ok = dets:close(TabRef),
true = FileSize1 > FileSize2,
file:delete(Fname),
check_pps(P0),
ok.
bulk_init(Ls, Type, N, Config, V) ->
bulk_init(Ls, Type, N, 256, Config, V).
bulk_init(Ls, Type, N, Est, Config, V) ->
T = init_table_test,
Fname = filename(T, Config),
file:delete(Fname),
Input = init(Ls),
Args = [{ram_file,false}, {type,Type},{keypos,1},{file,Fname},
{estimated_no_objects, Est},{version,V}],
{ok, T} = dets:open_file(T, Args),
ok = dets:init_table(T, Input),
All = sort(get_all_objects(T)),
FAll = get_all_objects_fast(T),
true = All =:= sort(FAll),
true = length(All) =:= N,
true = dets:info(T, size) =:= N,
ok = dets:close(T),
{ok, T} = dets:open_file(T, Args),
All2 = sort(get_all_objects(T)),
FAll2 = get_all_objects_fast(T),
true = All =:= All2,
true = All =:= sort(FAll2),
ok = dets:close(T),
file:delete(Fname).
init(L) ->
fun(close) ->
ok;
(read) when [] =:= L ->
end_of_input;
(read) ->
[E | Es] = L,
{E, init(Es)}
end.
init_fun(I, N) ->
fun(read) when I =:= N ->
end_of_input;
(read) ->
{NewN, Items} = items(I, N, 1000, []),
{Items, init_fun(NewN, N)};
(close) ->
ignored
end.
fast_init_table(Config) ->
V = 9,
TabRef = init_table_test,
Fname = filename(TabRef, Config),
file:delete(Fname),
P0 = pps(),
Args = [{file,Fname},{version,V},{auto_save,120000}],
Source = init_table_test_source,
SourceFname = filename(Source, Config),
file:delete(SourceFname),
SourceArgs = [{file,SourceFname},{version,V},{auto_save,120000}],
{ok, Source} = dets:open_file(Source, SourceArgs),
{ok, _} = dets:open_file(TabRef, Args),
{'EXIT', _} =
(catch dets:init_table(TabRef, fun(foo) -> bar end, {format,bchunk})),
dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
{'EXIT', _} = (catch dets:init_table(TabRef, fun() -> foo end,
{format,bchunk})),
dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
{'EXIT', {badarg, _}} =
(catch dets:init_table(TabRef, nofun, {format,bchunk})),
dets:close(TabRef),
{ok, _} = dets:open_file(TabRef, Args),
away = (catch dets:init_table(TabRef, fun(_) -> throw(away) end,
{format,bchunk})),
dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
{error, {init_fun, fopp}} =
dets:init_table(TabRef, fun(read) -> fopp end, {format,bchunk}),
dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
dets:safe_fixtable(TabRef, true),
{error, {fixed_table, TabRef}} =
dets:init_table(TabRef, init([]), {format,bchunk}),
dets:safe_fixtable(TabRef, false),
ok = dets:close(TabRef),
{ok, _} = dets:open_file(TabRef, [{access,read} | Args]),
{error, {access_mode, Fname}} =
dets:init_table(TabRef, init([]), {format,bchunk}),
ok = dets:close(TabRef),
{ok, _} = dets:open_file(TabRef, Args),
{error, {init_fun,{1,2}}} =
dets:init_table(TabRef, init([[{1,2},bad,{3,4}]]), {format,bchunk}),
_ = dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
{error, {init_fun, end_of_input}} =
dets:init_table(TabRef, init([]),{format,bchunk}),
_ = dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
{'EXIT', {badarg, _}} =
(catch dets:init_table(TabRef, init([]),{format,foppla})),
_ = dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
ok = ins(TabRef, 100),
[BParms | Objs] = collect_bchunk(TabRef, init_bchunk(TabRef)),
Parms = binary_to_term(BParms),
{error, {init_fun, <<"foobar">>}} =
dets:init_table(TabRef, init([[<<"foobar">>]]),{format,bchunk}),
_ = dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
Parms1 = setelement(1, Parms, foobar),
BParms1 = term_to_binary(Parms1),
{error, {init_fun, BParms1}} =
dets:init_table(TabRef, init([[BParms1 | Objs]]),{format,bchunk}),
_ = dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
[{Sz1,No1} | NoColls17] = element(tuple_size(Parms), Parms),
Parms2 = setelement(tuple_size(Parms), Parms, [{Sz1,No1+1} | NoColls17]),
BParms2 = term_to_binary(Parms2),
{error, invalid_objects_list} =
dets:init_table(TabRef, init([[BParms2 | Objs]]),{format,bchunk}),
_ = dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
[{LSize1,Slot1,Obj1} | ObjsRest] = Objs,
BadSize = byte_size(Obj1)-1,
<<BadSizeObj:BadSize/binary,_:1/binary>> = Obj1,
BadObjs = [{LSize1,Slot1,BadSizeObj} | ObjsRest],
{error, invalid_objects_list} =
dets:init_table(TabRef, init([[BParms | BadObjs]]),{format,bchunk}),
_ = dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
<<Size:32,BigObj0/binary>> = list_to_binary(lists:duplicate(16,Obj1)),
BigObj = <<(Size*16):32,BigObj0/binary>>,
BadColl = [BParms, {LSize1+4,Slot1,BigObj} | ObjsRest],
{error, invalid_objects_list} =
dets:init_table(TabRef, init([BadColl]),{format,bchunk}),
_ = dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, Args),
BadObj = <<"foobar">>,
{error, invalid_objects_list} =
dets:init_table(TabRef, init([[BParms, BadObj]]),{format,bchunk}),
_ = dets:close(TabRef),
file:delete(Fname),
{ok, _} = dets:open_file(TabRef, [{type,bag} | Args]),
{error, {init_fun, _}} =
dets:init_table(TabRef, init([[BParms]]),{format,bchunk}),
_ = dets:close(TabRef),
file:delete(Fname),
ok = dets:close(Source),
file:delete(SourceFname),
L1 = [{1,a},{2,b},{3,c},{4,d}],
fast_bulk_init(L1, set, 4, 4, Config, V),
L2 = [{1,a},{2,b},{2,q},{3,c},{4,d},{4,e},{2,q}],
fast_bulk_init(L2, set, 4, 4, Config, V),
fast_bulk_init(L2, bag, 6, 4, Config, V),
fast_bulk_init(L2, duplicate_bag, 7, 4, Config, V),
fast_bulk_init(L1, set, 4, 4, 512, Config, V),
fast_bulk_init([], set, 0, 0, 10000, Config, V),
file:delete(Fname),
%% Initiate a file that contains a lot of objects.
{ok, _} = dets:open_file(Source, [{min_no_slots,10000} | SourceArgs]),
Fun1 = init_fun(0, 10000),
ok = dets:init_table(Source, Fun1, {format,term}),
{ok, _} = dets:open_file(TabRef, [{min_no_slots,10000} | Args]),
ok = ins(TabRef, 6000),
Fun2 = init_bchunk(Source),
true =
dets:is_compatible_bchunk_format(TabRef,
dets:info(Source, bchunk_format)),
false = dets:is_compatible_bchunk_format(TabRef, <<"foobar">>),
ok = dets:init_table(TabRef, Fun2, {format, bchunk}),
ok = dets:close(Source),
file:delete(SourceFname),
All = sort(get_all_objects(TabRef)),
FAll = get_all_objects_fast(TabRef),
true = All =:= sort(FAll),
true = length(All) =:= 10000,
ok = dets:close(TabRef),
file:delete(Fname),
%% Initiate inserts fewer objects than the table contains.
{ok, _} = dets:open_file(Source, [{min_no_slots,1000} | SourceArgs]),
ok = ins(Source, 4000),
{ok, _} = dets:open_file(TabRef, [{min_no_slots,1000} | Args]),
ok = ins(TabRef, 6000),
FileSize1 = dets:info(TabRef, file_size),
Fun4 = init_bchunk(Source),
ok = dets:init_table(TabRef, Fun4, {format, bchunk}),
ok = dets:close(Source),
file:delete(SourceFname),
FileSize2 = dets:info(TabRef, file_size),
All_2 = sort(get_all_objects(TabRef)),
FAll_2 = get_all_objects_fast(TabRef),
true = All_2 =:= sort(FAll_2),
true = length(All_2) =:= 4000,
ok = dets:close(TabRef),
true = FileSize1 > FileSize2,
%% Bchunk and fixed table.
{ok, _} = dets:open_file(TabRef, Args),
NoItems = dets:info(TabRef, no_objects),
AllObjects1 = sort(get_all_objects_fast(TabRef)),
dets:safe_fixtable(TabRef, true),
true = dets:info(TabRef, fixed),
Cont1 = init_bchunk(TabRef),
NoDel =
dets:select_delete(TabRef, [{{'_',{item,'_','_'}},[],[true]}]),
true = (NoDel > 0),
AllObjects2 = sort(get_all_objects_fast(TabRef)),
true = dets:info(TabRef, fixed),
Cont2 = init_bchunk(TabRef),
NoItems2 = dets:info(TabRef, no_objects),
true = (NoItems =:= NoItems2 + NoDel),
NoDel2 = dets:select_delete(TabRef, [{'_',[],[true]}]),
true = (NoDel2 > 0),
AllObjects3 = sort(get_all_objects_fast(TabRef)),
NoItems3 = dets:info(TabRef, no_objects),
true = (NoItems3 =:= 0),
true = dets:info(TabRef, fixed),
true = (NoItems2 =:= NoItems3 + NoDel2),
Cont3 = init_bchunk(TabRef),
BinColl1 = collect_bchunk(TabRef, Cont1),
BinColl2 = collect_bchunk(TabRef, Cont2),
BinColl3 = collect_bchunk(TabRef, Cont3),
dets:safe_fixtable(TabRef, false),
ok = dets:close(TabRef),
file:delete(Fname),
%% Now check that the above collected binaries are correct.
{ok, _} = dets:open_file(TabRef, Args),
ok = dets:init_table(TabRef, init([BinColl1]),{format,bchunk}),
true = (AllObjects1 =:= sort(get_all_objects_fast(TabRef))),
true = (length(AllObjects1) =:= dets:info(TabRef, no_objects)),
ok = dets:init_table(TabRef, init([BinColl2]),{format,bchunk}),
true = (AllObjects2 =:= sort(get_all_objects_fast(TabRef))),
true = (length(AllObjects2) =:= dets:info(TabRef, no_objects)),
ok = dets:init_table(TabRef, init([BinColl3]),{format,bchunk}),
true = (AllObjects3 =:= sort(get_all_objects_fast(TabRef))),
true = (length(AllObjects3) =:= dets:info(TabRef, no_objects)),
ok = dets:close(TabRef),
file:delete(Fname),
check_pps(P0),
ok.
fast_bulk_init(L, Type, N, NoKeys, Config, V) ->
fast_bulk_init(L, Type, N, NoKeys, 256, Config, V).
fast_bulk_init(L, Type, N, NoKeys, Est, Config, V) ->
T = init_table_test,
Fname = filename(T, Config),
file:delete(Fname),
Args0 = [{ram_file,false}, {type,Type},{keypos,1},
{estimated_no_objects, Est},{version,V}],
Args = [{file,Fname} | Args0],
S = init_table_test_source,
SFname = filename(S, Config),
file:delete(SFname),
SArgs = [{file,SFname} | Args0],
{ok, S} = dets:open_file(S, SArgs),
ok = dets:insert(S, L),
Input = init_bchunk(S),
{ok, T} = dets:open_file(T, Args),
ok = dets:init_table(T, Input, [{format,bchunk}]),
All = sort(get_all_objects(T)),
FAll = get_all_objects_fast(T),
true = All =:= sort(FAll),
true = length(All) =:= N,
true = dets:info(T, size) =:= N,
true = dets:info(T, no_keys) =:= NoKeys,
ok = dets:close(T),
{ok, T} = dets:open_file(T, Args),
All2 = sort(get_all_objects(T)),
FAll2 = get_all_objects_fast(T),
true = All =:= All2,
true = All =:= sort(FAll2),
ok = dets:close(T),
file:delete(Fname),
ok = dets:close(S),
file:delete(SFname),
ok.
init_bchunk(T) ->
Start = dets:bchunk(T, start),
init_bchunk(T, Start).
init_bchunk(Tab, State) ->
fun(read) when State =:= '$end_of_table' ->
end_of_input;
(read) when element(1, State) =:= error ->
State;
(read) ->
{Cont, Objs} = State,
{Objs, init_bchunk(Tab, dets:bchunk(Tab, Cont))};
(close) ->
ok
end.
collect_bchunk(Tab, Fun) ->
collect_bchunk(Tab, Fun, []).
collect_bchunk(Tab, Fun, L) ->
case Fun(read) of
end_of_input ->
lists:append(lists:reverse(L));
{Objs, Fun2} when is_list(Objs) ->
collect_bchunk(Tab, Fun2, [Objs | L]);
Error ->
Error
end.
items(I, N, C, L) when I =:= N; C =:= 0 ->
{I, L};
items(I, N, C, L) ->
items(I+1, N, C-1, [{I, item(I)} | L]).
%% Test open_file and repair.
repair_v8(Config) when is_list(Config) ->
repair(Config, 8).
%% Test open_file and repair.
repair_v9(Config) when is_list(Config) ->
%% Convert from format 9 to format 8.
T = convert_98,
Fname = filename(T, Config),
file:delete(Fname),
{ok, _} = dets:open_file(T, [{file,Fname},{version,9},
{type,duplicate_bag}]),
9 = dets:info(T, version),
true = is_binary(dets:info(T, bchunk_format)),
ok = dets:insert(T, [{1,a},{2,b},{1,c},{2,c},{1,c},{2,a},{1,b}]),
dets:close(T),
{error, {version_mismatch, _}} =
dets:open_file(T, [{file,Fname},{version,8},{type,duplicate_bag}]),
{ok, _} = dets:open_file(T, [{file,Fname},{version,8},
{type,duplicate_bag},{repair,force}]),
8 = dets:info(T, version),
true = undefined =:= dets:info(T, bchunk_format),
[{1,a},{1,b},{1,c},{1,c}] = sort(dets:lookup(T, 1)),
[{2,a},{2,b},{2,c}] = sort(dets:lookup(T, 2)),
7 = dets:info(T, no_objects),
no_keys_test(T),
_ = histogram(T, silent),
ok = dets:close(T),
file:delete(Fname),
%% The short lived format 9(a).
%% Not very throughly tested here.
A9 = a9,
Version9aS = filename:join(?datadir(Config), "version_9a.dets"),
Version9aT = filename('v9a.dets', Config),
{ok, _} = file:copy(Version9aS, Version9aT),
{ok, A9} = dets:open_file(A9, [{file,Version9aT}]),
undefined = dets:info(A9, bchunk_format),
[{1,a},{2,b},{3,c}] = sort(dets:match_object(A9, '_')),
ok = dets:insert(A9, {4,d}),
ok = dets:close(A9),
{ok, A9} = dets:open_file(A9, [{file,Version9aT}]),
{error, old_version} = dets:bchunk(A9, start),
ok = dets:close(A9),
io:format("Expect forced repair:~n"),
{ok, A9} = dets:open_file(A9, [{file,Version9aT},{repair,force}]),
{_, _} = dets:bchunk(A9, start),
ok = dets:close(A9),
file:delete(Version9aT),
repair(Config, 9).
repair(Config, V) ->
TabRef = repair_test,
Fname = filename(TabRef, Config),
file:delete(Fname),
HeadSize = headsz(V),
P0 = pps(),
{'EXIT', {badarg, _}} =
(catch dets:open_file(TabRef, [{min_no_slots,1000},
{max_no_slots,500}])),
{error,{file_error,hoppla,enoent}} = dets:file_info(hoppla),
{error,{file_error,Fname,enoent}} =
dets:open_file(TabRef, [{file, Fname}, {access, read}]),
%% compacting, and some kind of test that free lists are saved OK on file
{ok, TabRef} = dets:open_file(TabRef, [{file,Fname},{version,V}]),
0 = dets:info(TabRef, size),
ok = ins(TabRef, 30000),
ok = del(TabRef, 30000, 3),
ok = dets:close(TabRef),
{error, {access_mode,Fname}} =
dets:open_file(foo, [{file,Fname},{repair,force},{access,read}]),
{ok, Ref3} = dets:open_file(Fname), % no repair!
20000 = dets:info(Ref3, size),
20000 = dets:foldl(fun(_, N) -> N+1 end, 0, Ref3),
20000 = count_objects_quite_fast(Ref3), % actually a test of match
no_keys_test(Ref3),
ok = dets:close(Ref3),
if
V =:= 8 ->
{ok, TabRef} = dets:open_file(TabRef,
[{file, Fname},{version,V},{access,read}]),
ok = dets:close(TabRef),
io:format("Expect compacting repair:~n"),
{ok, TabRef} = dets:open_file(TabRef,
[{file, Fname},{version,V}]),
20000 = dets:info(TabRef, size),
_ = histogram(TabRef, silent),
ok = dets:close(TabRef);
true ->
ok
end,
{error,{keypos_mismatch,Fname}} =
dets:open_file(TabRef, [{file, Fname},{keypos,17}]),
{error,{type_mismatch,Fname}} =
dets:open_file(TabRef, [{file, Fname},{type,duplicate_bag}]),
%% make one of the temporary files unwritable
TmpFile = if
V =:= 8 ->
Fname ++ ".TMP.10000";
true -> Fname ++ ".TMP.1"
end,
file:delete(TmpFile),
{ok, TmpFd} = file:open(TmpFile, [read,write]),
ok = file:close(TmpFd),
unwritable(TmpFile),
{error,{file_error,TmpFile,eacces}} = dets:fsck(Fname, V),
{ok, _} = dets:open_file(TabRef,
[{repair,false},{file, Fname},{version,V}]),
20000 = length(get_all_objects(TabRef)),
_ = histogram(TabRef, silent),
20000 = length(get_all_objects_fast(TabRef)),
ok = dets:close(TabRef),
writable(TmpFile),
file:delete(TmpFile),
truncate(Fname, HeadSize + 10),
{error,{not_closed, Fname}} =
dets:open_file(TabRef, [{file, Fname}, {access, read}]),
{error,{not_closed, Fname}} =
dets:open_file(TabRef, [{file, Fname}, {access, read},
{repair,force}]),
{error,{needs_repair, Fname}} =
dets:open_file(TabRef, [{file, Fname}, {repair, false}]),
file:delete(Fname),
%% truncated file header
{ok, TabRef} = dets:open_file(TabRef, [{file,Fname},{version,V}]),
ok = ins(TabRef, 100),
ok = dets:close(TabRef),
file:delete(Fname),
%% version bump (v8)
Version7S = filename:join(?datadir(Config), "version_r2d.dets"),
Version7T = filename('v2.dets', Config),
{ok, _} = file:copy(Version7S, Version7T),
{error,{version_bump, Version7T}} = dets:open_file(Version7T),
{error,{version_bump, Version7T}} =
dets:open_file(Version7T, [{file,Version7T},{repair,false}]),
{error,{version_bump, Version7T}} =
dets:open_file(Version7T, [{file, Version7T}, {access, read}]),
io:format("Expect upgrade:~n"),
{ok, _} = dets:open_file(Version7T,
[{file, Version7T},{version, V}]),
[{1,a},{2,b}] = sort(get_all_objects(Version7T)),
[{1,a},{2,b}] = sort(get_all_objects_fast(Version7T)),
Phash = if
V =:= 8 -> phash;
true -> phash2
end,
Phash = dets:info(Version7T, hash),
_ = histogram(Version7T, silent),
ok = dets:close(Version7T),
{ok, _} = dets:open_file(Version7T, [{file, Version7T}]),
Phash = dets:info(Version7T, hash),
ok = dets:close(Version7T),
file:delete(Version7T),
%% converting free lists
Version8aS = filename:join(?datadir(Config), "version_r3b02.dets"),
Version8aT = filename('v3.dets', Config),
{ok, _} = file:copy(Version8aS, Version8aT),
%% min_no_slots and max_no_slots are ignored - no repair is taking place
{ok, _} = dets:open_file(version_8a,
[{file, Version8aT},{min_no_slots,1000},
{max_no_slots,100000}]),
[{1,b},{2,a},{a,1},{b,2}] = sort(get_all_objects(version_8a)),
[{1,b},{2,a},{a,1},{b,2}] = sort(get_all_objects_fast(version_8a)),
ok = ins(version_8a, 1000),
1002 = dets:info(version_8a, size),
no_keys_test(version_8a),
All8a = sort(get_all_objects(version_8a)),
1002 = length(All8a),
FAll8a = sort(get_all_objects_fast(version_8a)),
true = sort(All8a) =:= sort(FAll8a),
ok = del(version_8a, 300, 3),
902 = dets:info(version_8a, size),
no_keys_test(version_8a),
All8a2 = sort(get_all_objects(version_8a)),
902 = length(All8a2),
FAll8a2 = sort(get_all_objects_fast(version_8a)),
true = sort(All8a2) =:= sort(FAll8a2),
_ = histogram(version_8a, silent),
ok = dets:close(version_8a),
file:delete(Version8aT),
%% will fail unless the slots are properly sorted when repairing (v8)
BArgs = [{file, Fname},{type,duplicate_bag},
{delayed_write,{3000,10000}},{version,V}],
{ok, TabRef} = dets:open_file(TabRef, BArgs),
Seq = seq(1, 500),
Small = map(fun(X) -> {X,X} end, Seq),
Big = map(fun(X) -> erlang:make_tuple(20, X) end, Seq),
ok = dets:insert(TabRef, Small),
ok = dets:insert(TabRef, Big),
ok = dets:insert(TabRef, Small),
ok = dets:insert(TabRef, Big),
All = sort(safe_get_all_objects(TabRef)),
ok = dets:close(TabRef),
io:format("Expect forced repair:~n"),
{ok, _} =
dets:open_file(TabRef, [{repair,force},{min_no_slots,2000} | BArgs]),
if
V =:= 9 ->
{MinNoSlots,_,MaxNoSlots} = dets:info(TabRef, no_slots),
ok = dets:close(TabRef),
io:format("Expect compaction:~n"),
{ok, _} =
dets:open_file(TabRef, [{repair,force},
{min_no_slots,MinNoSlots},
{max_no_slots,MaxNoSlots} | BArgs]);
true ->
ok
end,
All2 = get_all_objects(TabRef),
true = All =:= sort(All2),
FAll2 = get_all_objects_fast(TabRef),
true = All =:= sort(FAll2),
true = length(All) =:= dets:info(TabRef, size),
no_keys_test(TabRef),
Fun = fun(X) -> 4 = length(dets:lookup(TabRef, X)) end,
foreach(Fun, Seq),
_ = histogram(TabRef, silent),
ok = dets:close(TabRef),
file:delete(Fname),
%% object bigger than segments, the "hole" is taken care of
{ok, TabRef} = dets:open_file(TabRef, [{file, Fname},{version,V}]),
Tuple = erlang:make_tuple(1000, foobar), % > 2 kB
ok = dets:insert(TabRef, Tuple),
%% at least one full segment (objects smaller than 2 kB):
ins(TabRef, 2000),
ok = dets:close(TabRef),
if
V =:= 8 ->
%% first estimated number of objects is wrong, repair once more
{ok, Fd} = file:open(Fname, [read,write]),
NoPos = HeadSize - 8, % no_objects
file:pwrite(Fd, NoPos, <<0:32>>), % NoItems
ok = file:close(Fd),
dets:fsck(Fname, V),
{ok, _} =
dets:open_file(TabRef,
[{repair,false},{file, Fname},{version,V}]),
2001 = length(get_all_objects(TabRef)),
_ = histogram(TabRef, silent),
2001 = length(get_all_objects_fast(TabRef)),
ok = dets:close(TabRef);
true ->
ok
end,
{ok, _} =
dets:open_file(TabRef,
[{repair,false},{file, Fname},{version,V}]),
{ok, ObjPos} = dets:where(TabRef, {66,{item,number,66}}),
ok = dets:close(TabRef),
%% Damaged object.
Pos = 12, % v9: compaction fails, proper repair follows
crash(Fname, ObjPos+Pos),
io:format(
"Expect forced repair (possibly after attempted compaction):~n"),
{ok, _} =
dets:open_file(TabRef, [{repair,force},{file, Fname},{version,V}]),
true = dets:info(TabRef, size) < 2001,
ok = dets:close(TabRef),
file:delete(Fname),
%% The file is smaller than the padded object.
{ok, TabRef} = dets:open_file(TabRef, [{file,Fname},{version,V}]),
ok = dets:insert(TabRef, Tuple),
ok = dets:close(TabRef),
io:format("Expect forced repair or compaction:~n"),
{ok, _} =
dets:open_file(TabRef, [{repair,force},{file, Fname},{version,V}]),
true = 1 =:= dets:info(TabRef, size),
ok = dets:close(TabRef),
file:delete(Fname),
%% Damaged free lists.
{ok, TabRef} = dets:open_file(TabRef, [{file,Fname},{version,V}]),
ok = ins(TabRef, 300),
ok = dets:sync(TabRef),
ok = del(TabRef, 300, 3),
%% FileSize is approximately where the free lists will be written.
FileSize = dets:info(TabRef, memory),
ok = dets:close(TabRef),
crash(Fname, FileSize+20),
%% Used to return bad_freelists, but that changed in OTP-9622
{ok, TabRef} =
dets:open_file(TabRef, [{file,Fname},{version,V}]),
ok = dets:close(TabRef),
file:delete(Fname),
%% File not closed, opening with read and read_write access tried.
{ok, TabRef} = dets:open_file(TabRef, [{file,Fname},{version,V}]),
ok = ins(TabRef, 300),
ok = dets:close(TabRef),
crash(Fname, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED),
{error, {not_closed, Fname}} =
dets:open_file(foo, [{file,Fname},{version,V},{repair,force},
{access,read}]),
{error, {not_closed, Fname}} =
dets:open_file(foo, [{file,Fname},{version,V},{repair,true},
{access,read}]),
io:format("Expect repair:~n"),
{ok, TabRef} =
dets:open_file(TabRef, [{file,Fname},{version,V},{repair,true},
{access,read_write}]),
ok = dets:close(TabRef),
crash(Fname, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED),
io:format("Expect forced repair:~n"),
{ok, TabRef} =
dets:open_file(TabRef, [{file,Fname},{version,V},{repair,force},
{access,read_write}]),
ok = dets:close(TabRef),
file:delete(Fname),
%% The size of an object is huge.
{ok, TabRef} = dets:open_file(TabRef, [{file,Fname},{version,V}]),
ok = dets:insert(TabRef, [{1,2,3},{2,3,4}]),
{ok, ObjPos2} = dets:where(TabRef, {1,2,3}),
ok = dets:close(TabRef),
ObjPos3 = if
V =:= 8 -> ObjPos2 + 4;
V =:= 9 -> ObjPos2
end,
crash(Fname, ObjPos3, 255),
io:format("Expect forced repair:~n"),
{ok, TabRef} =
dets:open_file(TabRef, [{file,Fname},{version,V},{repair,force}]),
ok = dets:close(TabRef),
file:delete(Fname),
check_pps(P0),
ok.
%% Test the use of different hashing algorithms in v8b and v8c of the
%% Dets file format.
hash_v8b_v8c(Config) when is_list(Config) ->
Source =
filename:join(?datadir(Config), "dets_test_v8b.dets"),
%% Little endian version of old file (there is an endianess bug in
%% the old hash). This is all about version 8 of the dets file format.
P0 = pps(),
SourceLE =
filename:join(?datadir(Config),
"dets_test_v8b_little_endian.dets"),
Target1 = filename('oldhash1.dets', Config),
Target1LE = filename('oldhash1le.dets', Config),
Target2 = filename('oldhash2.dets', Config),
{ok, Bin} = file:read_file(Source),
{ok, BinLE} = file:read_file(SourceLE),
ok = file:write_file(Target1,Bin),
ok = file:write_file(Target1LE,BinLE),
ok = file:write_file(Target2,Bin),
{ok, d1} = dets:open_file(d1,[{file,Target1}]),
{ok, d1le} = dets:open_file(d1le,[{file,Target1LE}]),
{ok, d2} = dets:open_file(d2,[{file,Target2},{repair,force},
{version,8}]),
FF = fun(N,_F,_T) when N > 16#FFFFFFFFFFFFFFFF ->
ok;
(N,F,T) ->
V = integer_to_list(N),
case dets:lookup(T,N) of
[{N,V}] ->
F(N*2,F,T);
_Error ->
exit({failed,{lookup,T,N}})
end
end,
Mess = case (catch FF(1,FF,d1)) of
{'EXIT', {failed, {lookup,_,_}}} ->
ok = dets:close(d1),
FF(1,FF,d1le),
hash = dets:info(d1le,hash),
dets:insert(d1le,{33333333333,hejsan}),
[{33333333333,hejsan}] =
dets:lookup(d1le,33333333333),
ok = dets:close(d1le),
{ok, d1le} = dets:open_file(d1le,
[{file,Target1LE}]),
[{33333333333,hejsan}] =
dets:lookup(d1le,33333333333),
FF(1,FF,d1le),
ok = dets:close(d1le),
"Seems to be a little endian machine";
{'EXIT', Fault} ->
exit(Fault);
_ ->
ok = dets:close(d1le),
hash = dets:info(d1,hash),
dets:insert(d1,{33333333333,hejsan}),
[{33333333333,hejsan}] =
dets:lookup(d1,33333333333),
ok = dets:close(d1),
{ok, d1} = dets:open_file(d1,[{file,Target1}]),
[{33333333333,hejsan}] =
dets:lookup(d1,33333333333),
FF(1,FF,d1),
ok = dets:close(d1),
"Seems to be a big endian machine"
end,
FF(1,FF,d2),
phash = dets:info(d2,hash),
ok = dets:close(d2),
file:delete(Target1),
file:delete(Target1LE),
file:delete(Target2),
check_pps(P0),
{comment, Mess}.
%% Test version 9(b) with erlang:phash/2 as hash function.
phash(Config) when is_list(Config) ->
T = phash,
Phash_v9bS = filename:join(?datadir(Config), "version_9b_phash.dat"),
Fname = filename('v9b.dets', Config),
{ok, _} = file:copy(Phash_v9bS, Fname),
%% Deleting all objects changes the hash function.
%% A feature... (it's for free)
{ok, T} = dets:open_file(T, [{file, Fname}]),
phash = dets:info(T, hash),
dets:delete_all_objects(T),
phash2 = dets:info(T, hash),
[] = get_all_objects(T),
[] = get_all_objects_fast(T),
ok = dets:close(T),
%% The hash function is kept when compacting a table.
{ok, _} = file:copy(Phash_v9bS, Fname),
io:format("Expect compaction:~n"),
{ok, T} = dets:open_file(T, [{file, Fname},{repair,force}]),
phash = dets:info(T, hash),
[{1,a},{2,b},{3,c},{4,d},{5,e}] =
lists:sort(dets:lookup_keys(T, [1,2,3,4,5])),
ok = dets:close(T),
%% The hash function is updated when repairing a table (no cost).
{ok, _} = file:copy(Phash_v9bS, Fname),
crash(Fname, ?CLOSED_PROPERLY_POS+3, 0),
io:format("Expect repair:~n"),
{ok, T} = dets:open_file(T, [{file, Fname}]),
phash2 = dets:info(T, hash),
[{1,a},{2,b},{3,c},{4,d},{5,e}] =
lists:sort(dets:lookup_keys(T, [1,2,3,4,5])),
ok = dets:close(T),
%% One cannot use the bchunk format when copying between a phash
%% table and a phash2 table. (There is no test for the case an R9
%% (or later) node (using phash2) copies a table to an R8 node
%% (using phash).) See also the comment on HASH_PARMS in dets_v9.erl.
{ok, _} = file:copy(Phash_v9bS, Fname),
{ok, T} = dets:open_file(T, [{file, Fname}]),
Type = dets:info(T, type),
KeyPos = dets:info(T, keypos),
Input = init_bchunk(T),
T2 = phash_table,
Fname2 = filename(T2, Config),
Args = [{type,Type},{keypos,KeyPos},{version,9},{file,Fname2}],
{ok, T2} = dets:open_file(T2, Args),
{error, {init_fun, _}} =
dets:init_table(T2, Input, {format,bchunk}),
_ = dets:close(T2),
ok = dets:close(T),
file:delete(Fname2),
file:delete(Fname),
ok.
%% Test foldl, foldr, to_ets.
fold_v8(Config) when is_list(Config) ->
fold(Config, 8).
%% Test foldl, foldr, to_ets.
fold_v9(Config) when is_list(Config) ->
fold(Config, 9).
fold(Config, Version) ->
T = test_table,
N = 100,
Fname = filename(T, Config),
file:delete(Fname),
P0 = pps(),
Args = [{version, Version}, {file,Fname}, {estimated_no_objects, N}],
{ok, _} = dets:open_file(T, Args),
ok = ins(T, N),
Ets = ets:new(to_ets, [public]),
dets:to_ets(T, Ets),
true = N =:= ets:info(Ets, size),
ets:delete(Ets),
Ets2 = ets:new(to_ets, [private]),
dets:to_ets(T, Ets2),
true = N =:= ets:info(Ets2, size),
ets:delete(Ets2),
{'EXIT', {badarg, _}} = (catch dets:to_ets(T, not_an_ets_table)),
F0 = fun(X, A) -> [X | A] end,
true = N =:= length(dets:foldl(F0, [], T)),
true = N =:= length(dets:foldr(F0, [], T)),
F1 = fun(_X, _A) -> throw(away) end,
away = (catch dets:foldl(F1, [], T)),
away = (catch dets:foldr(F1, [], T)),
F2 = fun(X, A) -> X + A end,
{'EXIT', _} = (catch dets:foldl(F2, [], T)),
{'EXIT', _} = (catch dets:foldr(F2, [], T)),
F3 = fun(_X) -> throw(away) end,
away = (catch dets:traverse(T, F3)),
F4 = fun(X) -> X + 17 end,
{'EXIT', _} = (catch dets:traverse(T, F4)),
F5 = fun(_X) -> done end,
done = dets:traverse(T, F5),
{ok, ObjPos} = dets:where(T, {66,{item,number,66}}),
ok = dets:close(T),
%% Damaged object.
Pos = if
Version =:= 8 -> 12;
Version =:= 9 -> 8
end,
crash(Fname, ObjPos+Pos),
{ok, _} = dets:open_file(T, Args),
io:format("Expect corrupt table:~n"),
BadObject1 = dets:foldl(F0, [], T),
bad_object(BadObject1, Fname),
BadObject2 = dets:close(T),
bad_object(BadObject2, Fname),
file:delete(Fname),
check_pps(P0),
ok.
%% Add objects to a fixed table.
fixtable_v8(Config) when is_list(Config) ->
fixtable(Config, 8).
%% Add objects to a fixed table.
fixtable_v9(Config) when is_list(Config) ->
fixtable(Config, 9).
fixtable(Config, Version) when is_list(Config) ->
T = fixtable,
Fname = filename(fixtable, Config),
file:delete(Fname),
Args = [{version,Version},{file,Fname}],
P0 = pps(),
{ok, _} = dets:open_file(T, Args),
%% badarg
check_badarg(catch dets:safe_fixtable(no_table,true),
dets, safe_fixtable, [no_table,true]),
check_badarg(catch dets:safe_fixtable(T,undefined),
dets, safe_fixtable, [T,undefined]),
%% The table is not allowed to grow while the elements are inserted:
ok = ins(T, 500),
dets:safe_fixtable(T, false),
%% Now the table can grow. At the same time as elements are inserted,
%% the table tries to catch up with the previously inserted elements.
ok = ins(T, 1000),
1000 = dets:info(T, size),
ok = dets:close(T),
file:delete(Fname),
{ok, _} = dets:open_file(T, [{type, duplicate_bag} | Args]),
%% In a fixed table, delete and re-insert an object.
ok = dets:insert(T, {1, a, b}),
SysBefore = erlang:timestamp(),
MonBefore = erlang:monotonic_time(),
dets:safe_fixtable(T, true),
MonAfter = erlang:monotonic_time(),
SysAfter = erlang:timestamp(),
Self = self(),
{FixMonTime,[{Self,1}]} = dets:info(T,safe_fixed_monotonic_time),
{FixSysTime,[{Self,1}]} = dets:info(T,safe_fixed),
true = is_integer(FixMonTime),
true = MonBefore =< FixMonTime,
true = FixMonTime =< MonAfter,
{FstMs,FstS,FstUs} = FixSysTime,
true = is_integer(FstMs),
true = is_integer(FstS),
true = is_integer(FstUs),
case erlang:system_info(time_warp_mode) of
no_time_warp ->
true = timer:now_diff(FixSysTime, SysBefore) >= 0,
true = timer:now_diff(SysAfter, FixSysTime) >= 0;
_ ->
%% ets:info(Tab,safe_fixed) not timewarp safe...
ignore
end,
ok = dets:match_delete(T, {1, a, b}),
ok = dets:insert(T, {1, a, b}),
{FixMonTime,[{Self,1}]} = dets:info(T,safe_fixed_monotonic_time),
{FixSysTime,[{Self,1}]} = dets:info(T,safe_fixed),
dets:safe_fixtable(T, false),
1 = length(dets:match_object(T, '_')),
ok = dets:match_delete(T, '_'),
%% In a fixed table, delete and insert a smaller object.
ok = dets:insert(T, {1, duplicate(100, e)}),
dets:safe_fixtable(T, true),
ok = dets:match_delete(T, {1, '_'}),
ok = dets:insert(T, {1, a, b}),
dets:safe_fixtable(T, false),
1 = length(dets:match_object(T, '_')),
ok = dets:delete_all_objects(T),
%% Like the last one, but one extra object.
ok = dets:insert(T, {1, duplicate(100, e)}),
ok = dets:insert(T, {2, duplicate(100, e)}),
dets:safe_fixtable(T, true),
ok = dets:match_delete(T, {1, '_'}),
ok = dets:insert(T, {1, a, b}),
dets:safe_fixtable(T, false),
2 = length(dets:match_object(T, '_')),
dets:safe_fixtable(T, true),
ok = dets:delete_all_objects(T),
true = dets:info(T, fixed),
0 = length(dets:match_object(T, '_')),
ok = dets:close(T),
file:delete(Fname),
check_pps(P0),
ok.
%% Matching objects of a fixed table.
match_v8(Config) when is_list(Config) ->
match(Config, 8).
%% Matching objects of a fixed table.
match_v9(Config) when is_list(Config) ->
match(Config, 9).
match(Config, Version) ->
T = match,
Fname = filename(match, Config),
file:delete(Fname),
P0 = pps(),
Args = [{version, Version}, {file,Fname}, {type, duplicate_bag},
{estimated_no_objects,550}],
{ok, _} = dets:open_file(T, Args),
ok = dets:insert(T, {1, a, b}),
ok = dets:insert(T, {1, b, a}),
ok = dets:insert(T, {2, a, b}),
ok = dets:insert(T, {2, b, a}),
%% match, badarg
MSpec = [{'_',[],['$_']}],
check_badarg(catch dets:match(no_table, '_'),
dets, match, [no_table,'_']),
check_badarg(catch dets:match(T, '_', not_a_number),
dets, match, [T,'_',not_a_number]),
{EC1, _} = dets:select(T, MSpec, 1),
check_badarg(catch dets:match(EC1),
dets, match, [EC1]),
%% match_object, badarg
check_badarg(catch dets:match_object(no_table, '_'),
dets, match_object, [no_table,'_']),
check_badarg(catch dets:match_object(T, '_', not_a_number),
dets, match_object, [T,'_',not_a_number]),
{EC2, _} = dets:select(T, MSpec, 1),
check_badarg(catch dets:match_object(EC2),
dets, match_object, [EC2]),
dets:safe_fixtable(T, true),
{[_, _], C1} = dets:match_object(T, '_', 2),
{[_, _], C2} = dets:match_object(C1),
'$end_of_table' = dets:match_object(C2),
{[_, _], C3} = dets:match_object(T, {1, '_', '_'}, 100),
'$end_of_table' = dets:match_object(C3),
'$end_of_table' = dets:match_object(T, {'_'}, default),
dets:safe_fixtable(T, false),
dets:safe_fixtable(T, true),
{[_, _], C30} = dets:match(T, '$1', 2),
{[_, _], C31} = dets:match(C30),
'$end_of_table' = dets:match(C31),
{[_, _], C32} = dets:match(T, {1, '$1', '_'}, 100),
'$end_of_table' = dets:match(C32),
'$end_of_table' = dets:match(T, {'_'}, default),
dets:safe_fixtable(T, false),
[[1],[1],[2],[2]] = sort(dets:match(T, {'$1','_','_'})),
%% delete and insert while chunking
%% (this case almost worthless after changes in OTP-5232)
ok = dets:match_delete(T, '_'),
L500 = seq(1, 500),
Fun = fun(X) -> ok = dets:insert(T, {X, a, b, c, d}) end,
foreach(Fun, L500),
%% Select one object DI in L3 below to be deleted.
{_, TmpCont} = dets:match_object(T, '_', 200),
{_, TmpCont1} = dets:match_object(TmpCont),
{TTL, _} = dets:match_object(TmpCont1),
DI = if Version =:= 8 -> last(TTL); Version =:= 9 -> hd(TTL) end,
dets:safe_fixtable(T, true),
{L1, C20} = dets:match_object(T, '_', 200),
true = 200 =< length(L1),
ok = dets:match_delete(T, {'2','_','_'}), % no match
ok = dets:match_delete(T, DI), % last object
Tiny = {1050},
ok = dets:insert(T, Tiny),
true = member(Tiny, dets:match_object(T, '_')),
{_L2, C21} = dets:match_object(C20),
{_L3, _C22} = dets:match_object(C21),
%% It used to be that Tiny was not visible here, but since the
%% scanning of files was changed to inspect the free lists every
%% now and then it may very well be visible here.
%% false = member(Tiny, _L3),
%% DI used to visible here, but the above mentioned modification
%% has changed that; it may or may not be visible.
%% true = member(DI, _L3),
dets:safe_fixtable(T, false),
true = dets:member(T, 1050),
true = member(Tiny, dets:match_object(T, '_')),
false = member(DI, dets:match_object(T, '_')),
ok = dets:close(T),
file:delete(Fname),
N = 100,
{ok, _} = dets:open_file(T, [{estimated_no_objects,N} | Args]),
ok = ins(T, N),
Obj = {66,{item,number,66}},
Spec = {'_','_'},
{ok, ObjPos} = dets:where(T, Obj),
ok = dets:close(T),
%% Damaged object.
crash(Fname, ObjPos+12),
{ok, _} = dets:open_file(T, Args),
io:format("Expect corrupt table:~n"),
case ins(T, N) of
ok ->
bad_object(dets:sync(T), Fname);
Else ->
bad_object(Else, Fname)
end,
io:format("Expect corrupt table:~n"),
bad_object(dets:match(T, Spec), Fname),
io:format("Expect corrupt table:~n"),
bad_object(dets:match_delete(T, Spec), Fname),
bad_object(dets:close(T), Fname),
file:delete(Fname),
{ok, _} = dets:open_file(T, [{estimated_no_objects,N} | Args]),
ok = ins(T, N),
{ok, ObjPos2} = dets:where(T, Obj),
ok = dets:close(T),
%% Damaged size of object.
%% In v8, there is a next pointer before the size.
CrashPos = if Version =:= 8 -> 5; Version =:= 9 -> 1 end,
crash(Fname, ObjPos2+CrashPos),
{ok, _} = dets:open_file(T, Args),
case dets:insert_new(T, Obj) of % OTP-12024
ok ->
bad_object(dets:sync(T), Fname);
Else3 ->
bad_object(Else3, Fname)
end,
io:format("Expect corrupt table:~n"),
case ins(T, N) of
ok ->
bad_object(dets:sync(T), Fname);
Else2 ->
bad_object(Else2, Fname)
end,
%% Just echoes...
bad_object(dets:match(T, Spec), Fname),
bad_object(dets:match_delete(T, Spec), Fname),
bad_object(dets:close(T), Fname),
file:delete(Fname),
{ok, _} = dets:open_file(T, [{estimated_no_objects,N} | Args]),
ok = ins(T, N),
{ok, ObjPos3} = dets:where(T, Obj),
ok = dets:close(T),
%% match_delete finds an error
CrashPos3 = if Version =:= 8 -> 12; Version =:= 9 -> 16 end,
crash(Fname, ObjPos3+CrashPos3),
{ok, _} = dets:open_file(T, Args),
bad_object(dets:match_delete(T, Spec), Fname),
bad_object(dets:close(T), Fname),
file:delete(Fname),
%% The key is not fixed, but not all objects with the key are removed.
{ok, _} = dets:open_file(T, Args),
ok = dets:insert(T, [{1,a},{1,b},{1,c},{1,a},{1,b},{1,c}]),
6 = dets:info(T, size),
ok = dets:match_delete(T, {'_',a}),
4 = dets:info(T, size),
[{1,b},{1,b},{1,c},{1,c}] =
sort(dets:match_object(T,{'_','_'})),
ok = dets:close(T),
file:delete(Fname),
check_pps(P0),
ok.
%% Selecting objects of a fixed table.
select_v8(Config) when is_list(Config) ->
select(Config, 8).
%% Selecting objects of a fixed table.
select_v9(Config) when is_list(Config) ->
select(Config, 9).
select(Config, Version) ->
T = select,
Fname = filename(select, Config),
file:delete(Fname),
P0 = pps(),
Args = [{version,Version}, {file,Fname}, {type, duplicate_bag},
{estimated_no_objects,550}],
{ok, _} = dets:open_file(T, Args),
ok = dets:insert(T, {1, a, b}),
ok = dets:insert(T, {1, b, a}),
ok = dets:insert(T, {2, a, b}),
ok = dets:insert(T, {2, b, a}),
ok = dets:insert(T, {3, a, b}),
ok = dets:insert(T, {3, b, a}),
%% badarg
MSpec = [{'_',[],['$_']}],
check_badarg(catch dets:select(no_table, MSpec),
dets, select, [no_table,MSpec]),
check_badarg(catch dets:select(T, <<17>>),
dets, select, [T,<<17>>]),
check_badarg(catch dets:select(T, []),
dets, select, [T,[]]),
check_badarg(catch dets:select(T, MSpec, not_a_number),
dets, select, [T,MSpec,not_a_number]),
{EC, _} = dets:match(T, '_', 1),
check_badarg(catch dets:select(EC),
dets, select, [EC]),
AllSpec = [{'_',[],['$_']}],
dets:safe_fixtable(T, true),
{[_, _], C1} = dets:select(T, AllSpec, 2),
{[_, _], C2} = dets:select(C1),
{[_, _], C2a} = dets:select(C2),
'$end_of_table' = dets:select(C2a),
{[_, _], C3} = dets:select(T, [{{1,'_','_'},[],['$_']}], 100),
'$end_of_table' = dets:select(C3),
'$end_of_table' = dets:select(T, [{{'_'},[],['$_']}], default),
dets:safe_fixtable(T, false),
Sp1 = [{{1,'_','_'},[],['$_']},{{1,'_','_'},[],['$_']},
{{2,'_','_'},[],['$_']}],
[_,_,_,_] = dets:select(T, Sp1),
Sp2 = [{{1,'_','_'},[],['$_']},{{1,'_','_'},[],['$_']},
{{'_','_','_'},[],['$_']}],
[_,_,_,_,_,_] = dets:select(T, Sp2),
AllDeleteSpec = [{'_',[],[true]}],
%% delete and insert while chunking
%% (this case almost worthless after changes in OTP-5232)
6 = dets:select_delete(T, AllDeleteSpec),
L500 = seq(1, 500),
Fun = fun(X) -> ok = dets:insert(T, {X, a, b, c, d}) end,
foreach(Fun, L500),
%% Select one object DI in L3 below to be deleted.
{_, TmpCont} = dets:match_object(T, '_', 200),
{_, TmpCont1} = dets:match_object(TmpCont),
{TTL, _} = dets:match_object(TmpCont1),
DI = if Version =:= 8 -> last(TTL); Version =:= 9 -> hd(TTL) end,
dets:safe_fixtable(T, true),
{L1, C20} = dets:select(T, AllSpec, 200),
true = 200 =< length(L1),
0 = dets:select_delete(T, [{{2,'_','_'},[],[true]}]),
1 = dets:select_delete(T, [{DI,[],[true]}]), % last object
Tiny = {1050},
ok = dets:insert(T, Tiny),
true = member(Tiny, dets:select(T, AllSpec)),
{_L2, C21} = dets:select(C20),
{_L3, _C22} = dets:select(C21),
%% It used to be that Tiny was not visible here, but since the
%% scanning of files was changed to inspect the free lists every
%% now and then it may very well be visible here.
%% false = member(Tiny, _L3),
%% DI used to visible here, but the above mentioned modification
%% has changed that; it may or may not be visible.
%% true = member(DI, _L3),
true = dets:member(T, 1050),
true = member(Tiny, dets:select(T, AllSpec)),
false = member(DI, dets:select(T, AllSpec)),
dets:safe_fixtable(T, false),
true = dets:member(T, 1050),
true = member(Tiny, dets:select(T, AllSpec)),
false = member(DI, dets:select(T, AllSpec)),
ok = dets:close(T),
file:delete(Fname),
%% The key is not fixed, but not all objects with the key are removed.
{ok, _} = dets:open_file(T, Args),
ok = dets:insert(T, [{1,a},{1,b},{1,c},{1,a},{1,b},{1,c}]),
6 = dets:info(T, size),
2 = dets:select_delete(T, [{{'_',a},[],[true]}]),
4 = dets:info(T, size),
[{1,b},{1,b},{1,c},{1,c}] = sort(dets:select(T, AllSpec)),
ok = dets:close(T),
file:delete(Fname),
check_pps(P0),
ok.
%% Test update_counter/1.
update_counter(Config) when is_list(Config) ->
T = update_counter,
Fname = filename(select, Config),
file:delete(Fname),
P0 = pps(),
check_badarg(catch dets:update_counter(no_table, 1, 1),
dets, update_counter, [no_table,1,1]),
Args = [{file,Fname},{keypos,2}],
{ok, _} = dets:open_file(T, [{type,set} | Args]),
{'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)),
ok = dets:insert(T, {1,a}),
{'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)),
ok = dets:insert(T, {0,1}),
{'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)),
ok = dets:insert(T, {0,1,0}),
1 = dets:update_counter(T, 1, 1),
2 = dets:update_counter(T, 1, 1),
6 = dets:update_counter(T, 1, {3,4}),
{'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, {0,3})),
ok = dets:close(T),
file:delete(Fname),
{ok, _} = dets:open_file(T, [{type,bag} | Args]),
ok = dets:insert(T, {0,1,0}),
{'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)),
ok = dets:close(T),
file:delete(Fname),
check_pps(P0),
ok.
%% Call some functions with bad arguments.
badarg(Config) when is_list(Config) ->
T = badarg,
Fname = filename(select, Config),
file:delete(Fname),
P0 = pps(),
Args = [{file,Fname},{keypos,3}],
{ok, _} = dets:open_file(T, [{type,set} | Args]),
% dets:verbose(),
%% badargs are tested in match, select and fixtable too.
%% open
check_badarg(catch dets:open_file({a,tuple},[]),
dets, open_file, [{a,tuple},[]]),
check_badarg(catch dets:open_file({a,tuple}),
dets, open_file,[{a,tuple}]),
check_badarg(catch dets:open_file(file,[foo]),
dets, open_file, [file,[foo]]),
check_badarg(catch dets:open_file({hej,san},[{type,set}|3]),
dets, open_file, [{hej,san},[{type,set}|3]]),
%% insert
check_badarg(catch dets:insert(no_table, {1,2}),
dets, insert, [no_table,{1,2}]),
check_badarg(catch dets:insert(no_table, [{1,2}]),
dets, insert, [no_table,[{1,2}]]),
check_badarg(catch dets:insert(T, {1,2}),
dets, insert, [T,{1,2}]),
check_badarg(catch dets:insert(T, [{1,2}]),
dets, insert, [T,[{1,2}]]),
check_badarg(catch dets:insert(T, [{1,2,3} | 3]),
dets, insert, [T,[{1,2,3}|3]]),
%% lookup{_keys}
check_badarg(catch dets:lookup_keys(T, []),
dets, lookup_keys, [badarg,[]]),
check_badarg(catch dets:lookup(no_table, 1),
dets, lookup, [no_table,1]),
check_badarg(catch dets:lookup_keys(T, [1 | 2]),
dets, lookup_keys, [T,[1|2]]),
%% member
check_badarg(catch dets:member(no_table, 1),
dets, member, [no_table,1]),
%% sync
check_badarg(catch dets:sync(no_table),
dets, sync, [no_table]),
%% delete{_keys}
check_badarg(catch dets:delete(no_table, 1),
dets, delete, [no_table,1]),
%% delete_object
check_badarg(catch dets:delete_object(no_table, {1,2,3}),
dets, delete_object, [no_table,{1,2,3}]),
check_badarg(catch dets:delete_object(T, {1,2}),
dets, delete_object, [T,{1,2}]),
check_badarg(catch dets:delete_object(no_table, [{1,2,3}]),
dets, delete_object, [no_table,[{1,2,3}]]),
check_badarg(catch dets:delete_object(T, [{1,2}]),
dets, delete_object, [T,[{1,2}]]),
check_badarg(catch dets:delete_object(T, [{1,2,3} | 3]),
dets, delete_object, [T,[{1,2,3}|3]]),
%% first,next,slot
check_badarg(catch dets:first(no_table),
dets, first, [no_table]),
check_badarg(catch dets:next(no_table, 1),
dets, next, [no_table,1]),
check_badarg(catch dets:slot(no_table, 0),
dets, slot, [no_table,0]),
%% info
undefined = dets:info(no_table),
undefined = dets:info(no_table, foo),
undefined = dets:info(T, foo),
%% match_delete
check_badarg(catch dets:match_delete(no_table, '_'),
dets, match_delete, [no_table,'_']),
%% delete_all_objects
check_badarg(catch dets:delete_all_objects(no_table),
dets, delete_all_objects, [no_table]),
%% select_delete
MSpec = [{'_',[],['$_']}],
check_badarg(catch dets:select_delete(no_table, MSpec),
dets, select_delete, [no_table,MSpec]),
check_badarg(catch dets:select_delete(T, <<17>>),
dets, select_delete, [T, <<17>>]),
%% traverse, fold
TF = fun(_) -> continue end,
check_badarg(catch dets:traverse(no_table, TF),
dets, traverse, [no_table,TF]),
FF = fun(_, A) -> A end,
check_badarg(catch dets:foldl(FF, [], no_table),
dets, foldl, [FF,[],no_table]),
check_badarg(catch dets:foldr(FF, [], no_table),
dets, foldl, [FF,[],no_table]),
%% close
ok = dets:close(T),
{error, not_owner} = dets:close(T),
{error, not_owner} = dets:close(T),
%% init_table
IF = fun(X) -> X end,
check_badarg(catch dets:init_table(no_table, IF),
dets, init_table, [no_table,IF,[]]),
check_badarg(catch dets:init_table(no_table, IF, []),
dets, init_table, [no_table,IF,[]]),
%% from_ets
Ets = ets:new(ets,[]),
check_badarg(catch dets:from_ets(no_table, Ets),
dets, from_ets, [no_table,Ets]),
ets:delete(Ets),
{ok, T} = dets:open_file(T, Args),
{error,incompatible_arguments} =
dets:open_file(T, [{type,bag} | Args]),
ok = dets:close(T),
file:delete(Fname),
check_pps(P0),
ok.
%% Test the write cache for sets.
cache_sets_v8(Config) when is_list(Config) ->
cache_sets(Config, 8).
%% Test the write cache for sets.
cache_sets_v9(Config) when is_list(Config) ->
cache_sets(Config, 9).
cache_sets(Config, Version) ->
Small = 2,
cache_sets(Config, {0,0}, false, Small, Version),
cache_sets(Config, {0,0}, true, Small, Version),
cache_sets(Config, {5000,5000}, false, Small, Version),
cache_sets(Config, {5000,5000}, true, Small, Version),
%% Objects of size greater than 2 kB.
Big = 1200,
cache_sets(Config, {0,0}, false, Big, Version),
cache_sets(Config, {0,0}, true, Big, Version),
cache_sets(Config, {5000,5000}, false, Big, Version),
cache_sets(Config, {5000,5000}, true, Big, Version),
ok.
cache_sets(Config, DelayedWrite, Extra, Sz, Version) ->
%% Extra = bool(). Insert tuples until the tested key is not alone.
%% Sz = integer(). Size of the inserted tuples.
T = cache,
Fname = filename(cache, Config),
file:delete(Fname),
P0 = pps(),
{ok, _} =
dets:open_file(T,[{version, Version}, {file,Fname}, {type,set},
{delayed_write, DelayedWrite}]),
Dups = 1,
{Key, OtherKeys} =
if
Extra ->
%% Insert enough to get three keys in some slot.
dets:safe_fixtable(T, true),
insert_objs(T, 1, Sz, Dups);
true ->
{1,[]}
end,
Tuple = erlang:make_tuple(Sz, Key),
ok = dets:delete(T, Key),
ok = dets:sync(T),
%% The values of keys in the same slot as Key are checked.
OtherValues = sort(lookup_keys(T, OtherKeys)),
ok = dets:insert(T, Tuple),
[Tuple] = dets:lookup(T, Key),
true = dets:member(T, Key),
ok = dets:insert(T, [Tuple,Tuple]),
%% If no delay, the cache gets filled immediately, and written.
[Tuple] = dets:lookup_keys(T, [Key,a,b,c,d,e,f]),
true = dets:member(T, Key),
%% If delay, this happens without file access.
ok = dets:delete(T,Key),
ok = dets:insert(T,Tuple),
ok = dets:insert(T,Tuple),
[Tuple] = dets:lookup(T, Key),
true = dets:member(T, Key),
ok = dets:sync(T),
[Tuple] = dets:lookup(T, Key),
true = dets:member(T, Key),
%% Key's objects are is on file only,
%% key 'toto' in the cache (if there is one).
ok = dets:delete(T,toto),
ok = dets:insert(T,[{toto,b},{toto,b}]),
true = sort([Tuple,{toto,b}]) =:=
sort(dets:lookup_keys(T, [Key,toto])),
true = dets:member(T, toto),
ok = dets:delete(T, Key),
ok = dets:sync(T),
false = dets:member(T, Key),
Size = dets:info(T, size),
%% No object with the key on the file.
%% Delete, add one object.
Size1 = Size + 2,
del_and_ins(key, T, Size1, Tuple, Key, 1),
del_and_ins(object, T, Size1, Tuple, Key, 1),
del_and_ins(both, T, Size1, Tuple, Key, 1),
%% One object with the key on the file.
%% Delete it, add one object.
Size2 = Size + 2,
del_and_ins(key, T, Size2, Tuple, Key, 1),
del_and_ins(object, T, Size2, Tuple, Key, 1),
del_and_ins(both, T, Size2, Tuple, Key, 1),
%% Overwrite an old objekt with exactly the same size.
Element = case element(2, Tuple) of
255 -> 254;
E -> E + 1
end,
Tuple2 = setelement(2, Tuple, Element),
ok = dets:sync(T),
ok = dets:insert(T, Tuple2),
[Tuple2] = dets:lookup(T, Key),
true = dets:member(T, Key),
ok = dets:sync(T),
[Tuple2] = dets:lookup(T, Key),
true = dets:member(T, Key),
ok = dets:insert(T, {3,a}),
ok = dets:insert(T, {3,b}),
ok = dets:delete_object(T, {3,c}),
ok = dets:delete_object(T, {3,d}),
[{3,b}] = dets:lookup(T, 3),
ok = dets:delete(T, 3),
ok = dets:delete_object(T, {3,c}),
ok = dets:delete_object(T, {3,d}),
[] = dets:lookup(T, 3),
OtherValues = sort(lookup_keys(T, OtherKeys)),
if
Extra ->
%% Let the table grow a while, if it needs to.
All1 = get_all_objects(T),
dets:safe_fixtable(T, false),
timer:sleep(1000),
OtherValues = sort(lookup_keys(T, OtherKeys)),
dets:safe_fixtable(T, true),
All2 = get_all_objects(T),
FAll2 = get_all_objects_fast(T),
true = sort(All2) =:= sort(FAll2),
case symdiff(All1, All2) of
{[],[]} -> ok;
{X,Y} ->
NoBad = length(X) + length(Y),
ct:fail({sets,DelayedWrite,Extra,Sz,NoBad})
end;
true ->
ok
end,
ok = dets:close(T),
file:delete(Fname),
check_pps(P0),
ok.
%% Test the write cache for bags.
cache_bags_v8(Config) when is_list(Config) ->
cache_bags(Config, 8).
%% Test the write cache for bags.
cache_bags_v9(Config) when is_list(Config) ->
cache_bags(Config, 9).
cache_bags(Config, Version) ->
Small = 2,
cache_bags(Config, {0,0}, false, Small, Version),
cache_bags(Config, {0,0}, true, Small, Version),
cache_bags(Config, {5000,5000}, false, Small, Version),
cache_bags(Config, {5000,5000}, true, Small, Version),
%% Objects of size greater than 2 kB.
Big = 1200,
cache_bags(Config, {0,0}, false, Big, Version),
cache_bags(Config, {0,0}, true, Big, Version),
cache_bags(Config, {5000,5000}, false, Big, Version),
cache_bags(Config, {5000,5000}, true, Big, Version),
ok.
cache_bags(Config, DelayedWrite, Extra, Sz, Version) ->
%% Extra = bool(). Insert tuples until the tested key is not alone.
%% Sz = integer(). Size of the inserted tuples.
T = cache,
Fname = filename(cache, Config),
file:delete(Fname),
P0 = pps(),
{ok, _} =
dets:open_file(T,[{version, Version}, {file,Fname}, {type,bag},
{delayed_write, DelayedWrite}]),
Dups = 1,
{Key, OtherKeys} =
if
Extra ->
%% Insert enough to get three keys in some slot.
dets:safe_fixtable(T, true),
insert_objs(T, 1, Sz, Dups);
true ->
{1,[]}
end,
Tuple = erlang:make_tuple(Sz, Key),
ok = dets:delete(T, Key),
ok = dets:sync(T),
%% The values of keys in the same slot as Key are checked.
OtherValues = sort(lookup_keys(T, OtherKeys)),
ok = dets:insert(T, Tuple),
[Tuple] = dets:lookup(T, Key),
true = dets:member(T, Key),
ok = dets:insert(T, [Tuple,Tuple]),
%% If no delay, the cache gets filled immediately, and written.
[Tuple] = dets:lookup_keys(T, [Key,a,b,c,d,e,f]),
true = dets:member(T, Key),
%% If delay, this happens without file access.
%% (This is no longer true; cache lookup has been simplified.)
ok = dets:delete(T,Key),
ok = dets:insert(T,Tuple),
ok = dets:insert(T,Tuple),
[Tuple] = dets:lookup(T, Key),
true = dets:member(T, Key),
ok = dets:sync(T),
[Tuple] = dets:lookup(T, Key),
true = dets:member(T, Key),
%% Key's objects are is on file only,
%% key toto in the cache (if there is one).
ok = dets:delete(T,toto),
false = dets:member(T, toto),
ok = dets:insert(T,[{toto,b},{toto,b}]),
true = sort([Tuple,{toto,b}]) =:=
sort(dets:lookup_keys(T, [Key,toto])),
true = dets:member(T, toto),
ok = dets:delete(T, Key),
ok = dets:sync(T),
Size = dets:info(T, size),
%% No object with the key on the file.
%% Delete, add one object.
Size1 = Size + 2,
del_and_ins(key, T, Size1, Tuple, Key, 1),
del_and_ins(object, T, Size1, Tuple, Key, 1),
del_and_ins(both, T, Size1, Tuple, Key, 1),
%% One object with the key on the file.
%% Delete it, add one object.
Size2 = Size + 2,
del_and_ins(key, T, Size2, Tuple, Key, 1),
del_and_ins(object, T, Size2, Tuple, Key, 1),
del_and_ins(both, T, Size2, Tuple, Key, 1),
%% Overwrite an objekt on file with the same object.
ok = dets:insert(T, Tuple),
ok = dets:sync(T),
[Tuple2] = dets:lookup(T, Key),
true = dets:member(T, Key),
ok = dets:insert(T, Tuple),
ok = dets:sync(T),
[Tuple2] = dets:lookup(T, Key),
true = dets:member(T, Key),
%% A mix of insert and delete.
ok = dets:delete(T, Key),
ok = dets:sync(T),
ok = dets:delete(T, Key),
ok = dets:insert(T, {Key,foo}),
ok = dets:insert(T, {Key,bar}),
[{Key,bar},{Key,foo}] = sort(dets:lookup(T, Key)),
true = dets:member(T, Key),
ok = dets:delete_object(T, {Key,foo}),
ok = dets:insert(T, {Key,kar}),
[{Key,bar},{Key,kar}] = sort(dets:lookup(T, Key)),
true = dets:member(T, Key),
ok = dets:insert(T, [{Key,kar},{Key,kar}]),
[{Key,bar},{Key,kar}] = sort(dets:lookup(T, Key)),
true = dets:member(T, Key),
ok = dets:delete_object(T, {Key,bar}),
ok = dets:delete_object(T, {Key,kar}),
[] = dets:lookup(T, Key),
false = dets:member(T, Key),
ok = dets:sync(T),
[] = dets:lookup(T, Key),
false = dets:member(T, Key),
OtherValues = sort(lookup_keys(T, OtherKeys)),
if
Extra ->
%% Let the table grow for a while, if it needs to.
All1 = get_all_objects(T),
dets:safe_fixtable(T, false),
timer:sleep(1200),
OtherValues = sort(lookup_keys(T, OtherKeys)),
dets:safe_fixtable(T, true),
All2 = get_all_objects(T),
FAll2 = get_all_objects_fast(T),
true = sort(All2) =:= sort(FAll2),
case symdiff(All1, All2) of
{[],[]} -> ok;
{X,Y} ->
NoBad = length(X) + length(Y),
ct:fail({bags,DelayedWrite,Extra,Sz,NoBad})
end;
true ->
ok
end,
ok = dets:close(T),
file:delete(Fname),
%% Second object of a key added and looked up simultaneously.
R1 = {index_test,1,2,3,4},
R2 = {index_test,2,2,13,14},
R3 = {index_test,1,12,13,14},
{ok, _} = dets:open_file(T,[{version,Version},{type,bag},
{keypos,2},{file,Fname}]),
ok = dets:insert(T,R1),
ok = dets:sync(T),
ok = dets:insert(T,R2),
ok = dets:sync(T),
ok = dets:insert(T,R3),
[R1,R3] = sort(dets:lookup(T,1)),
true = dets:member(T, 1),
[R1,R3] = sort(dets:lookup(T,1)),
true = dets:member(T, 1),
ok = dets:close(T),
file:delete(Fname),
check_pps(P0),
ok.
%% Test the write cache for duplicate bags.
cache_duplicate_bags_v8(Config) when is_list(Config) ->
cache_duplicate_bags(Config, 8).
%% Test the write cache for duplicate bags.
cache_duplicate_bags_v9(Config) when is_list(Config) ->
cache_duplicate_bags(Config, 9).
cache_duplicate_bags(Config, Version) ->
Small = 2,
cache_dup_bags(Config, {0,0}, false, Small, Version),
cache_dup_bags(Config, {0,0}, true, Small, Version),
cache_dup_bags(Config, {5000,5000}, false, Small, Version),
cache_dup_bags(Config, {5000,5000}, true, Small, Version),
%% Objects of size greater than 2 kB.
Big = 1200,
cache_dup_bags(Config, {0,0}, false, Big, Version),
cache_dup_bags(Config, {0,0}, true, Big, Version),
cache_dup_bags(Config, {5000,5000}, false, Big, Version),
cache_dup_bags(Config, {5000,5000}, true, Big, Version).
cache_dup_bags(Config, DelayedWrite, Extra, Sz, Version) ->
%% Extra = bool(). Insert tuples until the tested key is not alone.
%% Sz = integer(). Size of the inserted tuples.
T = cache,
Fname = filename(cache, Config),
file:delete(Fname),
P0 = pps(),
{ok, _} =
dets:open_file(T,[{version, Version}, {file,Fname},
{type,duplicate_bag},
{delayed_write, DelayedWrite}]),
Dups = 2,
{Key, OtherKeys} =
if
Extra ->
%% Insert enough to get three keys in some slot.
dets:safe_fixtable(T, true),
insert_objs(T, 1, Sz, Dups);
true ->
{1,[]}
end,
Tuple = erlang:make_tuple(Sz, Key),
ok = dets:delete(T, Key),
ok = dets:sync(T),
false = dets:member(T, Key),
%% The values of keys in the same slot as Key are checked.
OtherValues = sort(lookup_keys(T, OtherKeys)),
ok = dets:insert(T, Tuple),
[Tuple] = dets:lookup(T, Key),
true = dets:member(T, Key),
ok = dets:insert(T, [Tuple,Tuple]),
%% If no delay, the cache gets filled immediately, and written.
[Tuple,Tuple,Tuple] = dets:lookup_keys(T, [Key,a,b,c,d,e,f]),
true = dets:member(T, Key),
%% If delay, this happens without file access.
%% (This is no longer true; cache lookup has been simplified.)
ok = dets:delete(T,Key),
ok = dets:insert(T,Tuple),
ok = dets:insert(T,Tuple),
[Tuple,Tuple] = dets:lookup(T, Key),
true = dets:member(T, Key),
ok = dets:sync(T),
[Tuple,Tuple] = dets:lookup(T, Key),
true = dets:member(T, Key),
%% One object in the cache, one on the file.
ok = dets:delete(T,Key),
ok = dets:insert(T,Tuple),
ok = dets:sync(T),
ok = dets:insert(T,Tuple),
true = dets:member(T, Key), % should not read the file, but it does..
%% Key's objects are is on file only,
%% key toto in the cache (if there is one).
ok = dets:delete(T,toto),
ok = dets:insert(T,[{toto,b},{toto,b}]),
true = sort([Tuple,Tuple,{toto,b},{toto,b}]) =:=
sort(dets:lookup_keys(T, [Key,toto])),
true = dets:member(T, toto),
Size = dets:info(T, size),
%% Two objects with the same key on the file.
%% Delete them, add two objects.
del_and_ins(key, T, Size, Tuple, Key, 2),
del_and_ins(object, T, Size, Tuple, Key, 2),
del_and_ins(both, T, Size, Tuple, Key, 2),
%% Two objects with the same key on the file.
%% Delete them, add three objects.
del_and_ins(key, T, Size, Tuple, Key, 3),
del_and_ins(object, T, Size, Tuple, Key, 3),
del_and_ins(both, T, Size, Tuple, Key, 3),
%% Two objects with the same key on the file.
%% Delete them, add one object.
del_and_ins(key, T, Size, Tuple, Key, 1),
del_and_ins(object, T, Size, Tuple, Key, 1),
del_and_ins(both, T, Size, Tuple, Key, 1),
OtherValues = sort(lookup_keys(T, OtherKeys)),
if
Extra ->
%% Let the table grow for a while, if it needs to.
All1 = get_all_objects(T),
dets:safe_fixtable(T, false),
timer:sleep(1200),
OtherValues = sort(lookup_keys(T, OtherKeys)),
dets:safe_fixtable(T, true),
All2 = get_all_objects(T),
FAll2 = get_all_objects_fast(T),
true = sort(All2) =:= sort(FAll2),
case symdiff(All1, All2) of
{[],[]} -> ok;
{X,Y} ->
NoBad = length(X) + length(Y),
ct:fail({dup_bags,DelayedWrite,Extra,Sz,NoBad})
end;
true ->
ok
end,
ok = dets:close(T),
file:delete(Fname),
check_pps(P0),
ok.
lookup_keys(_T, []) ->
[];
lookup_keys(T, Keys) ->
dets:lookup_keys(T, Keys).
del_and_ins(W, T, Size, Obj, Key, N) ->
case W of
object ->
ok = dets:delete_object(T, Obj);
key ->
ok = dets:delete(T, Key);
both ->
ok = dets:delete(T, Key),
ok = dets:delete_object(T, Obj)
end,
Objs = duplicate(N, Obj),
[] = dets:lookup(T, Key),
ok = dets:insert(T, Objs),
Objs = dets:lookup_keys(T, [snurrespratt,Key]),
true = Size + length(Objs)-2 =:= dets:info(T, size),
Objs = dets:lookup(T, Key).
insert_objs(T, N, Sz, Dups) ->
Seq = seq(N,N+255),
L0 = map(fun(I) -> erlang:make_tuple(Sz, I) end, Seq),
L = append(duplicate(Dups, L0)),
ok = dets:insert(T, L),
case search_slot(T, 0) of
false ->
insert_objs(T, N+256, Sz, Dups);
Keys ->
Keys
end.
search_slot(T, I) ->
case dets:slot(T, I) of
'$end_of_table' ->
false;
Objs ->
case usort(map(fun(X) -> element(1, X) end, Objs)) of
[_, Key, _ | _] = Keys0 ->
Keys = delete(Key, Keys0),
{Key, Keys};
_ ->
search_slot(T, I+1)
end
end.
symdiff(L1, L2) ->
{X, _, Y} =
sofs:symmetric_partition(sofs:set(L1), sofs:set(L2)),
{sofs:to_external(X), sofs:to_external(Y)}.
%% Test read-only tables and traversal caused crashes.
otp_4208(Config) when is_list(Config) ->
Tab = otp_4208,
FName = filename(Tab, Config),
Expected = sort([{3,ghi,12},{1,abc,10},{4,jkl,13},{2,def,11}]),
file:delete(FName),
{ok, Tab} = dets:open_file(Tab, [{file,FName}]),
ok = dets:insert(Tab, [{1,abc,10},{2,def,11},{3,ghi,12},{4,jkl,13}]),
Expected = sort(dets:traverse(Tab, fun(X) -> {continue, X} end)),
ok = dets:close(Tab),
{ok, Tab} = dets:open_file(Tab, [{access, read},{file,FName}]),
Expected = sort(dets:traverse(Tab, fun(X) -> {continue, X} end)),
ok = dets:close(Tab),
file:delete(FName),
ok.
%% Test read-only tables and growth.
otp_4989(Config) when is_list(Config) ->
Tab = otp_4989,
FName = filename(Tab, Config),
%% Do exactly as in the error report.
_Ets = ets:new(Tab, [named_table]),
ets_init(Tab, 100000),
{ok, Tab} =
dets:open_file(Tab, [{access, read_write}, {file,FName}, {keypos,2}]),
ok = dets:from_ets(Tab, Tab),
ok = dets:close(Tab),
%% Restore.
{ok, Tab} =
dets:open_file(Tab, [{access, read}, {keypos, 2}, {file, FName}]),
true = ets:delete_all_objects(Tab),
true = ets:from_dets(Tab, Tab),
ok = dets:close(Tab),
ets:delete(Tab),
file:delete(FName),
ok.
ets_init(_Tab, 0) ->
ok;
ets_init(Tab, N) ->
ets:insert(Tab, {N,N}),
ets_init(Tab, N - 1).
%% OTP-8898. Truncated Dets file.
otp_8898(Config) when is_list(Config) ->
Tab = otp_8898,
FName = filename(Tab, Config),
Server = self(),
file:delete(FName),
{ok, _} = dets:open_file(Tab,[{file, FName}]),
[P1,P2,P3] = new_clients(3, Tab),
Seq = [{P1,[sync]},{P2,[{lookup,1,[]}]},{P3,[{insert,{1,b}}]}],
atomic_requests(Server, Tab, [[]], Seq),
true = get_replies([{P1,ok},{P2,ok},{P3,ok}]),
ok = dets:close(Tab),
{ok, _} = dets:open_file(Tab,[{file, FName}]),
file:delete(FName),
ok.
%% OTP-8899. Several clients. Updated Head was ignored.
otp_8899(Config) when is_list(Config) ->
Tab = many_clients,
FName = filename(Tab, Config),
Server = self(),
file:delete(FName),
{ok, _} = dets:open_file(Tab,[{file, FName},{version,9}]),
[P1,P2,P3,P4] = new_clients(4, Tab),
MC = [Tab],
Seq6a = [{P1,[{insert,[{used_to_be_skipped_by,match}]},
{lookup,1,[{1,a}]}]},
{P2,[{verbose,true,MC}]},
{P3,[{lookup,1,[{1,a}]}]}, {P4,[{verbose,true,MC}]}],
atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq6a),
true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
[{1,a},{2,b},{3,c},{used_to_be_skipped_by,match}] =
lists:sort(dets:match_object(Tab, '_')),
_ = dets:close(Tab),
file:delete(FName),
ok.
%% Test several clients accessing a table simultaneously.
many_clients(Config) when is_list(Config) ->
Tab = many_clients,
FName = filename(Tab, Config),
Server = self(),
file:delete(FName),
P0 = pps(),
{ok, _} = dets:open_file(Tab,[{file, FName},{version,9}]),
[P1,P2,P3,P4] = new_clients(4, Tab),
%% dets:init_table/2 is used for making sure that all processes
%% start sending requests before the Dets process begins to handle
%% them; the requests arrive "in parallel".
%% Four processes accessing the same table at almost the same time.
%% One key is read, updated, and read again.
Seq1 = [{P1,[{lookup,1,[{1,a}]}]}, {P2,[{insert,{1,b}}]},
{P3,[{lookup,1,[{1,b}]}]}, {P4,[{lookup,1,[{1,b}]}]}],
atomic_requests(Server, Tab, [[{1,a}]], Seq1),
true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
%% Different keys read by different processes
Seq2 = [{P1,[{member,1,true}]}, {P2,[{lookup,2,[{2,b}]}]},
{P3,[{lookup,1,[{1,a}]}]}, {P4,[{lookup,3,[{3,c}]}]}],
atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq2),
true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
%% Reading deleted key.
Seq3 = [{P1,[{delete_key,2}]}, {P2,[{lookup,1,[{1,a}]}]},
{P3,[{lookup,1,[{1,a}]}]}, {P4,[{member,2,false}]}],
atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq3),
true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
%% Inserting objects.
Seq4 = [{P1,[{insert,[{1,a},{2,b}]}]}, {P2,[{insert,[{2,c},{3,a}]}]},
{P3,[{insert,[{3,b},{4,d}]}]},
{P4,[{lookup_keys,[1,2,3,4],[{1,a},{2,c},{3,b},{4,d}]}]}],
atomic_requests(Server, Tab, [], Seq4),
true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
%% Deleting objects.
Seq5 = [{P1,[{delete_object,{1,a}}]}, {P2,[{delete_object,{1,a}}]},
{P3,[{delete_object,{3,c}}]},
{P4,[{lookup_keys,[1,2,3,4],[{2,b}]}]}],
atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq5),
true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
%% Some request not streamed.
Seq6 = [{P1,[{lookup,1,[{1,a}]}]}, {P2,[{info,size,3}]},
{P3,[{lookup,1,[{1,a}]}]}, {P4,[{info,size,3}]}],
atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq6),
true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
%% Some request not streamed.
Seq7 = [{P1,[{insert,[{3,a}]}]}, {P2,[{insert,[{3,b}]}]},
{P3,[{delete_object,{3,c}}]},
{P4,[{lookup,3,[{3,b}]}]}],
atomic_requests(Server, Tab, [[{3,c}]], Seq7),
true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
put_requests(Server, [{P1,stop},{P2,stop},{P3,stop},{P4,stop}]),
ok = dets:close(Tab),
file:delete(FName),
%% Check that errors are handled correctly by the streaming operators.
{ok, _} = dets:open_file(Tab,[{file, FName},{version,9}]),
ok = ins(Tab, 100),
Obj = {66,{item,number,66}},
{ok, ObjPos} = dets:where(Tab, Obj),
ok = dets:close(Tab),
%% Damaged object.
crash(FName, ObjPos+12),
{ok, _} = dets:open_file(Tab,[{file, FName},{version,9}]),
BadObject1 = dets:lookup_keys(Tab, [65,66,67,68,69]),
bad_object(BadObject1, FName),
_Error = dets:close(Tab),
file:delete(FName),
check_pps(P0),
ok.
%% Tab is initiated with the objects in Objs. Objs = [[object()]].
atomic_requests(Server, Tab, Objs, Req) ->
ok = dets:init_table(Tab, atomic_requests(Server, Objs, Req)).
atomic_requests(Server, L, Req) ->
fun(close) ->
ok;
(read) when [] =:= L ->
put_requests(Server, Req),
end_of_input;
(read) ->
[E | Es] = L,
{E, atomic_requests(Server, Es, Req)}
end.
put_requests(Server, L) ->
lists:foreach(fun({Pid,R}) -> Pid ! {Server,R}, timer:sleep(1) end, L).
get_replies(L) ->
lists:all(fun({Pid,Reply}) -> Reply =:= get_reply(Pid) end, L).
get_reply(Pid) ->
receive {Pid, Reply} -> Reply end.
new_clients(0, _Tab) ->
[];
new_clients(N, Tab) ->
[new_client(Tab) | new_clients(N-1, Tab)].
new_client(Tab) ->
spawn(?MODULE, client, [self(), Tab]).
client(S, Tab) ->
receive
{S, stop} ->
exit(normal);
{S, ToDo} ->
Reply = eval(ToDo, Tab),
case Reply of
{error, _} -> io:format("~p: ~p~n", [self(), Reply]);
_ -> ok
end,
S ! {self(), Reply}
end,
client(S, Tab).
eval([], _Tab) ->
ok;
eval([{verbose,Bool,Expected} | L], Tab) ->
case dets:verbose(Bool) of
Expected -> eval(L, Tab);
Error -> {error, {verbose,Error}}
end;
eval([sync | L], Tab) ->
case dets:sync(Tab) of
ok -> eval(L, Tab);
Error -> {error, {sync,Error}}
end;
eval([{insert,Stuff} | L], Tab) ->
case dets:insert(Tab, Stuff) of
ok -> eval(L, Tab);
Error -> {error, {insert,Stuff,Error}}
end;
eval([{lookup,Key,Expected} | L], Tab) ->
case dets:lookup(Tab, Key) of
Expected -> eval(L, Tab);
Else -> {error, {lookup,Key,Expected,Else}}
end;
eval([{lookup_keys,Keys,Expected} | L], Tab) ->
%% Time order is destroyed...
case dets:lookup_keys(Tab, Keys) of
R when is_list(R) ->
case lists:sort(Expected) =:= lists:sort(R) of
true -> eval(L, Tab);
false -> {error, {lookup_keys,Keys,Expected,R}}
end;
Else -> {error, {lookup_keys,Keys,Expected,Else}}
end;
eval([{member,Key,Expected} | L], Tab) ->
case dets:member(Tab, Key) of
Expected -> eval(L, Tab);
Else -> {error, {member,Key,Expected,Else}}
end;
eval([{delete_key,Key} | L], Tab) ->
case dets:delete(Tab, Key) of
ok -> eval(L, Tab);
Else -> {error, {delete_key,Key,Else}}
end;
eval([{delete_object,Object} | L], Tab) ->
case dets:delete_object(Tab, Object) of
ok -> eval(L, Tab);
Else -> {error, {delete_object,Object,Else}}
end;
eval([{info,Tag,Expected} | L], Tab) ->
case dets:info(Tab, Tag) of
Expected -> eval(L, Tab);
Else -> {error, {info,Tag,Else,Expected}}
end;
eval(Else, _Tab) ->
{error, {bad_request,Else}}.
%% More than 128k keys caused crash.
otp_4906(Config) when is_list(Config) ->
N = 256*512 + 400,
Tab = otp_4906,
FName = filename(Tab, Config),
file:delete(FName),
{ok, Tab} = dets:open_file(Tab, [{file, FName}]),
ok = ins_small(Tab, 0, N),
ok = dets:close(Tab),
{ok, Tab} = dets:open_file(Tab, [{file, FName}]),
ok = read_4906(Tab, N-1),
ok = dets:close(Tab),
file:delete(FName),
%% If the (only) process fixing a table updates the table, the
%% process will no longer be punished with a 1 ms delay (hm, the
%% server is delayed, it should be the client...). In this example
%% the writing process *is* delayed.
{ok,Tab} = dets:open_file(Tab, [{file,FName}]),
Parent = self(),
FixPid = spawn_link(fun() ->
dets:safe_fixtable(Tab, true),
receive {Parent, stop} -> ok end
end),
ok = ins_small(Tab, 0, 1000),
FixPid ! {Parent, stop},
timer:sleep(1),
ok = dets:close(Tab),
file:delete(FName),
ok.
read_4906(_T, N) when N < 0 ->
ok;
read_4906(T, N) ->
[_] = dets:lookup(T, N),
read_4906(T, N-1).
ins_small(_T, I, N) when I =:= N ->
ok;
ins_small(T, I, N) ->
ok = dets:insert(T, {I}),
ins_small(T, I+1, N).
%% Unwritable ramfile caused crash.
otp_5402(Config) when is_list(Config) ->
Tab = otp_5402,
File = filename:join(["cannot", "write", "this", "file"]),
%% close
{ok, T} = dets:open_file(Tab, [{ram_file,true},
{file, File}]),
ok = dets:insert(T, {1,a}),
{error,{file_error,_,_}} = dets:close(T),
%% sync
{ok, T} = dets:open_file(Tab, [{ram_file,true},
{file, File}]),
ok = dets:insert(T, {1,a}),
{error,{file_error,_,_}} = dets:sync(T),
{error,{file_error,_,_}} = dets:close(T),
%% auto_save
{ok, T} = dets:open_file(Tab, [{ram_file,true},
{auto_save, 2000},
{file, File}]),
ok = dets:insert(T, {1,a}),
timer:sleep(5000),
{error,{file_error,_,_}} = dets:close(T),
ok.
%% Several clients open and close tables simultaneously.
simultaneous_open(Config) ->
Tab = sim_open,
File = filename(Tab, Config),
ok = monit(Tab, File),
case feasible() of
false -> {comment, "OK, but did not run all of the test"};
true ->
ok = kill_while_repairing(Tab, File),
ok = kill_while_init(Tab, File),
ok = open_ro(Tab, File),
ok = open_w(Tab, File, 0, Config),
ok = open_w(Tab, File, 100, Config)
end.
feasible() ->
LP = erlang:system_info(logical_processors),
(is_integer(LP)
andalso LP >= erlang:system_info(schedulers_online)
andalso not erlang:system_info(debug_compiled)
andalso not erlang:system_info(lock_checking)).
%% One process logs and another process closes the log. Before
%% monitors were used, this would make the client never return.
monit(Tab, File) ->
file:delete(File),
{ok, Tab} = dets:open_file(Tab, [{file,File}]),
F1 = fun() -> dets:close(Tab) end,
F2 = fun() -> {'EXIT', {badarg, _}} = do_log(Tab) end,
spawn(F2),
timer:sleep(100),
spawn(F1),
dets:close(Tab),
ok = file:delete(File).
do_log(Tab) ->
case catch dets:insert(Tab, {hej,san,sa}) of
ok -> do_log(Tab);
Else -> Else
end.
%% Kill the Dets process while repair is in progress.
kill_while_repairing(Tab, File) ->
create_opened_log(File),
Delay = 1000,
dets:start(),
Parent = self(),
F = fun() ->
R = (catch dets:open_file(Tab, [{file,File}])),
timer:sleep(Delay),
Parent ! {self(), R}
end,
%% One of these will open the file, the other will be pending
%% until the file has been repaired:
P1 = spawn(F),
P2 = spawn(F),
P3 = spawn(F),
DetsPid = find_dets_pid(),
exit(DetsPid, kill),
receive {P1,R1} -> R1 end,
receive {P2,R2} -> R2 end,
receive {P3,R3} -> R3 end,
io:format("Killed pid: ~p~n", [DetsPid]),
io:format("Remaining Dets-pids (should be nil): ~p~n",
[find_dets_pids()]),
{replies,[{'EXIT', {dets_process_died, _}}, {ok,_}, {ok, _}]} =
{replies,lists:sort([R1, R2, R3])},
timer:sleep(200),
case dets:info(Tab) of
undefined ->
ok;
_Info ->
timer:sleep(5000),
undefined = dets:info(Tab)
end,
file:delete(File),
ok.
find_dets_pid() ->
case find_dets_pids() of
[] ->
timer:sleep(100),
find_dets_pid();
[Pid] ->
Pid
end.
find_dets_pids() ->
lists:filter(fun(P) -> dets:pid2name(P) =/= undefined end,
erlang:processes()).
%% Kill the Dets process when there are users and an on-going
%% initiailization.
kill_while_init(Tab, File) ->
file:delete(File),
Parent = self(),
F = fun() ->
R = dets:open_file(Tab, [{file,File}]),
Parent ! {self(), R},
receive {Parent, die} -> ok end,
{error, not_owner} = dets:close(Tab)
end,
P1 = spawn(F),
P2 = spawn(F),
P3 = spawn(F),
IF = fun() ->
R = dets:open_file(Tab, [{file,File}]),
Parent ! {self(), R},
Fun = fun(_) -> timer:sleep(100000) end,
{'EXIT', {badarg, _}} = (catch dets:init_table(Tab, Fun)),
receive {Parent, die} -> ok end
end,
P4 = spawn(IF),
receive {P1,R1} -> {ok, _} = R1 end,
receive {P2,R2} -> {ok, _} = R2 end,
receive {P3,R3} -> {ok, _} = R3 end,
receive {P4,R4} -> {ok, _} = R4 end,
DetsPid = find_dets_pid(),
exit(DetsPid, kill),
timer:sleep(1000),
undefined = dets:info(Tab),
P1 ! {Parent, die},
P2 ! {Parent, die},
P3 ! {Parent, die},
P4 ! {Parent, die},
file:delete(File),
timer:sleep(100),
ok.
open_ro(Tab, File) ->
create_opened_log(File),
Delay = 1000,
Parent = self(),
F = fun() ->
R = dets:open_file(Tab, [{file,File},{access,read}]),
timer:sleep(Delay),
Parent ! {self(), R}
end,
P1 = spawn(F),
P2 = spawn(F),
P3 = spawn(F),
receive {P1,R1} -> {error,{not_closed,_}} = R1 end,
receive {P2,R2} -> {error,{not_closed,_}} = R2 end,
receive {P3,R3} -> {error,{not_closed,_}} = R3 end,
ok.
open_w(Tab, File, Delay, Config) ->
create_opened_log(File),
Tab2 = t2,
File2 = filename(Tab2, Config),
file:delete(File2),
{ok,Tab2} = dets:open_file(Tab2, [{file,File2}]),
ok = dets:close(Tab2),
Parent = self(),
F = fun() ->
R = dets:open_file(Tab, [{file,File}]),
timer:sleep(Delay),
Parent ! {self(), R}
end,
Pid1 = spawn(F),
Pid2 = spawn(F),
Pid3 = spawn(F),
ok = wait_for_repair_to_start(Tab),
%% It is assumed that it takes some time to repair the file.
{ok,Tab2} = dets:open_file(Tab2, [{file,File2}]),
%% The Dets server managed to handle to open_file request.
0 = qlen(), % still repairing
ok = dets:close(Tab2),
file:delete(File2),
receive {Pid1,R1} -> {ok, Tab} = R1 end,
receive {Pid2,R2} -> {ok, Tab} = R2 end,
receive {Pid3,R3} -> {ok, Tab} = R3 end,
timer:sleep(200),
case dets:info(Tab) of
undefined ->
ok;
_Info ->
timer:sleep(5000),
undefined = dets:info(Tab)
end,
file:delete(File),
ok.
wait_for_repair_to_start(Tab) ->
case catch dets_server:get_pid(Tab) of
{'EXIT', _} ->
timer:sleep(1),
wait_for_repair_to_start(Tab);
Pid when is_pid(Pid) ->
ok
end.
qlen() ->
{_, {_, N}} = lists:keysearch(message_queue_len, 1, process_info(self())),
N.
create_opened_log(File) ->
Tab = t,
file:delete(File),
{ok, Tab} = dets:open_file(Tab, [{file,File}]),
ok = ins(Tab, 60000),
ok = dets:close(Tab),
crash(File, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED),
ok.
%% OTP-5075. insert_new/2
insert_new(Config) ->
Tab = insert_new,
File = filename(Tab, Config),
file:delete(File),
{ok, T} = dets:open_file(Tab, [{file,File}]),
{'EXIT', {badarg, _}} = (catch dets:insert_new(Tab, 14)),
{'EXIT', {badarg, _}} = (catch dets:insert_new(Tab, {})),
true = dets:insert_new(Tab, {1,a}),
false = dets:insert_new(Tab, {1,a}),
true = dets:insert_new(Tab, [{2,b}, {3,c}]),
false = dets:insert_new(Tab, [{2,b}, {3,c}]),
false = dets:insert_new(Tab, [{1,a}, {4,d}]),
ok = dets:close(Tab),
file:delete(File),
{ok, T} = dets:open_file(Tab, [{file,File},{type,bag}]),
true = dets:insert_new(Tab, {1,a}),
false = dets:insert_new(Tab, {1,b}),
true = dets:insert_new(Tab, [{2,b}, {3,c}]),
false = dets:insert_new(Tab, [{2,a}, {3,d}]),
false = dets:insert_new(Tab, [{1,a}, {4,d}]),
ok = dets:close(Tab),
file:delete(File),
ok.
%% OTP-5126. repair_continuation/2
repair_continuation(Config) ->
Tab = repair_continuation_table,
Fname = filename(repair_cont, Config),
file:delete(Fname),
{ok, _} = dets:open_file(Tab, [{file,Fname}]),
ok = dets:insert(Tab, [{1,a},{2,b},{3,c}]),
MS = [{'_',[],[true]}],
{[true], C1} = dets:select(Tab, MS, 1),
C2 = binary_to_term(term_to_binary(C1)),
{'EXIT', {badarg, _}} = (catch dets:select(C2)),
C3 = dets:repair_continuation(C2, MS),
{[true], C4} = dets:select(C3),
C5 = dets:repair_continuation(C4, MS),
{[true], _} = dets:select(C5),
{'EXIT', {badarg, _}} = (catch dets:repair_continuation(Tab, bu)),
ok = dets:close(Tab),
file:delete(Fname),
ok.
%% OTP-5487. Growth of read-only table (again).
otp_5487(Config) ->
otp_5487(Config, 9),
otp_5487(Config, 8),
ok.
otp_5487(Config, Version) ->
Tab = otp_5487,
Fname = filename(otp_5487, Config),
file:delete(Fname),
Ets = ets:new(otp_5487, [public, set]),
lists:foreach(fun(I) -> ets:insert(Ets, {I,I+1}) end,
lists:seq(0,1000)),
{ok, _} = dets:open_file(Tab, [{file,Fname},{version,Version}]),
ok = dets:from_ets(Tab, Ets),
ok = dets:sync(Tab),
ok = dets:close(Tab),
{ok, _} = dets:open_file(Tab, [{file,Fname},{access,read}]),
[{1,2}] = dets:lookup(Tab, 1),
ok = dets:close(Tab),
ets:delete(Ets),
file:delete(Fname).
%% OTP-6206. Badly formed free lists.
otp_6206(Config) ->
Tab = otp_6206,
File = filename(Tab, Config),
file:delete(File),
Options = [{file,File}],
{ok, Tab} = dets:open_file(Tab, Options),
NObjs = 13006,
ok = ins(Tab, NObjs),
ok = del(Tab, NObjs, 2),
ok = dets:close(Tab),
%% Used to return {badmatch,{error,{bad_freelists,File}}.
{ok, Tab} = dets:open_file(Tab, [{repair,false}|Options]),
ok = dets:close(Tab),
file:delete(File),
ok.
%% OTP-6359. select and match never return the empty list.
otp_6359(Config) ->
Tab = otp_6359,
File = filename(Tab, Config),
file:delete(File),
{ok, _} = dets:open_file(Tab, [{file, File}]),
%% Used to return {[], Cont}:
'$end_of_table' = dets:match(Tab, '_', 100),
ok = dets:close(Tab),
file:delete(File),
ok.
%% OTP-4738. ==/2 and =:=/2.
otp_4738(Config) ->
%% Version 8 has not been corrected.
%% (The constant -12857447 is for version 9 only.)
otp_4738_set(9, Config),
otp_4738_bag(9, Config),
otp_4738_dupbag(9, Config),
ok.
otp_4738_dupbag(Version, Config) ->
Tab = otp_4738,
File = filename(Tab, Config),
file:delete(File),
I = -12857447,
F = float(I),
One = 1,
FOne = float(One),
Args = [{file,File},{type,duplicate_bag},{version,Version}],
{ok, Tab} = dets:open_file(Tab, Args),
ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]),
ok = dets:sync(Tab),
[{F,One},{F,FOne}] = dets:lookup(Tab, F),
[{I,One},{I,FOne}] = dets:lookup(Tab, I),
ok = dets:insert(Tab, [{F,One},{F,FOne}]),
[{I,One},{I,FOne},{F,One},{F,FOne},{F,One},{F,FOne}] =
dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:insert(Tab, [{F,FOne},{F,One}]),
[{I,One},{I,FOne},{F,One},{F,FOne},{F,One},
{F,FOne},{F,FOne},{F,One}] =
dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:delete_object(Tab, {I,FOne}),
[{I,One},{F,One},{F,FOne},{F,One},{F,FOne},{F,FOne},{F,One}] =
dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:insert(Tab, {I,FOne}),
[{I,One},{I,FOne},{F,One},{F,FOne},{F,One},
{F,FOne},{F,FOne},{F,One}] =
dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:delete_object(Tab, {F,FOne}),
[{I,One},{I,FOne},{F,One},{F,One},{F,One}] =
dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:delete(Tab, F),
[{I,One},{I,FOne}] = dets:match_object(Tab, '_'),
ok = dets:close(Tab),
file:delete(File),
Zero = 0,
FZero = float(Zero),
{ok, Tab} = dets:open_file(Tab, Args),
ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]),
ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]),
ok = dets:insert(Tab, [{I,Zero},{F,Zero},{I,FZero},{I,FZero}]),
Objs0 = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:close(Tab),
crash(File, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED),
io:format("Expect repair:~n"),
{ok, Tab} = dets:open_file(Tab, Args),
Objs1 = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:close(Tab),
Objs1 = Objs0,
file:delete(File),
ok.
otp_4738_bag(Version, Config) ->
Tab = otp_4738,
File = filename(Tab, Config),
file:delete(File),
I = -12857447,
F = float(I),
One = 1,
FOne = float(One),
Args = [{file,File},{type,bag},{version,Version}],
{ok, Tab} = dets:open_file(Tab, Args),
ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]),
ok = dets:sync(Tab),
[{F,One},{F,FOne}] = dets:lookup(Tab, F),
[{I,One},{I,FOne}] = dets:lookup(Tab, I),
ok = dets:insert(Tab, [{F,One},{F,FOne}]),
[{I,One},{I,FOne},{F,One},{F,FOne}] =
dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:insert(Tab, [{F,FOne},{F,One}]),
[{I,One},{I,FOne},{F,FOne},{F,One}] =
dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:delete_object(Tab, {I,FOne}),
[{I,One},{F,FOne},{F,One}] =
dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:insert(Tab, {I,FOne}),
[{I,One},{I,FOne},{F,FOne},{F,One}] =
dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:delete(Tab, F),
[{I,One},{I,FOne}] = dets:match_object(Tab, '_'),
ok = dets:close(Tab),
file:delete(File).
otp_4738_set(Version, Config) ->
Tab = otp_4738,
File = filename(Tab, Config),
file:delete(File),
Args = [{file,File},{type,set},{version,Version}],
%% I and F share the same slot.
I = -12857447,
F = float(I),
{ok, Tab} = dets:open_file(Tab, Args),
ok = dets:insert(Tab, [{I},{F}]),
ok = dets:sync(Tab),
[{F}] = dets:lookup(Tab, F),
[{I}] = dets:lookup(Tab, I),
ok = dets:insert(Tab, [{F}]),
[{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:close(Tab),
file:delete(File),
{ok, Tab} = dets:open_file(Tab, Args),
ok = dets:insert(Tab, [{I}]),
ok = dets:sync(Tab),
[] = dets:lookup(Tab, F),
[{I}] = dets:lookup(Tab, I),
ok = dets:insert(Tab, [{F}]),
[{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:close(Tab),
file:delete(File),
{ok, Tab} = dets:open_file(Tab, Args),
ok = dets:insert(Tab, [{I},{F}]),
%% {insert, ...} in the cache, try lookup:
[{F}] = dets:lookup(Tab, F),
[{I}] = dets:lookup(Tab, I),
%% Both were found, but that cannot be verified.
[{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:close(Tab),
file:delete(File),
{ok, Tab} = dets:open_file(Tab, Args),
ok = dets:insert(Tab, [{I}]),
ok = dets:sync(Tab),
ok = dets:insert(Tab, [{F}]),
%% {insert, ...} in the cache, try lookup:
[{F}] = dets:lookup(Tab, F),
[{I}] = dets:lookup(Tab, I),
[{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:close(Tab),
file:delete(File),
{ok, Tab} = dets:open_file(Tab, Args),
%% Both operations in the cache:
ok = dets:insert(Tab, [{I}]),
ok = dets:insert(Tab, [{F}]),
[{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
ok = dets:close(Tab),
file:delete(File),
ok.
%% OTP-7146. Bugfix: missing test when re-hashing.
otp_7146(Config) ->
Tab = otp_7146,
File = filename(Tab, Config),
file:delete(File),
Max = 2048,
{ok, Tab} = dets:open_file(Tab, [{max_no_slots,Max}, {file,File}]),
write_dets(Tab, Max),
ok = dets:close(Tab),
file:delete(File),
ok.
write_dets(Tab, Max) ->
write_dets(Tab, 0, Max).
write_dets(_Tab, N, Max) when N > Max ->
ok;
write_dets(Tab, N, Max) ->
ok = dets:insert(Tab,{ N, {entry,N}}),
write_dets(Tab, N+1, Max).
%% OTP-8070. Duplicated objects with insert_new() and duplicate_bag.
otp_8070(Config) when is_list(Config) ->
Tab = otp_8070,
File = filename(Tab, Config),
file:delete(File),
{ok, _} = dets:open_file(Tab, [{file,File},{type, duplicate_bag}]),
ok = dets:insert(Tab, [{3,0}]),
false = dets:insert_new(Tab, [{3,1},{3,1}]),
[{3,0}] = dets:lookup(Tab, 3),
ok = dets:close(Tab),
file:delete(File),
ok.
%% OTP-8856. insert_new() bug.
otp_8856(Config) when is_list(Config) ->
Tab = otp_8856,
File = filename(Tab, Config),
file:delete(File),
Me = self(),
{ok, _} = dets:open_file(Tab, [{type, bag}, {file, File}]),
spawn(fun()-> Me ! {1, dets:insert(Tab, [])} end),
spawn(fun()-> Me ! {2, dets:insert_new(Tab, [])} end),
ok = dets:close(Tab),
receive {1, ok} -> ok end,
receive {2, true} -> ok end,
file:delete(File),
{ok, _} = dets:open_file(Tab, [{type, set}, {file, File}]),
spawn(fun() -> dets:delete(Tab, 0) end),
spawn(fun() -> Me ! {3, dets:insert_new(Tab, {0,0})} end),
ok = dets:close(Tab),
receive {3, true} -> ok end,
file:delete(File),
ok.
%% OTP-8903. bchunk/match/select bug.
otp_8903(Config) when is_list(Config) ->
Tab = otp_8903,
File = filename(Tab, Config),
{ok,T} = dets:open_file(bug, [{file,File}]),
ok = dets:insert(T, [{1,a},{2,b},{3,c}]),
dets:safe_fixtable(T, true),
{[_],C1} = dets:match_object(T, '_', 1),
{BC1,_D} = dets:bchunk(T, start),
ok = dets:close(T),
{'EXIT', {badarg, _}} = (catch {foo,dets:match_object(C1)}),
{'EXIT', {badarg, _}} = (catch {foo,dets:bchunk(T, BC1)}),
{ok,T} = dets:open_file(bug, [{file,File}]),
false = dets:info(T, safe_fixed),
{'EXIT', {badarg, _}} = (catch {foo,dets:match_object(C1)}),
{'EXIT', {badarg, _}} = (catch {foo,dets:bchunk(T, BC1)}),
ok = dets:close(T),
file:delete(File),
ok.
%% OTP-8923. rehash due to lookup after initialization.
otp_8923(Config) when is_list(Config) ->
Tab = otp_8923,
File = filename(Tab, Config),
%% Create a file with more than 256 keys:
file:delete(File),
Bin = list_to_binary([ 0 || _ <- lists:seq(1, 400) ]),
BigBin = list_to_binary([ 0 ||_ <- lists:seq(1, 4000)]),
Ets = ets:new(temp, [{keypos,1}]),
[ true = ets:insert(Ets, {C,Bin}) || C <- lists:seq(1, 700) ],
true = ets:insert(Ets, {helper_data,BigBin}),
true = ets:insert(Ets, {prim_btree,BigBin}),
true = ets:insert(Ets, {sec_btree,BigBin}),
%% Note: too few slots; re-hash will take place
{ok, Tab} = dets:open_file(Tab, [{file,File}]),
Tab = ets:to_dets(Ets, Tab),
ok = dets:close(Tab),
true = ets:delete(Ets),
{ok,Ref} = dets:open_file(File),
[{1,_}] = dets:lookup(Ref, 1),
ok = dets:close(Ref),
{ok,Ref2} = dets:open_file(File),
[{helper_data,_}] = dets:lookup(Ref2, helper_data),
ok = dets:close(Ref2),
file:delete(File),
ok.
%% OTP-9282. The name of a table can be an arbitrary term.
otp_9282(Config) when is_list(Config) ->
some_calls(make_ref(), Config),
some_calls({a,typical,name}, Config),
some_calls(fun() -> a_funny_name end, Config),
ok.
some_calls(Tab, Config) ->
File = filename(ref, Config),
{ok,T} = dets:open_file(Tab, [{file,File}]),
T = Tab,
false = dets:info(T, safe_fixed),
File = dets:info(T, filename),
ok = dets:insert(Tab, [{3,0}]),
[{3,0}] = dets:lookup(Tab, 3),
[{3,0}] = dets:traverse(Tab, fun(X) -> {continue, X} end),
ok = dets:close(T),
file:delete(File).
%% OTP-11245. Tables remained fixed after traversal.
otp_11245(Config) when is_list(Config) ->
Tab = otp_11245,
File = filename(Tab, Config),
{ok, Tab} = dets:open_file(Tab, [{file,File}]),
N = 1024,
ins(Tab, N),
N = length(dets:match(Tab, '_')),
false = dets:info(Tab, safe_fixed),
dets:traverse(Tab, fun(_) -> continue end),
false = dets:info(Tab, safe_fixed),
N = dets:foldl(fun(_, N2) -> N2+1 end, 0, Tab),
false = dets:info(Tab, safe_fixed),
N = dets:foldr(fun(_, N2) -> N2+1 end, 0, Tab),
false = dets:info(Tab, safe_fixed),
ok = dets:close(Tab),
file:delete(File),
ok.
% OTP-11709. Bugfixes.
otp_11709(Config) when is_list(Config) ->
Short = <<"foo">>,
Long = <<"a sufficiently long text">>,
%% Bug: leaking file descriptor
P0 = pps(),
File = filename(otp_11709, Config),
ok = file:write_file(File, Long),
false = dets:is_dets_file(File),
check_pps(P0),
%% Bug: deleting file
Args = [[{access, A}, {repair, R}] ||
A <- [read, read_write],
R <- [true, false, force]],
Fun1 = fun(S, As) ->
P1 = pps(),
ok = file:write_file(File, S),
{error,{not_a_dets_file,File}} = dets:open_file(File, As),
{ok, S} = file:read_file(File),
check_pps(P1)
end,
Fun2 = fun(S) ->
_ = [Fun1(S, As) || As <- Args],
ok
end,
ok = Fun2(Long), % no change here
ok = Fun2(Short), % mimic the behaviour for longer files
%% open_file/1
ok = file:write_file(File, Long),
{error,{not_a_dets_file,File}} = dets:open_file(File), % no change
ok = file:write_file(File, Short),
{error,{not_a_dets_file,File}} = dets:open_file(File), % mimic
_ = file:delete(File),
ok.
%% OTP-13229. open_file() exits with badarg when given binary file name.
otp_13229(_Config) ->
F = <<"binfile.tab">>,
try dets:open_file(name, [{file, F}]) of
R ->
exit({open_succeeded, R})
catch
error:badarg ->
ok
end.
%% OTP-13260. Race when opening a table.
otp_13260(Config) ->
[ok] = lists:usort([otp_13260_1(Config) || _ <- lists:seq(1, 3)]),
ok.
otp_13260_1(Config) ->
Tab = otp_13260,
File = filename(Tab, Config),
N = 20,
P = self(),
Pids = [spawn_link(fun() -> counter(P, Tab, File) end) ||
_ <- lists:seq(1, N)],
Rs = rec(Pids),
true = lists:all(fun(R) -> is_integer(R) end, Rs),
wait_for_close(Tab).
rec([]) ->
[];
rec([Pid | Pids]) ->
receive {Pid, R} ->
[R | rec(Pids)]
end.
%% One may have to run the test several times to trigger the bug.
counter(P, Tab, File) ->
Key = key,
N = case catch dets:update_counter(Tab, Key, 1) of
{'EXIT', _} ->
{ok, Tab} = dets:open_file(Tab, [{file, File}]),
ok = dets:insert(Tab, {Key, 1}),
dets:update_counter(Tab, Key, 1);
N1 when is_integer(N1) ->
N1;
DetsBug ->
DetsBug
end,
P ! {self(), N}.
wait_for_close(Tab) ->
case dets:info(Tab, owner) of
undefined ->
ok;
_ ->
timer:sleep(100),
wait_for_close(Tab)
end.
%%
%% Parts common to several test cases
%%
crash(File, Where) ->
crash(File, Where, 10).
crash(File, Where, What) when is_integer(What) ->
{ok, Fd} = file:open(File, [read,write]),
file:position(Fd, Where),
ok = file:write(Fd, [What]),
ok = file:close(Fd).
args(Config) ->
{Sets, Bags, Dups} =
{[
[],
[{type, set}, {estimated_no_objects, 300},
{ram_file, true}],
[{type, set}, {estimated_no_objects, 300}],
[{type, set}, {estimated_no_objects, 300}],
[{auto_save,20}, {type, set},
{estimated_no_objects, 300}]
],
[
[{type, bag}, {estimated_no_objects, 300}, {ram_file, true}],
[{type, bag}],
[{type, bag}, {estimated_no_objects, 300}],
[{type, bag}, {estimated_no_objects, 300}],
[{type, bag},
{auto_save,20}, {estimated_no_objects, 300}],
[{type, bag}, {estimated_no_objects, 300}, {ram_file, true}]
],
[
[{type, duplicate_bag}, {estimated_no_objects, 300},
{ram_file, true}],
[{type, duplicate_bag}],
[{type, duplicate_bag}, {estimated_no_objects, 300}],
[{type, duplicate_bag}, {estimated_no_objects, 300}],
[{type, duplicate_bag},
{auto_save,20}, {estimated_no_objects, 300}],
[{type, duplicate_bag}, {estimated_no_objects, 300},
{ram_file, true}]
]
},
zip_filename(Sets, Bags, Dups, Config).
zip_filename(S, B, D, Conf) ->
zip_filename(S, B, D, [], [], [], 1, Conf).
zip_filename([H|T], B, D, S1, B1, D1, I, Conf) ->
zip_filename(T, B, D, [[{file, new_filename(I, Conf)} | H] | S1],
B1, D1, I+1, Conf);
zip_filename([], [H|B], D, S1, B1, D1, I, Conf) ->
zip_filename([], B, D, S1, [[{file, new_filename(I, Conf)} | H] | B1],
D1, I+1, Conf);
zip_filename([], [], [H|T], S1, B1, D1, I, Conf) ->
zip_filename([], [], T, S1, B1, [[{file, new_filename(I, Conf)} | H] | D1],
I+1, Conf);
zip_filename([], [], [], S1, B1, D1, _, _Conf) ->
{reverse(S1), reverse(B1), reverse(D1)}.
del_test(Tab) ->
?format("Deltest on ~p~n", [Tab]),
Objs = safe_get_all_objects(Tab),
Keys = map(fun(X) -> element(1, X) end, Objs),
foreach(fun(Key) -> dets:delete(Tab, Key) end, Keys),
0 = length(get_all_objects(Tab)),
[] = get_all_objects_fast(Tab),
0 = dets:info(Tab, size).
del_obj_test(Tab) ->
?format("Delobjtest on ~p~n", [Tab]),
Objs = safe_get_all_objects(Tab),
LL = length(Objs),
LL = dets:info(Tab, size),
foreach(fun(Obj) -> dets:delete_object(Tab, Obj) end, Objs),
0 = length(get_all_objects(Tab)),
[] = get_all_objects_fast(Tab),
0 = dets:info(Tab, size).
match_del_test(Tab) ->
?format("Match delete test on ~p~n", [Tab]),
ok = dets:match_delete(Tab, {'_','_','_'}),
Sz = dets:info(Tab, size),
true = Sz =:= length(dets:match_object(Tab, '_')),
ok = dets:match_delete(Tab, '_'),
0 = dets:info(Tab, size),
0 = length(get_all_objects(Tab)),
[] = get_all_objects_fast(Tab).
trav_test(_Data, Len, Tab) ->
?format("Travtest on ~p~n", [Tab]),
_X0 = dets:traverse(Tab, fun(_X) -> continue end),
XX = dets:traverse(Tab, fun(X) -> {continue, X} end),
case Len =:= length(XX) of
false -> ?format("DIFF ~p~n", [XX -- _Data]);
true -> ok
end,
1 = length(dets:traverse(Tab, fun(X) -> {done, X} end)).
match_test(Data, Tab) ->
?format("Match test on ~p~n", [Tab]),
Data1 = sort(filter(fun(X) when tuple_size(X) =:= 3 -> true;
(_X) -> false
end, Data)),
Data1 = sort(dets:match_object(Tab, {'$1', '$2', '$3'})),
Len = length(Data),
Len = length(dets:match(Tab, '_')),
Len2 = length(Data1),
Len2 = length(dets:match(Tab, {'$1', '_', '_'})),
Data3 =
filter(fun(X) ->
K = element(1, X),
if
tuple_size(X) =:= 3, tuple_size(K) =:= 2 -> true;
true -> false
end
end, Data),
Len3 = length(Data3),
Len3 = length(dets:match(Tab, {{'$1', '$2'}, '_', '_'})),
Len3 = length(dets:match_object(Tab, {{'$1', '$2'}, '_', '_'})),
R = make_ref(),
dets:insert(Tab, {{R, R}, 33 ,44}),
1 = length(dets:match(Tab, {{R, R}, '_', '_'})),
1 = length(dets:match_object(Tab, {{R, R}, '_', '_'})).
%%
%% Utilities
%%
headsz(8) ->
?HEADSZ_v8;
headsz(_) ->
?HEADSZ_v9.
unwritable(Fname) ->
{ok, Info} = file:read_file_info(Fname),
Mode = Info#file_info.mode - 8#00200,
file:write_file_info(Fname, Info#file_info{mode = Mode}).
writable(Fname) ->
{ok, Info} = file:read_file_info(Fname),
Mode = Info#file_info.mode bor 8#00200,
file:write_file_info(Fname, Info#file_info{mode = Mode}).
truncate(File, Where) ->
{ok, Fd} = file:open(File, [read,write]),
file:position(Fd, Where),
ok = file:truncate(Fd),
ok = file:close(Fd).
new_filename(Name, _Config) when is_integer(Name) ->
filename:join(?privdir(_Config),
integer_to_list(Name) ++ ".DETS").
filename(Name, Config) when is_atom(Name) ->
filename(atom_to_list(Name), Config);
filename(Name, _Config) ->
filename:join(?privdir(_Config), Name).
open_files(_Name, [], _Version) ->
[];
open_files(Name0, [Args | Tail], Version) ->
?format("init ~p~n", [Args]),
Name = list_to_atom(integer_to_list(Name0)),
{ok, Name} = dets:open_file(Name, [{version,Version} | Args]),
[Name | open_files(Name0+1, Tail, Version)].
close_all(Tabs) -> foreach(fun(Tab) -> ok = dets:close(Tab) end, Tabs).
delete_files(Args) ->
Fun = fun(F) ->
{value, {file, File}} = keysearch(file, 1, F),
file:delete(File),
File
end,
map(Fun, Args).
%% Initialize all tables
initialize(Tabs, Data) ->
foreach(fun(Tab) ->
Fun = fun(Obj) -> ok = dets:insert(Tab, Obj) end,
foreach(Fun, Data),
dets:sync(Tab)
end, Tabs).
%% need more than 512 objects to really trig overflow
make_data(Kp) ->
make_data(Kp, set).
make_data(Kp, Type) ->
dup(Type, make_data(Kp, Type, 520)).
dup(duplicate_bag, [H1, H2 |T]) ->
[H1,H2, H1, H2 | dup(duplicate_bag, T)];
dup(_, Other) ->
Other.
make_data(_Kp, Type, 0) ->
odd_keys(Type);
make_data(1, set, I) ->
[{I, q,w} | make_data(1, set, I-1)];
make_data(2, set, I) ->
[{hh, I, q,w} | make_data(2, set, I-1)];
make_data(1, bag, I) ->
[{I, q,w} , {I, hah, 77} | make_data(1, bag, I-1)];
make_data(2, bag, I) ->
[{hh, I, q,w} , {hh, I, lalal, 900} | make_data(2, bag, I-1)];
make_data(1, duplicate_bag, I) ->
[{I, q,w} , {I, hah, 77} | make_data(1, duplicate_bag, I-1)];
make_data(2, duplicate_bag, I) ->
[{hh, I, q,w} , {hh, I, lalal, 900} | make_data(2, duplicate_bag, I-1)].
odd_keys(_) ->
[{foo, 1 ,2},
{{foo, foo}, 2,3},
{"kakaka", {{{}}}, jj},
{{"kallll", "kkk", []}, 66.7777},
{make_ref(), 99, 66},
{{1},2,3,4,5,6,7,duplicate(50, 8)},
{self(), 7,8,88},
{[self()], 8, 11}].
ins(_T, 0) ->
ok;
ins(T, N) ->
case dets:insert(T, {N, item(N)}) of
ok -> ins(T, N-1);
Error -> Error
end.
item(N) when N rem 2 =:= 0 ->
{item, number, N};
item(N) ->
{item, number, N, a, much, bigger, one, i, think}.
del(_T, N, _I) when N =< 0 ->
ok;
del(T, N, I) ->
ok = dets:delete(T, N),
del(T, N-I, I).
ensure_node(0, _Node) ->
could_not_start_node;
ensure_node(N, Node) ->
case net_adm:ping(Node) of
pang ->
receive after 1000 ->
ok
end,
ensure_node(N-1,Node);
pong ->
ok
end.
size_test(Len, Tabs) ->
foreach(fun(Tab) ->
Len = dets:info(Tab, size)
end, Tabs).
no_keys_test([T | Ts]) ->
no_keys_test(T),
no_keys_test(Ts);
no_keys_test([]) ->
ok;
no_keys_test(T) ->
case dets:info(T, version) of
8 ->
ok;
9 ->
Kp = dets:info(T, keypos),
All = dets:match_object(T, '_'),
L = lists:map(fun(X) -> element(Kp, X) end, All),
NoKeys = length(lists:usort(L)),
case {dets:info(T, no_keys), NoKeys} of
{N, N} ->
ok;
{N1, N2} ->
exit({no_keys_test, N1, N2})
end
end.
safe_get_all_objects(Tab) ->
dets:safe_fixtable(Tab, true),
Objects = get_all_objects(Tab),
dets:safe_fixtable(Tab, false),
Objects.
%% Caution: unless the table has been fixed, strange results can be returned.
get_all_objects(Tab) -> get_all_objects(dets:first(Tab), Tab, []).
%% Assuming no key matches {error, Reason}...
get_all_objects('$end_of_table', _Tab, L) -> L;
get_all_objects({error, Reason}, _Tab, _L) ->
exit({get_all_objects, {error, Reason}});
get_all_objects(Key, Tab, L) ->
Objs = dets:lookup(Tab, Key),
get_all_objects(dets:next(Tab, Key), Tab, Objs ++ L).
count_objects_quite_fast(Tab) ->
R1 = dets:match_object(Tab, '_', 1),
count_objs_1(R1, 0).
count_objs_1('$end_of_table', N) ->
N;
count_objs_1({Ts,C}, N) when is_list(Ts) ->
count_objs_1(dets:match_object(C), length(Ts) + N).
get_all_objects_fast(Tab) ->
dets:match_object(Tab, '_').
%% Relevant for version 8.
histogram(Tab) ->
OnePercent = case dets:info(Tab, no_slots) of
undefined -> undefined;
{_, NoSlots, _} -> NoSlots/100
end,
histogram(Tab, OnePercent).
histogram(Tab, OnePercent) ->
E = ets:new(histo, []),
dets:safe_fixtable(Tab, true),
Hist = histo(Tab, E, 0, OnePercent, OnePercent),
dets:safe_fixtable(Tab, false),
case Hist of
ok ->
H = ets:tab2list(E),
true = ets:delete(E),
sort(H);
Error ->
ets:delete(E),
Error
end.
histo(T, E, I, One, Count) when is_number(Count), I > Count ->
io:format("."),
histo(T, E, I, One, Count+One);
histo(T, E, I, One, Count) ->
case dets:slot(T, I) of
'$end_of_table' when is_number(Count) ->
io:format("~n"),
ok;
'$end_of_table' ->
ok;
Objs when is_list(Objs) ->
L = length(Objs),
case catch ets:update_counter(E, L, 1) of
{'EXIT', _} ->
ets:insert(E, {L, 1});
_ ->
ok
end,
histo(T, E, I+1, One, Count);
Error ->
Error
end.
sum_histogram(H) ->
sum_histogram(H, 0).
sum_histogram([{S,N1} | H], N) ->
sum_histogram(H, N + S*N1);
sum_histogram([], N) ->
N.
ave_histogram(H) ->
ave_histogram(H, 0)/sum_histogram(H).
ave_histogram([{S,N1} | H], N) ->
ave_histogram(H, N + S*S*N1);
ave_histogram([], N) ->
N.
bad_object({error,{bad_object,FileName}}, FileName) ->
ok; % Version 8, no debug.
bad_object({error,{{bad_object,_,_},FileName}}, FileName) ->
ok; % Version 8, debug...
bad_object({error,{{bad_object,_}, FileName}}, FileName) ->
ok; % No debug.
bad_object({error,{{{bad_object,_,_},_,_,_}, FileName}}, FileName) ->
ok. % Debug.
check_badarg({'EXIT', {badarg, [{M,F,Args,_} | _]}}, M, F, Args) ->
true;
check_badarg({'EXIT', {badarg, [{M,F,A,_} | _]}}, M, F, Args) ->
true = test_server:is_native(M) andalso length(Args) =:= A.
check_pps({Ports0,Procs0} = P0) ->
ok = check_dets_tables(),
case pps() of
P0 ->
ok;
_ ->
%% On some (rare) occasions the dets process is still
%% running although the call to close() has returned, as
%% it seems...
timer:sleep(500),
case pps() of
P0 ->
ok;
{Ports1,Procs1} = P1 ->
case {Ports1 -- Ports0, Procs1 -- Procs0} of
{[], []} -> ok;
{PortsDiff,ProcsDiff} ->
io:format("failure, got ~p~n, expected ~p\n", [P1, P0]),
show("Old port", Ports0 -- Ports1),
show("New port", PortsDiff),
show("Old proc", Procs0 -- Procs1),
show("New proc", ProcsDiff),
ct:fail(failed)
end
end
end.
%% Copied from dets_server.erl:
-define(REGISTRY, dets_registry).
-define(OWNERS, dets_owners).
-define(STORE, dets).
check_dets_tables() ->
Store = [T ||
T <- ets:all(),
ets:info(T, name) =:= ?STORE,
owner(T) =:= dets],
S = case Store of
[Tab] -> ets:tab2list(Tab);
[] -> []
end,
case {ets:tab2list(?REGISTRY), ets:tab2list(?OWNERS), S} of
{[], [], []} -> ok;
{R, O, _} ->
io:format("Registry: ~p~n", [R]),
io:format("Owners: ~p~n", [O]),
io:format("Store: ~p~n", [S]),
not_ok
end.
owner(Tab) ->
Owner = ets:info(Tab, owner),
case process_info(Owner, registered_name) of
{registered_name, Name} -> Name;
_ -> Owner
end.
show(_S, []) ->
ok;
show(S, [{Pid, Name, InitCall}|Pids]) when is_pid(Pid) ->
io:format("~s: ~w (~w), ~w: ~p~n",
[S, Pid, proc_reg_name(Name), InitCall,
erlang:process_info(Pid)]),
show(S, Pids);
show(S, [{Port, _}|Ports]) when is_port(Port)->
io:format("~s: ~w: ~p~n", [S, Port, erlang:port_info(Port)]),
show(S, Ports).
pps() ->
dets:start(),
{port_list(), process_list()}.
port_list() ->
[{P,safe_second_element(erlang:port_info(P, name))} ||
P <- erlang:ports()].
process_list() ->
[{P,process_info(P, registered_name),
safe_second_element(process_info(P, initial_call))} ||
P <- processes()].
proc_reg_name({registered_name, Name}) -> Name;
proc_reg_name([]) -> no_reg_name.
safe_second_element({_,Info}) -> Info;
safe_second_element(Other) -> Other.