diff options
Diffstat (limited to 'lib/dialyzer/test')
55 files changed, 1359 insertions, 66 deletions
diff --git a/lib/dialyzer/test/dialyzer_SUITE.erl b/lib/dialyzer/test/dialyzer_SUITE.erl index 1b62291a00..8507525597 100644 --- a/lib/dialyzer/test/dialyzer_SUITE.erl +++ b/lib/dialyzer/test/dialyzer_SUITE.erl @@ -30,12 +30,12 @@ -export([init_per_testcase/2, end_per_testcase/2]). %% Test cases must be exported. --export([app_test/1, appup_test/1]). +-export([app_test/1, appup_test/1, beam_tests/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [app_test, appup_test]. + [app_test, appup_test, beam_tests]. groups() -> []. @@ -75,3 +75,38 @@ app_test(Config) when is_list(Config) -> %% Test that the .appup file does not contain any `basic' errors appup_test(Config) when is_list(Config) -> ok = ?t:appup_test(dialyzer). + +beam_tests(Config) when is_list(Config) -> + Prog = <<" + -module(no_auto_import). + + %% Copied from erl_lint_SUITE.erl, clash6 + + -export([size/1]). + + size([]) -> + 0; + size({N,_}) -> + N; + size([_|T]) -> + 1+size(T). + ">>, + Opts = [no_auto_import], + {ok, BeamFile} = compile(Config, Prog, no_auto_import, Opts), + [] = run_dialyzer([BeamFile]), + ok. + +compile(Config, Prog, Module, CompileOpts) -> + Source = lists:concat([Module, ".erl"]), + PrivDir = ?config(priv_dir,Config), + Filename = filename:join([PrivDir, Source]), + ok = file:write_file(Filename, Prog), + Opts = [{outdir, PrivDir}, debug_info | CompileOpts], + {ok, Module} = compile:file(Filename, Opts), + {ok, filename:join([PrivDir, lists:concat([Module, ".beam"])])}. + +run_dialyzer(Files) -> + dialyzer:run([{analysis_type, plt_build}, + {files, Files}, + {from, byte_code}, + {check_plt, false}]). diff --git a/lib/dialyzer/test/opaque_SUITE_data/dialyzer_options b/lib/dialyzer/test/opaque_SUITE_data/dialyzer_options index 3ff26b87db..44a65f6e90 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/opaque_SUITE_data/dialyzer_options @@ -1 +1 @@ -{dialyzer_options, [{warnings, [no_unused, no_return]}]}. +{dialyzer_options, [{warnings, [no_unused, no_return, no_unknown]}]}. diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/simple b/lib/dialyzer/test/opaque_SUITE_data/results/simple index 072ac9be8f..29864d6065 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/simple +++ b/lib/dialyzer/test/opaque_SUITE_data/results/simple @@ -14,12 +14,17 @@ is_rec.erl:53: The call erlang:is_record(A::simple1_adt:d1(),A::simple1_adt:d1() is_rec.erl:57: Guard test is_record(A::simple1_adt:d1(),'r',2) breaks the opaqueness of its argument is_rec.erl:61: The record #r{f1::simple1_adt:d1()} violates the declared type for #r{} is_rec.erl:65: The call erlang:is_record({simple1_adt:d1(),1},'r',2) contains an opaque term as 1st argument when terms of different types are expected in these positions -rec_api.erl:22: Record construction #r1{f1::10} violates the declared type of field f1::'undefined' | rec_api:a() -rec_api.erl:23: The pattern {'r1', 10} violates the declared type for #r1{} -rec_api.erl:27: The attempt to match a term of type rec_adt:r1() against the pattern {'r1', 'a'} breaks the opaqueness of the term -rec_api.erl:29: Invalid type specification for function rec_api:adt_t1/1. The success typing is (#r1{f1::'a'}) -> #r1{f1::'a'} -rec_api.erl:34: Invalid type specification for function rec_api:adt_r1/0. The success typing is () -> #r1{f1::'a'} -rec_api.erl:77: The attempt to match a term of type rec_api:f() against the variable _ breaks the opaqueness of the term +rec_api.erl:104: Matching of pattern {'r2', 10} tagged with a record name violates the declared type of #r2{f1::10} +rec_api.erl:113: The attempt to match a term of type #r3{f1::queue:queue(_)} against the pattern {'r3', 'a'} breaks the opaqueness of queue:queue(_) +rec_api.erl:118: Record construction #r3{f1::10} violates the declared type of field f1::queue:queue(_) +rec_api.erl:123: The attempt to match a term of type #r3{f1::10} against the pattern {'r3', 10} breaks the opaqueness of queue:queue(_) +rec_api.erl:24: Record construction #r1{f1::10} violates the declared type of field f1::'undefined' | rec_api:a() +rec_api.erl:29: Matching of pattern {'r1', 10} tagged with a record name violates the declared type of #r1{f1::10} +rec_api.erl:33: The attempt to match a term of type rec_adt:r1() against the pattern {'r1', 'a'} breaks the opaqueness of the term +rec_api.erl:35: Invalid type specification for function rec_api:adt_t1/1. The success typing is (#r1{f1::'a'}) -> #r1{f1::'a'} +rec_api.erl:40: Invalid type specification for function rec_api:adt_r1/0. The success typing is () -> #r1{f1::'a'} +rec_api.erl:85: The attempt to match a term of type rec_api:f() against the variable _ breaks the opaqueness of rec_adt:f() +rec_api.erl:99: Record construction #r2{f1::10} violates the declared type of field f1::rec_api:a() simple1_api.erl:113: The test simple1_api:d1() =:= simple1_api:d2() can never evaluate to 'true' simple1_api.erl:118: Guard test simple1_api:d2() =:= A::simple1_api:d1() can never succeed simple1_api.erl:142: Attempt to test for equality between a term of type simple1_adt:o2() and a term of opaque type simple1_adt:o1() @@ -58,7 +63,7 @@ simple1_api.erl:381: Invalid type specification for function simple1_api:bool_ad simple1_api.erl:407: The size simple1_adt:i1() breaks the opaqueness of A simple1_api.erl:418: The attempt to match a term of type non_neg_integer() against the variable A breaks the opaqueness of simple1_adt:i1() simple1_api.erl:425: The attempt to match a term of type non_neg_integer() against the variable B breaks the opaqueness of simple1_adt:i1() -simple1_api.erl:432: The attempt to match a term of type non_neg_integer() against the variable B breaks the opaqueness of simple1_api:o1() +simple1_api.erl:432: The pattern <<_:B/integer-unit:1>> can never match the type any() simple1_api.erl:448: The attempt to match a term of type non_neg_integer() against the variable Sz breaks the opaqueness of simple1_adt:i1() simple1_api.erl:460: The attempt to match a term of type simple1_adt:bit1() against the pattern <<_/binary-unit:8>> breaks the opaqueness of the term simple1_api.erl:478: The call 'foo':A(A::simple1_adt:a()) breaks the opaqueness of the term A :: simple1_adt:a() diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/timer b/lib/dialyzer/test/opaque_SUITE_data/results/timer index e917b76b08..b1cfcd4e9f 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/timer +++ b/lib/dialyzer/test/opaque_SUITE_data/results/timer @@ -1,4 +1,4 @@ timer_use.erl:16: The pattern 'gazonk' can never match the type {'error',_} | {'ok',timer:tref()} -timer_use.erl:17: The attempt to match a term of type {'ok',timer:tref()} against the pattern {'ok', 42} breaks the opaqueness of timer:tref() +timer_use.erl:17: The attempt to match a term of type {'error',_} | {'ok',timer:tref()} against the pattern {'ok', 42} breaks the opaqueness of timer:tref() timer_use.erl:18: The attempt to match a term of type {'error',_} | {'ok',timer:tref()} against the pattern {Tag, 'gazonk'} breaks the opaqueness of timer:tref() diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/modules/opaque_erl_scan.erl b/lib/dialyzer/test/opaque_SUITE_data/src/modules/opaque_erl_scan.erl index 9ecd4f92a1..24d0793a7c 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/modules/opaque_erl_scan.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/modules/opaque_erl_scan.erl @@ -1,4 +1,4 @@ -%% -*- coding: utf-8 -*- +%% %% %% %CopyrightBegin% %% diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/opaque/opaque_bug5.erl b/lib/dialyzer/test/opaque_SUITE_data/src/opaque/opaque_bug5.erl new file mode 100644 index 0000000000..28d739de8e --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/opaque/opaque_bug5.erl @@ -0,0 +1,10 @@ +%% Second arg of is_record call wasn't checked properly + +-module(opaque_bug5). + +-export([b/0]). + +b() -> + is_record(id({a}), id(a)). + +id(I) -> I. diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/simple/rec_api.erl b/lib/dialyzer/test/opaque_SUITE_data/src/simple/rec_api.erl index d9b1d59f0c..fb6d59d263 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/simple/rec_api.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/simple/rec_api.erl @@ -1,9 +1,9 @@ -module(rec_api). --export([t1/0, t2/0, adt_t1/0, adt_t1/1, adt_r1/0, - t/1, t_adt/0, r/0, r_adt/0]). +-export([t1/0, t2/0, t3/0, adt_t1/0, adt_t1/1, adt_r1/0, + t/1, t_adt/0, r/0, r_adt/0, u1/0, u2/0, u3/0, v1/0, v2/0, v3/0]). --export_type([{a,0},{r1,0}]). +-export_type([{a,0},{r1,0}, r2/0, r3/0]). -export_type([f/0, op_t/0, r/0, tup/0]). @@ -19,8 +19,14 @@ t1() -> {r1, a} = A. t2() -> - A = {r1, 10}, % violates the type of #r1{} - {r1, 10} = A. % violates the type of #r1{} + A = {r1, 10}, + {r1, 10} = A, + A = #r1{f1 = 10}, % violates the type of field f1 + #r1{f1 = 10} = A. + +t3() -> + A = {r1, 10}, + #r1{f1 = 10} = A. % violates the type of #r1{} adt_t1() -> R = rec_adt:r1(), @@ -66,7 +72,8 @@ t_adt() -> -spec r() -> _. r() -> - {r, f(), 2}. % OK, f() is a local opaque type + {{r, f(), 2}, + #r{f = f(), o = 2}}. % OK, f() is a local opaque type -spec f() -> f(). @@ -74,4 +81,43 @@ f() -> fun(_) -> 3 end. r_adt() -> - {r, rec_adt:f(), 2}. % breaks the opaqueness + {{r, rec_adt:f(), 2}, + #r{f = rec_adt:f(), o = 2}}. % breaks the opaqueness + +-record(r2, % like #r1{}, but with initial value + {f1 = a :: a()}). + +-opaque r2() :: #r2{}. + +u1() -> + A = #r2{f1 = a}, + {r2, a} = A. + +u2() -> + A = {r2, 10}, + {r2, 10} = A, + A = #r2{f1 = 10}, % violates the type of field f1 + #r2{f1 = 10} = A. + +u3() -> + A = {r2, 10}, + #r2{f1 = 10} = A. % violates the type of #r2{} + +-record(r3, % like #r1{}, but an opaque type + {f1 = queue:new():: queue:queue()}). + +-opaque r3() :: #r3{}. + +v1() -> + A = #r3{f1 = queue:new()}, + {r3, a} = A. % breaks the opaqueness + +v2() -> + A = {r3, 10}, + {r3, 10} = A, + A = #r3{f1 = 10}, % violates the type of field f1 + #r3{f1 = 10} = A. + +v3() -> + A = {r3, 10}, + #r3{f1 = 10} = A. % breaks the opaqueness diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl b/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl index 5135eb8e59..eef2074e0c 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl @@ -428,8 +428,8 @@ bit_adt_t3(A) -> bit_t5(A) -> B = o1(), - case none:none() of - <<A:B>> -> 1 % breaks the opaqueness + case none:none() of % the type is any(); should fix that XXX + <<A:B>> -> 1 % can never match (local opaque type is OK) end. -spec bit_t4(<<_:1>>) -> integer(). diff --git a/lib/dialyzer/test/options1_SUITE_data/dialyzer_options b/lib/dialyzer/test/options1_SUITE_data/dialyzer_options index c612e77d3e..65d233ac0d 100644 --- a/lib/dialyzer/test/options1_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/options1_SUITE_data/dialyzer_options @@ -1,2 +1,2 @@ -{dialyzer_options, [{include_dirs, ["my_include"]}, {defines, [{'COMPILER_VSN', 42}]}, {warnings, [no_improper_lists]}]}. +{dialyzer_options, [{include_dirs, ["my_include"]}, {defines, [{'COMPILER_VSN', 42}]}, {warnings, [no_improper_lists, no_unknown]}]}. {time_limit, 30}. diff --git a/lib/dialyzer/test/options1_SUITE_data/results/compiler b/lib/dialyzer/test/options1_SUITE_data/results/compiler index 6399e3e36b..30b6f4814a 100644 --- a/lib/dialyzer/test/options1_SUITE_data/results/compiler +++ b/lib/dialyzer/test/options1_SUITE_data/results/compiler @@ -4,7 +4,7 @@ beam_bool.erl:193: The pattern {[], _} can never match the type {[{_,_,_,_},...] beam_bool.erl:510: The pattern [{'set', [Dst], _, _}, {'%live', _}] can never match the type [{_,_,_,_}] beam_disasm.erl:537: The variable X can never match since previous clauses completely covered the type 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 beam_type.erl:284: The pattern <'pi', 0> can never match the type <_,1 | 2> -beam_validator.erl:396: The pattern <{'jump', {'f', _}}, Vst = {'vst', 'none', _}> can never match the type <_,#vst{current::#st{ct::[]}}> +beam_validator.erl:396: Matching of pattern {'vst', 'none', _} tagged with a record name violates the declared type of #vst{current::#st{ct::[]}} beam_validator.erl:690: The pattern <'term', OldT> can never match the type <{'tuple',[any(),...]},_> beam_validator.erl:693: Guard test 'or'('false','false') can never succeed beam_validator.erl:700: Guard test 'or'('false','false') can never succeed @@ -33,4 +33,4 @@ core_lint.erl:473: The pattern <{'c_atom', _, 'all'}, 'binary', _Def, St> can ne core_lint.erl:505: The pattern <_Req, 'unknown', St> can never match the type <non_neg_integer(),non_neg_integer(),_> v3_codegen.erl:1569: The call v3_codegen:load_reg_1(V::any(),I::0,Rs::any(),pos_integer()) will never return since it differs in the 4th argument from the success typing arguments: (any(),0,maybe_improper_list(),0) v3_codegen.erl:1571: The call v3_codegen:load_reg_1(V::any(),I::0,[],pos_integer()) will never return since it differs in the 4th argument from the success typing arguments: (any(),0,maybe_improper_list(),0) -v3_core.erl:646: The pattern <Prim = {'iprimop', _, _, _}, St> can never match the type <#c_nil{anno::[any(),...]} | {'c_atom' | 'c_char' | 'c_float' | 'c_int' | 'c_string' | 'c_tuple' | 'c_var' | 'ibinary' | 'icatch' | 'ireceive1',[any(),...] | {_,_,_,_},_} | #c_cons{anno::[any(),...]} | #c_fname{anno::[any(),...]} | #iletrec{anno::{_,_,_,_},defs::[any(),...],body::[any(),...]} | #icase{anno::{_,_,_,_},args::[any()],clauses::[any()],fc::{_,_,_,_,_,_}} | #ireceive2{anno::{_,_,_,_},clauses::[any()],action::[any()]} | #ifun{anno::{_,_,_,_},id::[any(),...],vars::[any()],clauses::[any(),...],fc::{_,_,_,_,_,_}} | #imatch{anno::{_,_,_,_},guard::[],fc::{_,_,_,_,_,_}} | #itry{anno::{_,_,_,_},args::[any()],vars::[any(),...],body::[any(),...],evars::[any(),...],handler::[any(),...]},_> +v3_core.erl:646: Matching of pattern {'iprimop', _, _, _} tagged with a record name violates the declared type of #c_nil{anno::[any(),...]} | {'c_atom' | 'c_char' | 'c_float' | 'c_int' | 'c_string' | 'c_tuple' | 'c_var' | 'ibinary' | 'icatch' | 'ireceive1',[any(),...] | {_,_,_,_},_} | #c_cons{anno::[any(),...]} | #c_fname{anno::[any(),...]} | #iletrec{anno::{_,_,_,_},defs::[any(),...],body::[any(),...]} | #icase{anno::{_,_,_,_},args::[any()],clauses::[any()],fc::{_,_,_,_,_,_}} | #ireceive2{anno::{_,_,_,_},clauses::[any()],action::[any()]} | #ifun{anno::{_,_,_,_},id::[any(),...],vars::[any()],clauses::[any(),...],fc::{_,_,_,_,_,_}} | #imatch{anno::{_,_,_,_},guard::[],fc::{_,_,_,_,_,_}} | #itry{anno::{_,_,_,_},args::[any()],vars::[any(),...],body::[any(),...],evars::[any(),...],handler::[any(),...]} diff --git a/lib/dialyzer/test/r9c_SUITE_data/dialyzer_options b/lib/dialyzer/test/r9c_SUITE_data/dialyzer_options index e00e23bb66..ba0e6b1ad7 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/r9c_SUITE_data/dialyzer_options @@ -1,2 +1,2 @@ -{dialyzer_options, [{defines, [{vsn, 42}]}]}. +{dialyzer_options, [{defines, [{vsn, 42}]}, {warnings, [no_unknown]}]}. {time_limit, 20}. diff --git a/lib/dialyzer/test/r9c_SUITE_data/results/asn1 b/lib/dialyzer/test/r9c_SUITE_data/results/asn1 index c11105b76d..1cf03346ee 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/results/asn1 +++ b/lib/dialyzer/test/r9c_SUITE_data/results/asn1 @@ -5,7 +5,7 @@ asn1ct.erl:1673: The pattern 'all' can never match the type 'asn1_module' | 'exc asn1ct.erl:672: The pattern <{'false', Result}, _, _> can never match the type <{'true','true'},atom() | binary() | [atom() | [any()] | char()],[any()]> asn1ct.erl:909: Guard test is_atom(Ext::[49 | 97 | 98 | 100 | 110 | 115]) can never succeed asn1ct_check.erl:1698: The pattern {'error', _} can never match the type [any()] -asn1ct_check.erl:2733: The pattern {'type', Tag, _, _, _, _} can never match the type 'ASN1_OPEN_TYPE' | {_,_} | {'fixedtypevaluefield',_,_} +asn1ct_check.erl:2733: Matching of pattern {'type', Tag, _, _, _, _} tagged with a record name violates the declared type of 'ASN1_OPEN_TYPE' | {_,_} | {'fixedtypevaluefield',_,_} asn1ct_check.erl:2738: The pattern <_S, _> can never match since previous clauses completely covered the type <#state{},#'ObjectClassFieldType'{class::#objectclass{fields::maybe_improper_list() | {_,_,_,_}},fieldname::{_,maybe_improper_list()},type::'ASN1_OPEN_TYPE' | {_,_} | {'fixedtypevaluefield',_,_}}> asn1ct_check.erl:2887: The variable Other can never match since previous clauses completely covered the type any() asn1ct_check.erl:3188: The pattern <_S, [], B> can never match the type <#state{},{'SingleValue',_},{'ValueRange',_}> diff --git a/lib/dialyzer/test/race_SUITE_data/dialyzer_options b/lib/dialyzer/test/race_SUITE_data/dialyzer_options index 44e1720715..6992fc6c40 100644 --- a/lib/dialyzer/test/race_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/race_SUITE_data/dialyzer_options @@ -1 +1 @@ -{dialyzer_options, [{warnings, [race_conditions]}]}. +{dialyzer_options, [{warnings, [race_conditions, no_unknown]}]}. diff --git a/lib/dialyzer/test/small_SUITE_data/dialyzer_options b/lib/dialyzer/test/small_SUITE_data/dialyzer_options index 50991c9bc5..0d91699e4d 100644 --- a/lib/dialyzer/test/small_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/small_SUITE_data/dialyzer_options @@ -1 +1 @@ -{dialyzer_options, []}. +{dialyzer_options, [{warnings, [no_unknown]}]}. diff --git a/lib/dialyzer/test/small_SUITE_data/results/confusing_record_warning b/lib/dialyzer/test/small_SUITE_data/results/confusing_record_warning deleted file mode 100644 index ac3d89b02b..0000000000 --- a/lib/dialyzer/test/small_SUITE_data/results/confusing_record_warning +++ /dev/null @@ -1,3 +0,0 @@ - -confusing_record_warning.erl:18: Function test/1 has no local return -confusing_record_warning.erl:18: Matching of pattern {'r', [_]} tagged with a record name violates the declared type of #r{field::'binary' | 'undefined'} diff --git a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes index bfa33cd296..fbdd182358 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes +++ b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes @@ -1,27 +1,28 @@ contracts_with_subtypes.erl:106: The call contracts_with_subtypes:rec_arg({'a','b'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) contracts_with_subtypes.erl:107: The call contracts_with_subtypes:rec_arg({'b','a'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:108: The call contracts_with_subtypes:rec_arg({'a',{'b','a'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) contracts_with_subtypes.erl:109: The call contracts_with_subtypes:rec_arg({'b',{'a','b'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:110: The call contracts_with_subtypes:rec_arg({'a',{'b',{'a','b'}}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:111: The call contracts_with_subtypes:rec_arg({'b',{'a',{'b','a'}}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:142: The pattern 1 can never match the type string() -contracts_with_subtypes.erl:145: The pattern 'alpha' can never match the type {'ok',_} | {'ok',_,string()} -contracts_with_subtypes.erl:147: The pattern 42 can never match the type {'ok',_} | {'ok',_,string()} -contracts_with_subtypes.erl:163: The pattern 'alpha' can never match the type {'ok',_} -contracts_with_subtypes.erl:165: The pattern 42 can never match the type {'ok',_} -contracts_with_subtypes.erl:183: The pattern 'alpha' can never match the type {'ok',_} -contracts_with_subtypes.erl:185: The pattern 42 can never match the type {'ok',_} -contracts_with_subtypes.erl:202: The pattern 1 can never match the type string() -contracts_with_subtypes.erl:205: The pattern {'ok', _} can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:206: The pattern 'alpha' can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:207: The pattern {'ok', 42} can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:208: The pattern 42 can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:234: Function flat_ets_new_t/0 has no local return -contracts_with_subtypes.erl:235: The call contracts_with_subtypes:flat_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,'set' | 'ordered_set' | 'bag' | 'duplicate_bag' | 'public' | 'protected' | 'private' | 'named_table' | {'keypos',integer()} | {'heir',pid(),term()} | {'heir','none'} | {'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed') +contracts_with_subtypes.erl:135: The call contracts_with_subtypes:rec2({'a','b'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:136: The call contracts_with_subtypes:rec2({'b','a'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:137: The call contracts_with_subtypes:rec2({'a',{'b','a'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:138: The call contracts_with_subtypes:rec2({'b',{'a','b'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:171: The pattern 1 can never match the type string() +contracts_with_subtypes.erl:174: The pattern 'alpha' can never match the type {'ok',_} | {'ok',_,string()} +contracts_with_subtypes.erl:176: The pattern 42 can never match the type {'ok',_} | {'ok',_,string()} +contracts_with_subtypes.erl:192: The pattern 'alpha' can never match the type {'ok',_} +contracts_with_subtypes.erl:194: The pattern 42 can never match the type {'ok',_} +contracts_with_subtypes.erl:212: The pattern 'alpha' can never match the type {'ok',_} +contracts_with_subtypes.erl:214: The pattern 42 can never match the type {'ok',_} +contracts_with_subtypes.erl:231: The pattern 1 can never match the type string() +contracts_with_subtypes.erl:234: The pattern {'ok', _} can never match the type {'ok',_,string()} +contracts_with_subtypes.erl:235: The pattern 'alpha' can never match the type {'ok',_,string()} +contracts_with_subtypes.erl:236: The pattern {'ok', 42} can never match the type {'ok',_,string()} +contracts_with_subtypes.erl:237: The pattern 42 can never match the type {'ok',_,string()} contracts_with_subtypes.erl:23: Invalid type specification for function contracts_with_subtypes:extract2/0. The success typing is () -> 'something' -contracts_with_subtypes.erl:261: Function factored_ets_new_t/0 has no local return -contracts_with_subtypes.erl:262: The call contracts_with_subtypes:factored_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,Type | Access | 'named_table' | {'keypos',Pos} | {'heir',Pid::pid(),HeirData} | {'heir','none'} | Tweaks), is_subtype(Type,type()), is_subtype(Access,access()), is_subtype(Tweaks,{'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed'), is_subtype(Pos,pos_integer()), is_subtype(HeirData,term()) +contracts_with_subtypes.erl:263: Function flat_ets_new_t/0 has no local return +contracts_with_subtypes.erl:264: The call contracts_with_subtypes:flat_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,'set' | 'ordered_set' | 'bag' | 'duplicate_bag' | 'public' | 'protected' | 'private' | 'named_table' | {'keypos',integer()} | {'heir',pid(),term()} | {'heir','none'} | {'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed') +contracts_with_subtypes.erl:290: Function factored_ets_new_t/0 has no local return +contracts_with_subtypes.erl:291: The call contracts_with_subtypes:factored_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,Type | Access | 'named_table' | {'keypos',Pos} | {'heir',Pid::pid(),HeirData} | {'heir','none'} | Tweaks), is_subtype(Type,type()), is_subtype(Access,access()), is_subtype(Tweaks,{'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed'), is_subtype(Pos,pos_integer()), is_subtype(HeirData,term()) contracts_with_subtypes.erl:77: The call contracts_with_subtypes:foo1(5) breaks the contract (Arg1) -> Res when is_subtype(Arg1,atom()), is_subtype(Res,atom()) contracts_with_subtypes.erl:78: The call contracts_with_subtypes:foo2(5) breaks the contract (Arg1) -> Res when is_subtype(Arg1,Arg2), is_subtype(Arg2,atom()), is_subtype(Res,atom()) contracts_with_subtypes.erl:79: The call contracts_with_subtypes:foo3(5) breaks the contract (Arg1) -> Res when is_subtype(Arg2,atom()), is_subtype(Arg1,Arg2), is_subtype(Res,atom()) diff --git a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes2 b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes2 new file mode 100644 index 0000000000..9f5433a13d --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes2 @@ -0,0 +1,3 @@ + +contracts_with_subtypes2.erl:18: Function t/0 has no local return +contracts_with_subtypes2.erl:19: The call contracts_with_subtypes2:t({'a',{'b',{'c',{'d',{'e',{'g',3}}}}}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A}), is_subtype(A,{'b',B}), is_subtype(B,{'c',C}), is_subtype(C,{'d',D}), is_subtype(D,{'e',E}), is_subtype(E,{'f',_}) diff --git a/lib/dialyzer/test/small_SUITE_data/results/funs_from_outside b/lib/dialyzer/test/small_SUITE_data/results/funs_from_outside new file mode 100644 index 0000000000..3e597ef1bc --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/funs_from_outside @@ -0,0 +1,7 @@ + +funs_from_outside.erl:18: The pattern 'error' can never match the type {'ok','nothing' | 'something'} +funs_from_outside.erl:32: Function run2/2 has no local return +funs_from_outside.erl:35: Function testb/3 has no local return +funs_from_outside.erl:41: The pattern 'error' can never match the type {'ok','nothing' | 'something'} +funs_from_outside.erl:78: Function test2/1 has no local return +funs_from_outside.erl:83: The pattern 'error' can never match the type 'ok' diff --git a/lib/dialyzer/test/small_SUITE_data/results/invalid_spec_2 b/lib/dialyzer/test/small_SUITE_data/results/invalid_spec_2 new file mode 100644 index 0000000000..4565112ea0 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/invalid_spec_2 @@ -0,0 +1,2 @@ + +scala_user.erl:5: Invalid type specification for function scala_user:is_list/2. The success typing is (maybe_improper_list() | tuple(),_) -> boolean() diff --git a/lib/dialyzer/test/small_SUITE_data/results/literals b/lib/dialyzer/test/small_SUITE_data/results/literals new file mode 100644 index 0000000000..03e161ca71 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/literals @@ -0,0 +1,14 @@ + +literals.erl:11: Function t1/0 has no local return +literals.erl:12: Record construction #r{id::'a'} violates the declared type of field id::'integer' | 'undefined' +literals.erl:14: Function t2/0 has no local return +literals.erl:15: Record construction #r{id::'a'} violates the declared type of field id::'integer' | 'undefined' +literals.erl:17: Function t3/0 has no local return +literals.erl:18: Record construction #r{id::'a'} violates the declared type of field id::'integer' | 'undefined' +literals.erl:21: Record construction #r{id::'a'} violates the declared type of field id::'integer' | 'undefined' +literals.erl:23: Function m1/1 has no local return +literals.erl:23: Matching of pattern {'r', 'a'} tagged with a record name violates the declared type of #r{id::'integer' | 'undefined'} +literals.erl:26: Function m2/1 has no local return +literals.erl:26: Matching of pattern {'r', 'a'} tagged with a record name violates the declared type of #r{id::'integer' | 'undefined'} +literals.erl:29: Function m3/1 has no local return +literals.erl:29: The pattern {{'r', 'a'}} can never match the type any() diff --git a/lib/dialyzer/test/small_SUITE_data/results/maps_difftype b/lib/dialyzer/test/small_SUITE_data/results/maps_difftype new file mode 100644 index 0000000000..8980321135 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/maps_difftype @@ -0,0 +1,3 @@ + +maps_difftype.erl:10: Function empty_mismatch/1 has no local return +maps_difftype.erl:11: The pattern ~{}~ can never match the type tuple() diff --git a/lib/dialyzer/test/small_SUITE_data/results/my_sofs b/lib/dialyzer/test/small_SUITE_data/results/my_sofs index bc97c08d62..0b933e6cd7 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/my_sofs +++ b/lib/dialyzer/test/small_SUITE_data/results/my_sofs @@ -1,3 +1,3 @@ -my_sofs.erl:34: The pattern {'Set', _, _} can never match the type #'OrdSet'{} -my_sofs.erl:54: The pattern {'Set', _, _} can never match the type #'OrdSet'{} +my_sofs.erl:34: Matching of pattern {'Set', _, _} tagged with a record name violates the declared type of #'OrdSet'{} +my_sofs.erl:54: Matching of pattern {'Set', _, _} tagged with a record name violates the declared type of #'OrdSet'{} diff --git a/lib/dialyzer/test/small_SUITE_data/results/pretty_bitstring b/lib/dialyzer/test/small_SUITE_data/results/pretty_bitstring new file mode 100644 index 0000000000..0ad6eee766 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/pretty_bitstring @@ -0,0 +1,3 @@ + +pretty_bitstring.erl:7: Function t/0 has no local return +pretty_bitstring.erl:8: The call binary:copy(#{#<1>(8, 1, 'integer', ['unsigned', 'big']), #<2>(8, 1, 'integer', ['unsigned', 'big']), #<3>(3, 1, 'integer', ['unsigned', 'big'])}#,2) breaks the contract (Subject,N) -> binary() when is_subtype(Subject,binary()), is_subtype(N,non_neg_integer()) diff --git a/lib/dialyzer/test/small_SUITE_data/results/record_pat b/lib/dialyzer/test/small_SUITE_data/results/record_pat index 9a3f925e42..a46be6c451 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/record_pat +++ b/lib/dialyzer/test/small_SUITE_data/results/record_pat @@ -1,2 +1,2 @@ -record_pat.erl:14: The pattern {'foo', 'baz'} violates the declared type for #foo{} +record_pat.erl:14: Matching of pattern {'foo', 'baz'} tagged with a record name violates the declared type of #foo{bar::'undefined' | integer()} diff --git a/lib/dialyzer/test/small_SUITE_data/results/record_test b/lib/dialyzer/test/small_SUITE_data/results/record_test index 9715f0dcfb..7060bfa200 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/record_test +++ b/lib/dialyzer/test/small_SUITE_data/results/record_test @@ -1,3 +1,3 @@ -record_test.erl:19: The pattern {'foo', _} can never match the type 'foo' +record_test.erl:19: Matching of pattern {'foo', _} tagged with a record name violates the declared type of 'foo' record_test.erl:21: The variable _ can never match since previous clauses completely covered the type 'foo' diff --git a/lib/dialyzer/test/small_SUITE_data/results/record_update b/lib/dialyzer/test/small_SUITE_data/results/record_update new file mode 100644 index 0000000000..ea52057adf --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/record_update @@ -0,0 +1,2 @@ + +record_update.erl:7: Invalid type specification for function record_update:quux/2. The success typing is (#foo{bar::atom()},atom()) -> #foo{bar::atom()} diff --git a/lib/dialyzer/test/small_SUITE_data/results/relevant_record_warning b/lib/dialyzer/test/small_SUITE_data/results/relevant_record_warning new file mode 100644 index 0000000000..2e417e1b2a --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/relevant_record_warning @@ -0,0 +1,3 @@ + +relevant_record_warning.erl:22: Function test/1 has no local return +relevant_record_warning.erl:23: Record construction #r{field::<<_:8>>} violates the declared type of field field::'binary' | 'undefined' diff --git a/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl index d72138d509..d7dfd9752e 100644 --- a/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl +++ b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl @@ -103,15 +103,44 @@ c(babb) -> rec_arg({b, {a, {b, b}}}); c(ababb) -> rec_arg({a, {b, {a, {b, b}}}}); c(babaa) -> rec_arg({b, {a, {b, {a, a}}}}). -w(ab) -> rec_arg({a, b}); -w(ba) -> rec_arg({b, a}); -w(aba) -> rec_arg({a, {b, a}}); -w(bab) -> rec_arg({b, {a, b}}); -w(abab) -> rec_arg({a, {b, {a, b}}}); -w(baba) -> rec_arg({b, {a, {b, a}}}); +w(ab) -> rec_arg({a, b}); % breaks the contract +w(ba) -> rec_arg({b, a}); % breaks the contract +w(aba) -> rec_arg({a, {b, a}}); % no longer breaks the contract +w(bab) -> rec_arg({b, {a, b}}); % breaks the contract +w(abab) -> rec_arg({a, {b, {a, b}}}); % no longer breaks the contract +w(baba) -> rec_arg({b, {a, {b, a}}}); % no longer breaks the contract w(ababa) -> rec_arg({a, {b, {a, {b, a}}}}); w(babab) -> rec_arg({b, {a, {b, {a, b}}}}). +%% For comparison: the same thing with types + +-type ab() :: {a, a()} | {b, b()}. +-type a() :: a | {b, b()}. +-type b() :: b | {a, a()}. + +-spec rec2(Arg) -> ok when + Arg :: ab(). + +rec2(X) -> get(X). + +d(aa) -> rec2({a, a}); +d(bb) -> rec2({b, b}); +d(abb) -> rec2({a, {b, b}}); +d(baa) -> rec2({b, {a, a}}); +d(abaa) -> rec2({a, {b, {a, a}}}); +d(babb) -> rec2({b, {a, {b, b}}}); +d(ababb) -> rec2({a, {b, {a, {b, b}}}}); +d(babaa) -> rec2({b, {a, {b, {a, a}}}}). + +q(ab) -> rec2({a, b}); % breaks the contract +q(ba) -> rec2({b, a}); % breaks the contract +q(aba) -> rec2({a, {b, a}}); % breaks the contract +q(bab) -> rec2({b, {a, b}}); % breaks the contract +q(abab) -> rec2({a, {b, {a, b}}}); +q(baba) -> rec2({b, {a, {b, a}}}); +q(ababa) -> rec2({a, {b, {a, {b, a}}}}); +q(babab) -> rec2({b, {a, {b, {a, b}}}}). + %=============================================================================== -type dublo(X) :: {X, X}. @@ -143,7 +172,7 @@ st(X) when is_atom(X) -> _Other -> ok end; alpha -> bad; - {ok, 42} -> bad; + {ok, 42} -> ok; 42 -> bad end. @@ -161,7 +190,7 @@ dt(X) when is_atom(X) -> err2 -> ok; {ok, X} -> ok; alpha -> bad; - {ok, 42} -> bad; + {ok, 42} -> ok; 42 -> bad end. @@ -181,7 +210,7 @@ dt2(X) when is_atom(X) -> err2 -> ok; {ok, X} -> ok; alpha -> bad; - {ok, 42} -> bad; + {ok, 42} -> ok; 42 -> bad end. diff --git a/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes2.erl b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes2.erl new file mode 100644 index 0000000000..d2f945b284 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes2.erl @@ -0,0 +1,40 @@ +-module(contracts_with_subtypes2). + +-compile(export_all). + +-behaviour(supervisor). + +-spec t(Arg) -> ok when + Arg :: {a, A}, + A :: {b, B}, + B :: {c, C}, + C :: {d, D}, + D :: {e, E}, + E :: {f, _}. + +t(X) -> + get(X). + +t() -> + t({a, {b, {c, {d, {e, {g, 3}}}}}}). % breaks the contract + +%% This one should possibly result in warnings about unused variables. +-spec l() -> ok when + X :: Y, + Y :: X. + +l() -> + ok. + +%% This is the example from seq12547 (ticket OTP-11798). +%% There used to be a warning. + +-spec init(term()) -> Result when + Result :: {ok, {{supervisor:strategy(), + non_neg_integer(), + pos_integer()}, + [supervisor:child_spec()]}} + | ignore. + +init(_) -> + foo:bar(). diff --git a/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/common_types.hrl b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/common_types.hrl new file mode 100644 index 0000000000..f362a06bca --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/common_types.hrl @@ -0,0 +1,6 @@ +-type host() :: nonempty_string(). +-type path() :: nonempty_string(). +-type url() :: binary(). + +% The host portion of a url, if available. +-type url_host() :: host() | none. diff --git a/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/config.hrl b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/config.hrl new file mode 100644 index 0000000000..8cab65fc9c --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/config.hrl @@ -0,0 +1,148 @@ + +-define(SECOND, 1000). +-define(MINUTE, (60 * ?SECOND)). +-define(HOUR, (60 * ?MINUTE)). +-define(DAY, (24 * ?HOUR)). +-define(MB, (1024 * 1024)). + +% Maximum length of tag/blob prefix +-define(NAME_MAX, 511). + +% How long ddfs node startup can take. The most time-consuming part +% is the scanning of the tag objects in the node's DDFS volumes. +-define(NODE_STARTUP, (1 * ?MINUTE)). + +% How long to wait on the master for replies from nodes. +-define(NODE_TIMEOUT, (10 * ?SECOND)). + +% How long to wait for a reply from an operation coordinated by the +% master that accesses nodes. This value should be larger than +% NODE_TIMEOUT. +-define(NODEOP_TIMEOUT, (1 * ?MINUTE)). + +% The minimum amount of free space a node must have, to be considered +% a primary candidate host for a new blob. +-define(MIN_FREE_SPACE, (1024 * ?MB)). + +% The maximum number of active HTTP connections on a system (this +% applies separately for GET and PUT operations). +-define(HTTP_MAX_ACTIVE, 3). + +% The maximum number of waiting HTTP connections to queue up on a busy system. +-define(HTTP_QUEUE_LENGTH, 100). + +% The maximum number of simultaneous HTTP connections. Note that +% HTTP_MAX_CONNS * 2 * 2 + 32 < Maximum number of file descriptors, where +% 2 = Get and put, 2 = two FDs required for each connection (connection +% itself + a file it accesses), 32 = a guess how many extra fds is needed. +-define(HTTP_MAX_CONNS, 128). + +% How long to keep a PUT request in queue if the system is busy. +-define(PUT_WAIT_TIMEOUT, (1 * ?MINUTE)). + +% How long to keep a GET request in queue if the system is busy. +-define(GET_WAIT_TIMEOUT, (1 * ?MINUTE)). + +% An unused loaded tag expires in TAG_EXPIRES milliseconds. Note that +% if TAG_EXPIRES is not smaller than GC_INTERVAL, tags will never +% expire from the memory cache and will always take up memory. +-define(TAG_EXPIRES, (10 * ?HOUR)). + +% How often the master's cache of all known tag names is refreshed. +% This refresh is only needed to purge deleted tags eventually from +% the tag cache. It doesn't harm to have a long interval. +-define(TAG_CACHE_INTERVAL, (10 * ?MINUTE)). + +% How soon a tag object initialized in memory expires if it's content +% cannot be fetched from the cluster. +-define(TAG_EXPIRES_ONERROR, (1 * ?SECOND)). + +% How often a DDFS node should refresh its tag cache from disk. +-define(FIND_TAGS_INTERVAL, ?DAY). + +% How often buffered (delayed) updates to a tag need to be +% flushed. Tradeoff: The longer the interval, the more updates are +% bundled in a single commit. On the other hand, in the worst case +% the requester has to wait for the full interval before getting a +% reply. A long interval also increases the likelihood that the server +% crashes before the commit has finished successfully, making requests +% more unreliable. +-define(DELAYED_FLUSH_INTERVAL, (1 * ?SECOND)). + +% How long to wait between garbage collection runs. +-define(GC_INTERVAL, ?DAY). + +% Max duration for a GC run. This should be smaller than +% min(ORPHANED_{BLOB,TAG}_EXPIRES). +-define(GC_MAX_DURATION, (3 * ?DAY)). + +% How long to wait after startup for cluster to stabilize before +% starting the first GC run. +-define(GC_DEFAULT_INITIAL_WAIT, (5 * ?MINUTE)). + +% The longest potential interval between messages in the GC protocol; +% used to ensure GC makes forward progress. This can be set to the +% estimated time to traverse all the volumes on a DDFS node. +-define(GC_PROGRESS_INTERVAL, (30 * ?MINUTE)). + +% Number of extra replicas (i.e. lost replicas recovered during GC) to +% allow before deleting extra replicas. +-define(NUM_EXTRA_REPLICAS, 1). + +% Permissions for files backing blobs and tags. +-define(FILE_MODE, 8#00400). + +% How often to check available disk space in ddfs_node. +-define(DISKSPACE_INTERVAL, (10 * ?SECOND)). + +% The maximum size of payloads of HTTP requests to the /ddfs/tag/ +% prefix. +-define(MAX_TAG_BODY_SIZE, (512 * ?MB)). + +% Tag attribute names and values have a limited size, and there +% can be only a limited number of them. +-define(MAX_TAG_ATTRIB_NAME_SIZE, 1024). +-define(MAX_TAG_ATTRIB_VALUE_SIZE, 1024). +-define(MAX_NUM_TAG_ATTRIBS, 1000). + +% How long HTTP requests that perform tag updates should wait to +% finish (a long time). +-define(TAG_UPDATE_TIMEOUT, ?DAY). + +% Timeout for re-replicating a single blob over HTTP PUT. This +% depends on the largest blobs hosted by DDFS, and the speed of the +% cluster network. +-define(GC_PUT_TIMEOUT, (180 * ?MINUTE)). + +% Delete !partial files after this many milliseconds. +-define(PARTIAL_EXPIRES, ?DAY). + +% When orphaned blob can be deleted. This should be large enough that +% you can upload all the new blobs of a tag and perform the tag update +% within this time. +-define(ORPHANED_BLOB_EXPIRES, (5 * ?DAY)). + +% When orphaned tag can be deleted. +-define(ORPHANED_TAG_EXPIRES, (5 * ?DAY)). + +% How long a tag has to stay on the deleted list before +% we can permanently forget it, after all known instances +% of the tag object have been removed. This quarantine period +% ensures that a node that was temporarily unavailable +% and reactivates can't resurrect deleted tags. You +% must ensure that all temporarily inactive nodes +% are reactivated (or cleaned) within the ?DELETED_TAG_EXPIRES +% time frame. +% +% This value _must_ be larger than the other time-related DDFS +% parameters listed in this file. In particular, it must be larger +% than ORPHANED_TAG_EXPIRES. +-define(DELETED_TAG_EXPIRES, (30 * ?DAY)). + +% How many times a tag operation should be retried before aborting. +-define(MAX_TAG_OP_RETRIES, 3). + +% How long to wait before timing out a tag retrieval. This should be +% large enough to read a large tag object off the disk and send it +% over the network. +-define(GET_TAG_TIMEOUT, (5 * ?MINUTE)). diff --git a/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs.hrl b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs.hrl new file mode 100644 index 0000000000..e43ec23fe1 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs.hrl @@ -0,0 +1,9 @@ +-type volume_name() :: nonempty_string(). + +% Diskinfo is {FreeSpace, UsedSpace}. +-type diskinfo() :: {non_neg_integer(), non_neg_integer()}. +-type volume() :: {diskinfo(), volume_name()}. + +-type object_type() :: 'blob' | 'tag'. +-type object_name() :: binary(). +-type taginfo() :: {erlang:timestamp(), volume_name()}. diff --git a/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs_gc.hrl b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs_gc.hrl new file mode 100644 index 0000000000..dc43f7586b --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs_gc.hrl @@ -0,0 +1,17 @@ +-type local_object() :: {object_name(), node()}. +-type phase() :: 'start' | 'build_map' | 'map_wait' | 'gc' + | 'rr_blobs' | 'rr_blobs_wait' | 'rr_tags'. +-type protocol_msg() :: {'check_blob', object_name()} | 'start_gc' | 'end_rr'. + +-type blob_update() :: {object_name(), 'filter' | [url()]}. + +-type check_blob_result() :: 'false' | {'true', volume_name()}. + +% GC statistics + +% {Files, Bytes} +-type gc_stat() :: {non_neg_integer(), non_neg_integer()}. +% {Kept, Deleted} +-type obj_stats() :: {gc_stat(), gc_stat()}. +% {Tags, Blobs}. +-type gc_run_stats() :: {obj_stats(), obj_stats()}. diff --git a/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs_master.erl b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs_master.erl new file mode 100644 index 0000000000..2be2773dc5 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs_master.erl @@ -0,0 +1,531 @@ +-module(ddfs_master). +-behaviour(gen_server). + +-export([start_link/0]). +-export([get_tags/1, get_tags/3, + get_nodeinfo/1, + get_read_nodes/0, + get_hosted_tags/1, + gc_blacklist/0, gc_blacklist/1, + gc_stats/0, + choose_write_nodes/3, + new_blob/4, new_blob/5, + safe_gc_blacklist/0, safe_gc_blacklist/1, + refresh_tag_cache/0, + tag_notify/2, + tag_operation/2, tag_operation/3, + update_gc_stats/1, + update_nodes/1 + ]). +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-define(WEB_PORT, 8011). + +-compile(nowarn_deprecated_type). + +-include("common_types.hrl"). +-include("gs_util.hrl"). +-include("config.hrl"). +-include("ddfs.hrl"). +-include("ddfs_tag.hrl"). +-include("ddfs_gc.hrl"). + +-type node_info() :: {node(), {non_neg_integer(), non_neg_integer()}}. +-type gc_stats() :: none | gc_run_stats(). + +-record(state, {tags = gb_trees:empty() :: gb_trees:tree(), + tag_cache = false :: false | gb_sets:set(), + cache_refresher :: pid(), + + nodes = [] :: [node_info()], + write_blacklist = [] :: [node()], + read_blacklist = [] :: [node()], + gc_blacklist = [] :: [node()], + safe_gc_blacklist = gb_sets:empty() :: gb_sets:set(), + gc_stats = none :: none | {gc_stats(), erlang:timestamp()}}). +-type state() :: #state{}. +-type replyto() :: {pid(), reference()}. + +-export_type([gc_stats/0, node_info/0]). + +%% =================================================================== +%% API functions + +-spec start_link() -> {ok, pid()}. +start_link() -> + lager:info("DDFS master starts"), + case gen_server:start_link({local, ?MODULE}, ?MODULE, [], []) of + {ok, Server} -> {ok, Server}; + {error, {already_started, Server}} -> {ok, Server} + end. + +-spec tag_operation(term(), tagname()) -> term(). +tag_operation(Op, Tag) -> + gen_server:call(?MODULE, {tag, Op, Tag}). +-spec tag_operation(term(), tagname(), non_neg_integer() | infinity) -> + term(). +tag_operation(Op, Tag, Timeout) -> + gen_server:call(?MODULE, {tag, Op, Tag}, Timeout). + +-spec tag_notify(term(), tagname()) -> ok. +tag_notify(Op, Tag) -> + gen_server:cast(?MODULE, {tag_notify, Op, Tag}). + +-spec get_nodeinfo(all) -> {ok, [node_info()]}. +get_nodeinfo(all) -> + gen_server:call(?MODULE, {get_nodeinfo, all}). + +-spec get_read_nodes() -> {ok, [node()], non_neg_integer()} | {error, term()}. +get_read_nodes() -> + gen_server:call(?MODULE, get_read_nodes, infinity). + +-spec gc_blacklist() -> {ok, [node()]}. +gc_blacklist() -> + gen_server:call(?MODULE, gc_blacklist). + +-spec gc_blacklist([node()]) -> ok. +gc_blacklist(Nodes) -> + gen_server:cast(?MODULE, {gc_blacklist, Nodes}). + +-spec gc_stats() -> {ok, none | {gc_stats(), erlang:timestamp()}} | {error, term()}. +gc_stats() -> + gen_server:call(?MODULE, gc_stats). + +-spec get_hosted_tags(host()) -> {ok, [tagname()]} | {error, term()}. +get_hosted_tags(Host) -> + gen_server:call(?MODULE, {get_hosted_tags, Host}). + +-spec choose_write_nodes(non_neg_integer(), [node()], [node()]) -> {ok, [node()]}. +choose_write_nodes(K, Include, Exclude) -> + gen_server:call(?MODULE, {choose_write_nodes, K, Include, Exclude}). + +-spec get_tags(gc) -> {ok, [tagname()], [node()]} | too_many_failed_nodes; + (safe) -> {ok, [binary()]} | too_many_failed_nodes. +get_tags(Mode) -> + get_tags(?MODULE, Mode, ?GET_TAG_TIMEOUT). + +-spec get_tags(server(), gc, non_neg_integer()) -> + {ok, [tagname()], [node()]} | too_many_failed_nodes; + (server(), safe, non_neg_integer()) -> + {ok, [binary()]} | too_many_failed_nodes. +get_tags(Server, Mode, Timeout) -> + disco_profile:timed_run( + fun() -> gen_server:call(Server, {get_tags, Mode}, Timeout) end, + get_tags). + +-spec new_blob(string()|object_name(), non_neg_integer(), [node()], [node()]) -> + too_many_replicas | {ok, [nonempty_string()]}. +new_blob(Obj, K, Include, Exclude) -> + gen_server:call(?MODULE, {new_blob, Obj, K, Include, Exclude}, infinity). + +-spec new_blob(server(), string()|object_name(), non_neg_integer(), [node()], [node()]) -> + too_many_replicas | {ok, [nonempty_string()]}. +new_blob(Master, Obj, K, Include, Exclude) -> + gen_server:call(Master, {new_blob, Obj, K, Include, Exclude}, infinity). + +-spec safe_gc_blacklist() -> {ok, [node()]} | {error, term()}. +safe_gc_blacklist() -> + gen_server:call(?MODULE, safe_gc_blacklist). + +-spec safe_gc_blacklist(gb_sets:set()) -> ok. +safe_gc_blacklist(SafeGCBlacklist) -> + gen_server:cast(?MODULE, {safe_gc_blacklist, SafeGCBlacklist}). + +-spec update_gc_stats(gc_run_stats()) -> ok. +update_gc_stats(Stats) -> + gen_server:cast(?MODULE, {update_gc_stats, Stats}). + +-type nodes_update() :: [{node(), boolean(), boolean()}]. +-spec update_nodes(nodes_update()) -> ok. +update_nodes(DDFSNodes) -> + gen_server:cast(?MODULE, {update_nodes, DDFSNodes}). + +-spec update_nodestats(gb_trees:tree()) -> ok. +update_nodestats(NewNodes) -> + gen_server:cast(?MODULE, {update_nodestats, NewNodes}). + +-spec update_tag_cache(gb_sets:set()) -> ok. +update_tag_cache(TagCache) -> + gen_server:cast(?MODULE, {update_tag_cache, TagCache}). + +-spec refresh_tag_cache() -> ok. +refresh_tag_cache() -> + gen_server:cast(?MODULE, refresh_tag_cache). + +%% =================================================================== +%% gen_server callbacks + +-spec init(_) -> gs_init(). +init(_Args) -> + _ = [disco_profile:new_histogram(Name) + || Name <- [get_tags, do_get_tags_all, do_get_tags_filter, + do_get_tags_safe, do_get_tags_gc]], + spawn_link(fun() -> monitor_diskspace() end), + spawn_link(fun() -> ddfs_gc:start_gc(disco:get_setting("DDFS_DATA")) end), + Refresher = spawn_link(fun() -> refresh_tag_cache_proc() end), + put(put_port, disco:get_setting("DDFS_PUT_PORT")), + {ok, #state{cache_refresher = Refresher}}. + +-type choose_write_nodes_msg() :: {choose_write_nodes, non_neg_integer(), [node()], [node()]}. +-type new_blob_msg() :: {new_blob, string() | object_name(), non_neg_integer(), [node()]}. +-type tag_msg() :: {tag, ddfs_tag:call_msg(), tagname()}. +-spec handle_call(dbg_state_msg(), from(), state()) -> + gs_reply(state()); + ({get_nodeinfo, all}, from(), state()) -> + gs_reply({ok, [node_info()]}); + (get_read_nodes, from(), state()) -> + gs_reply({ok, [node()], non_neg_integer}); + (gc_blacklist, from(), state()) -> + gs_reply({ok, [node()]}); + (gc_stats, from(), state()) -> + gs_reply({ok, gc_stats(), erlang:timestamp()}); + (choose_write_nodes_msg(), from(), state()) -> + gs_reply({ok, [node()]}); + (new_blob_msg(), from(), state()) -> + gs_reply(new_blob_result()); + (tag_msg(), from(), state()) -> + gs_reply({error, nonodes}) | gs_noreply(); + ({get_tags, gc | safe}, from(), state()) -> + gs_noreply(); + ({get_hosted_tags, host()}, from(), state()) -> + gs_noreply(); + (safe_gc_blacklist, from(), state()) -> + gs_reply({ok, [node()]}). +handle_call(dbg_get_state, _, S) -> + {reply, S, S}; + +handle_call({get_nodeinfo, all}, _From, #state{nodes = Nodes} = S) -> + {reply, {ok, Nodes}, S}; + +handle_call(get_read_nodes, _F, #state{nodes = Nodes, read_blacklist = RB} = S) -> + {reply, do_get_readable_nodes(Nodes, RB), S}; + +handle_call(gc_blacklist, _F, #state{gc_blacklist = Nodes} = S) -> + {reply, {ok, Nodes}, S}; + +handle_call(gc_stats, _F, #state{gc_stats = Stats} = S) -> + {reply, {ok, Stats}, S}; + +handle_call({choose_write_nodes, K, Include, Exclude}, _, + #state{nodes = N, write_blacklist = WBL, gc_blacklist = GBL} = S) -> + BL = lists:umerge(WBL, GBL), + {reply, do_choose_write_nodes(N, K, Include, Exclude, BL), S}; + +handle_call({new_blob, Obj, K, Include, Exclude}, _, + #state{nodes = N, gc_blacklist = GBL, write_blacklist = WBL} = S) -> + BL = lists:umerge(WBL, GBL), + {reply, do_new_blob(Obj, K, Include, Exclude, BL, N), S}; + +handle_call({tag, _M, _Tag}, _From, #state{nodes = []} = S) -> + {reply, {error, no_nodes}, S}; + +handle_call({tag, M, Tag}, From, S) -> + {noreply, do_tag_request(M, Tag, From, S)}; + +handle_call({get_tags, Mode}, From, #state{nodes = Nodes} = S) -> + spawn(fun() -> + gen_server:reply(From, do_get_tags(Mode, [N || {N, _} <- Nodes])) + end), + {noreply, S}; + +handle_call({get_hosted_tags, Host}, From, S) -> + spawn(fun() -> gen_server:reply(From, ddfs_gc:hosted_tags(Host)) end), + {noreply, S}; + +handle_call(safe_gc_blacklist, _From, #state{safe_gc_blacklist = SBL} = S) -> + {reply, {ok, gb_sets:to_list(SBL)}, S}. + +-spec handle_cast({tag_notify, ddfs_tag:cast_msg(), tagname()} + | {gc_blacklist, [node()]} + | {safe_gc_blacklist, gb_sets:set()} + | {update_gc_stats, gc_stats()} + | {update_tag_cache, gb_sets:set()} + | refresh_tag_cache + | {update_nodes, nodes_update()} + | {update_nodestats, gb_trees:tree()}, + state()) -> gs_noreply(). +handle_cast({tag_notify, M, Tag}, S) -> + {noreply, do_tag_notify(M, Tag, S)}; + +handle_cast({gc_blacklist, Nodes}, #state{safe_gc_blacklist = SBL} = S) -> + BLSet = gb_sets:from_list(Nodes), + NewSBL = gb_sets:intersection(BLSet, SBL), + {noreply, S#state{gc_blacklist = gb_sets:to_list(BLSet), + safe_gc_blacklist = NewSBL}}; + +handle_cast({safe_gc_blacklist, SafeBlacklist}, #state{gc_blacklist = BL} = S) -> + SBL = gb_sets:intersection(SafeBlacklist, gb_sets:from_list(BL)), + {noreply, S#state{safe_gc_blacklist = SBL}}; + +handle_cast({update_gc_stats, Stats}, S) -> + {noreply, S#state{gc_stats = {Stats, now()}}}; + +handle_cast({update_tag_cache, TagCache}, S) -> + {noreply, S#state{tag_cache = TagCache}}; + +handle_cast(refresh_tag_cache, #state{cache_refresher = Refresher} = S) -> + Refresher ! refresh, + {noreply, S}; + +handle_cast({update_nodes, NewNodes}, S) -> + {noreply, do_update_nodes(NewNodes, S)}; + +handle_cast({update_nodestats, NewNodes}, S) -> + {noreply, do_update_nodestats(NewNodes, S)}. + +-spec handle_info({'DOWN', _, _, pid(), _}, state()) -> gs_noreply(). +handle_info({'DOWN', _, _, Pid, _}, S) -> + {noreply, do_tag_exit(Pid, S)}. + +%% =================================================================== +%% gen_server callback stubs + +-spec terminate(term(), state()) -> ok. +terminate(Reason, _State) -> + lager:warning("DDFS master died: ~p", [Reason]). + +-spec code_change(term(), state(), term()) -> {ok, state()}. +code_change(_OldVsn, State, _Extra) -> {ok, State}. + +%% =================================================================== +%% internal functions + +-spec do_get_readable_nodes([node_info()], [node()]) -> + {ok, [node()], non_neg_integer()}. +do_get_readable_nodes(Nodes, ReadBlacklist) -> + NodeSet = gb_sets:from_ordset(lists:sort([Node || {Node, _} <- Nodes])), + BlackSet = gb_sets:from_ordset(ReadBlacklist), + ReadableNodeSet = gb_sets:subtract(NodeSet, BlackSet), + {ok, gb_sets:to_list(ReadableNodeSet), gb_sets:size(BlackSet)}. + +-spec do_choose_write_nodes([node_info()], non_neg_integer(), [node()], [node()], [node()]) -> + {ok, [node()]}. +do_choose_write_nodes(Nodes, K, Include, Exclude, BlackList) -> + % Include is the list of nodes that must be included + % + % Node selection algorithm: + % 1. try to choose K nodes randomly from all the nodes which have + % more than ?MIN_FREE_SPACE bytes free space available and which + % are not excluded or blacklisted. + % 2. if K nodes cannot be found this way, choose the K emptiest + % nodes which are not excluded or blacklisted. + Primary = ([N || {N, {Free, _Total}} <- Nodes, Free > ?MIN_FREE_SPACE / 1024] + -- (Exclude ++ BlackList)), + if length(Primary) >= K -> + {ok, Include ++ disco_util:choose_random(Primary -- Include , K - length(Include))}; + true -> + Preferred = [N || {N, _} <- lists:reverse(lists:keysort(2, Nodes))], + Secondary = Include ++ lists:sublist(Preferred -- (Include ++ Exclude ++ BlackList), + K - length(Include)), + {ok, Secondary} + end. + +-type new_blob_result() :: too_many_replicas | {ok, [nonempty_string()]}. +-spec do_new_blob(string()|object_name(), non_neg_integer(), [node()], [node()], [node()], [node_info()]) -> + new_blob_result(). +do_new_blob(_Obj, K, _Include, _Exclude, _BlackList, Nodes) when K > length(Nodes) -> + too_many_replicas; +do_new_blob(Obj, K, Include, Exclude, BlackList, Nodes) -> + {ok, WriteNodes} = do_choose_write_nodes(Nodes, K, Include, Exclude, BlackList), + Urls = [["http://", disco:host(N), ":", get(put_port), "/ddfs/", Obj] + || N <- WriteNodes], + {ok, Urls}. + +% Tag request: Start a new tag server if one doesn't exist already. Forward +% the request to the tag server. + +-spec get_tag_pid(tagname(), gb_trees:tree(), false | gb_sets:set()) -> + {pid(), gb_trees:tree()}. +get_tag_pid(Tag, Tags, Cache) -> + case gb_trees:lookup(Tag, Tags) of + none -> + NotFound = (Cache =/= false + andalso not gb_sets:is_element(Tag, Cache)), + {ok, Server} = ddfs_tag:start(Tag, NotFound), + erlang:monitor(process, Server), + {Server, gb_trees:insert(Tag, Server, Tags)}; + {value, P} -> + {P, Tags} + end. + +-spec do_tag_request(term(), tagname(), replyto(), state()) -> + state(). +do_tag_request(M, Tag, From, #state{tags = Tags, tag_cache = Cache} = S) -> + {Pid, TagsN} = get_tag_pid(Tag, Tags, Cache), + gen_server:cast(Pid, {M, From}), + S#state{tags = TagsN, + tag_cache = Cache =/= false andalso gb_sets:add(Tag, Cache)}. + +-spec do_tag_notify(term(), tagname(), state()) -> state(). +do_tag_notify(M, Tag, #state{tags = Tags, tag_cache = Cache} = S) -> + {Pid, TagsN} = get_tag_pid(Tag, Tags, Cache), + gen_server:cast(Pid, {notify, M}), + S#state{tags = TagsN, + tag_cache = Cache =/= false andalso gb_sets:add(Tag, Cache)}. + +-spec do_update_nodes(nodes_update(), state()) -> state(). +do_update_nodes(NewNodes, #state{nodes = Nodes, tags = Tags} = S) -> + WriteBlacklist = lists:sort([Node || {Node, false, _} <- NewNodes]), + ReadBlacklist = lists:sort([Node || {Node, _, false} <- NewNodes]), + OldNodes = gb_trees:from_orddict(Nodes), + UpdatedNodes = lists:keysort(1, [case gb_trees:lookup(Node, OldNodes) of + none -> + {Node, {0, 0}}; + {value, OldStats} -> + {Node, OldStats} + end || {Node, _WB, _RB} <- NewNodes]), + if + UpdatedNodes =/= Nodes -> + _ = [gen_server:cast(Pid, {die, none}) || Pid <- gb_trees:values(Tags)], + spawn(fun() -> + {ok, ReadableNodes, RBSize} = + do_get_readable_nodes(UpdatedNodes, ReadBlacklist), + refresh_tag_cache(ReadableNodes, RBSize) + end), + S#state{nodes = UpdatedNodes, + write_blacklist = WriteBlacklist, + read_blacklist = ReadBlacklist, + tag_cache = false, + tags = gb_trees:empty()}; + true -> + S#state{write_blacklist = WriteBlacklist, + read_blacklist = ReadBlacklist} + end. + +-spec do_update_nodestats(gb_trees:tree(), state()) -> state(). +do_update_nodestats(NewNodes, #state{nodes = Nodes} = S) -> + UpdatedNodes = [case gb_trees:lookup(Node, NewNodes) of + none -> + {Node, Stats}; + {value, NewStats} -> + {Node, NewStats} + end || {Node, Stats} <- Nodes], + S#state{nodes = UpdatedNodes}. + +-spec do_tag_exit(pid(), state()) -> state(). +do_tag_exit(Pid, S) -> + NewTags = [X || {_, V} = X <- gb_trees:to_list(S#state.tags), V =/= Pid], + S#state{tags = gb_trees:from_orddict(NewTags)}. + +-spec do_get_tags(all | filter, [node()]) -> {[node()], [node()], [binary()]}; + (safe, [node()]) -> {ok, [binary()]} | too_many_failed_nodes; + (gc, [node()]) -> {ok, [binary()], [node()]} | too_many_failed_nodes. +do_get_tags(all, Nodes) -> + disco_profile:timed_run( + fun() -> + {Replies, Failed} = + gen_server:multi_call(Nodes, ddfs_node, get_tags, ?NODE_TIMEOUT), + {OkNodes, Tags} = lists:unzip(Replies), + {OkNodes, Failed, lists:usort(lists:flatten(Tags))} + end, do_get_tags_all); + +do_get_tags(filter, Nodes) -> + disco_profile:timed_run( + fun() -> + {OkNodes, Failed, Tags} = do_get_tags(all, Nodes), + case tag_operation(get_tagnames, <<"+deleted">>, ?NODEOP_TIMEOUT) of + {ok, Deleted} -> + TagSet = gb_sets:from_ordset(Tags), + DelSet = gb_sets:insert(<<"+deleted">>, Deleted), + NotDeleted = gb_sets:to_list(gb_sets:subtract(TagSet, DelSet)), + {OkNodes, Failed, NotDeleted}; + E -> + E + end + end, do_get_tags_filter); + +do_get_tags(safe, Nodes) -> + disco_profile:timed_run( + fun() -> + TagMinK = list_to_integer(disco:get_setting("DDFS_TAG_MIN_REPLICAS")), + case do_get_tags(filter, Nodes) of + {_OkNodes, Failed, Tags} when length(Failed) < TagMinK -> + {ok, Tags}; + _ -> + too_many_failed_nodes + end + end, do_get_tags_safe); + +% The returned tag list may include +deleted. +do_get_tags(gc, Nodes) -> + disco_profile:timed_run( + fun() -> + {OkNodes, Failed, Tags} = do_get_tags(all, Nodes), + TagMinK = list_to_integer(disco:get_setting("DDFS_TAG_MIN_REPLICAS")), + case length(Failed) < TagMinK of + false -> + too_many_failed_nodes; + true -> + case tag_operation(get_tagnames, <<"+deleted">>, ?NODEOP_TIMEOUT) of + {ok, Deleted} -> + TagSet = gb_sets:from_ordset(Tags), + NotDeleted = gb_sets:subtract(TagSet, Deleted), + {ok, gb_sets:to_list(NotDeleted), OkNodes}; + E -> + E + end + end + end, do_get_tags_gc). + +% Timeouts in this call by the below processes can cause ddfs_master +% itself to crash, since the processes are linked to it. +-spec safe_get_read_nodes() -> {ok, [node()], non_neg_integer()} | error. +safe_get_read_nodes() -> + try get_read_nodes() of + {ok, _ReadableNodes, _RBSize} = RN -> + RN; + E -> + lager:error("unexpected response retrieving readable nodes: ~p", [E]), + error + catch + K:E -> + lager:error("error retrieving readable nodes: ~p:~p", [K, E]), + error + end. + +-spec monitor_diskspace() -> no_return(). +monitor_diskspace() -> + case safe_get_read_nodes() of + {ok, ReadableNodes, _RBSize} -> + {Space, _F} = gen_server:multi_call(ReadableNodes, + ddfs_node, + get_diskspace, + ?NODE_TIMEOUT), + update_nodestats(gb_trees:from_orddict(lists:keysort(1, Space))); + error -> + ok + end, + timer:sleep(?DISKSPACE_INTERVAL), + monitor_diskspace(). + +-spec refresh_tag_cache_proc() -> no_return(). +refresh_tag_cache_proc() -> + case safe_get_read_nodes() of + {ok, ReadableNodes, RBSize} -> + refresh_tag_cache(ReadableNodes, RBSize); + error -> + ok + end, + receive + refresh -> + ok + after ?TAG_CACHE_INTERVAL -> + ok + end, + refresh_tag_cache_proc(). + +-spec refresh_tag_cache([node()], non_neg_integer()) -> ok. +refresh_tag_cache(Nodes, BLSize) -> + TagMinK = list_to_integer(disco:get_setting("DDFS_TAG_MIN_REPLICAS")), + {Replies, Failed} = + gen_server:multi_call(Nodes, ddfs_node, get_tags, ?NODE_TIMEOUT), + if Nodes =/= [], length(Failed) + BLSize < TagMinK -> + {_OkNodes, Tags} = lists:unzip(Replies), + update_tag_cache(gb_sets:from_list(lists:flatten(Tags))); + true -> ok + end. diff --git a/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs_tag.hrl b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs_tag.hrl new file mode 100644 index 0000000000..2920b67fc5 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/ddfs_tag.hrl @@ -0,0 +1,19 @@ + +-type tokentype() :: 'read' | 'write'. +-type user_attr() :: [{binary(), binary()}]. +% An 'internal' token is also used by internal consumers, but never stored. +-type token() :: 'null' | binary(). + +-type tagname() :: binary(). +-type tagid() :: binary(). + +-type attrib() :: 'urls' | 'read_token' | 'write_token' | {'user', binary()}. + +-record(tagcontent, {id :: tagid(), + last_modified :: binary(), + read_token = null :: token(), + write_token = null :: token(), + urls = [] :: [[binary()]], + user = [] :: user_attr()}). + +-type tagcontent() :: #tagcontent{}. diff --git a/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/gs_util.hrl b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/gs_util.hrl new file mode 100644 index 0000000000..d579e9a7d7 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/ddfs_master/gs_util.hrl @@ -0,0 +1,16 @@ +% This is a set of type utilities to be used when spec-cing the +% callbacks of a gen_server implementation. It should be included in +% the impl module, which needs to define the state() type. + +-type gs_init() :: {ok, state()}. +-type gs_reply(T) :: {reply, (T), state()}. +-type gs_noreply() :: {noreply, state()}. +-type gs_noreply_t() :: {noreply, state(), non_neg_integer()}. +-type gs_stop(T) :: {stop, (T), state()}. + +% Generic utilities. + +-type server() :: pid() | atom() | {atom(), node()}. +-type from() :: {pid(), term()}. + +-type dbg_state_msg() :: dbg_get_state. diff --git a/lib/dialyzer/test/small_SUITE_data/src/fun2ms.erl b/lib/dialyzer/test/small_SUITE_data/src/fun2ms.erl new file mode 100644 index 0000000000..9e7df85e4c --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/fun2ms.erl @@ -0,0 +1,21 @@ +-module(fun2ms). +-export([return/0]). +-include_lib("stdlib/include/ms_transform.hrl"). + +-record(snapshot, {id :: integer(), arg1 :: atom(), arg2 :: tuple()}). + +return() -> + TableId = ets:new(table, [public, {keypos, #snapshot.id}]), + + ets:insert(TableId, [#snapshot{id = 1, arg1 = hard, arg2 = {1,2}}, + #snapshot{id = 2, arg1 = rock, arg2 = {1,2}}, + #snapshot{id = 3, arg1 = hallelujah, arg2 = + {1,2}}]), + + + Example = ets:fun2ms( + fun(#snapshot{id = Arg1, arg1 = Arg2}) -> + {Arg1, Arg2} + end), + + ets:select(TableId, Example). diff --git a/lib/dialyzer/test/small_SUITE_data/src/funs_from_outside.erl b/lib/dialyzer/test/small_SUITE_data/src/funs_from_outside.erl new file mode 100644 index 0000000000..f4cbf31160 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/funs_from_outside.erl @@ -0,0 +1,83 @@ +-module(funs_from_outside).
+
+-export([run1/2, run2/2, run3/2]).
+-export([test1/1, test2/1]).
+
+%%------------------------------------------------------------------------------
+
+run1(X, Y) ->
+ testa(fun do_something/1, X, Y).
+
+testa(Fun, X, Y) ->
+ F = case even(X) of
+ true -> Fun;
+ false -> fun do_nothing/1
+ end,
+ case F(Y) of
+ {ok, _} -> ok;
+ error -> error
+ end.
+
+do_nothing(_) -> {ok, nothing}.
+
+do_something(_) -> {ok, something}.
+
+even(X) ->
+ X rem 2 =:= 0.
+
+%%------------------------------------------------------------------------------
+
+%% Duplicating code since we are monovariant...
+
+run2(X, Y) ->
+ testb(fun do_something/1, X, Y).
+
+testb(Fun, X, Y) ->
+ F = case even(X) of
+ true -> Fun;
+ false -> fun do_nothing/1
+ end,
+ case F(Y) of
+ error -> error
+ end.
+
+%%------------------------------------------------------------------------------
+
+%% Duplicating code since we are monovariant...
+
+run3(X, Y) ->
+ testc(fun do_something_2/1, X, Y).
+
+testc(Fun, X, Y) ->
+ F = case even(X) of
+ true -> Fun;
+ false -> fun do_nothing/1
+ end,
+ case F(Y) of
+ {ok, _} -> ok;
+ %% This pattern can match.
+ error -> error
+ end.
+
+do_something_2(foo) -> {ok, something};
+do_something_2(_) -> error.
+
+%%------------------------------------------------------------------------------
+
+test1(Fun) ->
+ F = case get(test1) of
+ test1_t -> Fun;
+ test1_f -> fun fok/0
+ end,
+ error = F().
+
+fok() -> ok.
+
+%%------------------------------------------------------------------------------
+
+test2(Fun) ->
+ F = case get(test1) of
+ test1_t -> fun fok/0;
+ test1_f -> fun fok/0
+ end,
+ error = F().
diff --git a/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_data.erl b/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_data.erl new file mode 100644 index 0000000000..c26787fe24 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_data.erl @@ -0,0 +1,5 @@ +-module(scala_data). + +-export_type([data/0]). + +-opaque data() :: {'data', term()}. diff --git a/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_user.erl b/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_user.erl new file mode 100644 index 0000000000..4e981f3b74 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_user.erl @@ -0,0 +1,8 @@ +-module(scala_user). + +-export([is_list/2]). + +-spec is_list(atom(), scala_data:data()) -> boolean(). + +is_list( List,Data) when is_list(List) -> true; +is_list(Tuple,Data) when is_tuple(Tuple) -> false. diff --git a/lib/dialyzer/test/small_SUITE_data/src/limit.erl b/lib/dialyzer/test/small_SUITE_data/src/limit.erl new file mode 100644 index 0000000000..97ee585b77 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/limit.erl @@ -0,0 +1,20 @@ +%% Misc cases where Dialyzer would fail with system_limit or crash + +-module(limit). + +-export([tu/0, big/1, b2/0]). + +tu() -> + erlang:make_tuple(1 bsl 24, def, [{5,e},{1,a},{3,c}]). + +big(<<Int:1152921504606846976/unit:128,0,_/binary>>) -> {5,Int}. + +b2() -> + Maxbig = maxbig(), + _ = bnot Maxbig, + ok. + +maxbig() -> + %% We assume that the maximum arity is (1 bsl 19) - 1. + Ws = erlang:system_info(wordsize), + (((1 bsl ((16777184 * (Ws div 4))-1)) - 1) bsl 1) + 1. diff --git a/lib/dialyzer/test/small_SUITE_data/src/literals.erl b/lib/dialyzer/test/small_SUITE_data/src/literals.erl new file mode 100644 index 0000000000..abd7033712 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/literals.erl @@ -0,0 +1,33 @@ +-module(literals). + +%% Bad records inside structures used to be ignored. The reason: +%% v3_core:unfold() does not annotate the parts of a literal. +%% This example does not work perfectly yet, in particular Maps. + +-export([t1/0, t2/0, t3/0, t4/0, m1/1, m2/1, m3/1, m4/1]). + +-record(r, {id :: integer}). + +t1() -> + #r{id = a}. % violation + +t2() -> + [#r{id = a}]. % violation + +t3() -> + {#r{id = a}}. % violation + +t4() -> + #{a => #r{id = a}}. % violation found, but t4() returns... (bug) + +m1(#r{id = a}) -> % violation + ok. + +m2([#r{id = a}]) -> % violation + ok. + +m3({#r{id = a}}) -> % can never match; not so good + ok. + +m4(#{a := #r{id = a}}) -> % violation not found + ok. diff --git a/lib/dialyzer/test/small_SUITE_data/src/maps1.erl b/lib/dialyzer/test/small_SUITE_data/src/maps1.erl index 06ced5b69e..228ffe2c22 100644 --- a/lib/dialyzer/test/small_SUITE_data/src/maps1.erl +++ b/lib/dialyzer/test/small_SUITE_data/src/maps1.erl @@ -10,6 +10,7 @@ -export([recv/3, decode/1]). +-export([get_my_map/0,is_my_map/1]). %-record(can_pkt, {id, data :: binary(), timestamp}). @@ -39,3 +40,38 @@ t2() -> ok. update(#{ id := Id, val := Val } = M, X) when is_integer(Id) -> M#{ val := [Val,X] }. + +%% key coalescing + +-spec get_my_map() -> map(). + +get_my_map() -> + #{labels => [one, two], + number => 27, + [1,2,3] => wer, + {4,5,6} => sdf, + kvok => #{ + <<"wat">> => v, + a => qwe, + 2 => asd, + [1,2,3] => wer, + {4,5,6} => sdf, + "abc" => zxc + } + }. + +-spec is_my_map(map()) -> 'ok'. + +is_my_map(#{labels := [one, two], + number := 27, + [1,2,3] := wer, + {4,5,6} := sdf, + kvok := #{ + <<"wat">> := v, + a := qwe, + 2 := asd, + [1,2,3] := wer, + {4,5,6} := sdf, + "abc" := zxc + } + }) -> ok. diff --git a/lib/dialyzer/test/small_SUITE_data/src/maps_difftype.erl b/lib/dialyzer/test/small_SUITE_data/src/maps_difftype.erl new file mode 100644 index 0000000000..19e61a7944 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/maps_difftype.erl @@ -0,0 +1,11 @@ +%% +%% File: maps_difftype.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-04-29 +%% +-module(maps_difftype). + +-export([empty_mismatch/1]). + +empty_mismatch(Tuple) when is_tuple(Tuple) -> + case Tuple of #{} -> ok end. diff --git a/lib/dialyzer/test/small_SUITE_data/src/maps_redef.erl b/lib/dialyzer/test/small_SUITE_data/src/maps_redef.erl new file mode 100644 index 0000000000..70059f73b6 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/maps_redef.erl @@ -0,0 +1,12 @@ +-module(maps_redef). + +-export([t/0]). + +%% OK in Erlang/OTP 17, at least. + +-type map() :: atom(). % redefine built-in type + +-spec t() -> map(). + +t() -> + a. % OK diff --git a/lib/dialyzer/test/small_SUITE_data/src/maps_redef2.erl b/lib/dialyzer/test/small_SUITE_data/src/maps_redef2.erl new file mode 100644 index 0000000000..945b2a9144 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/maps_redef2.erl @@ -0,0 +1,23 @@ +%% In 17, the linter says that map(A) redefines 'type map', which is +%% allowed until next release. However, Dialyzer used to replace +%% map(A) with #{}, which resulted in warnings. + +-module(maps_redef2). + +-export([t/0]). + +-type map(_A) :: integer(). + +t() -> + M = new(), + t1(M). + +-spec t1(map(_)) -> map(_). + +t1(A) -> + A + A. + +-spec new() -> map(_). + +new() -> + 3. diff --git a/lib/dialyzer/test/small_SUITE_data/src/pretty_bitstring.erl b/lib/dialyzer/test/small_SUITE_data/src/pretty_bitstring.erl new file mode 100644 index 0000000000..3dbf5ab7a7 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/pretty_bitstring.erl @@ -0,0 +1,8 @@ +%% Prettyprint bitstrings. + +-module(pretty_bitstring). + +-export([t/0]). + +t() -> + binary:copy(<<1,2,3:3>>,2). diff --git a/lib/dialyzer/test/small_SUITE_data/src/record_update.erl b/lib/dialyzer/test/small_SUITE_data/src/record_update.erl new file mode 100644 index 0000000000..bad7a0a929 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/record_update.erl @@ -0,0 +1,10 @@ +-module(record_update). + +-export([quux/2]). + +-record(foo, {bar :: atom()}). + +-spec quux(#foo{}, string()) -> #foo{}. + +quux(Foo, NotBar) -> + Foo#foo{ bar = NotBar }. diff --git a/lib/dialyzer/test/small_SUITE_data/src/confusing_record_warning.erl b/lib/dialyzer/test/small_SUITE_data/src/relevant_record_warning.erl index 8af74e0914..3ff65458df 100644 --- a/lib/dialyzer/test/small_SUITE_data/src/confusing_record_warning.erl +++ b/lib/dialyzer/test/small_SUITE_data/src/relevant_record_warning.erl @@ -1,3 +1,7 @@ +%% Formerly confusing_record_warning.erl. +%% The warning output is relevant as of Erlang/OTP 17.1. +%% The original comment kept below. + %%--------------------------------------------------------------------- %% A user complained that dialyzer produces a weird warning for the %% following program. I explained to him that there is an implicit @@ -9,7 +13,7 @@ %% The pattern {'r', [_]} can never match the type any() %% We should clearly give some less confusing warning in this case. %%--------------------------------------------------------------------- --module(confusing_record_warning). +-module(relevant_record_warning). -export([test/1]). diff --git a/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl b/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl new file mode 100644 index 0000000000..c34fa1b9dd --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl @@ -0,0 +1,11 @@ +-module(remote_field). + +-type f(T) :: {ssl:sslsocket(), T}. + +-record(r1, { f1 :: f(_) }). +-type r1(T) :: #r1{ f1 :: fun((ssl:sslsocket(), T) -> any()) }. + +-record(state, { + r :: r1(T), + arg :: T + }). diff --git a/lib/dialyzer/test/small_SUITE_data/src/remote_field2.erl b/lib/dialyzer/test/small_SUITE_data/src/remote_field2.erl new file mode 100644 index 0000000000..35687e22ec --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/remote_field2.erl @@ -0,0 +1,17 @@ +-module(remote_field2). + +-export([handle_cast/2]). + +-record(state, {tcp_socket :: inet:socket()}). + +-spec handle_cast(_,_) -> + {noreply,_} | + {stop,{shutdown,connection_closed}, + #state{tcp_socket :: port()}}. +handle_cast({send, Message}, #state{tcp_socket = TCPSocket} = State) -> + case gen_tcp:send(TCPSocket, Message) of + ok -> + {noreply, State}; + {error, closed} -> + {stop, {shutdown, connection_closed}, State} + end. diff --git a/lib/dialyzer/test/underspecs_SUITE_data/dialyzer_options b/lib/dialyzer/test/underspecs_SUITE_data/dialyzer_options index f7197ac30f..6843119b9d 100644 --- a/lib/dialyzer/test/underspecs_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/underspecs_SUITE_data/dialyzer_options @@ -1 +1 @@ -{dialyzer_options, [{warnings, [underspecs]}]}. +{dialyzer_options, [{warnings, [underspecs, no_unknown]}]}. diff --git a/lib/dialyzer/test/underspecs_SUITE_data/results/arr b/lib/dialyzer/test/underspecs_SUITE_data/results/arr new file mode 100644 index 0000000000..9497d12eec --- /dev/null +++ b/lib/dialyzer/test/underspecs_SUITE_data/results/arr @@ -0,0 +1,4 @@ + +arr.erl:14: Type specification arr:test2(array:array(T),non_neg_integer(),T) -> array:array(T) is a supertype of the success typing: arr:test2(array:array(_),pos_integer(),_) -> array:array(_) +arr.erl:24: Type specification arr:test4(array:array(T),non_neg_integer(),_) -> array:array(T) is a supertype of the success typing: arr:test4(array:array(_),pos_integer(),_) -> array:array(_) +arr.erl:29: Type specification arr:test5(array:array(T),non_neg_integer(),T) -> array:array(T) is a supertype of the success typing: arr:test5(array:array(_),non_neg_integer(),integer()) -> array:array(_) diff --git a/lib/dialyzer/test/underspecs_SUITE_data/src/arr.erl b/lib/dialyzer/test/underspecs_SUITE_data/src/arr.erl new file mode 100644 index 0000000000..3b265ccec2 --- /dev/null +++ b/lib/dialyzer/test/underspecs_SUITE_data/src/arr.erl @@ -0,0 +1,41 @@ +-module(arr). + +%% http://erlang.org/pipermail/erlang-questions/2014-August/080445.html + +-define(A, array). + +-export([test/3, test2/3, test3/3, test4/3, test5/3, test6/3]). + +-spec test(?A:array(T), non_neg_integer(), T) -> ?A:array(T). + +test(Array, N, Value) -> + ?A:set(N, Value, Array). + +-spec test2(?A:array(T), non_neg_integer(), T) -> ?A:array(T). + +test2(Array, N, Value) when N > 0 -> + ?A:set(N, Value, Array). + +-spec test3(?A:array(T), non_neg_integer(), _) -> ?A:array(T). + +test3(Array, N, Value) -> + ?A:set(N, Value, Array). + +-spec test4(?A:array(T), non_neg_integer(), _) -> ?A:array(T). + +test4(Array, N, Value) when N > 0 -> + ?A:set(N, Value, Array). + +-spec test5(?A:array(T), non_neg_integer(), T) -> ?A:array(T). + +test5(Array, N, Value) when is_integer(Value) -> + ?A:set(N, Value, Array). + +%% One would ideally want a warning also for test6(), but the current +%% analysis of parametrized opaque types is not strong enough to +%% discover this. +-spec test6(?A:array(integer()), non_neg_integer(), integer()) -> + ?A:array(any()). + +test6(Array, N, Value) -> + ?A:set(N, Value, Array). diff --git a/lib/dialyzer/test/user_SUITE_data/dialyzer_options b/lib/dialyzer/test/user_SUITE_data/dialyzer_options index 513ed7752b..d20ecd389f 100644 --- a/lib/dialyzer/test/user_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/user_SUITE_data/dialyzer_options @@ -1,2 +1,2 @@ -{dialyzer_options, []}. +{dialyzer_options, [{warnings, [no_unknown]}]}. {time_limit, 3}.
\ No newline at end of file |