aboutsummaryrefslogtreecommitdiffstats
path: root/lib/stdlib/test
diff options
context:
space:
mode:
authorRaimo Niskanen <[email protected]>2018-02-12 09:41:45 +0100
committerRaimo Niskanen <[email protected]>2018-02-21 10:47:15 +0100
commit66b3383675cb9090469cf8f4b281642691eb40cd (patch)
treebccb31b5cd618fcea467660b40d5a895b5e9d76e /lib/stdlib/test
parent98b4b5de3b8acb040413085989c09284063934a9 (diff)
downloadotp-66b3383675cb9090469cf8f4b281642691eb40cd.tar.gz
otp-66b3383675cb9090469cf8f4b281642691eb40cd.tar.bz2
otp-66b3383675cb9090469cf8f4b281642691eb40cd.zip
Repeat stat tests to only fail for systematic errors
Diffstat (limited to 'lib/stdlib/test')
-rw-r--r--lib/stdlib/test/rand_SUITE.erl49
1 files changed, 33 insertions, 16 deletions
diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl
index 174a34001f..743676fc13 100644
--- a/lib/stdlib/test/rand_SUITE.erl
+++ b/lib/stdlib/test/rand_SUITE.erl
@@ -52,7 +52,9 @@ all() ->
[seed, interval_int, interval_float,
api_eq,
reference,
- {group, basic_stats}, uniform_real_conv,
+ {group, basic_stats},
+ {group, distr_stats},
+ uniform_real_conv,
plugin, measure,
{group, reference_jump}
].
@@ -60,8 +62,9 @@ all() ->
groups() ->
[{basic_stats, [parallel],
[basic_stats_uniform_1, basic_stats_uniform_2,
- basic_stats_standard_normal,
- stats_standard_normal_box_muller,
+ basic_stats_standard_normal]},
+ {distr_stats, [parallel],
+ [stats_standard_normal_box_muller,
stats_standard_normal_box_muller_2,
stats_standard_normal]},
{reference_jump, [parallel],
@@ -70,6 +73,9 @@ groups() ->
group(basic_stats) ->
%% valgrind needs a lot of time
[{timetrap,{minutes,10}}];
+group(distr_stats) ->
+ %% valgrind needs a lot of time
+ [{timetrap,{minutes,10}}];
group(reference_jump) ->
%% valgrind needs a lot of time
[{timetrap,{minutes,10}}].
@@ -437,7 +443,7 @@ stats_standard_normal_box_muller(Config) when is_list(Config) ->
{Z, [S]}
end,
State = [rand:seed(exrop)],
- stats_standard_normal(NormalS, State)
+ stats_standard_normal(NormalS, State, 3)
catch error:_ ->
{skip, "math:erfc/1 not supported"}
end.
@@ -462,7 +468,7 @@ stats_standard_normal_box_muller_2(Config) when is_list(Config) ->
{Z, [S]}
end,
State = [rand:seed(exrop)],
- stats_standard_normal(NormalS, State)
+ stats_standard_normal(NormalS, State, 3)
catch error:_ ->
{skip, "math:erfc/1 not supported"}
end.
@@ -472,12 +478,12 @@ stats_standard_normal(Config) when is_list(Config) ->
try math:erfc(1.0) of
_ ->
stats_standard_normal(
- fun rand:normal_s/1, rand:seed_s(exrop))
+ fun rand:normal_s/1, rand:seed_s(exrop), 3)
catch error:_ ->
{skip, "math:erfc/1 not supported"}
end.
%%
-stats_standard_normal(Fun, S) ->
+stats_standard_normal(Fun, S, Retries) ->
%%%
%%% ct config:
%%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.2}]}]}.
@@ -509,7 +515,7 @@ stats_standard_normal(Fun, S) ->
"Running standard normal test against ~w std devs for ~w seconds...",
[StdDevs, Seconds]),
StopTime = erlang:monotonic_time(second) + Seconds,
- {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds} =
+ {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds, NewS} =
stats_standard_normal(
InvDelta, Buckets, Histogram, Histogram, 0.0,
Fun, S, Rounds, StopTime, Rounds, 0),
@@ -529,15 +535,26 @@ stats_standard_normal(Fun, S) ->
check_histogram(
W, TotalRounds, StdDevs, NegativeHistogram, Buckets)}
of
+ {_, _, [], []} when InvOP < 100 ->
+ {comment, {tp, TopPrecision, op, InvOP}};
{_, _, [], []} ->
- ok;
+ %% If the probability for getting this Outlier is lower than
+ %% 1/100, then this is fishy!
+ stats_standard_normal(
+ Fun, NewS, Retries, {outlier_fishy, InvOP});
BucketErrors ->
- ct:fail(BucketErrors)
- end,
- %% If the probability for getting this Outlier is lower than 1/100,
- %% then this is fishy!
- (InvOP < 100) orelse ct:fail({outlier_fishy, InvOP}),
- {comment, {tp, TopPrecision, op, InvOP}}.
+ stats_standard_normal(
+ Fun, NewS, Retries, BucketErrors)
+ end.
+%%
+stats_standard_normal(Fun, S, Retries, Failure) ->
+ case Retries - 1 of
+ 0 ->
+ ct:fail(Failure);
+ NewRetries ->
+ ct:pal("Retry due to TC glitch: ~p", [Failure]),
+ stats_standard_normal(Fun, S, NewRetries)
+ end.
%%
stats_standard_normal(
InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier,
@@ -550,7 +567,7 @@ stats_standard_normal(
Fun, S, Rounds, StopTime, Rounds, TotalRounds + Rounds);
_ ->
{PositiveHistogram, NegativeHistogram,
- Outlier, TotalRounds + Rounds}
+ Outlier, TotalRounds + Rounds, S}
end;
stats_standard_normal(
InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier,