aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRaimo Niskanen <[email protected]>2018-02-22 14:50:19 +0100
committerRaimo Niskanen <[email protected]>2018-02-22 14:50:19 +0100
commit83c39a8c38e68347f3fb19a3f83113deadb532e0 (patch)
tree2e376be6b27716fe0c6790cb7c0d668c68b48922
parent4c21bcd5bd08d67f00471455f9489867cc1766fc (diff)
parent510bb79a7ccd5d32db8f531042824b6b95a1bebe (diff)
downloadotp-83c39a8c38e68347f3fb19a3f83113deadb532e0.tar.gz
otp-83c39a8c38e68347f3fb19a3f83113deadb532e0.tar.bz2
otp-83c39a8c38e68347f3fb19a3f83113deadb532e0.zip
Merge branch 'raimo/stdlib/rand-polish-stat-tests'
* raimo/stdlib/rand-polish-stat-tests: Decrease tolerance a wee bit Repeat stat tests to only fail for systematic errors Fix histogram and outlier checks
-rw-r--r--lib/stdlib/test/rand_SUITE.erl70
1 files changed, 45 insertions, 25 deletions
diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl
index 15ae4fa2d7..d753d929f5 100644
--- a/lib/stdlib/test/rand_SUITE.erl
+++ b/lib/stdlib/test/rand_SUITE.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2000-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2018. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -52,7 +52,9 @@ all() ->
[seed, interval_int, interval_float,
api_eq,
reference,
- {group, basic_stats}, uniform_real_conv,
+ {group, basic_stats},
+ {group, distr_stats},
+ uniform_real_conv,
plugin, measure,
{group, reference_jump}
].
@@ -60,8 +62,9 @@ all() ->
groups() ->
[{basic_stats, [parallel],
[basic_stats_uniform_1, basic_stats_uniform_2,
- basic_stats_standard_normal,
- stats_standard_normal_box_muller,
+ basic_stats_standard_normal]},
+ {distr_stats, [parallel],
+ [stats_standard_normal_box_muller,
stats_standard_normal_box_muller_2,
stats_standard_normal]},
{reference_jump, [parallel],
@@ -70,6 +73,9 @@ groups() ->
group(basic_stats) ->
%% valgrind needs a lot of time
[{timetrap,{minutes,10}}];
+group(distr_stats) ->
+ %% valgrind needs a lot of time
+ [{timetrap,{minutes,10}}];
group(reference_jump) ->
%% valgrind needs a lot of time
[{timetrap,{minutes,10}}].
@@ -437,7 +443,7 @@ stats_standard_normal_box_muller(Config) when is_list(Config) ->
{Z, [S]}
end,
State = [rand:seed(exrop)],
- stats_standard_normal(NormalS, State)
+ stats_standard_normal(NormalS, State, 3)
catch error:_ ->
{skip, "math:erfc/1 not supported"}
end.
@@ -462,7 +468,7 @@ stats_standard_normal_box_muller_2(Config) when is_list(Config) ->
{Z, [S]}
end,
State = [rand:seed(exrop)],
- stats_standard_normal(NormalS, State)
+ stats_standard_normal(NormalS, State, 3)
catch error:_ ->
{skip, "math:erfc/1 not supported"}
end.
@@ -472,21 +478,21 @@ stats_standard_normal(Config) when is_list(Config) ->
try math:erfc(1.0) of
_ ->
stats_standard_normal(
- fun rand:normal_s/1, rand:seed_s(exrop))
+ fun rand:normal_s/1, rand:seed_s(exrop), 3)
catch error:_ ->
{skip, "math:erfc/1 not supported"}
end.
%%
-stats_standard_normal(Fun, S) ->
+stats_standard_normal(Fun, S, Retries) ->
%%%
%%% ct config:
-%%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.2}]}]}.
+%%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.0}]}]}.
%%%
Seconds = ct:get_config({?MODULE, ?FUNCTION_NAME, seconds}, 8),
StdDevs =
ct:get_config(
{?MODULE, ?FUNCTION_NAME, std_devs},
- 4.2), % probability erfc(4.2/sqrt(2)) (1/37465) to fail a bucket
+ 4.0), % probability erfc(4.0/sqrt(2)) (1/15787) to fail a bucket
%%%
ct:timetrap({seconds, Seconds + 120}),
%% Buckets is chosen to get a range where the the probability to land
@@ -505,11 +511,11 @@ stats_standard_normal(Fun, S) ->
P0 = math:erf(1 / W),
Rounds = TargetHits * ceil(1.0 / P0),
Histogram = array:new({default, 0}),
- StopTime = erlang:monotonic_time(second) + Seconds,
ct:pal(
"Running standard normal test against ~w std devs for ~w seconds...",
[StdDevs, Seconds]),
- {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds} =
+ StopTime = erlang:monotonic_time(second) + Seconds,
+ {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds, NewS} =
stats_standard_normal(
InvDelta, Buckets, Histogram, Histogram, 0.0,
Fun, S, Rounds, StopTime, Rounds, 0),
@@ -522,16 +528,33 @@ stats_standard_normal(Fun, S) ->
"Total rounds: ~w, tolerance: 1/~.2f..1/~.2f, "
"outlier: ~.2f, probability 1/~.2f.",
[TotalRounds, Precision, TopPrecision, Outlier, InvOP]),
- {TotalRounds, [], []} =
- {TotalRounds,
+ case
+ {bucket_error, TotalRounds,
check_histogram(
W, TotalRounds, StdDevs, PositiveHistogram, Buckets),
check_histogram(
- W, TotalRounds, StdDevs, NegativeHistogram, Buckets)},
- %% If the probability for getting this Outlier is lower than 1/50,
- %% then this is fishy!
- true = (1/50 =< OutlierProbability),
- {comment, {tp, TopPrecision, op, InvOP}}.
+ W, TotalRounds, StdDevs, NegativeHistogram, Buckets)}
+ of
+ {_, _, [], []} when InvOP < 100 ->
+ {comment, {tp, TopPrecision, op, InvOP}};
+ {_, _, [], []} ->
+ %% If the probability for getting this Outlier is lower than
+ %% 1/100, then this is fishy!
+ stats_standard_normal(
+ Fun, NewS, Retries, {outlier_fishy, InvOP});
+ BucketErrors ->
+ stats_standard_normal(
+ Fun, NewS, Retries, BucketErrors)
+ end.
+%%
+stats_standard_normal(Fun, S, Retries, Failure) ->
+ case Retries - 1 of
+ 0 ->
+ ct:fail(Failure);
+ NewRetries ->
+ ct:pal("Retry due to TC glitch: ~p", [Failure]),
+ stats_standard_normal(Fun, S, NewRetries)
+ end.
%%
stats_standard_normal(
InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier,
@@ -544,7 +567,7 @@ stats_standard_normal(
Fun, S, Rounds, StopTime, Rounds, TotalRounds + Rounds);
_ ->
{PositiveHistogram, NegativeHistogram,
- Outlier, TotalRounds + Rounds}
+ Outlier, TotalRounds + Rounds, S}
end;
stats_standard_normal(
InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier,
@@ -571,9 +594,6 @@ increment_bucket(Bucket, Array) ->
array:set(Bucket, array:get(Bucket, Array) + 1, Array).
check_histogram(W, Rounds, StdDevs, Histogram, Buckets) ->
- %%PrevBucket = 512,
- %%Bucket = PrevBucket - 1,
- %%P = 0.5 * math:erfc(PrevBucket / W),
TargetP = 0.5 * math:erfc(Buckets / W),
P = 0.0,
N = 0,
@@ -592,7 +612,7 @@ check_histogram(
P = 0.5 * math:erfc(Bucket / W),
BucketP = P - PrevP,
if
- TargetP =< BucketP ->
+ BucketP < TargetP ->
check_histogram(
W, Rounds, StdDevs, Histogram, TargetP,
Bucket - 1, PrevBucket, PrevP, N);
@@ -604,7 +624,7 @@ check_histogram(
UpperLimit = ceil(Exp + Threshold),
if
N < LowerLimit; UpperLimit < N ->
- [#{bucket => {Bucket, PrevBucket}, n => N, exp => Exp,
+ [#{bucket => {Bucket, PrevBucket}, n => N,
lower => LowerLimit, upper => UpperLimit} |
check_histogram(
W, Rounds, StdDevs, Histogram, TargetP,