From 66b3383675cb9090469cf8f4b281642691eb40cd Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Mon, 12 Feb 2018 09:41:45 +0100
Subject: Repeat stat tests to only fail for systematic errors

---
 lib/stdlib/test/rand_SUITE.erl | 49 ++++++++++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 16 deletions(-)

(limited to 'lib/stdlib')

diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl
index 174a34001f..743676fc13 100644
--- a/lib/stdlib/test/rand_SUITE.erl
+++ b/lib/stdlib/test/rand_SUITE.erl
@@ -52,7 +52,9 @@ all() ->
     [seed, interval_int, interval_float,
      api_eq,
      reference,
-     {group, basic_stats}, uniform_real_conv,
+     {group, basic_stats},
+     {group, distr_stats},
+     uniform_real_conv,
      plugin, measure,
      {group, reference_jump}
     ].
@@ -60,8 +62,9 @@ all() ->
 groups() ->
     [{basic_stats, [parallel],
       [basic_stats_uniform_1, basic_stats_uniform_2,
-       basic_stats_standard_normal,
-       stats_standard_normal_box_muller,
+       basic_stats_standard_normal]},
+     {distr_stats, [parallel],
+      [stats_standard_normal_box_muller,
        stats_standard_normal_box_muller_2,
        stats_standard_normal]},
      {reference_jump, [parallel],
@@ -70,6 +73,9 @@ groups() ->
 group(basic_stats) ->
     %% valgrind needs a lot of time
     [{timetrap,{minutes,10}}];
+group(distr_stats) ->
+    %% valgrind needs a lot of time
+    [{timetrap,{minutes,10}}];
 group(reference_jump) ->
     %% valgrind needs a lot of time
     [{timetrap,{minutes,10}}].
@@ -437,7 +443,7 @@ stats_standard_normal_box_muller(Config) when is_list(Config) ->
                         {Z, [S]}
                 end,
             State = [rand:seed(exrop)],
-            stats_standard_normal(NormalS, State)
+            stats_standard_normal(NormalS, State, 3)
     catch error:_ ->
             {skip, "math:erfc/1 not supported"}
     end.
@@ -462,7 +468,7 @@ stats_standard_normal_box_muller_2(Config) when is_list(Config) ->
                         {Z, [S]}
                 end,
             State = [rand:seed(exrop)],
-            stats_standard_normal(NormalS, State)
+            stats_standard_normal(NormalS, State, 3)
     catch error:_ ->
             {skip, "math:erfc/1 not supported"}
     end.
@@ -472,12 +478,12 @@ stats_standard_normal(Config) when is_list(Config) ->
     try math:erfc(1.0) of
         _ ->
             stats_standard_normal(
-              fun rand:normal_s/1, rand:seed_s(exrop))
+              fun rand:normal_s/1, rand:seed_s(exrop), 3)
     catch error:_ ->
             {skip, "math:erfc/1 not supported"}
     end.
 %%
-stats_standard_normal(Fun, S) ->
+stats_standard_normal(Fun, S, Retries) ->
 %%%
 %%% ct config:
 %%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.2}]}]}.
@@ -509,7 +515,7 @@ stats_standard_normal(Fun, S) ->
       "Running standard normal test against ~w std devs for ~w seconds...",
       [StdDevs, Seconds]),
     StopTime = erlang:monotonic_time(second) + Seconds,
-    {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds} =
+    {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds, NewS} =
         stats_standard_normal(
           InvDelta, Buckets, Histogram, Histogram, 0.0,
           Fun, S, Rounds, StopTime, Rounds, 0),
@@ -529,15 +535,26 @@ stats_standard_normal(Fun, S) ->
          check_histogram(
            W, TotalRounds, StdDevs, NegativeHistogram, Buckets)}
     of
+        {_, _, [], []} when InvOP < 100 ->
+            {comment, {tp, TopPrecision, op, InvOP}};
         {_, _, [], []} ->
-            ok;
+            %% If the probability for getting this Outlier is lower than
+            %% 1/100, then this is fishy!
+            stats_standard_normal(
+              Fun, NewS, Retries, {outlier_fishy, InvOP});
         BucketErrors ->
-            ct:fail(BucketErrors)
-    end,
-    %% If the probability for getting this Outlier is lower than 1/100,
-    %% then this is fishy!
-    (InvOP < 100) orelse ct:fail({outlier_fishy, InvOP}),
-    {comment, {tp, TopPrecision, op, InvOP}}.
+            stats_standard_normal(
+              Fun, NewS, Retries, BucketErrors)
+    end.
+%%
+stats_standard_normal(Fun, S, Retries, Failure) ->
+    case Retries - 1 of
+        0 ->
+            ct:fail(Failure);
+        NewRetries ->
+            ct:pal("Retry due to TC glitch: ~p", [Failure]),
+            stats_standard_normal(Fun, S, NewRetries)
+    end.
 %%
 stats_standard_normal(
   InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier,
@@ -550,7 +567,7 @@ stats_standard_normal(
               Fun, S, Rounds, StopTime, Rounds, TotalRounds + Rounds);
         _ ->
             {PositiveHistogram, NegativeHistogram,
-             Outlier, TotalRounds + Rounds}
+             Outlier, TotalRounds + Rounds, S}
     end;
 stats_standard_normal(
   InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier,
-- 
cgit v1.2.3