Merge branch 'hasse/fix_erl_anno_use'

* hasse/fix_erl_anno_use:
  parsetools: Fix handling of locations and annotations
  diameter: Fix handling of locations and annotations
  kernel: Fix handling of locations and annotations
  compiler: Fix handling of locations and annotations
  eunit: Fix handling of locations and annotations
  Update preloaded
  erts: Fix handling of locations and annotations
  dialyzer: Fix handling of annotations in a test
  debugger: Fix handling of locations and annotations
  stdlib: Extend functions in erl_parse to handle form_info()
  stdlib: Fix handling of locations and annotations
  erts: Correct the documentation of abstract end-of-file
  stdlib: Add debug tests to the erl_parse module
  stdlib: improve the erl_anno module's debug tests
  stdlib: Improve the  erl_pp module's debug tests

12 files changed, 82 insertions, 40 deletions

diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl
index 52df2319dd..bb7b485490 100644
--- a/lib/stdlib/src/c.erl
+++ b/lib/stdlib/src/c.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -400,7 +400,7 @@ split_def([], Res) -> {d, list_to_atom(reverse(Res))}.
 make_term(Str) ->
     case erl_scan:string(Str) of
 	{ok, Tokens, _} ->
-	    case erl_parse:parse_term(Tokens ++ [{dot, 1}]) of
+	    case erl_parse:parse_term(Tokens ++ [{dot, erl_anno:new(1)}]) of
 		{ok, Term} -> Term;
 		{error, {_,_,Reason}} ->
 		    io:format("~ts: ~ts~n", [Reason, Str]),
diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl
index 40eba4ad67..61d755ba55 100644
--- a/lib/stdlib/src/epp.erl
+++ b/lib/stdlib/src/epp.erl
@@ -286,7 +286,7 @@ parse_file(Epp) ->
 	{warning,W} ->
 	    [{warning,W}|parse_file(Epp)];
 	{eof,Location} ->
-	    [{eof,erl_anno:new(Location)}]
+	    [{eof,Location}]
     end.
 
 -spec default_encoding() -> source_encoding().
diff --git a/lib/stdlib/src/erl_anno.erl b/lib/stdlib/src/erl_anno.erl
index d32c34dabd..d0310f52e2 100644
--- a/lib/stdlib/src/erl_anno.erl
+++ b/lib/stdlib/src/erl_anno.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2015. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -42,7 +42,7 @@
 
 %% Debug: define DEBUG to make sure that annotations are handled as an
 %% opaque type. Note that all abstract code need to be compiled with
-%% DEBUG=true. See also ./erl_pp.erl.
+%% DEBUG=true. See also ./erl_pp.erl and ./erl_parse.yrl.
 
 %-define(DEBUG, true).
 
@@ -52,7 +52,11 @@
                     | {'record', record()}
                     | {'text', string()}.
 
+-ifdef(DEBUG).
+-opaque anno() :: [annotation(), ...].
+-else.
 -opaque anno() :: location() | [annotation(), ...].
+-endif.
 -type anno_term() :: term().
 
 -type column() :: pos_integer().
diff --git a/lib/stdlib/src/erl_compile.erl b/lib/stdlib/src/erl_compile.erl
index a6ae398d03..76db2eeacd 100644
--- a/lib/stdlib/src/erl_compile.erl
+++ b/lib/stdlib/src/erl_compile.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -337,7 +337,7 @@ file_or_directory(Name) ->
 make_term(Str) -> 
     case erl_scan:string(Str) of
 	{ok, Tokens, _} ->		  
-	    case erl_parse:parse_term(Tokens ++ [{dot, 1}]) of
+	    case erl_parse:parse_term(Tokens ++ [{dot, erl_anno:new(1)}]) of
 		{ok, Term} -> Term;
 		{error, {_,_,Reason}} ->
 		    io:format(?STDERR, "~ts: ~ts~n", [Reason, Str]),
diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl
index 1b84234fac..0ffca0886f 100644
--- a/lib/stdlib/src/erl_lint.erl
+++ b/lib/stdlib/src/erl_lint.erl
@@ -2,7 +2,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -737,7 +737,12 @@ start_state({attribute,_,module,M}, St0) ->
     St1 = St0#lint{module=M},
     St1#lint{state=attribute};
 start_state(Form, St) ->
-    St1 = add_error(element(2, Form), undefined_module, St),
+    Anno = case Form of
+               {eof, L} -> erl_anno:new(L);
+               %% {warning, Warning} and {error, Error} not possible here.
+               _ -> element(2, Form)
+           end,
+    St1 = add_error(Anno, undefined_module, St),
     attribute_state(Form, St1#lint{state=attribute}).
 
 %% attribute_state(Form, State) ->
diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl
index 922455a6f2..2dcddeb8c2 100644
--- a/lib/stdlib/src/erl_parse.yrl
+++ b/lib/stdlib/src/erl_parse.yrl
@@ -981,6 +981,16 @@ Erlang code.
 %% keep track of annotation info in tokens
 -define(anno(Tup), element(2, Tup)).
 
+%-define(DEBUG, true).
+
+-ifdef(DEBUG).
+%% Assumes that erl_anno has been compiled with DEBUG=true.
+-define(ANNO_CHECK(Tokens),
+        [] = [T || T <- Tokens, not is_list(element(2, T))]).
+-else.
+-define(ANNO_CHECK(Tokens), ok).
+-endif.
+
 %% Entry points compatible to old erl_parse.
 %% These really suck and are only here until Calle gets multiple
 %% entry points working.
@@ -990,10 +1000,15 @@ Erlang code.
       AbsForm :: abstract_form(),
       ErrorInfo :: error_info().
 parse_form([{'-',A1},{atom,A2,spec}|Tokens]) ->
-    parse([{'-',A1},{'spec',A2}|Tokens]);
+    NewTokens = [{'-',A1},{'spec',A2}|Tokens],
+    ?ANNO_CHECK(NewTokens),
+    parse(NewTokens);
 parse_form([{'-',A1},{atom,A2,callback}|Tokens]) ->
-    parse([{'-',A1},{'callback',A2}|Tokens]);
+    NewTokens = [{'-',A1},{'callback',A2}|Tokens],
+    ?ANNO_CHECK(NewTokens),
+    parse(NewTokens);
 parse_form(Tokens) ->
+    ?ANNO_CHECK(Tokens),
     parse(Tokens).
 
 -spec parse_exprs(Tokens) -> {ok, ExprList} | {error, ErrorInfo} when
@@ -1001,6 +1016,7 @@ parse_form(Tokens) ->
       ExprList :: [abstract_expr()],
       ErrorInfo :: error_info().
 parse_exprs(Tokens) ->
+    ?ANNO_CHECK(Tokens),
     A = erl_anno:new(0),
     case parse([{atom,A,f},{'(',A},{')',A},{'->',A}|Tokens]) of
 	{ok,{function,_Lf,f,0,[{clause,_Lc,[],[],Exprs}]}} ->
@@ -1013,6 +1029,7 @@ parse_exprs(Tokens) ->
       Term :: term(),
       ErrorInfo :: error_info().
 parse_term(Tokens) ->
+    ?ANNO_CHECK(Tokens),
     A = erl_anno:new(0),
     case parse([{atom,A,f},{'(',A},{')',A},{'->',A}|Tokens]) of
 	{ok,{function,_Af,f,0,[{clause,_Ac,[],[],[Expr]}]}} ->
@@ -1531,8 +1548,8 @@ type_preop_prec('#') -> {700,800}.
       Fun :: fun((Anno) -> NewAnno),
       Anno :: erl_anno:anno(),
       NewAnno :: erl_anno:anno(),
-      Abstr :: erl_parse_tree(),
-      NewAbstr :: erl_parse_tree().
+      Abstr :: erl_parse_tree() | form_info(),
+      NewAbstr :: erl_parse_tree() | form_info().
 
 map_anno(F0, Abstr) ->
     F = fun(A, Acc) -> {F0(A), Acc} end,
@@ -1546,7 +1563,7 @@ map_anno(F0, Abstr) ->
       Acc1 :: term(),
       AccIn :: term(),
       AccOut :: term(),
-      Abstr :: erl_parse_tree().
+      Abstr :: erl_parse_tree() | form_info().
 
 fold_anno(F0, Acc0, Abstr) ->
     F = fun(A, Acc) -> {A, F0(A, Acc)} end,
@@ -1561,15 +1578,15 @@ fold_anno(F0, Acc0, Abstr) ->
       Acc1 :: term(),
       AccIn :: term(),
       AccOut :: term(),
-      Abstr :: erl_parse_tree(),
-      NewAbstr :: erl_parse_tree().
+      Abstr :: erl_parse_tree() | form_info(),
+      NewAbstr :: erl_parse_tree() | form_info().
 
 mapfold_anno(F, Acc0, Abstr) ->
     modify_anno1(Abstr, Acc0, F).
 
 -spec new_anno(Term) -> Abstr when
       Term :: term(),
-      Abstr :: erl_parse_tree().
+      Abstr :: erl_parse_tree() | form_info().
 
 new_anno(Term) ->
     F = fun(L, Acc) -> {erl_anno:new(L), Acc} end,
@@ -1577,14 +1594,14 @@ new_anno(Term) ->
     NewAbstr.
 
 -spec anno_to_term(Abstr) -> term() when
-      Abstr :: erl_parse_tree().
+      Abstr :: erl_parse_tree() | form_info().
 
 anno_to_term(Abstract) ->
     F = fun(Anno, Acc) -> {erl_anno:to_term(Anno), Acc} end,
     {NewAbstract, []} = modify_anno1(Abstract, [], F),
     NewAbstract.
 
--spec anno_from_term(Term) -> erl_parse_tree() when
+-spec anno_from_term(Term) -> erl_parse_tree() | form_info() when
       Term :: term().
 
 anno_from_term(Term) ->
@@ -1629,6 +1646,8 @@ modify_anno1({warning,W}, Ac, _Mf) ->
     {{warning,W},Ac};
 modify_anno1({error,W}, Ac, _Mf) ->
     {{error,W},Ac};
+modify_anno1({eof,L}, Ac, _Mf) ->
+    {{eof,L},Ac};
 %% Expressions.
 modify_anno1({clauses,Cs}, Ac, Mf) ->
     {Cs1,Ac1} = modify_anno1(Cs, Ac, Mf),
diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl
index d30cd508c1..6068afb293 100644
--- a/lib/stdlib/src/erl_pp.erl
+++ b/lib/stdlib/src/erl_pp.erl
@@ -51,6 +51,15 @@
 %-define(DEBUG, true).
 
 -ifdef(DEBUG).
+-define(FORM_TEST(T),
+        _ = case T of
+                {eof, _Line} -> ok;
+                {warning, _W} -> ok;
+                {error, _E} -> ok;
+                _ -> ?TEST(T)
+            end).
+-define(EXPRS_TEST(L),
+        [?TEST(E) || E <- L]).
 -define(TEST(T),
         %% Assumes that erl_anno has been compiled with DEBUG=true.
         %% erl_pp does not use the annoations, but test it anyway.
@@ -62,6 +71,8 @@
                     erlang:error(badarg, [T])
             end).
 -else.
+-define(FORM_TEST(T), ok).
+-define(EXPRS_TEST(T), ok).
 -define(TEST(T), ok).
 -endif.
 
@@ -80,7 +91,7 @@ form(Thing) ->
       Options :: options()).
 
 form(Thing, Options) ->
-    ?TEST(Thing),
+    ?FORM_TEST(Thing),
     State = state(Options),
     frmt(lform(Thing, options(Options)), State).
 
@@ -124,7 +135,7 @@ guard(Gs) ->
       Options :: options()).
 
 guard(Gs, Options) ->
-    ?TEST(Gs),
+    ?EXPRS_TEST(Gs),
     frmt(lguard(Gs, options(Options)), state(Options)).
 
 -spec(exprs(Expressions) -> io_lib:chars() when
@@ -146,7 +157,7 @@ exprs(Es, Options) ->
       Options :: options()).
 
 exprs(Es, I, Options) ->
-    ?TEST(Es),
+    ?EXPRS_TEST(Es),
     frmt({seq,[],[],[$,],lexprs(Es, options(Options))}, I, state(Options)).
 
 -spec(expr(Expression) -> io_lib:chars() when
diff --git a/lib/stdlib/src/escript.erl b/lib/stdlib/src/escript.erl
index c42ae981e7..6e8f780f7c 100644
--- a/lib/stdlib/src/escript.erl
+++ b/lib/stdlib/src/escript.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2007-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -629,8 +629,7 @@ parse_source(S, File, Fd, StartLine, HeaderSz, CheckOnly) ->
                     {error, _} ->
                         epp_parse_file2(Epp, S2, [FileForm], OptModRes);
                     {eof, LastLine} ->
-                        Anno = anno(LastLine),
-                        S#state{forms_or_bin = [FileForm, {eof, Anno}]}
+                        S#state{forms_or_bin = [FileForm, {eof, LastLine}]}
                 end,
             ok = epp:close(Epp),
             ok = file:close(Fd),
@@ -728,8 +727,7 @@ epp_parse_file2(Epp, S, Forms, Parsed) ->
                       [S#state.file,Ln,Mod:format_error(Args)]),
             epp_parse_file(Epp, S#state{n_errors = S#state.n_errors + 1}, [Form | Forms]);
         {eof, LastLine} ->
-            Anno = anno(LastLine),
-            S#state{forms_or_bin = lists:reverse([{eof, Anno} | Forms])}
+            S#state{forms_or_bin = lists:reverse([{eof, LastLine} | Forms])}
     end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
diff --git a/lib/stdlib/src/qlc_pt.erl b/lib/stdlib/src/qlc_pt.erl
index 28221ea75f..4a39f8ae9d 100644
--- a/lib/stdlib/src/qlc_pt.erl
+++ b/lib/stdlib/src/qlc_pt.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2004-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2004-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -439,7 +439,7 @@ compile_forms(Forms0, Options) ->
                  (_) -> false
               end,
     Forms = ([F || F <- Forms0, not Exclude(element(1, F))]
-             ++ [{eof,anno0()}]),
+             ++ [{eof,0}]),
     try 
         case compile:noenv_forms(Forms, compile_options(Options)) of
             {ok, _ModName, Ws0} ->
diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl
index c7dcd9ae16..df38edf393 100644
--- a/lib/stdlib/test/erl_lint_SUITE.erl
+++ b/lib/stdlib/test/erl_lint_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -2549,7 +2549,7 @@ otp_5878(Config) when is_list(Config) ->
                    {function,9,t,0,[{clause,9,[],[],[{record,10,r,[]}]}]},
                    {eof,11}],
     {error,[{"rec.erl",[{7,erl_lint,old_abstract_code}]}],[]} =
-        compile:forms(OldAbstract, [return, report]),
+        compile_forms(OldAbstract, [return, report]),
 
     ok.
 
@@ -3848,9 +3848,13 @@ otp_11879(_Config) ->
              [{1,erl_lint,{spec_fun_undefined,{f,1}}},
               {2,erl_lint,spec_wrong_arity},
               {22,erl_lint,callback_wrong_arity}]}],
-     []} = compile:forms(Fs, [return,report]),
+     []} = compile_forms(Fs, [return,report]),
     ok.
 
+compile_forms(Terms, Opts) ->
+    Forms = [erl_parse:anno_from_term(Term) || Term <- Terms],
+    compile:forms(Forms, Opts).
+
 %% OTP-13230: -deprecated without -module.
 otp_13230(Config) when is_list(Config) ->
     Abstr = <<"-deprecated([{frutt,0,next_version}]).">>,
diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl
index 31ea3210a8..1a028204b4 100644
--- a/lib/stdlib/test/erl_pp_SUITE.erl
+++ b/lib/stdlib/test/erl_pp_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2006-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -1068,10 +1068,10 @@ otp_11100(Config) when is_list(Config) ->
     %% There are a few places where the added code ("options(none)")
     %% doesn't make a difference (pp:bit_elem_type/1 is an example).
 
+    A1 = erl_anno:new(1),
     %% Cannot trigger the use of the hook function with export/import.
     "-export([{fy,a}/b]).\n" =
-        pf({attribute,1,export,[{{fy,a},b}]}),
-    A1 = erl_anno:new(1),
+        pf({attribute,A1,export,[{{fy,a},b}]}),
     "-type foo() :: integer(INVALID-FORM:{foo,bar}:).\n" =
         pf({attribute,A1,type,{foo,{type,A1,integer,[{foo,bar}]},[]}}),
     pf({attribute,A1,type,
@@ -1100,10 +1100,11 @@ otp_11100(Config) when is_list(Config) ->
 
 %% OTP-11861. behaviour_info() and -callback.
 otp_11861(Config) when is_list(Config) ->
+    A3 = erl_anno:new(3),
     "-optional_callbacks([bar/0]).\n" =
-        pf({attribute,3,optional_callbacks,[{bar,0}]}),
+        pf({attribute,A3,optional_callbacks,[{bar,0}]}),
     "-optional_callbacks([{bar,1,bad}]).\n" =
-        pf({attribute,4,optional_callbacks,[{bar,1,bad}]}),
+        pf({attribute,A3,optional_callbacks,[{bar,1,bad}]}),
     ok.
 
 pf(Form) ->
diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl
index 7d0ba967f9..aca5b1e54f 100644
--- a/lib/stdlib/test/erl_scan_SUITE.erl
+++ b/lib/stdlib/test/erl_scan_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1998-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -904,9 +904,9 @@ more_chars() ->
 otp_10302(Config) when is_list(Config) ->
     %% From unicode():
     {ok,[{atom,1,'aсb'}],1} =
-        erl_scan:string("'a"++[1089]++"b'", 1),
+        erl_scan_string("'a"++[1089]++"b'", 1),
     {ok,[{atom,{1,1},'qaપ'}],{1,12}} =
-        erl_scan:string("'qa\\x{aaa}'",{1,1}),
+        erl_scan_string("'qa\\x{aaa}'",{1,1}),
 
     {ok,[{char,1,1089}],1} = erl_scan_string([$$,1089], 1),
     {ok,[{char,1,1089}],1} = erl_scan_string([$$,$\\,1089],1),