Store abstract code in the Dbgi chunk

The new Dbgi chunk returns data in the following format:

    {debug_info_v1, Backend, Data}

This allows compilers to store the debug info in different
formats. In order to retrieve a particular format, for
instance, Erlang Abstract Format, one may invoke:

    Backend:debug_info(erlang_v1, Module, Data, Opts)

Besides introducing the chunk above, this commit also:

  * Changes beam_lib:chunk(Beam, [:abstract_code]) to
    read from the new Dbgi chunk while keeping backwards
    compatibility with old .beams

  * Adds the {debug_info, {Backend, Data}} option to
    compile:file/2 and friends that are stored in the
    Dbgi chunk. This allows the debug info encryption
    mechanism to work across compilers

  * Improves dialyzer to work directly on Core Erlang,
    allowing languages that do not have the Erlang
    Abstract Format to be dialyzer as long as they emit
    the new chunk and their backend implementation is
    available

Backwards compatibility is kept across the board except
for those calling beam_lib:chunk(Beam, ["Abst"]), as the
old chunk is no longer available. Note however the "Abst"
chunk has always been optional.

Future OTP versions may remove parsing the "Abst" chunk
altogether from beam_lib once Erlang 19 and earlier is no
longer supported.

The current Dialyzer implementation still supports earlier
.beam files and such may also be removed in future versions.

6 files changed, 113 insertions, 61 deletions

diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml
index ed04dac1c0..94bda0d5e3 100644
--- a/lib/compiler/doc/src/compile.xml
+++ b/lib/compiler/doc/src/compile.xml
@@ -132,12 +132,10 @@
           <tag><c>debug_info</c></tag>
           <item>
             <marker id="debug_info"></marker>
-            <p>Includes debug information in the form of abstract code
-              (see
-              <seealso marker="erts:absform">The Abstract Format</seealso>
-              in ERTS User's Guide) in the compiled beam module. Tools
-	      such as Debugger, Xref, and Cover require
-	      the debug information to be included.</p>
+            <p>Includes debug information in the form of <seealso marker="erts:absform">
+              Erlang Abstract Format</seealso> in the <c>debug_info</c>
+              chunk of the compiled beam module. Tools such as Debugger,
+              Xref, and Cover require the debug information to be included.</p>
 
             <p><em>Warning</em>: Source code can be reconstructed from
               the debug information. Use encrypted debug information
@@ -147,6 +145,21 @@
               <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p>
           </item>
 
+          <tag><c>{debug_info, {Backend, Data}}</c></tag>
+          <item>
+            <marker id="debug_info"></marker>
+            <p>Includes custom debug information in the form of a
+              <c>Backend</c> module with custom <c>Data</c> in the compiled beam module.
+              The given module must implement a <c>debug_info/4</c> function
+              and is responsible for generating different code representations,
+              as described in the <c>debug_info</c> under
+              <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p>
+
+            <p><em>Warning</em>: Source code can be reconstructed from
+              the debug information. Use encrypted debug information
+              (<c>encrypt_debug_info</c>) to prevent this.</p>
+          </item>
+
           <tag><c>{debug_info_key,KeyString}</c></tag>
           <item></item>
           <tag><c>{debug_info_key,{Mode,KeyString}}</c></tag>
diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl
index 019d8ba864..b106f856ee 100644
--- a/lib/compiler/src/compile.erl
+++ b/lib/compiler/src/compile.erl
@@ -198,9 +198,9 @@ expand_opts(Opts0) ->
     %% {debug_info_key,Key} implies debug_info.
     Opts = case {proplists:get_value(debug_info_key, Opts0),
 		 proplists:get_value(encrypt_debug_info, Opts0),
-		 proplists:get_bool(debug_info, Opts0)} of
+		 proplists:get_value(debug_info, Opts0)} of
 	       {undefined,undefined,_} -> Opts0;
-	       {_,_,false} -> [debug_info|Opts0];
+	       {_,_,undefined} -> [debug_info|Opts0];
 	       {_,_,_} -> Opts0
 	   end,
     foldr(fun expand_opt/2, [], Opts).
@@ -302,7 +302,7 @@ format_error_reason(Reason) ->
 		  ofile=""    :: file:filename(),
 		  module=[]   :: module() | [],
 		  core_code=[] :: cerl:c_module() | [],
-		  abstract_code=[] :: binary() | [], %Abstract code for debugger.
+		  abstract_code=[] :: abstract_code(), %Abstract code for debugger.
 		  options=[]  :: [option()],  %Options for compilation
 		  mod_options=[]  :: [option()], %Options for module_info
                   encoding=none :: none | epp:source_encoding(),
@@ -1315,44 +1315,43 @@ core_inline_module(Code0, #compile{options=Opts}=St) ->
     Code = cerl_inline:core_transform(Code0, Opts),
     {ok,Code,St}.
 
-save_abstract_code(Code, #compile{ifile=File}=St) ->
-    case abstract_code(Code, St) of
-	{ok,Abstr} ->
-	    {ok,Code,St#compile{abstract_code=Abstr}};
-	{error,Es} ->
-	    {error,St#compile{errors=St#compile.errors ++ [{File,Es}]}}
-    end.
+save_abstract_code(Code, St) ->
+    {ok,Code,St#compile{abstract_code=erl_parse:anno_to_term(Code)}}.
+
+debug_info(#compile{module=Module,mod_options=Opts0,ofile=OFile,abstract_code=Abst}) ->
+    AbstOpts = cleanup_compile_options(Opts0),
+    Opts1 = proplists:delete(debug_info, Opts0),
+    {Backend,Metadata,Opts2} =
+	case proplists:get_value(debug_info, Opts0, false) of
+	    {OptBackend,OptMetadata} when is_atom(OptBackend) -> {OptBackend,OptMetadata,Opts1};
+	    false -> {erl_abstract_code,{none,AbstOpts},Opts1};
+	    true -> {erl_abstract_code,{Abst,AbstOpts},[debug_info | Opts1]}
+	end,
+    DebugInfo = erlang:term_to_binary({debug_info_v1,Backend,Metadata}, [compressed]),
 
-abstract_code(Code0, #compile{options=Opts,ofile=OFile}) ->
-    Code = erl_parse:anno_to_term(Code0),
-    Abstr = erlang:term_to_binary({raw_abstract_v1,Code}, [compressed]),
-    case member(encrypt_debug_info, Opts) of
+    case member(encrypt_debug_info, Opts2) of
 	true ->
-	    case keyfind(debug_info_key, 1, Opts) of
-		{_,Key} ->
-		    encrypt_abs_code(Abstr, Key);
+	    case lists:keytake(debug_info_key, 1, Opts2) of
+		{value,{_, Key},Opts3} ->
+		    encrypt_debug_info(DebugInfo, Key, [{debug_info_key,'********'} | Opts3]);
 		false ->
-		    %% Note: #compile.module has not been set yet.
-		    %% Here is an approximation that should work for
-		    %% all valid cases.
-		    Module = list_to_atom(filename:rootname(filename:basename(OFile))),
-		    Mode = proplists:get_value(crypto_mode, Opts, des3_cbc),
+		    Mode = proplists:get_value(crypto_mode, Opts2, des3_cbc),
 		    case beam_lib:get_crypto_key({debug_info, Mode, Module, OFile}) of
 			error ->
 			    {error, [{none,?MODULE,no_crypto_key}]};
 			Key ->
-			    encrypt_abs_code(Abstr, {Mode, Key})
+			    encrypt_debug_info(DebugInfo, {Mode, Key}, Opts2)
 		    end
 	    end;
 	false ->
-	    {ok,Abstr}
+	    {ok,DebugInfo,Opts2}
     end.
 
-encrypt_abs_code(Abstr, Key0) ->
+encrypt_debug_info(DebugInfo, Key, Opts) ->
     try
-	RealKey = generate_key(Key0),
+	RealKey = generate_key(Key),
 	case start_crypto() of
-	    ok -> {ok,encrypt(RealKey, Abstr)};
+	    ok -> {ok,encrypt(RealKey, DebugInfo),Opts};
 	    {error,_}=E -> E
 	end
     catch
@@ -1360,6 +1359,20 @@ encrypt_abs_code(Abstr, Key0) ->
 	    {error,[{none,?MODULE,bad_crypto_key}]}
     end.
 
+cleanup_compile_options(Opts) ->
+    lists:filter(fun keep_compile_option/1, Opts).
+
+%% We are storing abstract, not asm or core.
+keep_compile_option(from_asm) -> false;
+keep_compile_option(from_core) -> false;
+%% Parse transform and macros have already been applied.
+keep_compile_option({parse_transform, _}) -> false;
+keep_compile_option({d, _, _}) -> false;
+%% Do not affect compilation result on future calls.
+keep_compile_option({outdir, _}) -> false;
+keep_compile_option(warnings_as_errors) -> false;
+keep_compile_option(Option) -> effects_code_generation(Option).
+
 start_crypto() ->
     try crypto:start() of
 	{error,{already_started,crypto}} -> ok;
@@ -1386,16 +1399,16 @@ encrypt({des3_cbc=Type,Key,IVec,BlockSize}, Bin0) ->
 save_core_code(Code, St) ->
     {ok,Code,St#compile{core_code=cerl:from_records(Code)}}.
 
-beam_asm(Code0, #compile{ifile=File,abstract_code=Abst,extra_chunks=ExtraChunks,
-			 options=CompilerOpts,mod_options=Opts0}=St) ->
-    Source = paranoid_absname(File),
-    Opts1 = lists:map(fun({debug_info_key,_}) -> {debug_info_key,'********'};
-			 (Other) -> Other
-		      end, Opts0),
-    Opts2 = [O || O <- Opts1, effects_code_generation(O)],
-    Chunks = [{<<"Abst">>, Abst} | ExtraChunks],
-    case beam_asm:module(Code0, Chunks, Source, Opts2, CompilerOpts) of
-	{ok,Code} -> {ok,Code,St#compile{abstract_code=[]}}
+beam_asm(Code0, #compile{ifile=File,extra_chunks=ExtraChunks,options=CompilerOpts}=St) ->
+    case debug_info(St) of
+	{ok,DebugInfo,Opts0} ->
+	    Source = paranoid_absname(File),
+	    Opts1 = [O || O <- Opts0, effects_code_generation(O)],
+	    Chunks = [{<<"Dbgi">>, DebugInfo} | ExtraChunks],
+	    {ok,Code} = beam_asm:module(Code0, Chunks, Source, Opts1, CompilerOpts),
+	    {ok,Code,St#compile{abstract_code=[]}};
+	{error,Es} ->
+	    {error,St#compile{errors=St#compile.errors ++ [{File,Es}]}}
     end.
 
 paranoid_absname(""=File) ->
@@ -1479,7 +1492,7 @@ embed_native_code(Code, {Architecture,NativeCode}) ->
 %%  errors will be reported).
 
 effects_code_generation(Option) ->
-    case Option of 
+    case Option of
 	beam -> false;
 	report_warnings -> false;
 	report_errors -> false;
diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl
index 8dea7ec03a..fcfc1a4076 100644
--- a/lib/compiler/src/v3_core.erl
+++ b/lib/compiler/src/v3_core.erl
@@ -184,11 +184,8 @@ form({function,_,_,_,_}=F0, Module, Opts) ->
 form({attribute,_,module,Mod}, Module, _Opts) ->
     true = is_atom(Mod),
     Module#imodule{name=Mod};
-form({attribute,_,file,{File,_Line}}, Module, _Opts) ->
-    Module#imodule{file=File};
-form({attribute,_,compile,_}, Module, _Opts) ->
-    %% Ignore compilation options.
-    Module;
+form({attribute,_,file,{File,_Line}}=F, #imodule{attrs=As}=Module, _Opts) ->
+    Module#imodule{file=File, attrs=[attribute(F)|As]};
 form({attribute,_,import,_}, Module, _Opts) ->
     %% Ignore. We have no futher use for imports.
     Module;
@@ -201,9 +198,8 @@ form(_, Module, _Opts) ->
     %% Ignore uninteresting forms such as 'eof'.
     Module.
 
-attribute(Attribute) ->
-    Fun = fun(A) ->  [erl_anno:location(A)] end,
-    {attribute,Line,Name,Val0} = erl_parse:map_anno(Fun, Attribute),
+attribute({attribute,A,Name,Val0}) ->
+    Line = [erl_anno:location(A)],
     Val = if
 	      is_list(Val0) -> Val0;
 	      true -> [Val0]
diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl
index 4b5d7d919c..7a0a63d286 100644
--- a/lib/compiler/src/v3_kernel.erl
+++ b/lib/compiler/src/v3_kernel.erl
@@ -148,6 +148,8 @@ include_attribute(opaque) -> false;
 include_attribute(export_type) -> false;
 include_attribute(record) -> false;
 include_attribute(optional_callbacks) -> false;
+include_attribute(file) -> false;
+include_attribute(compile) -> false;
 include_attribute(_) -> true.
 
 function({#c_var{name={F,Arity}=FA},Body}, St0) ->
diff --git a/lib/compiler/test/compilation_SUITE.erl b/lib/compiler/test/compilation_SUITE.erl
index cd1bc099e9..c98aa07a4f 100644
--- a/lib/compiler/test/compilation_SUITE.erl
+++ b/lib/compiler/test/compilation_SUITE.erl
@@ -319,7 +319,7 @@ self_compile_1(Config, Prefix, Opts) ->
     %% Compile the compiler. (In this node to get better coverage.)
     CompA = make_compiler_dir(Priv, Prefix++"compiler_a"),
     VsnA = Version ++ ".0",
-    compile_compiler(compiler_src(), CompA, VsnA, [clint0,clint|Opts]),
+    compile_compiler(compiler_src(), CompA, VsnA, Opts),
 
     %% Compile the compiler again using the newly compiled compiler.
     %% (In another node because reloading the compiler would disturb cover.)
diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl
index 474c3e2ca0..11a62ce753 100644
--- a/lib/compiler/test/compile_SUITE.erl
+++ b/lib/compiler/test/compile_SUITE.erl
@@ -27,6 +27,7 @@
 -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
 	 init_per_group/2,end_per_group/2,
 	 app_test/1,appup_test/1,
+	 debug_info/4, custom_debug_info/1,
 	 file_1/1, forms_2/1, module_mismatch/1, big_file/1, outdir/1,
 	 binary/1, makedep/1, cond_and_ifdef/1, listings/1, listings_big/1,
 	 other_output/1, kernel_listing/1, encrypted_abstr/1,
@@ -51,7 +52,7 @@ all() ->
      strict_record, utf8_atoms, extra_chunks,
      cover, env, core, core_roundtrip, asm, optimized_guards,
      sys_pre_attributes, dialyzer, warnings, pre_load_check,
-     env_compiler_options].
+     env_compiler_options, custom_debug_info].
 
 groups() -> 
     [].
@@ -504,17 +505,23 @@ encrypted_abstr_1(Simple, Target) ->
     {ok,simple} = compile:file(Simple,
 				     [debug_info,{debug_info_key,Key},
 				      {outdir,TargetDir}]),
-    verify_abstract(Target),
+    verify_abstract(Target, erl_abstract_code),
 
     {ok,simple} = compile:file(Simple,
 				     [{debug_info_key,Key},
 				      {outdir,TargetDir}]),
-    verify_abstract(Target),
+    verify_abstract(Target, erl_abstract_code),
 
     {ok,simple} = compile:file(Simple,
 				     [debug_info,{debug_info_key,{des3_cbc,Key}},
 				      {outdir,TargetDir}]),
-    verify_abstract(Target),
+    verify_abstract(Target, erl_abstract_code),
+
+    {ok,simple} = compile:file(Simple,
+				     [{debug_info,{?MODULE,ok}},
+				      {debug_info_key,Key},
+				      {outdir,TargetDir}]),
+    verify_abstract(Target, ?MODULE),
 
     {ok,{simple,[{compile_info,CInfo}]}} =
 	beam_lib:chunks(Target, [compile_info]),
@@ -539,7 +546,7 @@ encrypted_abstr_1(Simple, Target) ->
     NewKey = "better use another key here",
     write_crypt_file(["[{debug_info,des3_cbc,simple,\"",NewKey,"\"}].\n"]),
     {ok,simple} = compile:file(Simple, [encrypt_debug_info,report]),
-    verify_abstract("simple.beam"),
+    verify_abstract("simple.beam", erl_abstract_code),
     ok = file:delete(".erlang.crypt"),
     beam_lib:clear_crypto_key_fun(),
     {error,beam_lib,{key_missing_or_invalid,"simple.beam",abstract_code}} =
@@ -572,9 +579,10 @@ encrypted_abstr_no_crypto(Simple, Target) ->
 				{outdir,TargetDir},report]),
     ok.
     
-verify_abstract(Target) ->
-    {ok,{simple,[Chunk]}} = beam_lib:chunks(Target, [abstract_code]),
-    {abstract_code,{raw_abstract_v1,_}} = Chunk.
+verify_abstract(Beam, Backend) ->
+    {ok,{simple,[Abst, Dbgi]}} = beam_lib:chunks(Beam, [abstract_code, debug_info]),
+    {abstract_code,{raw_abstract_v1,_}} = Abst,
+    {debug_info,{debug_info_v1,Backend,_}} = Dbgi.
 
 has_crypto() ->
     try
@@ -593,6 +601,26 @@ install_crypto_key(Key) ->
     ok = beam_lib:crypto_key_fun(F).
 
 %% Miscellanous tests, mainly to get better coverage.
+debug_info(erlang_v1, Module, ok, _Opts) ->
+    {ok, [Module]};
+debug_info(erlang_v1, Module, error, _Opts) ->
+    {error, unknown_format}.
+
+custom_debug_info(Config) when is_list(Config) ->
+    {Simple,_} = get_files(Config, simple, "file_1"),
+
+    {ok,simple,OkBin} = compile:file(Simple, [binary, {debug_info,{?MODULE,ok}}]), %Coverage
+    {ok,{simple,[{abstract_code,{raw_abstract_v1,[simple]}}]}} =
+	beam_lib:chunks(OkBin, [abstract_code]),
+    {ok,{simple,[{debug_info,{debug_info_v1,?MODULE,ok}}]}} =
+	beam_lib:chunks(OkBin, [debug_info]),
+
+    {ok,simple,ErrorBin} = compile:file(Simple, [binary, {debug_info,{?MODULE,error}}]), %Coverage
+    {ok,{simple,[{abstract_code,no_abstract_code}]}} =
+	beam_lib:chunks(ErrorBin, [abstract_code]),
+    {ok,{simple,[{debug_info,{debug_info_v1,?MODULE,error}}]}} =
+	beam_lib:chunks(ErrorBin, [debug_info]).
+
 cover(Config) when is_list(Config) ->
     io:format("~p\n", [compile:options()]),
     ok.