Merge pull request #1367 from josevalim/jv-dbgi-chunk

Introduce new "Dbgi" chunk

OTP-14369

6 files changed, 113 insertions, 61 deletions

diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml
index ed04dac1c0..94bda0d5e3 100644
--- a/lib/compiler/doc/src/compile.xml
+++ b/lib/compiler/doc/src/compile.xml
@@ -132,12 +132,10 @@
           <tag><c>debug_info</c></tag>
           <item>
             <marker id="debug_info"></marker>
-            <p>Includes debug information in the form of abstract code
-              (see
-              <seealso marker="erts:absform">The Abstract Format</seealso>
-              in ERTS User's Guide) in the compiled beam module. Tools
-	      such as Debugger, Xref, and Cover require
-	      the debug information to be included.</p>
+            <p>Includes debug information in the form of <seealso marker="erts:absform">
+              Erlang Abstract Format</seealso> in the <c>debug_info</c>
+              chunk of the compiled beam module. Tools such as Debugger,
+              Xref, and Cover require the debug information to be included.</p>
 
             <p><em>Warning</em>: Source code can be reconstructed from
               the debug information. Use encrypted debug information
@@ -147,6 +145,21 @@
               <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p>
           </item>
 
+          <tag><c>{debug_info, {Backend, Data}}</c></tag>
+          <item>
+            <marker id="debug_info"></marker>
+            <p>Includes custom debug information in the form of a
+              <c>Backend</c> module with custom <c>Data</c> in the compiled beam module.
+              The given module must implement a <c>debug_info/4</c> function
+              and is responsible for generating different code representations,
+              as described in the <c>debug_info</c> under
+              <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p>
+
+            <p><em>Warning</em>: Source code can be reconstructed from
+              the debug information. Use encrypted debug information
+              (<c>encrypt_debug_info</c>) to prevent this.</p>
+          </item>
+
           <tag><c>{debug_info_key,KeyString}</c></tag>
           <item></item>
           <tag><c>{debug_info_key,{Mode,KeyString}}</c></tag>
diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl
index 019d8ba864..be0f45323f 100644
--- a/lib/compiler/src/compile.erl
+++ b/lib/compiler/src/compile.erl
@@ -198,9 +198,9 @@ expand_opts(Opts0) ->
     %% {debug_info_key,Key} implies debug_info.
     Opts = case {proplists:get_value(debug_info_key, Opts0),
 		 proplists:get_value(encrypt_debug_info, Opts0),
-		 proplists:get_bool(debug_info, Opts0)} of
+		 proplists:get_value(debug_info, Opts0)} of
 	       {undefined,undefined,_} -> Opts0;
-	       {_,_,false} -> [debug_info|Opts0];
+	       {_,_,undefined} -> [debug_info|Opts0];
 	       {_,_,_} -> Opts0
 	   end,
     foldr(fun expand_opt/2, [], Opts).
@@ -302,7 +302,7 @@ format_error_reason(Reason) ->
 		  ofile=""    :: file:filename(),
 		  module=[]   :: module() | [],
 		  core_code=[] :: cerl:c_module() | [],
-		  abstract_code=[] :: binary() | [], %Abstract code for debugger.
+		  abstract_code=[] :: abstract_code(), %Abstract code for debugger.
 		  options=[]  :: [option()],  %Options for compilation
 		  mod_options=[]  :: [option()], %Options for module_info
                   encoding=none :: none | epp:source_encoding(),
@@ -1315,44 +1315,43 @@ core_inline_module(Code0, #compile{options=Opts}=St) ->
     Code = cerl_inline:core_transform(Code0, Opts),
     {ok,Code,St}.
 
-save_abstract_code(Code, #compile{ifile=File}=St) ->
-    case abstract_code(Code, St) of
-	{ok,Abstr} ->
-	    {ok,Code,St#compile{abstract_code=Abstr}};
-	{error,Es} ->
-	    {error,St#compile{errors=St#compile.errors ++ [{File,Es}]}}
-    end.
+save_abstract_code(Code, St) ->
+    {ok,Code,St#compile{abstract_code=erl_parse:anno_to_term(Code)}}.
+
+debug_info(#compile{module=Module,mod_options=Opts0,ofile=OFile,abstract_code=Abst}) ->
+    AbstOpts = cleanup_compile_options(Opts0),
+    Opts1 = proplists:delete(debug_info, Opts0),
+    {Backend,Metadata,Opts2} =
+	case proplists:get_value(debug_info, Opts0, false) of
+	    {OptBackend,OptMetadata} when is_atom(OptBackend) -> {OptBackend,OptMetadata,Opts1};
+	    false -> {erl_abstract_code,{none,AbstOpts},Opts1};
+	    true -> {erl_abstract_code,{Abst,AbstOpts},[debug_info | Opts1]}
+	end,
+    DebugInfo = erlang:term_to_binary({debug_info_v1,Backend,Metadata}, [compressed]),
 
-abstract_code(Code0, #compile{options=Opts,ofile=OFile}) ->
-    Code = erl_parse:anno_to_term(Code0),
-    Abstr = erlang:term_to_binary({raw_abstract_v1,Code}, [compressed]),
-    case member(encrypt_debug_info, Opts) of
+    case member(encrypt_debug_info, Opts2) of
 	true ->
-	    case keyfind(debug_info_key, 1, Opts) of
-		{_,Key} ->
-		    encrypt_abs_code(Abstr, Key);
+	    case lists:keytake(debug_info_key, 1, Opts2) of
+		{value,{_, Key},Opts3} ->
+		    encrypt_debug_info(DebugInfo, Key, [{debug_info_key,'********'} | Opts3]);
 		false ->
-		    %% Note: #compile.module has not been set yet.
-		    %% Here is an approximation that should work for
-		    %% all valid cases.
-		    Module = list_to_atom(filename:rootname(filename:basename(OFile))),
-		    Mode = proplists:get_value(crypto_mode, Opts, des3_cbc),
+		    Mode = proplists:get_value(crypto_mode, Opts2, des3_cbc),
 		    case beam_lib:get_crypto_key({debug_info, Mode, Module, OFile}) of
 			error ->
 			    {error, [{none,?MODULE,no_crypto_key}]};
 			Key ->
-			    encrypt_abs_code(Abstr, {Mode, Key})
+			    encrypt_debug_info(DebugInfo, {Mode, Key}, Opts2)
 		    end
 	    end;
 	false ->
-	    {ok,Abstr}
+	    {ok,DebugInfo,Opts2}
     end.
 
-encrypt_abs_code(Abstr, Key0) ->
+encrypt_debug_info(DebugInfo, Key, Opts) ->
     try
-	RealKey = generate_key(Key0),
+	RealKey = generate_key(Key),
 	case start_crypto() of
-	    ok -> {ok,encrypt(RealKey, Abstr)};
+	    ok -> {ok,encrypt(RealKey, DebugInfo),Opts};
 	    {error,_}=E -> E
 	end
     catch
@@ -1360,6 +1359,18 @@ encrypt_abs_code(Abstr, Key0) ->
 	    {error,[{none,?MODULE,bad_crypto_key}]}
     end.
 
+cleanup_compile_options(Opts) ->
+    lists:filter(fun keep_compile_option/1, Opts).
+
+%% We are storing abstract, not asm or core.
+keep_compile_option(from_asm) -> false;
+keep_compile_option(from_core) -> false;
+%% Parse transform and macros have already been applied.
+keep_compile_option({parse_transform, _}) -> false;
+keep_compile_option({d, _, _}) -> false;
+%% Do not affect compilation result on future calls.
+keep_compile_option(Option) -> effects_code_generation(Option).
+
 start_crypto() ->
     try crypto:start() of
 	{error,{already_started,crypto}} -> ok;
@@ -1386,16 +1397,16 @@ encrypt({des3_cbc=Type,Key,IVec,BlockSize}, Bin0) ->
 save_core_code(Code, St) ->
     {ok,Code,St#compile{core_code=cerl:from_records(Code)}}.
 
-beam_asm(Code0, #compile{ifile=File,abstract_code=Abst,extra_chunks=ExtraChunks,
-			 options=CompilerOpts,mod_options=Opts0}=St) ->
-    Source = paranoid_absname(File),
-    Opts1 = lists:map(fun({debug_info_key,_}) -> {debug_info_key,'********'};
-			 (Other) -> Other
-		      end, Opts0),
-    Opts2 = [O || O <- Opts1, effects_code_generation(O)],
-    Chunks = [{<<"Abst">>, Abst} | ExtraChunks],
-    case beam_asm:module(Code0, Chunks, Source, Opts2, CompilerOpts) of
-	{ok,Code} -> {ok,Code,St#compile{abstract_code=[]}}
+beam_asm(Code0, #compile{ifile=File,extra_chunks=ExtraChunks,options=CompilerOpts}=St) ->
+    case debug_info(St) of
+	{ok,DebugInfo,Opts0} ->
+	    Source = paranoid_absname(File),
+	    Opts1 = [O || O <- Opts0, effects_code_generation(O)],
+	    Chunks = [{<<"Dbgi">>, DebugInfo} | ExtraChunks],
+	    {ok,Code} = beam_asm:module(Code0, Chunks, Source, Opts1, CompilerOpts),
+	    {ok,Code,St#compile{abstract_code=[]}};
+	{error,Es} ->
+	    {error,St#compile{errors=St#compile.errors ++ [{File,Es}]}}
     end.
 
 paranoid_absname(""=File) ->
@@ -1479,15 +1490,17 @@ embed_native_code(Code, {Architecture,NativeCode}) ->
 %%  errors will be reported).
 
 effects_code_generation(Option) ->
-    case Option of 
+    case Option of
 	beam -> false;
 	report_warnings -> false;
 	report_errors -> false;
 	return_errors-> false;
 	return_warnings-> false;
+	warnings_as_errors -> false;
 	binary -> false;
 	verbose -> false;
 	{cwd,_} -> false;
+	{outdir, _} -> false;
 	_ -> true
     end.
 
diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl
index 8dea7ec03a..fcfc1a4076 100644
--- a/lib/compiler/src/v3_core.erl
+++ b/lib/compiler/src/v3_core.erl
@@ -184,11 +184,8 @@ form({function,_,_,_,_}=F0, Module, Opts) ->
 form({attribute,_,module,Mod}, Module, _Opts) ->
     true = is_atom(Mod),
     Module#imodule{name=Mod};
-form({attribute,_,file,{File,_Line}}, Module, _Opts) ->
-    Module#imodule{file=File};
-form({attribute,_,compile,_}, Module, _Opts) ->
-    %% Ignore compilation options.
-    Module;
+form({attribute,_,file,{File,_Line}}=F, #imodule{attrs=As}=Module, _Opts) ->
+    Module#imodule{file=File, attrs=[attribute(F)|As]};
 form({attribute,_,import,_}, Module, _Opts) ->
     %% Ignore. We have no futher use for imports.
     Module;
@@ -201,9 +198,8 @@ form(_, Module, _Opts) ->
     %% Ignore uninteresting forms such as 'eof'.
     Module.
 
-attribute(Attribute) ->
-    Fun = fun(A) ->  [erl_anno:location(A)] end,
-    {attribute,Line,Name,Val0} = erl_parse:map_anno(Fun, Attribute),
+attribute({attribute,A,Name,Val0}) ->
+    Line = [erl_anno:location(A)],
     Val = if
 	      is_list(Val0) -> Val0;
 	      true -> [Val0]
diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl
index 4b5d7d919c..7a0a63d286 100644
--- a/lib/compiler/src/v3_kernel.erl
+++ b/lib/compiler/src/v3_kernel.erl
@@ -148,6 +148,8 @@ include_attribute(opaque) -> false;
 include_attribute(export_type) -> false;
 include_attribute(record) -> false;
 include_attribute(optional_callbacks) -> false;
+include_attribute(file) -> false;
+include_attribute(compile) -> false;
 include_attribute(_) -> true.
 
 function({#c_var{name={F,Arity}=FA},Body}, St0) ->
diff --git a/lib/compiler/test/compilation_SUITE.erl b/lib/compiler/test/compilation_SUITE.erl
index cd1bc099e9..c98aa07a4f 100644
--- a/lib/compiler/test/compilation_SUITE.erl
+++ b/lib/compiler/test/compilation_SUITE.erl
@@ -319,7 +319,7 @@ self_compile_1(Config, Prefix, Opts) ->
     %% Compile the compiler. (In this node to get better coverage.)
     CompA = make_compiler_dir(Priv, Prefix++"compiler_a"),
     VsnA = Version ++ ".0",
-    compile_compiler(compiler_src(), CompA, VsnA, [clint0,clint|Opts]),
+    compile_compiler(compiler_src(), CompA, VsnA, Opts),
 
     %% Compile the compiler again using the newly compiled compiler.
     %% (In another node because reloading the compiler would disturb cover.)
diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl
index 474c3e2ca0..11a62ce753 100644
--- a/lib/compiler/test/compile_SUITE.erl
+++ b/lib/compiler/test/compile_SUITE.erl
@@ -27,6 +27,7 @@
 -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
 	 init_per_group/2,end_per_group/2,
 	 app_test/1,appup_test/1,
+	 debug_info/4, custom_debug_info/1,
 	 file_1/1, forms_2/1, module_mismatch/1, big_file/1, outdir/1,
 	 binary/1, makedep/1, cond_and_ifdef/1, listings/1, listings_big/1,
 	 other_output/1, kernel_listing/1, encrypted_abstr/1,
@@ -51,7 +52,7 @@ all() ->
      strict_record, utf8_atoms, extra_chunks,
      cover, env, core, core_roundtrip, asm, optimized_guards,
      sys_pre_attributes, dialyzer, warnings, pre_load_check,
-     env_compiler_options].
+     env_compiler_options, custom_debug_info].
 
 groups() -> 
     [].
@@ -504,17 +505,23 @@ encrypted_abstr_1(Simple, Target) ->
     {ok,simple} = compile:file(Simple,
 				     [debug_info,{debug_info_key,Key},
 				      {outdir,TargetDir}]),
-    verify_abstract(Target),
+    verify_abstract(Target, erl_abstract_code),
 
     {ok,simple} = compile:file(Simple,
 				     [{debug_info_key,Key},
 				      {outdir,TargetDir}]),
-    verify_abstract(Target),
+    verify_abstract(Target, erl_abstract_code),
 
     {ok,simple} = compile:file(Simple,
 				     [debug_info,{debug_info_key,{des3_cbc,Key}},
 				      {outdir,TargetDir}]),
-    verify_abstract(Target),
+    verify_abstract(Target, erl_abstract_code),
+
+    {ok,simple} = compile:file(Simple,
+				     [{debug_info,{?MODULE,ok}},
+				      {debug_info_key,Key},
+				      {outdir,TargetDir}]),
+    verify_abstract(Target, ?MODULE),
 
     {ok,{simple,[{compile_info,CInfo}]}} =
 	beam_lib:chunks(Target, [compile_info]),
@@ -539,7 +546,7 @@ encrypted_abstr_1(Simple, Target) ->
     NewKey = "better use another key here",
     write_crypt_file(["[{debug_info,des3_cbc,simple,\"",NewKey,"\"}].\n"]),
     {ok,simple} = compile:file(Simple, [encrypt_debug_info,report]),
-    verify_abstract("simple.beam"),
+    verify_abstract("simple.beam", erl_abstract_code),
     ok = file:delete(".erlang.crypt"),
     beam_lib:clear_crypto_key_fun(),
     {error,beam_lib,{key_missing_or_invalid,"simple.beam",abstract_code}} =
@@ -572,9 +579,10 @@ encrypted_abstr_no_crypto(Simple, Target) ->
 				{outdir,TargetDir},report]),
     ok.
     
-verify_abstract(Target) ->
-    {ok,{simple,[Chunk]}} = beam_lib:chunks(Target, [abstract_code]),
-    {abstract_code,{raw_abstract_v1,_}} = Chunk.
+verify_abstract(Beam, Backend) ->
+    {ok,{simple,[Abst, Dbgi]}} = beam_lib:chunks(Beam, [abstract_code, debug_info]),
+    {abstract_code,{raw_abstract_v1,_}} = Abst,
+    {debug_info,{debug_info_v1,Backend,_}} = Dbgi.
 
 has_crypto() ->
     try
@@ -593,6 +601,26 @@ install_crypto_key(Key) ->
     ok = beam_lib:crypto_key_fun(F).
 
 %% Miscellanous tests, mainly to get better coverage.
+debug_info(erlang_v1, Module, ok, _Opts) ->
+    {ok, [Module]};
+debug_info(erlang_v1, Module, error, _Opts) ->
+    {error, unknown_format}.
+
+custom_debug_info(Config) when is_list(Config) ->
+    {Simple,_} = get_files(Config, simple, "file_1"),
+
+    {ok,simple,OkBin} = compile:file(Simple, [binary, {debug_info,{?MODULE,ok}}]), %Coverage
+    {ok,{simple,[{abstract_code,{raw_abstract_v1,[simple]}}]}} =
+	beam_lib:chunks(OkBin, [abstract_code]),
+    {ok,{simple,[{debug_info,{debug_info_v1,?MODULE,ok}}]}} =
+	beam_lib:chunks(OkBin, [debug_info]),
+
+    {ok,simple,ErrorBin} = compile:file(Simple, [binary, {debug_info,{?MODULE,error}}]), %Coverage
+    {ok,{simple,[{abstract_code,no_abstract_code}]}} =
+	beam_lib:chunks(ErrorBin, [abstract_code]),
+    {ok,{simple,[{debug_info,{debug_info_v1,?MODULE,error}}]}} =
+	beam_lib:chunks(ErrorBin, [debug_info]).
+
 cover(Config) when is_list(Config) ->
     io:format("~p\n", [compile:options()]),
     ok.