Simplify and extend diameter_make interface

In particular, make codec/2 flexible as to what's generated, the formats
(erl, hrl, parse, forms and beam) being passed in the options list and
defaulting to [erl, hrl]. The 'parse' format is the internal format to
which dictionaries are parsed, which can be manipulated by flatten/1
before being passed back to codec/2 or format/1.

Remove the (undocumented) dict/1,2 since codec/2 now subsumes it with
the 'parse' option.

4 files changed, 200 insertions, 143 deletions

diff --git a/lib/diameter/bin/diameterc b/lib/diameter/bin/diameterc
index d4dd3153c5..d31f341c36 100755
--- a/lib/diameter/bin/diameterc
+++ b/lib/diameter/bin/diameterc
@@ -54,9 +54,6 @@ usage() ->
       [?MODULE]).
 
 main(Args) ->
-    %% Add the ebin directory relative to the script path.
-    BinDir = filename:dirname(escript:script_name()),
-    code:add_path(filename:join([BinDir, "..", "ebin"])),
     halt(gen(Args)).
 
 gen(Args) ->
@@ -72,15 +69,12 @@ gen(Args) ->
             1
     end.
 
-compile(#argv{file = File, options = Opts} = A) ->
-    try diameter_dict_util:parse({path, File}, Opts) of
-        {ok, Dict} ->
-            maybe_output(A, Dict, Opts, dict), %% parsed dictionary
-            maybe_output(A, Dict, Opts, erl),  %% the erl file
-            maybe_output(A, Dict, Opts, hrl),  %% The hrl file
+compile(#argv{file = File, options = Opts, output = Out}) ->
+    try diameter_make:codec({path, File}, Opts ++ Out) of
+        ok ->
             0;
         {error, Reason} ->
-            error_msg(diameter_dict_util:format_error(Reason), []),
+            error_msg(Reason, []),
             1
     catch
         error: Reason ->
@@ -88,10 +82,6 @@ compile(#argv{file = File, options = Opts} = A) ->
             2
     end.
 
-maybe_output(#argv{file = File, output = Output}, Spec, Opts, Mode) ->
-    lists:member(Mode, Output)
-        andalso diameter_codegen:from_dict(File, Spec, Opts, Mode).
-
 error_msg({Fmt, Args}) ->
     error_msg(Fmt, Args).
 
@@ -119,8 +109,9 @@ arg(["-o", Dir | Args], #argv{options = Opts} = A) ->
     true = dir_exists(Dir),
     arg(Args, A#argv{options = [{outdir, Dir} | Opts]});
 
-arg(["-i", Dir | Args], #argv{options = Opts} = A) ->
-    arg(Args, A#argv{options = Opts ++ [{include, Dir}]});
+arg(["-i", Dir | Args], #argv{} = A) ->
+    code:add_patha(Dir), %% Set path here instead of passing an include
+    arg(Args, A);        %% option so it's set before calling diameter_make.
 
 arg(["--name", Name | Args], #argv{options = Opts} = A) ->
     arg(Args, A#argv{options = [{name, Name} | Opts]});
@@ -138,7 +129,7 @@ arg(["-H" | Args], #argv{output = Output} = A) ->
     arg(Args, A#argv{output = lists:delete(hrl, Output)});
 
 arg(["-d" | Args], #argv{output = Output} = A) ->
-    arg(Args, A#argv{output = [dict, forms | Output]});
+    arg(Args, A#argv{output = [parse, forms | Output -- [parse, forms]]});
 
 arg([[$- = M, C, H | T] | Args], A)  %% clustered options
   when C /= $i, C /= $o, C /= $- ->
diff --git a/lib/diameter/src/compiler/diameter_codegen.erl b/lib/diameter/src/compiler/diameter_codegen.erl
index 91bab2205c..47da193457 100644
--- a/lib/diameter/src/compiler/diameter_codegen.erl
+++ b/lib/diameter/src/compiler/diameter_codegen.erl
@@ -33,11 +33,6 @@
 
 -export([from_dict/4]).
 
-%% Internal exports (for test).
--export([file/1,
-         file/2,
-         file/3]).
-
 -include("diameter_forms.hrl").
 -include("diameter_vsn.hrl").
 
@@ -50,11 +45,11 @@
 
 -spec from_dict(File, ParseD, Opts, Mode)
    -> ok
-    | list()
+    | term()
  when File :: string(),
       ParseD :: orddict:orddict(),
       Opts :: list(),
-      Mode :: dict | forms | erl | hrl.
+      Mode :: parse | forms | erl | hrl | beam.
 
 from_dict(File, ParseD, Opts, Mode) ->
     Outdir = proplists:get_value(outdir, Opts, "."),
@@ -74,11 +69,12 @@ mod(_, {ok, Mod}) ->
 
 maybe_write(true, _, _, _, T) ->
     T;
+
 maybe_write(_, Mode, Outdir, Mod, T) ->
     Path = filename:join(Outdir, Mod),  %% minus extension
     do_write(Mode, [Path, $., ext(Mode)], T).
 
-ext(dict) ->
+ext(parse) ->
     "D";
 ext(forms) ->
     "F";
@@ -86,9 +82,11 @@ ext(T) ->
     ?S(T).
 
 do_write(M, Path, T)
-  when M == dict;
+  when M == parse;
        M == forms ->
     write_term(Path, T);
+do_write(beam, Path, {_Mod, Beam}) ->
+    write(Path, Beam);
 do_write(_, Path, T) ->
     write(Path, T).
 
@@ -125,36 +123,35 @@ eraser(Key) ->
 %% ===========================================================================
 %% ===========================================================================
 
-%% Generate from parsed dictionary in a file.
-
-file(F) ->
-    file(F, dict).
-
-file(F, Mode) ->
-    file(F, ".", Mode).
-
-file(F, Outdir, Mode) ->
-    {ok, [ParseD]} = file:consult(F),
-    from_dict(F, ParseD, Outdir, Mode).
-
-%% ===========================================================================
-%% ===========================================================================
-
 get_value(Key, Plist) ->
     proplists:get_value(Key, Plist, []).
 
-gen(dict, ParseD, _Mod) ->
+gen(parse, ParseD, _Mod) ->
     [?VERSION | ParseD];
 
-gen(hrl, ParseD, Mod) ->
-    gen_hrl(Mod, ParseD);
-
 gen(forms, ParseD, Mod) ->
     erl_forms(Mod, ParseD);
 
+gen(beam, ParseD, Mod) ->
+    compile(pp(erl_forms(Mod, ParseD)));
+
+gen(hrl, ParseD, Mod) ->
+    gen_hrl(Mod, ParseD);
+
 gen(erl, ParseD, Mod) ->
     [header(), prettypr(erl_forms(Mod, ParseD)), $\n].
 
+compile({ok, Forms}) ->
+    case compile:forms(Forms, [debug_info, return]) of
+        {ok, Mod, Beam, _Warnings} ->
+            {Mod, Beam};
+        {error, Errors, _Warnings} ->
+            erlang:error({compile, Errors})
+    end;
+
+compile({error, Reason}) ->
+    erlang:error(Reason).
+
 erl_forms(Mod, ParseD) ->
     Forms = [[{?attribute, module, Mod},
               {?attribute, compile, {parse_transform, diameter_exprecs}},
@@ -851,3 +848,70 @@ prefix(ParseD) ->
 
 rec_name(Name, Prefix) ->
     Prefix ++ Name.
+
+%% ===========================================================================
+%% pp/1
+%%
+%% Preprocess forms as generated by 'forms' option. In particular,
+%% replace the include_lib attributes in generated forms by the
+%% corresponding forms, extracting the latter from an existing
+%% dictionary (diameter_gen_relay). The resulting forms can be
+%% compiled to beam using compile:forms/2 (which does no preprocessing
+%% or it's own; DiY currently appears to be the only way to preprocess
+%% a forms list).
+
+pp(Forms) ->
+    {_, Beam, _} = code:get_object_code(diameter_gen_relay),
+    pp(Forms, abstract_code(Beam)).
+
+pp(Forms, {ok, Code}) ->
+    Files = files(Code, []),
+    {ok, lists:flatmap(fun(T) -> include(T, Files) end, Forms)};
+
+pp(_, {error, _} = No) ->
+    No.
+
+include({attribute, _, include_lib, Path}, Files) ->
+    Inc = filename:basename(Path),
+    [{Inc, Forms}] = [T || {F, _} = T <- Files, F == Inc], %% expect one
+    lists:flatmap(fun filter/1, Forms);
+
+include(T, _) ->
+    [T].
+
+abstract_code(Beam) ->
+    case beam_lib:chunks(Beam, [abstract_code]) of
+        {ok, {_Mod, [{abstract_code, {_Vsn, Code}}]}} ->
+            {ok, Code};
+        {ok, {_Mod, [{abstract_code, no_abstract_code = No}]}} ->
+            {error, No};
+        {error = E, beam_lib, Reason} ->
+            {E, Reason}
+    end.
+
+files([{attribute, _, file, {Path, _}} | T], Acc) ->
+    {Body, Rest} = lists:splitwith(fun({attribute, _, file, _}) -> false;
+                                      (_) -> true
+                                   end,
+                                   T),
+    files(Rest, [{filename:basename(Path), Body} | Acc]);
+
+files([], Acc) ->
+    Acc.
+
+%% Only retain record diameter_avp and functions not generated by
+%% diameter_exprecs.
+
+filter({attribute, _, record, {diameter_avp, _}} = T) ->
+    [T];
+
+filter({function, _, Name, _, _} = T) ->
+    case ?S(Name) of
+        [$#|_] ->  %% generated by diameter_exprecs
+            [];
+        _ ->
+            [T]
+    end;
+
+filter(_) ->
+    [].
diff --git a/lib/diameter/src/compiler/diameter_make.erl b/lib/diameter/src/compiler/diameter_make.erl
index 0d8cdec7f2..bf337544a6 100644
--- a/lib/diameter/src/compiler/diameter_make.erl
+++ b/lib/diameter/src/compiler/diameter_make.erl
@@ -30,27 +30,35 @@
 
 -module(diameter_make).
 
--export([codec/1,
-         codec/2,
-         dict/1,
-         dict/2,
+-export([codec/2,
+         codec/1,
          format/1,
          flatten/1]).
 
 -export_type([opt/0]).
 
+-include("diameter_vsn.hrl").
+
 %% Options passed to codec/2.
 -type opt() :: {include|outdir|name|prefix|inherits, string()}
              | return
              | verbose
-             | debug.
+             | parse  %% internal parsed form
+             | forms  %% abstract format from which erl is generated
+             | beam   %% compiled directly from preprocessed forms
+             | erl
+             | hrl.
+
+%% Internal parsed format with a version tag.
+-type parsed() :: maybe_improper_list(integer(), orddict:orddict()).
 
 %% Literal dictionary or path. A NL of CR identifies the former.
 -type dict() :: iolist()
-              | binary().
+              | binary()
+              | parsed().  %% as returned by codec/2
 
 %% Name of a literal dictionary if otherwise unspecified.
--define(DEFAULT_DICT_NAME, "dictionary.dia").
+-define(DEFAULT_DICT_FILE, "dictionary.dia").
 
 %% ===========================================================================
 
@@ -63,98 +71,45 @@
 
 -spec codec(File, [opt()])
    -> ok
-    | {ok, Ret}
+    | {ok, list()}   %% with option 'return', one element for each output
     | {error, Reason}
- when File :: dict(),
-      Ret  :: list(),  %% [Erl, Hrl | Debug], Debug = [] | [ParseD, Forms]
+ when File :: dict()
+            | {path, file:name_all()},
       Reason :: string().
 
 codec(File, Opts) ->
-    case to_dict(File, Opts) of
-        {ok, {Dict, Dictish}} ->
-            make(file(Dictish), Opts, Dict);
-        {error, _} = E ->
-            E
-    end.
-
-codec(File) ->
-    codec(File, []).
-
-file({path, Path}) ->
-    Path;
-file(_) ->
-    ?DEFAULT_DICT_NAME.
-
-%% dict/2
-%%
-%% Parse a dictionary file and return the orddict that a codec module
-%% returns from dict/0.
-
--spec dict(File, [opt()])
-   -> {ok, orddict:orddict()}
-    | {error, string()}
-  when File :: dict().
-
-dict(File, Opts) ->
-    case to_dict(File, Opts) of
-        {ok, {Dict, _}} ->
-            {ok, Dict};
-        {error, _} = E ->
-            E
-    end.
-
-dict(File) ->
-    dict(File, []).
-
-%% to_dict/2
-
-to_dict(File, Opts) ->
-    Dictish = maybe_path(File),
-    case diameter_dict_util:parse(Dictish, Opts) of
-        {ok, Dict} ->
-            {ok, {Dict, Dictish}};
+    {Dict, Path} = identify(File),
+    case parse(Dict, Opts) of
+        {ok, ParseD} ->
+            make(Path, default(Opts), ParseD);
         {error = E, Reason} ->
             {E, diameter_dict_util:format_error(Reason)}
     end.
 
-maybe_path(File) ->
-    Bin = iolist_to_binary([File]),
-    case is_path(Bin) of
-        true  -> {path, File};
-        false -> Bin
-    end.
-
-%% Interpret anything containing \n or \r as a literal dictionary,
-%% otherwise a path. (Which might be the wrong guess in the worst case.)
-is_path(Bin) ->
-    try
-        [throw(C) || <<C>> <= Bin, $\n == C orelse $\r == C],
-        true
-    catch
-        throw:_ -> false
-    end.
+codec(File) ->
+    codec(File, []).
 
 %% format/1
 %%
 %% Turn an orddict returned by dict/1-2 back into a dictionary.
 
--spec format(orddict:orddict())
+-spec format(parsed())
    -> iolist().
 
-format(Dict) ->
+format([?VERSION | Dict]) ->
     diameter_dict_util:format(Dict).
 
 %% flatten/1
 %%
 %% Reconstitute a dictionary without @inherits.
 
--spec flatten(orddict:orddict())
-   -> orddict:orddict().
+-spec flatten(parsed())
+   -> parsed().
 
-flatten(Dict) ->
-    lists:foldl(fun flatten/2, Dict, [[avp_types, import_avps],
-                                      [grouped, import_groups],
-                                      [enum, import_enums]]).
+flatten([?VERSION = V | Dict]) ->
+    [V | lists:foldl(fun flatten/2, Dict, [[avp_types, import_avps],
+                                           [grouped, import_groups],
+                                           [enum, import_enums]])].
 
 flatten([_,_] = Keys, Dict) ->
     [Values, Imports] = [orddict:fetch(K, Dict) || K <- Keys],
@@ -168,20 +123,57 @@ store({Key, Value}, Dict) ->
 
 %% ===========================================================================
 
+parse({dict, ParseD}, _) ->
+    {ok, ParseD};
+parse(File, Opts) ->
+    diameter_dict_util:parse(File, Opts).
+
+default(Opts) ->
+    def(modes(Opts), Opts).
+
+def([], Opts) ->
+    [erl, hrl | Opts];
+def(_, Opts) ->
+    Opts.
+
+modes(Opts) ->
+    lists:filter(fun is_mode/1, Opts).
+
+is_mode(T) ->
+    lists:member(T, [erl, hrl, parse, forms, beam]).
+
+identify([Vsn | [T|_] = ParseD])
+  when is_tuple(T) ->
+    ?VERSION == Vsn orelse erlang:error({version, {Vsn, ?VERSION}}),
+    {{dict, ParseD}, ?DEFAULT_DICT_FILE};
+identify({path, File} = T) ->
+    {T, File};
+identify(File) ->
+    Bin = iolist_to_binary([File]),
+    case is_path(Bin) of
+        true  -> {{path, File}, File};
+        false -> {Bin, ?DEFAULT_DICT_FILE}
+    end.
+
+%% Interpret anything containing \n or \r as a literal dictionary,
+%% otherwise a path. (Which might be the wrong guess in the worst case.)
+is_path(Bin) ->
+    try
+        [throw(C) || <<C>> <= Bin, $\n == C orelse $\r == C],
+        true
+    catch
+        throw:_ -> false
+    end.
+
 make(File, Opts, Dict) ->
     ok(lists:foldl(fun(M,A) -> [make(File, Opts, Dict, M) | A] end,
                    [],
-                   lists:append([[dict, forms] || lists:member(debug, Opts)])
-                   ++ [erl, hrl])).
-%% The order in which results are generated (dict/forms/erl/hrl) is
-%% intentional, in order of more processing (except for hrl, which
-%% isn't needed by diameter itself), since an error raises an
-%% exception. The order of return results is the reverse.
-
-ok([ok,_|_]) ->
+                   modes(Opts))).
+
+ok([ok|_]) ->
     ok;
-ok([_,_|_] = L) ->
-    {ok, L}.
+ok([_|_] = L) ->
+    {ok, lists:reverse(L)}.
 
 make(File, Opts, Dict, Mode) ->
     try
diff --git a/lib/diameter/test/diameter_compiler_SUITE.erl b/lib/diameter/test/diameter_compiler_SUITE.erl
index 66c671b52d..943cac1446 100644
--- a/lib/diameter/test/diameter_compiler_SUITE.erl
+++ b/lib/diameter/test/diameter_compiler_SUITE.erl
@@ -366,8 +366,8 @@ format(Mods, Bin) ->
     {Dict, Dict} = {Dict, D}.
 
 make(File, Opts) ->
-    case diameter_make:codec(File, [return, debug | Opts]) of
-        {ok, [_E,_H,_F,[_Vsn|Dict]]} ->
+    case diameter_make:codec(File, [parse, hrl, return | Opts]) of
+        {ok, [Dict, _]} ->
             {ok, Dict};
         {error, _} = E ->
             E
@@ -408,20 +408,30 @@ generate(Config) ->
     [] = ?util:run([{?MODULE, [generate, M, Bin, N, T]}
                     || {E,N} <- Rs,
                        {ok, M} <- [norm(E)],
-                       T <- [erl, hrl, dict]]).
+                       T <- [erl, hrl, parse, forms]]).
 
 generate(Mods, Bin, N, Mode) ->
     B = modify(Bin, Mods ++ [{"@name .*", "@name dict" ++ ?L(N)}]),
     {ok, Dict} = make(B, []),
     File = "dict" ++ integer_to_list(N),
-    {_, ok} = {Dict, diameter_codegen:from_dict("dict",
-                                                Dict,
-                                                [{name, File},
-                                                 {prefix, "base"},
-                                                 debug],
-                                                Mode)},
-    Mode == erl
-        andalso ({ok, _} = compile:file(File ++ ".erl", [return_errors])).
+    {_, ok} = {Dict, diameter_make:codec(Dict,
+                                         [{name, File},
+                                          {prefix, "base"},
+                                          Mode])},
+    generate(Mode, File, Dict).
+
+generate(erl, File, _) ->
+    {ok, _} = compile:file(File ++ ".erl", [return_errors]);
+
+generate(forms, File, _) ->
+    {ok, [_]} = file:consult(File ++ ".F");
+
+generate(parse, File, Dict) ->
+    {ok, [Dict]} = file:consult(File ++ ".D"),  %% assert
+    {ok, [_]} = diameter_make:codec(Dict, [beam, return]);
+
+generate(hrl, _, _) ->
+    ok.
 
 %% ===========================================================================