beam_asm: Strenghten the calculation of Uniq for funs

Funs are identified by a triple, <Module,Uniq,Index>, where Module is
the module name, Uniq is a 27 bit hash value of some intermediate
representation of the code for the fun, and index is a small integer.

When a fun is loaded, the triple for the fun will be compared to
previously loaded funs. If all elements in the triple in the newly
loaded fun are the same, the newly loaded fun will replace the previous
fun. The idea is that if Uniq are the same, the code for the fun is also
the same.

The problem is that Uniq is only based on the intermediate representation
of the fun itself. If the fun calls local functions in the same module,
Uniq may remain the same even if the behavior of the fun has been changed.

See

  http://erlang.org/pipermail/erlang-bugs/2007-June/000368.htlm

for an example.

As a long-term plan to fix this problem, the NewIndex and NewUniq
fields was added to each fun in the R8 release (where NewUniq is the
MD5 of the BEAM code for the module). Unfortunately, it turns
out that the compiler does not assign unique value to NewIndex (if it
isn't tested, it doesn't work), so we cannot use the
<Module,NewUniq,NewIndex> triple as identification.

It would be possible to use <Module,NewUniq,Index>, but that seems
ugly. Therefore, fix the problem by making Uniq more unique by
taking 27 bits from the MD5 for the BEAM code. That only requires
a change to the compiler.

Also update a test case for cover, which now fails because of the
stronger Uniq calculation. (The comment in test case about why the
Pid2 process survived is not correct.)

2 files changed, 43 insertions, 22 deletions

diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl
index 6e63c4d0f2..9b46abe697 100644
--- a/lib/compiler/src/beam_asm.erl
+++ b/lib/compiler/src/beam_asm.erl
@@ -146,8 +146,10 @@ build_file(Code, Attr, Dict, NumLabels, NumFuncs, Abst, SourceFile, Opts) ->
 
     Essentials0 = [AtomChunk,CodeChunk,StringChunk,ImportChunk,
 		   ExpChunk,LambdaChunk,LiteralChunk],
-    Essentials = [iolist_to_binary(C) || C <- Essentials0],
-    {Attributes,Compile} = build_attributes(Opts, SourceFile, Attr, Essentials),
+    Essentials1 = [iolist_to_binary(C) || C <- Essentials0],
+    MD5 = module_md5(Essentials1),
+    Essentials = finalize_fun_table(Essentials1, MD5),
+    {Attributes,Compile} = build_attributes(Opts, SourceFile, Attr, MD5),
     AttrChunk = chunk(<<"Attr">>, Attributes),
     CompileChunk = chunk(<<"CInf">>, Compile),
 
@@ -166,6 +168,24 @@ build_file(Code, Attr, Dict, NumLabels, NumFuncs, Abst, SourceFile, Opts) ->
 	     end,
     build_form(<<"BEAM">>, Chunks).
 
+%% finalize_fun_table(Essentials, MD5) -> FinalizedEssentials
+%%  Update the 'old_uniq' field in the entry for each fun in the
+%%  'FunT' chunk. We'll use part of the MD5 for the module as a
+%%  unique value.
+
+finalize_fun_table(Essentials, MD5) ->
+    [finalize_fun_table_1(E, MD5) || E <- Essentials].
+
+finalize_fun_table_1(<<"FunT",Keep:8/binary,Table0/binary>>, MD5) ->
+    <<Uniq:27,_:101/bits>> = MD5,
+    Table = finalize_fun_table_2(Table0, Uniq, <<>>),
+    <<"FunT",Keep/binary,Table/binary>>;
+finalize_fun_table_1(Chunk, _) -> Chunk.
+
+finalize_fun_table_2(<<Keep:20/binary,0:32,T/binary>>, Uniq, Acc) ->
+    finalize_fun_table_2(T, Uniq, <<Acc/binary,Keep/binary,Uniq:32>>);
+finalize_fun_table_2(<<>>, _, Acc) -> Acc.
+
 %% Build an IFF form.
 
 build_form(Id, Chunks0) when byte_size(Id) =:= 4, is_list(Chunks0) ->
@@ -202,7 +222,7 @@ flatten_exports(Exps) ->
 flatten_imports(Imps) ->
     list_to_binary(map(fun({M,F,A}) -> <<M:32,F:32,A:32>> end, Imps)).
 
-build_attributes(Opts, SourceFile, Attr, Essentials) ->
+build_attributes(Opts, SourceFile, Attr, MD5) ->
     Misc = case member(slim, Opts) of
 	       false ->
 		   {{Y,Mo,D},{H,Mi,S}} = erlang:universaltime(),
@@ -210,7 +230,7 @@ build_attributes(Opts, SourceFile, Attr, Essentials) ->
 	       true -> []
 	   end,
     Compile = [{options,Opts},{version,?COMPILER_VSN}|Misc],
-    {term_to_binary(calc_vsn(Attr, Essentials)),term_to_binary(Compile)}.
+    {term_to_binary(set_vsn_attribute(Attr, MD5)),term_to_binary(Compile)}.
 
 build_line_table(Dict) ->
     {NumLineInstrs,NumFnames0,Fnames0,NumLines,Lines0} =
@@ -243,32 +263,30 @@ encode_line_items([], _) -> [].
 %% We'll not change an existing 'vsn' attribute.
 %%
 
-calc_vsn(Attr, Essentials0) ->
+set_vsn_attribute(Attr, MD5) ->
     case keymember(vsn, 1, Attr) of
 	true -> Attr;
 	false ->
-	    Essentials = filter_essentials(Essentials0),
-	    <<Number:128>> = erlang:md5(Essentials),
+	    <<Number:128>> = MD5,
 	    [{vsn,[Number]}|Attr]
     end.
 
+module_md5(Essentials0) ->
+    Essentials = filter_essentials(Essentials0),
+    erlang:md5(Essentials).
+
 %% filter_essentials([Chunk]) -> [Chunk']
 %%  Filter essentials so that we obtain the same MD5 as code:module_md5/1 and
-%%  beam_lib:md5/1 would calculate for this module.
+%%  beam_lib:md5/1 would calculate for this module.  Note that at this
+%%  point, the 'old_uniq' entry for each fun in the 'FunT' chunk is zeroed,
+%%  so there is no need to go through the 'FunT' chunk.
 
-filter_essentials([<<"FunT",_Sz:4/binary,Entries:4/binary,Table0/binary>>|T]) ->
-    Table = filter_funtab(Table0, <<0:32>>),
-    [Entries,Table|filter_essentials(T)];
 filter_essentials([<<_Tag:4/binary,Sz:32,Data:Sz/binary,_Padding/binary>>|T]) ->
     [Data|filter_essentials(T)];
 filter_essentials([<<>>|T]) ->
     filter_essentials(T);
 filter_essentials([]) -> [].
 
-filter_funtab(<<Important:20/binary,_OldUniq:4/binary,T/binary>>, Zero) ->
-    [Important,Zero|filter_funtab(T, Zero)];
-filter_funtab(<<>>, _) -> [].
-
 bif_type(fnegate, 1) -> {op,fnegate};
 bif_type(fadd, 2)   -> {op,fadd};
 bif_type(fsub, 2)   -> {op,fsub};
@@ -310,8 +328,8 @@ make_op({test,Cond,Fail,Ops}, Dict) when is_list(Ops) ->
     encode_op(Cond, [Fail|Ops], Dict);
 make_op({test,Cond,Fail,Live,[Op|Ops],Dst}, Dict) when is_list(Ops) ->
     encode_op(Cond, [Fail,Op,Live|Ops++[Dst]], Dict);
-make_op({make_fun2,{f,Lbl},Index,OldUniq,NumFree}, Dict0) ->
-    {Fun,Dict} = beam_dict:lambda(Lbl, Index, OldUniq, NumFree, Dict0),
+make_op({make_fun2,{f,Lbl},Index,_OldUniq,NumFree}, Dict0) ->
+    {Fun,Dict} = beam_dict:lambda(Lbl, Index, NumFree, Dict0),
     make_op({make_fun2,Fun}, Dict);
 make_op({kill,Y}, Dict) ->
     make_op({init,Y}, Dict);
diff --git a/lib/compiler/src/beam_dict.erl b/lib/compiler/src/beam_dict.erl
index ee76623976..f82a8f2e3e 100644
--- a/lib/compiler/src/beam_dict.erl
+++ b/lib/compiler/src/beam_dict.erl
@@ -22,7 +22,7 @@
 
 -export([new/0,opcode/2,highest_opcode/1,
 	 atom/2,local/4,export/4,import/4,
-	 string/2,lambda/5,literal/2,line/2,fname/2,
+	 string/2,lambda/4,literal/2,line/2,fname/2,
 	 atom_table/1,local_table/1,export_table/1,import_table/1,
 	 string_table/1,lambda_table/1,literal_table/1,
 	 line_table/1]).
@@ -133,13 +133,16 @@ string(Str, Dict) when is_list(Str) ->
 	    {NextOffset-Offset,Dict}
     end.
 
-%% Returns the index for a funentry (adding it to the table if necessary).
-%%    lambda(Lbl, Index, Uniq, NumFree, Dict) -> {Index,Dict'}
--spec lambda(label(), non_neg_integer(), integer(), non_neg_integer(), bdict()) ->
+%% Returns the index for a fun entry.
+%%    lambda(Lbl, Index, NumFree, Dict) -> {Index,Dict'}
+-spec lambda(label(), non_neg_integer(), non_neg_integer(), bdict()) ->
         {non_neg_integer(), bdict()}.
 
-lambda(Lbl, Index, OldUniq, NumFree, #asm{lambdas=Lambdas0}=Dict) ->
+lambda(Lbl, Index, NumFree, #asm{lambdas=Lambdas0}=Dict) ->
     OldIndex = length(Lambdas0),
+    %% Initialize OldUniq to 0. It will be set to an unique value
+    %% based on the MD5 checksum of the BEAM code for the module.
+    OldUniq = 0,
     Lambdas = [{Lbl,{OldIndex,Lbl,Index,NumFree,OldUniq}}|Lambdas0],
     {OldIndex,Dict#asm{lambdas=Lambdas}}.