Fix object set duplication test

Wrong fields in the record where checked when sorting, which caused
duplicate objects to exist in constructed object sets and later caused
an error.

6 files changed, 309 insertions, 37 deletions

diff --git a/lib/asn1/src/asn1ct.erl b/lib/asn1/src/asn1ct.erl
index df341e5aab..cd7900f919 100644
--- a/lib/asn1/src/asn1ct.erl
+++ b/lib/asn1/src/asn1ct.erl
@@ -34,7 +34,8 @@
 %% Application internal exports
 -export([compile_asn/3,compile_asn1/3,compile_py/3,compile/3,
 	 vsn/0,
-	 get_name_of_def/1,get_pos_of_def/1]).
+	 get_name_of_def/1,get_pos_of_def/1,
+	 unset_pos_mod/1]).
 -export([read_config_data/1,get_gen_state_field/1,
 	 partial_inc_dec_toptype/1,update_gen_state/2,
 	 get_tobe_refed_func/1,reset_gen_state/0,is_function_generated/1,
@@ -559,7 +560,10 @@ unset_pos_mod(Def) when is_record(Def,pvaluesetdef) ->
 unset_pos_mod(Def) when is_record(Def,pobjectdef) ->
     Def#pobjectdef{pos=undefined};
 unset_pos_mod(Def) when is_record(Def,pobjectsetdef) ->
-    Def#pobjectsetdef{pos=undefined}.
+    Def#pobjectsetdef{pos=undefined};
+unset_pos_mod(#'ComponentType'{} = Def) ->
+    Def#'ComponentType'{pos=undefined};
+unset_pos_mod(Def) -> Def.
 
 get_pos_of_def(#typedef{pos=Pos}) ->
     Pos;
diff --git a/lib/asn1/src/asn1ct_check.erl b/lib/asn1/src/asn1ct_check.erl
index cc03cd68fe..e883ee28bd 100644
--- a/lib/asn1/src/asn1ct_check.erl
+++ b/lib/asn1/src/asn1ct_check.erl
@@ -914,7 +914,7 @@ check_object(S,
 	    Unknown ->
 		exit({error,{unknown_object_set,Unknown},S})
 	end,
-    NewSet2 = remove_duplicate_objects(NewObjSet#'ObjectSet'.set),
+    NewSet2 = remove_duplicate_objects(S, NewObjSet#'ObjectSet'.set),
     NewObjSet2 = NewObjSet#'ObjectSet'{set=NewSet2},
     Gen = gen_incl_set(S,NewObjSet2#'ObjectSet'.set,
 		       ClassDef),
@@ -924,14 +924,101 @@ check_object(S,
 %% remove_duplicate_objects/1 remove duplicates of objects.
 %% For instance may Set contain objects of same class from
 %% different object sets that in fact might be duplicates.
-remove_duplicate_objects(Set) when is_list(Set) ->
-    Pred = fun({A,B,_},{A,C,_}) when B =< C -> true;
-	      ({A,_,_},{B,_,_}) when A < B -> true;
-	      ('EXTENSIONMARK','EXTENSIONMARK') -> true;
-	      (T,A) when is_tuple(T),is_atom(A) -> true;% EXTENSIONMARK last in list
-	      (_,_) -> false 
-	   end,
-    lists:usort(Pred,Set).
+remove_duplicate_objects(S, Set0) when is_list(Set0) ->
+    Set1 = lists:map(fun({_,Id,_}=Orig) ->
+			     {{a,Id},Orig};
+			('EXTENSIONMARK'=Ext) ->
+			     {{z,Ext},Ext}
+		     end, Set0),
+    Set2 = sofs:relation(Set1),
+    Set3 = sofs:relation_to_family(Set2),
+    Set = sofs:to_external(Set3),
+    remove_duplicate_objects_1(S, Set).
+
+remove_duplicate_objects_1(S, [{{a,no_unique_value},Objs}|T]) ->
+    Objs ++ remove_duplicate_objects_1(S, T);
+remove_duplicate_objects_1(S, [{_,[_]=Objs}|T]) ->
+    Objs ++ remove_duplicate_objects_1(S, T);
+remove_duplicate_objects_1(S, [{{_,Id},[_|_]=Objs}|T]) ->
+    MakeSortable = fun(What) -> sortable_type(S, What) end,
+    Tagged = order_tag_set(Objs, MakeSortable),
+    case lists:ukeysort(1, Tagged) of
+	[{_,Obj}] ->
+	    [Obj|remove_duplicate_objects_1(S, T)];
+	[_|_] ->
+	    asn1_error(S, S#state.type, {non_unique_object,Id})
+    end;
+remove_duplicate_objects_1(_, []) ->
+    [].
+
+order_tag_set([{_, _, Fields}=Orig|Fs], Fun) ->
+    Pair = {[{FId, traverse(F, Fun)} || {FId, F} <- Fields], Orig},
+    [Pair|order_tag_set(Fs, Fun)];
+order_tag_set([], _) -> [].
+
+sortable_type(S, #'Externaltypereference'{}=ERef) ->
+    try get_referenced_type(S, ERef) of
+	 {_,#typedef{}=OI} ->
+	    OI#typedef{pos=undefined,name=undefined}
+    catch
+	_:_ ->
+	    ERef
+    end;
+sortable_type(_, #typedef{}=TD) ->
+    asn1ct:unset_pos_mod(TD#typedef{name=undefined});
+sortable_type(_, Type) ->
+    asn1ct:unset_pos_mod(Type).
+
+traverse(Structure0, Fun) ->
+    Structure = Fun(Structure0),
+    traverse_1(Structure, Fun).
+
+traverse_1(#typedef{typespec=TS0} = TD, Fun) ->
+    TS = traverse(TS0, Fun),
+    TD#typedef{typespec=TS};
+traverse_1(#valuedef{type=TS0} = VD, Fun) ->
+    TS = traverse(TS0, Fun),
+    VD#valuedef{type=TS};
+traverse_1(#type{def=TS0} = TD, Fun) ->
+    TS = traverse(TS0, Fun),
+    TD#type{def=TS};
+traverse_1(#'SEQUENCE'{components=Cs0} = Seq, Fun) ->
+    Cs = traverse_seq_set(Cs0, Fun),
+    Seq#'SEQUENCE'{components=Cs};
+traverse_1({'SEQUENCE OF',Type0}, Fun) ->
+    Type = traverse(Type0, Fun),
+    {'SEQUENCE OF',Type};
+traverse_1({'SET OF',Type0}, Fun) ->
+    Type = traverse(Type0, Fun),
+    {'SET OF',Type};
+traverse_1(#'SET'{components=Cs0} = Set, Fun) ->
+    Cs = traverse_seq_set(Cs0, Fun),
+    Set#'SET'{components=Cs};
+traverse_1({'CHOICE', Cs0}, Fun) ->
+    Cs = traverse_seq_set(Cs0, Fun),
+    {'CHOICE', Cs};
+traverse_1(Leaf, _) ->
+    Leaf.
+
+traverse_seq_set(List, Fun) when is_list(List) ->
+    traverse_seq_set_1(List, Fun);
+traverse_seq_set({Set, Ext}, Fun) ->
+    {traverse_seq_set_1(Set, Fun), traverse_seq_set_1(Ext, Fun)};
+traverse_seq_set({Set1, Set2, Set3}, Fun) ->
+    {traverse_seq_set_1(Set1, Fun),
+     traverse_seq_set_1(Set2, Fun),
+     traverse_seq_set_1(Set3, Fun)}.
+
+traverse_seq_set_1([#'ComponentType'{} = CT0|Cs], Fun) ->
+    CT = #'ComponentType'{typespec=TS0} = Fun(CT0),
+    TS = traverse(TS0, Fun),
+    [CT#'ComponentType'{typespec=TS}|traverse_seq_set_1(Cs, Fun)];
+traverse_seq_set_1([{'COMPONENTS OF', _} = CO0|Cs], Fun) ->
+    {'COMPONENTS OF', TS0} = Fun(CO0),
+    TS = traverse(TS0, Fun),
+    [{'COMPONENTS OF', TS}|traverse_seq_set_1(Cs, Fun)];
+traverse_seq_set_1([], _) ->
+    [].
 
 %%
 extensionmark(L,true) ->
@@ -1002,11 +1089,12 @@ object_set_from_objects(S,RefedObjMod,ClassDef,FieldName,[O|Os],InterSect,Acc) -
 	Obj ->
 	    object_set_from_objects(S,RefedObjMod,ClassDef,FieldName,Os,InterSect,[Obj|Acc])
     end;
-object_set_from_objects(_S,_RefedObjMod,_ClassDef,_FieldName,[],InterSect,Acc) ->
+object_set_from_objects(S,_RefedObjMod,_ClassDef,_FieldName,[],InterSect,Acc) ->
     %% For instance may Acc contain objects of same class from
     %% different object sets that in fact might be duplicates.
-    remove_duplicate_objects(osfo_intersection(InterSect,Acc)).
-%%    Acc.
+    Set = osfo_intersection(InterSect,Acc),
+    remove_duplicate_objects(S, Set).
+
 object_set_from_objects2(S,RefedObjMod,ClassDef,[{valuefieldreference,OName}],
 			 Fields,_InterSect) ->
     %% this is an object
@@ -1364,15 +1452,8 @@ get_unique_valuelist(S,ObjSet,{UFN,Opt}) ->
 
 get_unique_vlist(_S,[],_,_,[]) ->
     ['EXTENSIONMARK'];
-get_unique_vlist(S,[],_,Opt,Acc) ->
-    case catch check_uniqueness(remove_duplicate_objects(Acc)) of
-	{asn1_error,_} when Opt =/= 'OPTIONAL' ->
-	    error({'ObjectSet',"not unique objects in object set",S});
-	{asn1_error,_} ->
-	    lists:reverse(Acc);
-	_ ->
-	    lists:reverse(Acc)
-    end;
+get_unique_vlist(_, [], _, _Opt, Acc) ->
+    lists:reverse(Acc);
 get_unique_vlist(S,['EXTENSIONMARK'|Rest],UniqueFieldName,Opt,Acc) ->
     get_unique_vlist(S,Rest,UniqueFieldName,Opt,Acc);
 get_unique_vlist(S,[{ObjName,Obj}|Rest],UniqueFieldName,Opt,Acc) ->
@@ -1432,18 +1513,6 @@ get_unique_value(S,Fields,UniqueFieldName) ->
 	    end
     end.
 
-check_uniqueness(NameValueList) ->
-    check_uniqueness1(lists:keysort(2,NameValueList)).
-
-check_uniqueness1([]) ->
-    true;
-check_uniqueness1([_]) ->
-    true;
-check_uniqueness1([{_,N,_},{_,N,_}|_Rest]) ->
-    throw({asn1_error,{'objects in set must have unique values in UNIQUE fields',N}});
-check_uniqueness1([_|Rest]) ->
-    check_uniqueness1(Rest).
-
 %% instantiate_po/4
 %% ClassDef is the class of Object,
 %% Object is the Parameterized object, which is referenced,
@@ -6561,6 +6630,8 @@ format_error({undefined_import,Ref,Module}) ->
     io_lib:format("'~s' is not exported from ~s", [Ref,Module]);
 format_error({value_reused,Val}) ->
     io_lib:format("the value '~p' is used more than once", [Val]);
+format_error({non_unique_object,Id}) ->
+    io_lib:format("object set with a UNIQUE field value of '~p' is used more than once", [Id]);
 format_error(Other) ->
     io_lib:format("~p", [Other]).
 
diff --git a/lib/asn1/test/Makefile b/lib/asn1/test/Makefile
index ff2ae204dc..f4f55bbf52 100644
--- a/lib/asn1/test/Makefile
+++ b/lib/asn1/test/Makefile
@@ -109,6 +109,7 @@ MODULES= \
 	test_x691 \
 	testWSParamClass \
 	testValueTest \
+	testUniqueObjectSets \
 	asn1_test_lib \
 	asn1_app_test \
 	asn1_appup_test \
diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl
index 668a8a2648..2eff210ee2 100644
--- a/lib/asn1/test/asn1_SUITE.erl
+++ b/lib/asn1/test/asn1_SUITE.erl
@@ -135,6 +135,7 @@ groups() ->
        testChoiceIndefinite,
        per_open_type,
        testInfObjectClass,
+       testUniqueObjectSets,
        testInfObjExtract,
        testParam,
        testFragmented,
@@ -767,6 +768,11 @@ testInfObjectClass(Config, Rule, Opts) ->
     testInfObjectClass:main(Rule),
     testInfObj:main(Rule).
 
+testUniqueObjectSets(Config) -> test(Config, fun testUniqueObjectSets/3).
+testUniqueObjectSets(Config, Rule, Opts) ->
+    CaseDir = ?config(case_dir, Config),
+    testUniqueObjectSets:main(CaseDir, Rule, Opts).
+
 testInfObjExtract(Config) -> test(Config, fun testInfObjExtract/3).
 testInfObjExtract(Config, Rule, Opts) ->
     asn1_test_lib:compile("InfObjExtract", Config, [Rule|Opts]),
diff --git a/lib/asn1/test/error_SUITE.erl b/lib/asn1/test/error_SUITE.erl
index 35ca9a2f5e..49ba47aed5 100644
--- a/lib/asn1/test/error_SUITE.erl
+++ b/lib/asn1/test/error_SUITE.erl
@@ -22,8 +22,8 @@
 	 already_defined/1,bitstrings/1,
 	 classes/1,constraints/1,enumerated/1,
 	 imports/1,instance_of/1,integers/1,objects/1,
-	 object_field_extraction/1,parameterization/1,
-	 syntax/1,values/1]).
+	 object_field_extraction/1, object_sets/1,
+	 parameterization/1, syntax/1,values/1]).
 
 -include_lib("test_server/include/test_server.hrl").
 
@@ -44,6 +44,7 @@ groups() ->
        integers,
        objects,
        object_field_extraction,
+       object_sets,
        parameterization,
        syntax,
        values]}].
@@ -297,6 +298,20 @@ object_field_extraction(Config) ->
     } = run(P, Config),
     ok.
 
+object_sets(Config) ->
+    M = 'ObjectSets',
+    P = {M, <<"ObjectSets DEFINITIONS AUTOMATIC TAGS ::= BEGIN\n"
+	      "TEST-UNIQ ::= CLASS { &id INTEGER UNIQUE,  &test INTEGER }\n"
+	      "UniqSet TEST-UNIQ ::= { { &id 1, &test 1 } | {&id 1, &test 2} }\n"
+
+	      "END\n">>},
+    {error,
+     [{structured_error,{M,3},asn1ct_check,{non_unique_object,1}}
+     ]
+    } = run(P, Config),
+    ok.
+
+
 parameterization(Config) ->
     M = 'Parameterization',
     P = {M,
diff --git a/lib/asn1/test/testUniqueObjectSets.erl b/lib/asn1/test/testUniqueObjectSets.erl
new file mode 100644
index 0000000000..1ef61a885a
--- /dev/null
+++ b/lib/asn1/test/testUniqueObjectSets.erl
@@ -0,0 +1,175 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%
+
+-module(testUniqueObjectSets).
+-export([main/3]).
+
+%% Run-time function called by the generated code.
+seq_roundtrip(I, D0) ->
+    M = 'UniqueObjectSets',
+    try
+	{ok,Enc} = M:encode('Seq', {'Seq',I,D0}),
+	{ok,{'Seq',I,D}} = M:decode('Seq', Enc),
+	D
+    catch C:E ->
+	    Stk = erlang:get_stacktrace(),
+	    io:format("FAILED: ~p ~p\n", [I,D0]),
+	    erlang:raise(C, E, Stk)
+    end.
+
+types() ->
+    [{"CHOICE { a INTEGER, b BIT STRING }", {b,<<42:3>>}},
+     {"INTEGER",42},
+     {"SEQUENCE {a OCTET STRING}",{'_',<<"abc">>}},
+     {"SEQUENCE {b BOOLEAN, ...}",{'_',true}},
+     {"SEQUENCE {b BOOLEAN, ..., s IA5String, ..., e ENUMERATED { x, y, z}}",
+      {'_',false,"string",y}},
+     {"SET {a BIT STRING}",{'_',<<1:17>>}},
+     {"SEQUENCE OF INTEGER",[-19,0,555,777]},
+     {"SET OF BOOLEAN",[true,false,true]},
+     {"SEQUENCE OF SEQUENCE {x INTEGER (0..7)}",[{'_',7},{'_',0}]},
+     {"SET OF SEQUENCE {x INTEGER (0..7)}",[{'_',7},{'_',0}]}
+    ].
+
+main(CaseDir, Rule, Opts) ->
+    D0 = types(),
+    {D1,_} = lists:mapfoldl(fun({T,S}, I) ->
+				    {{I,T,S},I+1}
+			    end, 1, D0),
+    Types = [gen_types(I, Type) || {I,Type,_} <- D1],
+    Set = [gen_set_items(I, T) || {I,T,_} <- D1],
+    Objs = [gen_obj(I) || {I,_,_} <- D1],
+    DupObjs = [gen_dup_obj(I, T) || {I,T,_} <- D1],
+    DupObjRefs0 = [gen_dup_obj_refs(I) || {I,_,_} <- D1],
+    DupObjRefs = string:join(DupObjRefs0, " |\n"),
+    Asn1Spec = 'UniqueObjectSets',
+    A = ["UniqueObjectSets DEFINITIONS AUTOMATIC TAGS ::=\n",
+	 "BEGIN\n\n",
+	 "TEST-UNIQUE ::= CLASS {\n"
+	 " &id   INTEGER UNIQUE,\n"
+	 " &Type OPTIONAL\n"
+	 "}\n"
+	 "WITH SYNTAX {IDENTIFIED BY &id [TYPE &Type]}\n",
+	 $\n,
+	 "DUP-CONTAINER ::= CLASS {\n"
+	 " &id   INTEGER UNIQUE,\n"
+	 " &data TEST-UNIQUE\n"
+	 "} WITH SYNTAX {\n"
+	 " ID &id, &data\n"
+	 "}\n",
+	 $\n,
+	 Types,$\n,
+	 "UniqSet TEST-UNIQUE ::= {\n",
+	 Set,
+	 " DupSet-1 |\n",
+	 " DupSet-2, ...\n",
+	 "}\n\n",
+	 Objs,$\n,
+	 DupObjs,$\n,
+	 "DupSet-1 TEST-UNIQUE ::= {\n",
+	 DupObjRefs,$\n,
+	 "}\n\n",
+	 "DupSet-2 TEST-UNIQUE ::= {\n",
+	 DupObjRefs,",...\n",
+	 "}\n\n",
+	 "Seq ::= SEQUENCE {\n"
+	 "  id TEST-UNIQUE.&id ({UniqSet}),\n"
+	 "  type TEST-UNIQUE.&Type ({UniqSet}{@id})\n"
+	 "}\n"
+	 "END\n"],
+    Asn1File = filename:join(CaseDir, atom_to_list(Asn1Spec)++".asn1"),
+    ok = file:write_file(Asn1File, A),
+
+    TestModule = 'unique_object_sets',
+    Test0 = [gen_test(I, Data) || {I,_,Data} <- D1],
+    Test = ["-module(",atom_to_list(TestModule),").\n"
+	    "-export([main/1]).\n"
+	    "\n"
+	    "main(SeqRoundtrip) ->\n",
+	    "  ",atom_to_list(Rule)," = '",atom_to_list(Asn1Spec),
+	    "':encoding_rule(),\n",
+	    Test0,
+	    "  ok.\n"
+	    ],
+    ErlFile = filename:join(CaseDir, atom_to_list(TestModule)++".erl"),
+    ok = file:write_file(ErlFile, Test),
+
+    io:format("~s\n~s\n", [Asn1File,ErlFile]),
+    case Rule of
+	per ->
+	    io:put_chars([A,$\n,Test,$\n]);
+	_ ->
+	    ok
+    end,
+
+    ok = asn1ct:compile(Asn1File, [Rule,{outdir,CaseDir}|Opts]),
+    {ok,TestModule} = c:c(ErlFile, [{outdir,CaseDir}]),
+    TestModule:main(fun seq_roundtrip/2),
+    ok.
+
+gen_types(I, Type) ->
+    io_lib:format("AType~p ::= ~s\n", [I,Type]).
+
+gen_set_items(I, T) ->
+    io_lib:format(" {IDENTIFIED BY ~p TYPE AType~p} |\n"
+		  " {IDENTIFIED BY ~p TYPE AType~p} |\n"
+		  " {IDENTIFIED BY ~p TYPE ~s} |\n"
+		  " obj-~p |\n\n",
+		  [I,I,I,I,I,T,I]).
+
+gen_obj(I) ->
+    io_lib:format("obj-~p TEST-UNIQUE ::= {IDENTIFIED BY ~p TYPE AType~p}\n",
+		  [I,I,I]).
+
+gen_dup_obj(I, T) ->
+    io_lib:format("dup-obj-~p DUP-CONTAINER ::= "
+		  "{ID ~p, {IDENTIFIED BY ~p TYPE ~s}}\n",
+		  [I,I,I+1000,T]).
+
+gen_dup_obj_refs(I) ->
+    io_lib:format("dup-obj-~p.&data", [I]).
+
+gen_test(I, Data) ->
+    io_lib:format("  ~s = SeqRoundtrip(~p, ~p),\n",
+		  [match_term(Data),I,Data]).
+
+match_term('_') ->
+    "_";
+match_term([H|T]=L) ->
+    case is_intlist(L) of
+	true ->
+	    io_lib:format("~p", [L]);
+	false ->
+	    ["[",match_term(H),"|",match_term(T),"]"]
+    end;
+match_term(Tuple) when is_tuple(Tuple) ->
+    ["{",match_term_tuple(Tuple, 1),"}"];
+match_term(Other) ->
+    io_lib:format("~p", [Other]).
+
+match_term_tuple(T, I) when I =< tuple_size(T) ->
+    [match_term(element(I, T)),
+     if I < tuple_size(T) -> ",";
+	true -> "" end|match_term_tuple(T, I+1)];
+match_term_tuple(_, _) ->
+    [].
+
+is_intlist(L) ->
+    lists:all(fun is_integer/1, L).