Merge branch 'anders/diameter/UTF8String_encode/OTP-11172' into maint

* anders/diameter/UTF8String_encode/OTP-11172:
  Simplify Address encode/decode
  Use module unicode for UTF8String encode/decode
  Fix UTF8String encode

3 files changed, 111 insertions, 128 deletions

diff --git a/lib/diameter/doc/src/diameter_dict.xml b/lib/diameter/doc/src/diameter_dict.xml
index 4fcde495b3..8bf4a14240 100644
--- a/lib/diameter/doc/src/diameter_dict.xml
+++ b/lib/diameter/doc/src/diameter_dict.xml
@@ -609,7 +609,9 @@ UTF8String() = [integer()]
 <p>
 List elements are the UTF-8 encodings of the individual characters
 in the string.
-Invalid codepoints will result in encode/decode failure.</p>
+Invalid codepoints will result in encode/decode failure.
+On encode, a UTF8String() can be specified as a binary, or as a nested
+list of binaries and codepoints.</p>
 
 <marker id="DiameterIdentity"/>
 <pre>
diff --git a/lib/diameter/src/base/diameter_types.erl b/lib/diameter/src/base/diameter_types.erl
index 9ae289034c..8c07e84777 100644
--- a/lib/diameter/src/base/diameter_types.erl
+++ b/lib/diameter/src/base/diameter_types.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2013. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -250,13 +250,10 @@
 'Address'(encode, zero) ->
     <<0:48>>;
 
-'Address'(decode, <<1:16, B/binary>>)
-  when size(B) == 4 ->
-    list_to_tuple(binary_to_list(B));
-
-'Address'(decode, <<2:16, B/binary>>)
-  when size(B) == 16 ->
-    list_to_tuple(v6dec(B, []));
+'Address'(decode, <<A:16, B/binary>>)
+  when 1 == A,  4 == size(B);
+       2 == A, 16 == size(B) ->
+    list_to_tuple([N || <<N:A/unit:8>> <= B]);
 
 'Address'(decode, <<A:16, _/binary>> = B)
   when 1 == A;
@@ -264,30 +261,10 @@
     ?INVALID_LENGTH(B);
 
 'Address'(encode, T) ->
-    ipenc(diameter_lib:ipaddr(T)).
-
-ipenc(T)
-  when is_tuple(T), size(T) == 4 ->
-    B = list_to_binary(tuple_to_list(T)),
-    <<1:16, B/binary>>;
-
-ipenc(T)
-  when is_tuple(T), size(T) == 8 ->
-    B = v6enc(lists:reverse(tuple_to_list(T)), <<>>),
-    <<2:16, B/binary>>.
-
-v6dec(<<N:16, B/binary>>, Acc) ->
-    v6dec(B, [N | Acc]);
-
-v6dec(<<>>, Acc) ->
-    lists:reverse(Acc).
-
-v6enc([N | Rest], B)
-  when ?UINT(16,N) ->
-    v6enc(Rest, <<N:16, B/binary>>);
-
-v6enc([], B) ->
-    B.
+    Ns = tuple_to_list(diameter_lib:ipaddr(T)),  %% length 4 or 8
+    A = length(Ns) div 4,                        %% 1 or 2
+    B = << <<N:A/unit:8>> || N <- Ns >>,
+    <<A:16, B/binary>>.
 
 %% --------------------
 
@@ -354,36 +331,13 @@ v6enc([], B) ->
 %% --------------------
 
 'UTF8String'(decode, Bin) ->
-    udec(Bin, []);
+    tl([0|_] = unicode:characters_to_list([0, Bin])); %% assert list return
 
 'UTF8String'(encode = M, zero) ->
     'UTF8String'(M, []);
 
 'UTF8String'(encode, S) ->
-    uenc(S, []).
-
-udec(<<>>, Acc) ->
-    lists:reverse(Acc);
-
-udec(<<C/utf8, Rest/binary>>, Acc) ->
-    udec(Rest, [C | Acc]).
-
-uenc(E, Acc)
-  when E == [];
-       E == <<>> ->
-    list_to_binary(lists:reverse(Acc));
-
-uenc(<<C/utf8, Rest/binary>>, Acc) ->
-    uenc(Rest, [<<C/utf8>> | Acc]);
-
-uenc([[] | Rest], Acc) ->
-    uenc(Rest, Acc);
-
-uenc([[H|T] | Rest], Acc) ->
-    uenc([H, T | Rest], Acc);
-
-uenc([C | Rest], Acc) ->
-    uenc(Rest, [<<C/utf8>> | Acc]).
+    <<_/binary>> = unicode:characters_to_binary(S).   %% assert binary return
 
 %% --------------------
 
diff --git a/lib/diameter/test/diameter_codec_test.erl b/lib/diameter/test/diameter_codec_test.erl
index 24d4c7665e..295d23912b 100644
--- a/lib/diameter/test/diameter_codec_test.erl
+++ b/lib/diameter/test/diameter_codec_test.erl
@@ -1,3 +1,4 @@
+%% coding: utf-8
 %%
 %% %CopyrightBegin%
 %%
@@ -19,7 +20,9 @@
 
 -module(diameter_codec_test).
 
--compile(export_all).
+-export([base/0,
+         gen/1,
+         lib/0]).
 
 %%
 %% Test encode/decode of dictionary-related modules.
@@ -38,37 +41,34 @@
 %% Interface.
 
 base() ->
-    [] = run([{?MODULE, [base, T]} || T <- [zero, decode]]).
+    [] = run([[fun base/1, T] || T <- [zero, decode]]).
 
 gen(Mod) ->
     Fs = [{Mod, F, []} || F <- [name, id, vendor_id, vendor_name]],
-    [] = run(Fs ++ [{?MODULE, [gen, Mod, T]} || T <- [messages,
-                                                      command_codes,
-                                                      avp_types,
-                                                      grouped,
-                                                      enum,
-                                                      import_avps,
-                                                      import_groups,
-                                                      import_enums]]).
+    [] = run(Fs ++ [[fun gen/2, Mod, T] || T <- [messages,
+                                                 command_codes,
+                                                 avp_types,
+                                                 grouped,
+                                                 enum,
+                                                 import_avps,
+                                                 import_groups,
+                                                 import_enums]]).
 
 lib() ->
-    Vs = {_,_} = values('Address'),
-    [] = run([[fun lib/2, N, Vs] || N <- [1,2]]).
+    Vs = {_,_,_} = values('Address'),
+    [] = run([[fun lib/2, N, Vs] || N <- [{1, true}, {3, false}]]).
 
 %% ===========================================================================
 %% Internal functions.
 
-lib(N, {_,_} = T) ->
-    B = 1 == N rem 2,
-    [] = run([[fun lib/2, A, B] || A <- element(N,T)]);
+lib({N,B}, {_,_,_} = T) ->
+    [] = run([[fun lib/2, A, B] || A <- element(N,T), is_tuple(A)]);
 
 lib(IP, B) ->
-    LA = tuple_to_list(IP),
-    {SA,Fun} = ip(LA),
-    [] = run([[fun lib/4, IP, B, Fun, A] || A <- [IP, SA]]).
+    [] = run([[fun lib/3, IP, B, A] || A <- [IP, ntoa(tuple_to_list(IP))]]).
 
-lib(IP, B, Fun, A) ->
-    try Fun(A) of
+lib(IP, B, A) ->
+    try diameter_lib:ipaddr(A) of
         IP when B ->
             ok
     catch
@@ -76,12 +76,12 @@ lib(IP, B, Fun, A) ->
             ok
     end.
 
-ip([_,_,_,_] = A) ->
+ntoa([_,_,_,_] = A) ->
     [$.|S] = lists:append(["." ++ integer_to_list(N) || N <- A]),
-    {S, fun diameter_lib:ipaddr/1};
-ip([_,_,_,_,_,_,_,_] = A) ->
+    S;
+ntoa([_,_,_,_,_,_,_,_] = A) ->
     [$:|S] = lists:flatten([":" ++ io_lib:format("~.16B", [N]) || N <- A]),
-    {S, fun diameter_lib:ipaddr/1}.
+    S.
 
 %% ------------------------------------------------------------------------
 %% base/1
@@ -90,7 +90,7 @@ ip([_,_,_,_,_,_,_,_] = A) ->
 %% ------------------------------------------------------------------------
 
 base(T) ->
-    [] = run([{?MODULE, [base, T, F]} || F <- types()]).
+    [] = run([[fun base/2, T, F] || F <- types()]).
 
 %% Ensure that 'zero' values encode only zeros.
 base(zero = T, F) ->
@@ -100,32 +100,23 @@ base(zero = T, F) ->
 %% Ensure that we can decode what we encode and vice-versa, and that
 %% we can't decode invalid values.
 base(decode, F) ->
-    {Eq, Vs, Ns} = b(values(F)),
-    [] = run([{?MODULE, [base_decode, F, Eq, V]}  || V <- Vs]),
-    [] = run([{?MODULE, [base_invalid, F, Eq, V]} || V <- Ns]).
+    {Ts, Fs, Is} = values(F),
+    [] = run([[fun base_decode/3, F, true, V]  || V <- Ts]),
+    [] = run([[fun base_decode/3, F, false, V] || V <- Fs]),
+    [] = run([[fun base_invalid/2, F, V]       || V <- Is]).
 
 base_decode(F, Eq, Value) ->
     d(fun(X,V) -> diameter_types:F(X,V) end, Eq, Value).
 
-base_invalid(F, Eq, Value) ->
+base_invalid(F, Value) ->
     try
-        base_decode(F, Eq, Value),
+        base_decode(F, false, Value),
         exit(nok)
     catch
         error: _ ->
             ok
     end.
 
-b({_,_,_} = T) ->
-    T;
-b({B,Vs})
-  when is_atom(B) ->
-    {B,Vs,[]};
-b({Vs,Ns}) ->
-    {true, Vs, Ns};
-b(Vs) ->
-    {true, Vs, []}.
-
 types() ->
     [F || {F,2} <- diameter_types:module_info(exports)].
 
@@ -136,7 +127,7 @@ types() ->
 %% ------------------------------------------------------------------------
 
 gen(M, T) ->
-    [] = run(lists:map(fun(X) -> {?MODULE, [gen, M, T, X]} end,
+    [] = run(lists:map(fun(X) -> [fun gen/3, M, T, X] end,
                        fetch(T, dict(M)))).
 
 fetch(T, Spec) ->
@@ -197,18 +188,20 @@ gen(M, enum = T, {Name, ED})
     gen(M, T, {?A(Name), lists:map(fun({E,D}) -> {?A(E), D} end, ED)});
 
 gen(M, enum, {Name, ED}) ->
-    [] = run([{?MODULE, [enum, M, Name, T]} || T <- ED]);
+    [] = run([[fun enum/3, M, Name, T] || T <- ED]);
 
 gen(M, Tag, {_Mod, L}) ->
     T = retag(Tag),
-    [] = run([{?MODULE, [gen, M, T, I]} || I <- L]).
+    [] = run([[fun gen/3, M, T, I] || I <- L]).
 
 %% avp_decode/3
 
 avp_decode(Mod, Type, Name) ->
-    {Eq, Vs, _} = b(values(Type, Name, Mod)),
-    [] = run([{?MODULE, [avp_decode, Mod, Name, Type, Eq, V]}
-              || V <- v(Vs)]).
+    {Ts, Fs, _} = values(Type, Name, Mod),
+    [] = run([[fun avp_decode/5, Mod, Name, Type, true, V]
+              || V <- v(Ts)]),
+    [] = run([[fun avp_decode/5, Mod, Name, Type, false, V]
+              || V <- v(Fs)]).
 
 avp_decode(Mod, Name, Type, Eq, Value) ->
     d(fun(X,V) -> avp(Mod, X, V, Name, Type) end, Eq, Value).
@@ -250,7 +243,7 @@ v(N, Ord, E, Acc) ->
 
 arity(M, Name, Rname) ->
     Rec = M:'#new-'(Rname),
-    [] = run([{?MODULE, [arity, M, Name, F, Rec]}
+    [] = run([[fun arity/4, M, Name, F, Rec]
               || F <- M:'#info-'(Rname, fields)]).
 
 arity(M, Name, AvpName, Rec) ->
@@ -299,68 +292,93 @@ z(B) ->
 %% tested.)
 
 values('OctetString' = T) ->
-    {["", atom_to_list(T)], [-1, 256]};
+    {["", atom_to_list(T)],
+     [],
+     [-1, 256]};
 
 values('Integer32') ->
     Mx = (1 bsl 31) - 1,
     Mn = -1*Mx,
-    {[Mn, 0, random(Mn,Mx), Mx], [Mn - 1, Mx + 1]};
+    {[Mn, 0, random(Mn,Mx), Mx],
+     [],
+     [Mn - 1, Mx + 1]};
 
 values('Integer64') ->
     Mx = (1 bsl 63) - 1,
     Mn = -1*Mx,
-    {[Mn, 0, random(Mn,Mx), Mx], [Mn - 1, Mx + 1]};
+    {[Mn, 0, random(Mn,Mx), Mx],
+     [],
+     [Mn - 1, Mx + 1]};
 
 values('Unsigned32') ->
     M = (1 bsl 32) - 1,
-    {[0, random(M), M], [-1, M + 1]};
+    {[0, random(M), M],
+     [],
+     [-1, M + 1]};
 
 values('Unsigned64') ->
     M = (1 bsl 64) - 1,
-    {[0, random(M), M], [-1, M + 1]};
+    {[0, random(M), M],
+     [],
+     [-1, M + 1]};
 
 values('Float32') ->
     E = (1 bsl  8) - 2,
     F = (1 bsl 23) - 1,
     <<Mx:32/float>> = <<0:1, E:8, F:23>>,
     <<Mn:32/float>> = <<1:1, E:8, F:23>>,
-    {[0.0, infinity, '-infinity', Mx, Mn], [0]};
+    {[0.0, infinity, '-infinity', Mx, Mn],
+     [],
+     [0]};
 
 values('Float64') ->
     E = (1 bsl 11) - 2,
     F = (1 bsl 52) - 1,
     <<Mx:64/float>> = <<0:1, E:11, F:52>>,
     <<Mn:64/float>> = <<1:1, E:11, F:52>>,
-    {[0.0, infinity, '-infinity', Mx, Mn], [0]};
+    {[0.0, infinity, '-infinity', Mx, Mn],
+     [],
+     [0]};
 
 values('Address') ->
     {[{255,0,random(16#FF),1}, {65535,0,0,random(16#FFFF),0,0,0,1}],
-     [{256,0,0,1}, {65536,0,0,0,0,0,0,1}]};
+     ["127.0.0.1", "FFFF:FF::1.2.3.4"],
+     [{256,0,0,1}, {65536,0,0,0,0,0,0,1}, "256.0.0.1", "10000::1"]};
 
 values('DiameterIdentity') ->
-    {["x", "diameter.com"], [""]};
+    {["x", "diameter.com"],
+     [],
+     [""]};
 
 values('DiameterURI') ->
-    {false, ["aaa" ++ S ++ "://diameter.se" ++ P ++ Tr ++ Pr
-             || S  <- ["", "s"],
-                P  <- ["", ":1234"],
-                Tr <- ["" | [";transport=" ++ X
-                             || X <- ["tcp", "sctp", "udp"]]],
-                Pr <- ["" | [";protocol=" ++ X
-                             || X <- ["diameter","radius","tacacs+"]]]]};
+    {[],
+     ["aaa" ++ S ++ "://diameter.se" ++ P ++ Tr ++ Pr
+      || S  <- ["", "s"],
+         P  <- ["", ":1234"],
+         Tr <- ["" | [";transport=" ++ X
+                      || X <- ["tcp", "sctp", "udp"]]],
+         Pr <- ["" | [";protocol=" ++ X
+                      || X <- ["diameter","radius","tacacs+"]]]],
+     []};
 
 values(T)
   when T == 'IPFilterRule';
        T == 'QoSFilterRule' ->
-    ["deny in 0 from 127.0.0.1 to 10.0.0.1"];
+    {["deny in 0 from 127.0.0.1 to 10.0.0.1"],
+     [],
+     []};
 
 %% RFC 3629 defines the UTF-8 encoding of U+0000 through U+10FFFF with the
 %% exception of U+D800 through U+DFFF.
 values('UTF8String') ->
+    S = "ᚠᚢᚦᚨᚱᚲ",
+    B = unicode:characters_to_binary(S),
     {[[],
+      S,
       lists:seq(0,16#1FF),
       [0,16#D7FF,16#E000,16#10FFFF],
       [random(16#D7FF), random(16#E000,16#10FFFF)]],
+     [B, [B, S, hd(S)], [S, B]],
      [[-1],
       [16#D800],
       [16#DFFF],
@@ -372,6 +390,7 @@ values('Time') ->
       {{2036,2,7},{6,28,15}},
       {{2036,2,7},{6,28,16}},    %% 19000101T000000 + 2 bsl 31
       {{2104,2,26},{9,42,23}}],
+     [],
      [{{1968,1,20},{3,14,7}},
       {{2104,2,26},{9,42,24}}]}. %% 19000101T000000 + 3 bsl 31
 
@@ -382,18 +401,24 @@ values('Time') ->
 
 values('Enumerated', Name, Mod) ->
     {_Name, Vals} = lists:keyfind(?S(Name), 1, types(enum, Mod)),
-    lists:map(fun({_,N}) -> N end, Vals);
+    {lists:map(fun({_,N}) -> N end, Vals),
+     [],
+     []};
 
 values('Grouped', Name, Mod) ->
     Rname = Mod:name2rec(Name),
     Rec = Mod:'#new-'(Rname),
     Avps = Mod:'#info-'(Rname, fields),
-    Enum = diameter_enum:combine(lists:map(fun({_,Vs,_}) -> to_enum(Vs) end,
+    Enum = diameter_enum:combine(lists:map(fun({Vs,_,_}) -> to_enum(Vs) end,
                                            [values(F, Mod) || F <- Avps])),
-    {false, diameter_enum:append(group(Mod, Name, Rec, Avps, Enum))};
+    {[],
+     diameter_enum:append(group(Mod, Name, Rec, Avps, Enum)),
+     []};
 
 values(_, 'Framed-IP-Address', _) ->
-    [{127,0,0,1}];
+    {[{127,0,0,1}],
+     [],
+     []};
 
 values(Type, _, _) ->
     values(Type).
@@ -407,12 +432,14 @@ to_enum(E) ->
 %% values/2
 
 values('AVP', _) ->
-    {true, [#diameter_avp{code = 0, data = <<0>>}], []};
+    {[#diameter_avp{code = 0, data = <<0>>}],
+     [],
+     []};
 
 values(Name, Mod) ->
     Avps = types(avp_types, Mod),
     {_Name, _Code, Type, _Flags} = lists:keyfind(?S(Name), 1, Avps),
-    b(values(?A(Type), Name, Mod)).
+    values(?A(Type), Name, Mod).
 
 %% group/5
 %%