stdlib: Fix normalization function in uri_string

- Fix parsing of hostnames that start with a number.
- Update uri_string:parse/1 to be only responsible for parsing
  input URIs into URI components. Implicit percent-encoding
  normalization has been removed.
- Implement percent-encoding normalization.
- Update uri_string:normalize/{1,2} to include percent-encoding
  normalization.
- Update test suites according to the new semantics.
- Add new property test: normalize

Change-Id: I6f37dcae2b3fcb4b29d286dbb0dfc563e8f211ae

1 files changed, 235 insertions, 25 deletions

diff --git a/lib/stdlib/test/property_test/uri_string_recompose.erl b/lib/stdlib/test/property_test/uri_string_recompose.erl
index e51a671172..35b3a50b9c 100644
--- a/lib/stdlib/test/property_test/uri_string_recompose.erl
+++ b/lib/stdlib/test/property_test/uri_string_recompose.erl
@@ -65,15 +65,29 @@
 -define(QUERY, {query, query_map()}).
 -define(FRAGMENT, {fragment, fragment_map()}).
 
+%% Non-unicode
+-define(USER_NU, {userinfo, non_unicode()}).
+-define(HOST_NU, {host, host_map_nu()}).
+-define(PATH_ABE_NU, {path, path_abempty_map_nu()}).
+-define(PATH_ABS_NU, {path, path_absolute_map_nu()}).
+-define(PATH_NOS_NU, {path, path_noscheme_map_nu()}).
+-define(PATH_ROO_NU, {path, path_rootless_map_nu()}).
+-define(QUERY_NU, {query, query_map_nu()}).
+-define(FRAGMENT_NU, {fragment, fragment_map_nu()}).
 
 %%%========================================================================
 %%% Properties
 %%%========================================================================
 
 prop_recompose() ->
+    ?FORALL(Map, map_no_unicode(),
+           Map =:= uri_string:parse(uri_string:recompose(Map))).
+
+prop_normalize() ->
     ?FORALL(Map, map(),
-            Map =:= uri_string:parse(uri_string:recompose(Map))
-           ).
+            uri_string:normalize(Map, [return_map]) =:=
+                uri_string:normalize(uri_string:parse(uri_string:recompose(Map)),
+                                     [return_map])).
 
 %% Stats
 prop_map_key_length_collect() ->
@@ -96,6 +110,9 @@ prop_scheme_collect() ->
 map() ->
     ?LET(Gen, comp_proplist(), proplist_to_map(Gen)).
 
+map_no_unicode() ->
+    ?LET(Gen, comp_proplist_nu(), proplist_to_map(Gen)).
+
 comp_proplist() ->
     frequency([
                {2, [?SCHEME,?PATH_ABS]},
@@ -166,6 +183,76 @@ comp_proplist() ->
                {2, [?USER,?HOST,?PORT,?PATH_ABE,?QUERY,?FRAGMENT]}
               ]).
 
+comp_proplist_nu() ->
+    frequency([
+               {2, [?SCHEME,?PATH_ABS_NU]},
+               {2, [?SCHEME,?PATH_ROO_NU]},
+               {2, [?SCHEME,?PATH_EMP]},
+               {2, [?SCHEME,?HOST_NU,?PATH_ABE_NU]},
+               {2, [?SCHEME,?USER_NU,?HOST_NU,?PATH_ABE_NU]},
+               {2, [?SCHEME,?HOST_NU,?PORT,?PATH_ABE_NU]},
+               {2, [?SCHEME,?USER_NU,?HOST_NU,?PORT,?PATH_ABE_NU]},
+
+               {2, [?PATH_ABS_NU]},
+               {2, [?PATH_NOS_NU]},
+               {2, [?PATH_EMP]},
+               {2, [?HOST_NU,?PATH_ABE_NU]},
+               {2, [?USER_NU,?HOST_NU,?PATH_ABE_NU]},
+               {2, [?HOST_NU,?PORT,?PATH_ABE_NU]},
+               {2, [?USER_NU,?HOST_NU,?PORT,?PATH_ABE_NU]},
+
+
+               {2, [?SCHEME,?PATH_ABS_NU,?QUERY_NU]},
+               {2, [?SCHEME,?PATH_ROO_NU,?QUERY_NU]},
+               {2, [?SCHEME,?PATH_EMP,?QUERY_NU]},
+               {2, [?SCHEME,?HOST_NU,?PATH_ABE_NU,?QUERY_NU]},
+               {2, [?SCHEME,?USER_NU,?HOST_NU,?PATH_ABE_NU,?QUERY_NU]},
+               {2, [?SCHEME,?HOST_NU,?PORT,?PATH_ABE_NU,?QUERY_NU]},
+               {2, [?SCHEME,?USER_NU,?HOST_NU,?PORT,?PATH_ABE_NU,?QUERY_NU]},
+
+               {2, [?PATH_ABS_NU,?QUERY_NU]},
+               {2, [?PATH_NOS_NU,?QUERY_NU]},
+               {2, [?PATH_EMP,?QUERY_NU]},
+               {2, [?HOST_NU,?PATH_ABE_NU,?QUERY_NU]},
+               {2, [?USER_NU,?HOST_NU,?PATH_ABE_NU,?QUERY_NU]},
+               {2, [?HOST_NU,?PORT,?PATH_ABE_NU,?QUERY_NU]},
+               {2, [?USER_NU,?HOST_NU,?PORT,?PATH_ABE_NU,?QUERY_NU]},
+
+
+               {2, [?SCHEME,?PATH_ABS_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?PATH_ROO_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?PATH_EMP,?FRAGMENT_NU]},
+               {2, [?SCHEME,?HOST_NU,?PATH_ABE_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?USER_NU,?HOST_NU,?PATH_ABE_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?HOST_NU,?PORT,?PATH_ABE_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?USER_NU,?HOST_NU,?PORT,?PATH_ABE_NU,?FRAGMENT_NU]},
+
+               {2, [?PATH_ABS_NU,?FRAGMENT_NU]},
+               {2, [?PATH_NOS_NU,?FRAGMENT_NU]},
+               {2, [?PATH_EMP,?FRAGMENT_NU]},
+               {2, [?HOST_NU,?PATH_ABE_NU,?FRAGMENT_NU]},
+               {2, [?USER_NU,?HOST_NU,?PATH_ABE_NU,?FRAGMENT_NU]},
+               {2, [?HOST_NU,?PORT,?PATH_ABE_NU,?FRAGMENT_NU]},
+               {2, [?USER_NU,?HOST_NU,?PORT,?PATH_ABE_NU,?FRAGMENT_NU]},
+
+
+               {2, [?SCHEME,?PATH_ABS_NU,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?PATH_ROO_NU,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?PATH_EMP,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?HOST_NU,?PATH_ABE_NU,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?USER_NU,?HOST_NU,?PATH_ABE_NU,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?HOST_NU,?PORT,?PATH_ABE_NU,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?SCHEME,?USER_NU,?HOST_NU,?PORT,?PATH_ABE_NU,?QUERY_NU,?FRAGMENT_NU]},
+
+               {2, [?PATH_ABS_NU,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?PATH_NOS_NU,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?PATH_EMP,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?HOST_NU,?PATH_ABE_NU,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?USER_NU,?HOST_NU,?PATH_ABE_NU,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?HOST_NU,?PORT,?PATH_ABE_NU,?QUERY_NU,?FRAGMENT_NU]},
+               {2, [?USER_NU,?HOST_NU,?PORT,?PATH_ABE_NU,?QUERY_NU,?FRAGMENT_NU]}
+              ]).
+
 
 %%-------------------------------------------------------------------------
 %% Path
@@ -174,6 +261,11 @@ path_abempty_map() ->
     frequency([{90, path_abe_map()},
                {10, path_empty_map()}]).
 
+path_abempty_map_nu() ->
+    frequency([{90, path_abe_map_nu()},
+               {10, path_empty_map()}]).
+
+
 path_abe_map() ->
     ?SIZED(Length, path_abe_map(Length, [])).
 %%
@@ -182,6 +274,14 @@ path_abe_map(0, Segments) ->
 path_abe_map(N, Segments) ->
     path_abe_map(N-1, [slash(),segment()|Segments]).
 
+path_abe_map_nu() ->
+    ?SIZED(Length, path_abe_map_nu(Length, [])).
+%%
+path_abe_map_nu(0, Segments) ->
+    ?LET(Gen, Segments, lists:append(Gen));
+path_abe_map_nu(N, Segments) ->
+    path_abe_map_nu(N-1, [slash(),segment_nu()|Segments]).
+
 
 path_absolute_map() ->
     ?SIZED(Length, path_absolute_map(Length, [])).
@@ -191,6 +291,14 @@ path_absolute_map(0, Segments) ->
 path_absolute_map(N, Segments) ->
     path_absolute_map(N-1, [slash(),segment()|Segments]).
 
+path_absolute_map_nu() ->
+    ?SIZED(Length, path_absolute_map_nu(Length, [])).
+%%
+path_absolute_map_nu(0, Segments) ->
+    ?LET(Gen, [slash(),segment_nz_nu()|Segments], lists:append(Gen));
+path_absolute_map_nu(N, Segments) ->
+    path_absolute_map_nu(N-1, [slash(),segment_nu()|Segments]).
+
 
 path_noscheme_map() ->
     ?SIZED(Length, path_noscheme_map(Length, [])).
@@ -200,6 +308,15 @@ path_noscheme_map(0, Segments) ->
 path_noscheme_map(N, Segments) ->
     path_noscheme_map(N-1, [slash(),segment()|Segments]).
 
+path_noscheme_map_nu() ->
+    ?SIZED(Length, path_noscheme_map_nu(Length, [])).
+%%
+path_noscheme_map_nu(0, Segments) ->
+    ?LET(Gen, [segment_nz_nc_nu()|Segments], lists:append(Gen));
+path_noscheme_map_nu(N, Segments) ->
+    path_noscheme_map_nu(N-1, [slash(),segment_nu()|Segments]).
+
+
 path_rootless_map() ->
     ?SIZED(Length, path_rootless_map(Length, [])).
 %%
@@ -208,24 +325,59 @@ path_rootless_map(0, Segments) ->
 path_rootless_map(N, Segments) ->
     path_rootless_map(N-1, [slash(),segment()|Segments]).
 
+path_rootless_map_nu() ->
+    ?SIZED(Length, path_rootless_map_nu(Length, [])).
+%%
+path_rootless_map_nu(0, Segments) ->
+    ?LET(Gen, [segment_nz_nu()|Segments], lists:append(Gen));
+path_rootless_map_nu(N, Segments) ->
+    path_rootless_map_nu(N-1, [slash(),segment_nu()|Segments]).
+
 
 segment_nz() ->
     non_empty(segment()).
 
-segment_nz_nc() ->
-    non_empty(list(frequency([{30, unreserved()},
-                              {10, sub_delims()},
-                              {10, unicode_char()},
-                              {5, oneof([$@])}
-                             ]))).
+segment_nz_nu() ->
+    non_empty(segment_nu()).
 
 
+segment_nz_nc() ->
+    ?LET(Gen,
+         non_empty(list(frequency([{30, unreserved()},
+                                   {10, ptc_encoded_reserved()},
+                                   {10, sub_delims()},
+                                   {10, unicode_char()},
+                                   {5, oneof([$@])}
+                                  ]))),
+         lists:flatten(Gen)).
+
+segment_nz_nc_nu() ->
+    ?LET(Gen,
+         non_empty(list(frequency([{30, unreserved()},
+                                   {10, ptc_encoded_reserved()},
+                                   {10, sub_delims()},
+                                   {5, oneof([$@])}
+                                  ]))),
+         lists:flatten(Gen)).
+
 segment() ->
-    list(frequency([{30, unreserved()},
-                    {10, sub_delims()},
-                    {10, unicode_char()},
-                    {5, oneof([$:, $@])}
-                   ])).
+    ?LET(Gen,
+         list(frequency([{30, unreserved()},
+                         {10, ptc_encoded_reserved()},
+                         {10, sub_delims()},
+                         {10, unicode_char()},
+                         {5, oneof([$:, $@])}
+                        ])),
+         lists:flatten(Gen)).
+
+segment_nu() ->
+    ?LET(Gen,
+         list(frequency([{30, unreserved()},
+                         {10, ptc_encoded_reserved()},
+                         {10, sub_delims()},
+                         {5, oneof([$:, $@])}
+                        ])),
+         lists:flatten(Gen)).
 
 slash() ->
     "/".
@@ -235,19 +387,35 @@ path_empty_map() ->
 
 
 %%-------------------------------------------------------------------------
-%% Path
+%% Host
 %%-------------------------------------------------------------------------
 host_map() ->
     frequency([{30, reg_name()},
                {30, ip_address()}
               ]).
 
+host_map_nu() ->
+    frequency([{30, reg_name_nu()},
+               {30, ip_address()}
+              ]).
 
 reg_name() ->
-    list(frequency([{30, alpha()},
-                              {10, sub_delims()},
-                              {10, unicode_char()}
-                             ])).
+    ?LET(Gen,
+         list(frequency([{30, alpha()},
+                         {10, sub_delims()},
+                         {10, ptc_encoded_reserved()},
+                         {10, unicode_char()}
+                        ])),
+         lists:flatten(Gen)).
+
+reg_name_nu() ->
+    ?LET(Gen,
+         list(frequency([{30, alpha()},
+                         {10, sub_delims()},
+                         {10, ptc_encoded_reserved()}
+                        ])),
+         lists:flatten(Gen)).
+
 
 ip_address() ->
     oneof(["127.0.0.1", "::127.0.0.1",
@@ -258,10 +426,13 @@ ip_address() ->
 
 %% Generating only reg-names
 host_uri() ->
-    non_empty(list(frequency([{30, unreserved()},
-                              {10, sub_delims()},
-                              {10, pct_encoded()}
-                             ]))).
+    ?LET(Gen,
+         non_empty(list(frequency([{30, unreserved()},
+                                   {10, sub_delims()},
+                                   {10, ptc_encoded_reserved()},
+                                   {10, pct_encoded()}
+                                  ]))),
+         lists:flatten(Gen)).
 
 %%-------------------------------------------------------------------------
 %% Port, Query, Fragment
@@ -274,6 +445,9 @@ port() ->
 query_map() ->
     unicode().
 
+query_map_nu() ->
+    non_unicode().
+
 
 query_uri() ->
     [$?| non_empty(list(frequency([{20, pchar()},
@@ -283,6 +457,10 @@ query_uri() ->
 fragment_map() ->
     unicode().
 
+fragment_map_nu() ->
+    non_unicode().
+
+
 fragment_uri() ->
     [$?| non_empty(list(frequency([{20, pchar()},
                                    {5, oneof([$/, $?])} % punctuation
@@ -311,9 +489,14 @@ scheme(N, L) ->
 %%-------------------------------------------------------------------------
 unicode() ->
     list(frequency([{20, alpha()},                    % alpha
-               {10, digit()},                    % digit
-               {10, unicode_char()}              % unicode
-              ])).
+                    {10, digit()},                    % digit
+                    {10, unicode_char()}              % unicode
+                   ])).
+
+non_unicode() ->
+    list(frequency([{20, alpha()},                   % alpha
+                    {10, digit()}                    % digit
+                   ])).
 
 scheme_char() ->
     frequency([{20, alpha()},                    % alpha
@@ -327,6 +510,7 @@ sub_delims() ->
 
 pchar() ->
     frequency([{20, unreserved()},
+               {5, ptc_encoded_reserved()},
                {5, pct_encoded()},
                {5, sub_delims()},
                {1, oneof([$:, $@])}              % punctuation
@@ -351,6 +535,22 @@ digit() ->
 pct_encoded() ->
     oneof(["%C3%A4", "%C3%A5", "%C3%B6"]).
 
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 2.2. Reserved Characters]
+%%
+%%   reserved    = gen-delims / sub-delims
+%%
+%%   gen-delims  = ":" / "/" / "?" / "#" / "[" / "]" / "@"
+%%                 3A    2F    3F    23    5B    5D    40
+%%   sub-delims  = "!" / "$" / "&" / "'" / "(" / ")"
+%%                 21    24    26    27    28    29
+%%               / "*" / "+" / "," / ";" / "="
+%%                 2A    2B    2C    3B    3D
+%%-------------------------------------------------------------------------
+ptc_encoded_reserved() ->
+    oneof(["%3A","%2F","%3F","%23","%5B","%5D","%40",
+           "%21","%24","%26","%27","%28","%29",
+           "%2A","%2B","%2C","%3B","3D"]).
 
 %%%========================================================================
 %%% Helpers
@@ -359,3 +559,13 @@ proplist_to_map(L) ->
     lists:foldl(fun({K,V},M) -> M#{K => V};
                   (_,M) -> M
                end, #{}, L).
+
+map_scheme_host_to_lower(Map) ->
+    Fun = fun (scheme,V) ->
+                  string:to_lower(V);
+              (host,V) ->
+                  string:to_lower(V);
+              (_,V) ->
+                  V
+          end,
+    maps:map(Fun, Map).