[xmerl] Correct bug handling multiple documents on a stream

Change how to interpret end of document to comply with Tim Brays
comment on the standard. This makes it possible to handle more
than one doc on a stream, the standard makes it impossible to
know when the document is ended without waiting for the next
document (and not always even that).

Tim Brays comment about the trailing "Misc" rule:

The fact that you're allowed some trailing junk after the root
element, I decided (but unfortunately too late) is a real design
error in XML. If I'm writing a network client, I'm probably going
to close the link as soon as a I see the root element end-tag, and
not depend on the other end closing it down properly.
Furthermore, if I want to send a succession of XML documents over
a network link, if I find a processing instruction after a root
element, is it a trailer on the previous document, or part of the
prolog of the next?

6 files changed, 320 insertions, 4 deletions

diff --git a/lib/xmerl/test/Makefile b/lib/xmerl/test/Makefile
index 7a326e334f..b13fee05b3 100644
--- a/lib/xmerl/test/Makefile
+++ b/lib/xmerl/test/Makefile
@@ -55,7 +55,8 @@ SUITE_FILES= \
 	xmerl_xsd_SUITE.erl \
 	xmerl_xsd_MS2002-01-16_SUITE.erl \
 	xmerl_xsd_NIST2002-01-16_SUITE.erl \
-	xmerl_xsd_Sun2002-01-16_SUITE.erl
+	xmerl_xsd_Sun2002-01-16_SUITE.erl \
+        xmerl_sax_stream_SUITE.erl
 
 XML_FILES= \
 	testcases.dtd \
@@ -125,4 +126,5 @@ release_tests_spec: opt
 	@tar cfh - xmerl_xsd_MS2002-01-16_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -)
 	@tar cfh - xmerl_xsd_NIST2002-01-16_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -)
 	@tar cfh - xmerl_xsd_Sun2002-01-16_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -)
+	@tar cfh - xmerl_sax_stream_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -)
 	chmod -R u+w "$(RELSYSDIR)"
diff --git a/lib/xmerl/test/xmerl_sax_SUITE.erl b/lib/xmerl/test/xmerl_sax_SUITE.erl
index f5c0a783c4..7d1a70905c 100644
--- a/lib/xmerl/test/xmerl_sax_SUITE.erl
+++ b/lib/xmerl/test/xmerl_sax_SUITE.erl
@@ -85,17 +85,17 @@ ticket_11551(_Config) ->
 <a>hej</a>
 <?xml version=\"1.0\" encoding=\"utf-8\" ?>
 <a>hej</a>">>,
-    {ok, undefined, <<"<?xml",  _/binary>>} = xmerl_sax_parser:stream(Stream1, []),
+    {ok, undefined, <<"\n<?xml",  _/binary>>} = xmerl_sax_parser:stream(Stream1, []),
     Stream2= <<"<?xml version=\"1.0\" encoding=\"utf-8\" ?>
 <a>hej</a>
 
 
 <?xml version=\"1.0\" encoding=\"utf-8\" ?>
 <a>hej</a>">>,
-    {ok, undefined, <<"<?xml",  _/binary>>} = xmerl_sax_parser:stream(Stream2, []),
+    {ok, undefined, <<"\n\n\n<?xml",  _/binary>>} = xmerl_sax_parser:stream(Stream2, []),
     Stream3= <<"<a>hej</a>
 
 <?xml version=\"1.0\" encoding=\"utf-8\" ?>
 <a>hej</a>">>,
-    {ok, undefined, <<"<?xml",  _/binary>>} = xmerl_sax_parser:stream(Stream3, []),
+    {ok, undefined, <<"\n\n<?xml",  _/binary>>} = xmerl_sax_parser:stream(Stream3, []),
     ok.
diff --git a/lib/xmerl/test/xmerl_sax_stream_SUITE.erl b/lib/xmerl/test/xmerl_sax_stream_SUITE.erl
new file mode 100644
index 0000000000..a306eb66a2
--- /dev/null
+++ b/lib/xmerl/test/xmerl_sax_stream_SUITE.erl
@@ -0,0 +1,245 @@
+%%-*-erlang-*-
+%%----------------------------------------------------------------------
+%% %CopyrightBegin%
+%% 
+%% Copyright Ericsson AB 2017. All Rights Reserved.
+%% 
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%% 
+%% %CopyrightEnd%
+%%----------------------------------------------------------------------
+%% File    : xmerl_sax_stream_SUITE.erl
+%%----------------------------------------------------------------------
+-module(xmerl_sax_stream_SUITE).
+-compile(export_all).
+
+%%----------------------------------------------------------------------
+%% Include files
+%%----------------------------------------------------------------------
+-include_lib("common_test/include/ct.hrl").
+-include_lib("kernel/include/file.hrl").
+
+%%======================================================================
+%% External functions
+%%======================================================================
+
+%%----------------------------------------------------------------------
+%% Initializations
+%%----------------------------------------------------------------------
+all() ->
+    [
+     one_document,
+     two_documents,
+     one_document_and_junk
+    ].
+
+%%----------------------------------------------------------------------
+%% Initializations
+%%----------------------------------------------------------------------
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_Func, _Config) ->
+    ok.
+
+%%----------------------------------------------------------------------
+%% Tests
+%%----------------------------------------------------------------------
+one_document(Config) ->
+    Port = 11111,
+
+    {ok, ListenSocket} = listen(Port),
+    Self = self(),
+    
+    spawn(
+      fun() ->
+              case catch gen_tcp:accept(ListenSocket) of
+                  {ok, S} ->
+                      Result = xmerl_sax_parser:stream(<<>>, 
+                                                      [{continuation_state, S},
+                                                       {continuation_fun, 
+                                                        fun(Sd) ->
+                                                                io:format("Continuation called!!", []),
+                                                                case gen_tcp:recv(Sd, 0) of
+                                                                    {ok, Packet} ->
+                                                                        io:format("Packet: ~p\n", [Packet]),
+                                                                        {Packet, Sd};
+                                                                    {error, Reason} ->
+                                                                        throw({error, Reason})
+                                                                end
+                                                        end}]),
+                      Self ! {xmerl_sax, Result},
+                      close(S);
+                   Error ->
+                      Self ! {xmerl_sax, {error, {accept, Error}}}
+              end
+      end),
+
+    {ok, SendSocket} = connect(localhost, Port),
+   
+    {ok, Binary} = file:read_file(filename:join([datadir(Config), "xmerl_sax_stream_one.xml"])),
+    
+    send_chunks(SendSocket, Binary),
+
+    receive
+        {xmerl_sax, {ok, undefined, Rest}} ->
+            <<"\n">> = Rest,
+            io:format("Ok Rest: ~p\n", [Rest])        
+       after 5000 ->
+               ct:fail("Timeout")
+    end,
+    ok.
+    
+two_documents(Config) ->
+    Port = 11111,
+
+    {ok, ListenSocket} = listen(Port),
+    Self = self(),
+    
+    spawn(
+      fun() ->
+              case catch gen_tcp:accept(ListenSocket) of
+                  {ok, S} ->
+                      Result = xmerl_sax_parser:stream(<<>>, 
+                                                      [{continuation_state, S},
+                                                       {continuation_fun, 
+                                                        fun(Sd) ->
+                                                                io:format("Continuation called!!", []),
+                                                                case gen_tcp:recv(Sd, 0) of
+                                                                    {ok, Packet} ->
+                                                                        io:format("Packet: ~p\n", [Packet]),
+                                                                        {Packet, Sd};
+                                                                    {error, Reason} ->
+                                                                        throw({error, Reason})
+                                                                end
+                                                        end}]),
+                      Self ! {xmerl_sax, Result},
+                      close(S);
+                   Error ->
+                      Self ! {xmerl_sax, {error, {accept, Error}}}
+              end
+      end),
+
+    {ok, SendSocket} = connect(localhost, Port),
+   
+    {ok, Binary} = file:read_file(filename:join([datadir(Config), "xmerl_sax_stream_two.xml"])),
+    
+    send_chunks(SendSocket, Binary),
+
+    receive
+        {xmerl_sax, {ok, undefined, Rest}} ->
+            <<"\n<?x", _R/binary>> = Rest,
+            io:format("Ok Rest: ~p\n", [Rest])        
+       after 5000 ->
+               ct:fail("Timeout")
+    end,
+    ok.
+    
+one_document_and_junk(Config) ->
+    Port = 11111,
+
+    {ok, ListenSocket} = listen(Port),
+    Self = self(),
+    
+    spawn(
+      fun() ->
+              case catch gen_tcp:accept(ListenSocket) of
+                  {ok, S} ->
+                      Result = xmerl_sax_parser:stream(<<>>, 
+                                                      [{continuation_state, S},
+                                                       {continuation_fun, 
+                                                        fun(Sd) ->
+                                                                io:format("Continuation called!!", []),
+                                                                case gen_tcp:recv(Sd, 0) of
+                                                                    {ok, Packet} ->
+                                                                        io:format("Packet: ~p\n", [Packet]),
+                                                                        {Packet, Sd};
+                                                                    {error, Reason} ->
+                                                                        throw({error, Reason})
+                                                                end
+                                                        end}]),
+                      Self ! {xmerl_sax, Result},
+                      close(S);
+                   Error ->
+                      Self ! {xmerl_sax, {error, {accept, Error}}}
+              end
+      end),
+
+    {ok, SendSocket} = connect(localhost, Port),
+   
+    {ok, Binary} = file:read_file(filename:join([datadir(Config), "xmerl_sax_stream_one_junk.xml"])),
+    
+    send_chunks(SendSocket, Binary),
+
+    receive
+        {xmerl_sax, {ok, undefined, Rest}} ->
+            <<"\nth", _R/binary>> = Rest,
+            io:format("Ok Rest: ~p\n", [Rest])        
+       after 10000 ->
+               ct:fail("Timeout")
+    end,
+    ok.
+    
+%%----------------------------------------------------------------------
+%% Utility functions
+%%----------------------------------------------------------------------
+listen(Port) ->
+    case catch gen_tcp:listen(Port, [{active, false}, 
+				     binary, 
+				     {keepalive, true},
+				     {reuseaddr,true}]) of
+	{ok, ListenSocket} ->
+            {ok, ListenSocket};
+	{error, Reason} ->
+	    {error, {listen, Reason}}
+    end.
+
+close(Socket) ->
+    (catch gen_tcp:close(Socket)).
+
+connect(Host, Port) ->
+     Timeout = 5000,
+     % Options1 = check_options(Options),
+     Options = [binary],
+     case catch gen_tcp:connect(Host, Port, Options, Timeout) of
+ 	{ok, Socket} ->
+ 	    {ok, Socket};
+ 	{error, Reason} ->
+	    {error, Reason}
+     end.
+
+send_chunks(Socket, Binary) ->
+    BSize = erlang:size(Binary),
+    if 
+        BSize > 25 ->
+            <<Head:25/binary, Tail/binary>> = Binary,
+            case gen_tcp:send(Socket, Head) of
+                ok ->
+                    timer:sleep(1000),
+                    send_chunks(Socket, Tail);
+                {error,closed} ->
+                    ok
+            end;
+        true ->
+            gen_tcp:send(Socket, Binary)
+    end.
+
+datadir(Config) ->
+    proplists:get_value(data_dir, Config).
diff --git a/lib/xmerl/test/xmerl_sax_stream_SUITE_data/xmerl_sax_stream_one.xml b/lib/xmerl/test/xmerl_sax_stream_SUITE_data/xmerl_sax_stream_one.xml
new file mode 100644
index 0000000000..30328bb188
--- /dev/null
+++ b/lib/xmerl/test/xmerl_sax_stream_SUITE_data/xmerl_sax_stream_one.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<person>
+<name>
+Arne Andersson
+</name>
+<address>
+<street>
+ Old Road 456
+</street>
+<zip>
+12323
+</zip>
+<city>
+Small City
+</city>
+</address>
+</person>
diff --git a/lib/xmerl/test/xmerl_sax_stream_SUITE_data/xmerl_sax_stream_one_junk.xml b/lib/xmerl/test/xmerl_sax_stream_SUITE_data/xmerl_sax_stream_one_junk.xml
new file mode 100644
index 0000000000..f730a95865
--- /dev/null
+++ b/lib/xmerl/test/xmerl_sax_stream_SUITE_data/xmerl_sax_stream_one_junk.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<person>
+<name>
+Arne Andersson
+</name>
+<address>
+<street>
+ Old Road 456
+</street>
+<zip>
+12323
+</zip>
+<city>
+Small City
+</city>
+</address>
+</person>
+this is junk ......
diff --git a/lib/xmerl/test/xmerl_sax_stream_SUITE_data/xmerl_sax_stream_two.xml b/lib/xmerl/test/xmerl_sax_stream_SUITE_data/xmerl_sax_stream_two.xml
new file mode 100644
index 0000000000..e241a02190
--- /dev/null
+++ b/lib/xmerl/test/xmerl_sax_stream_SUITE_data/xmerl_sax_stream_two.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<person>
+<name>
+Arne Andersson
+</name>
+<address>
+<street>
+ Old Road 456
+</street>
+<zip>
+12323
+</zip>
+<city>
+Small City
+</city>
+</address>
+</person>
+<?xml version="1.0"?>
+<person>
+<name>
+Bertil Bengtson
+</name>
+<address>
+<street>
+ New Road 4
+</street>
+<zip>
+12328
+</zip>
+<city>
+Small City
+</city>
+</address>
+</person>