Enable Autobahn Test Suite by default

Includes a variety of small changes that are a first step to
improving the test system heavily.

1 files changed, 67 insertions, 87 deletions

diff --git a/test/ws_SUITE.erl b/test/ws_SUITE.erl
index cdb336a..77a7c76 100644
--- a/test/ws_SUITE.erl
+++ b/test/ws_SUITE.erl
@@ -13,75 +13,41 @@
 %% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 -module(ws_SUITE).
+-compile(export_all).
 
--include_lib("common_test/include/ct.hrl").
-
-%% ct.
--export([all/0]).
--export([groups/0]).
--export([init_per_suite/1]).
--export([end_per_suite/1]).
--export([init_per_group/2]).
--export([end_per_group/2]).
-
-%% Tests.
--export([ws0/1]).
--export([ws8/1]).
--export([ws8_init_shutdown/1]).
--export([ws8_single_bytes/1]).
--export([ws13/1]).
--export([ws_deflate/1]).
--export([ws_deflate_chunks/1]).
--export([ws_deflate_fragments/1]).
--export([ws_send_close/1]).
--export([ws_send_close_payload/1]).
--export([ws_send_many/1]).
--export([ws_text_fragments/1]).
--export([ws_timeout_hibernate/1]).
--export([ws_timeout_cancel/1]).
--export([ws_timeout_reset/1]).
--export([ws_upgrade_with_opts/1]).
+-import(cowboy_test, [config/2]).
 
 %% ct.
 
 all() ->
-	[{group, ws}].
+	[{group, autobahn}, {group, ws}].
 
 groups() ->
-	BaseTests = [
-		ws0,
-		ws8,
-		ws8_init_shutdown,
-		ws8_single_bytes,
-		ws13,
-		ws_deflate,
-		ws_deflate_chunks,
-		ws_deflate_fragments,
-		ws_send_close,
-		ws_send_close_payload,
-		ws_send_many,
-		ws_text_fragments,
-		ws_timeout_hibernate,
-		ws_timeout_cancel,
-		ws_timeout_reset,
-		ws_upgrade_with_opts
-	],
-	[{ws, [parallel], BaseTests}].
+	BaseTests = cowboy_test:all(?MODULE) -- [autobahn_fuzzingclient],
+	[{autobahn, [], [autobahn_fuzzingclient]}, {ws, [parallel], BaseTests}].
 
 init_per_suite(Config) ->
-	application:start(crypto),
-	application:start(cowlib),
-	application:start(ranch),
-	application:start(cowboy),
+	cowboy_test:start([cowboy]),
 	Config.
 
 end_per_suite(_Config) ->
-	application:stop(cowboy),
-	application:stop(ranch),
-	application:stop(cowlib),
-	application:stop(crypto),
-	ok.
-
+	cowboy_test:stop([cowboy]).
+
+init_per_group(autobahn, Config) ->
+	%% Some systems have it named pip2.
+	Out = os:cmd("pip show autobahntestsuite ; pip2 show autobahntestsuite"),
+	case string:str(Out, "autobahntestsuite") of
+		0 ->
+			ct:print("Skipping the autobahn group because the "
+				"Autobahn Test Suite is not installed.~nTo install it, "
+				"please follow the instructions on this page:~n~n    "
+				"http://autobahn.ws/testsuite/installation.html"),
+			{skip, "Autobahn Test Suite not installed."};
+		_ ->
+			{ok, _} = cowboy:start_http(autobahn, 100, [{port, 33080}], [
+				{env, [{dispatch, init_dispatch()}]}]),
+			Config
+	end;
 init_per_group(ws, Config) ->
 	cowboy:start_http(ws, 100, [{port, 0}], [
 		{env, [{dispatch, init_dispatch()}]},
@@ -99,8 +65,8 @@ end_per_group(Listener, _Config) ->
 init_dispatch() ->
 	cowboy_router:compile([
 		{"localhost", [
-			{"/ws_echo_timer", ws_echo_timer, []},
 			{"/ws_echo", ws_echo, []},
+			{"/ws_echo_timer", ws_echo_timer, []},
 			{"/ws_init_shutdown", ws_init_shutdown, []},
 			{"/ws_send_many", ws_send_many, [
 				{sequence, [
@@ -127,7 +93,21 @@ init_dispatch() ->
 		]}
 	]).
 
-%% ws and wss.
+%% Tests.
+
+autobahn_fuzzingclient(Config) ->
+	Out = os:cmd("cd " ++ config(priv_dir, Config)
+		++ " && wstest -m fuzzingclient -s "
+		++ config(data_dir, Config) ++ "client.json"),
+	Report = config(priv_dir, Config) ++ "reports/servers/index.html",
+	ct:log("<h2><a href=\"~s\">Full report</a></h2>~n", [Report]),
+	ct:print("Autobahn Test Suite report: file://~s~n", [Report]),
+	ct:log("~s~n", [Out]),
+	{ok, HTML} = file:read_file(Report),
+	case length(binary:matches(HTML, <<"case_failed">>)) > 2 of
+		true -> error(failed);
+		false -> ok
+	end.
 
 %% We do not support hixie76 anymore.
 ws0(Config) ->
@@ -163,7 +143,7 @@ ws8(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -223,7 +203,7 @@ ws8_single_bytes(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -283,7 +263,7 @@ ws13(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -335,7 +315,7 @@ ws_deflate(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -346,7 +326,7 @@ ws_deflate(Config) ->
 
 	Mask = 16#11223344,
 	Hello = << 242, 72, 205, 201, 201, 7, 0 >>,
-	MaskedHello = websocket_mask(Hello, Mask, <<>>),
+	MaskedHello = do_mask(Hello, Mask, <<>>),
 
 	% send compressed text frame containing the Hello string
 	ok = gen_tcp:send(Socket, << 1:1, 1:1, 0:2, 1:4, 1:1, 7:7, Mask:32,
@@ -377,7 +357,7 @@ ws_deflate_chunks(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -388,7 +368,7 @@ ws_deflate_chunks(Config) ->
 
 	Mask = 16#11223344,
 	Hello = << 242, 72, 205, 201, 201, 7, 0 >>,
-	MaskedHello = websocket_mask(Hello, Mask, <<>>),
+	MaskedHello = do_mask(Hello, Mask, <<>>),
 
 	% send compressed text frame containing the Hello string
 	ok = gen_tcp:send(Socket, << 1:1, 1:1, 0:2, 1:4, 1:1, 7:7, Mask:32,
@@ -422,7 +402,7 @@ ws_deflate_fragments(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -437,9 +417,9 @@ ws_deflate_fragments(Config) ->
 	% send compressed text frame containing the Hello string
 	% as 2 separate fragments
 	ok = gen_tcp:send(Socket, << 0:1, 1:1, 0:2, 1:4, 1:1, 4:7, Mask:32,
-		(websocket_mask(binary:part(Hello, 0, 4), Mask, <<>>))/binary >>),
+		(do_mask(binary:part(Hello, 0, 4), Mask, <<>>))/binary >>),
 	ok = gen_tcp:send(Socket, << 1:1, 1:1, 0:2, 0:4, 1:1, 3:7, Mask:32,
-		(websocket_mask(binary:part(Hello, 4, 3), Mask, <<>>))/binary >>),
+		(do_mask(binary:part(Hello, 4, 3), Mask, <<>>))/binary >>),
 	% receive compressed text frame containing the Hello string
 	{ok, << 1:1, 1:1, 0:2, 1:4, 0:1, 7:7, Hello/binary >>}
 		= gen_tcp:recv(Socket, 0, 6000),
@@ -465,7 +445,7 @@ ws_send_close(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -494,7 +474,7 @@ ws_send_close_payload(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -523,7 +503,7 @@ ws_send_many(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -555,7 +535,7 @@ ws_text_fragments(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -609,7 +589,7 @@ ws_timeout_hibernate(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -636,7 +616,7 @@ ws_timeout_cancel(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -663,7 +643,7 @@ ws_timeout_reset(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -696,7 +676,7 @@ ws_upgrade_with_opts(Config) ->
 	{ok, Handshake} = gen_tcp:recv(Socket, 0, 6000),
 	{ok, {http_response, {1, 1}, 101, "Switching Protocols"}, Rest}
 		= erlang:decode_packet(http, Handshake, []),
-	[Headers, <<>>] = websocket_headers(
+	[Headers, <<>>] = do_decode_headers(
 		erlang:decode_packet(httph, Rest, []), []),
 	{'Connection', "Upgrade"} = lists:keyfind('Connection', 1, Headers),
 	{'Upgrade', "websocket"} = lists:keyfind('Upgrade', 1, Headers),
@@ -711,27 +691,27 @@ ws_upgrade_with_opts(Config) ->
 
 %% Internal.
 
-websocket_headers({ok, http_eoh, Rest}, Acc) ->
+do_decode_headers({ok, http_eoh, Rest}, Acc) ->
 	[Acc, Rest];
-websocket_headers({ok, {http_header, _I, Key, _R, Value}, Rest}, Acc) ->
+do_decode_headers({ok, {http_header, _I, Key, _R, Value}, Rest}, Acc) ->
 	F = fun(S) when is_atom(S) -> S; (S) -> string:to_lower(S) end,
-	websocket_headers(erlang:decode_packet(httph, Rest, []),
+	do_decode_headers(erlang:decode_packet(httph, Rest, []),
 		[{F(Key), Value}|Acc]).
 
-websocket_mask(<<>>, _, Unmasked) ->
-	Unmasked;
-websocket_mask(<< O:32, Rest/bits >>, MaskKey, Acc) ->
+do_mask(<<>>, _, Acc) ->
+	Acc;
+do_mask(<< O:32, Rest/bits >>, MaskKey, Acc) ->
 	T = O bxor MaskKey,
-	websocket_mask(Rest, MaskKey, << Acc/binary, T:32 >>);
-websocket_mask(<< O:24 >>, MaskKey, Acc) ->
+	do_mask(Rest, MaskKey, << Acc/binary, T:32 >>);
+do_mask(<< O:24 >>, MaskKey, Acc) ->
 	<< MaskKey2:24, _:8 >> = << MaskKey:32 >>,
 	T = O bxor MaskKey2,
 	<< Acc/binary, T:24 >>;
-websocket_mask(<< O:16 >>, MaskKey, Acc) ->
+do_mask(<< O:16 >>, MaskKey, Acc) ->
 	<< MaskKey2:16, _:16 >> = << MaskKey:32 >>,
 	T = O bxor MaskKey2,
 	<< Acc/binary, T:16 >>;
-websocket_mask(<< O:8 >>, MaskKey, Acc) ->
+do_mask(<< O:8 >>, MaskKey, Acc) ->
 	<< MaskKey2:8, _:24 >> = << MaskKey:32 >>,
 	T = O bxor MaskKey2,
 	<< Acc/binary, T:8 >>.