From 7b5da290199fa0d668b8d9d8267e188b66da1eb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Tue, 20 Nov 2018 13:23:59 +0100 Subject: Don't run long test suites by default The examples test suite is only useful once in a while in order to know whether examples were broken, for example before issuing a release. The new ws_autobahn test suite isolates the autobahn test suite so that it can be ignored by default. It's only useful to run it when working on the Websocket code or before issuing a release. --- Makefile | 6 +++ test/ws_SUITE.erl | 53 ++------------------ test/ws_SUITE_data/client.json | 14 ------ test/ws_autobahn_SUITE.erl | 87 +++++++++++++++++++++++++++++++++ test/ws_autobahn_SUITE_data/client.json | 14 ++++++ 5 files changed, 111 insertions(+), 63 deletions(-) delete mode 100644 test/ws_SUITE_data/client.json create mode 100644 test/ws_autobahn_SUITE.erl create mode 100644 test/ws_autobahn_SUITE_data/client.json diff --git a/Makefile b/Makefile index 40e653d..9c3e54a 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,12 @@ AUTO_CI_WINDOWS ?= OTP-19+ include erlang.mk +# Don't run the examples test suite by default. + +ifndef FULL +CT_SUITES := $(filter-out examples ws_autobahn,$(CT_SUITES)) +endif + # Compile options. ERLC_OPTS += +warn_missing_spec +warn_untyped_record diff --git a/test/ws_SUITE.erl b/test/ws_SUITE.erl index af1be05..4e35ec7 100644 --- a/test/ws_SUITE.erl +++ b/test/ws_SUITE.erl @@ -22,29 +22,12 @@ %% ct. all() -> - [{group, ws}, {group, autobahn}]. + [{group, ws}]. groups() -> - BaseTests = ct_helper:all(?MODULE) -- [autobahn_fuzzingclient], - [{autobahn, [], [autobahn_fuzzingclient]}, {ws, [parallel], BaseTests}]. - -init_per_group(Name = autobahn, Config) -> - %% Some systems have it named pip2. - Out = os:cmd("pip show autobahntestsuite ; pip2 show autobahntestsuite"), - case string:str(Out, "autobahntestsuite") of - 0 -> - ct:print("Skipping the autobahn group because the " - "Autobahn Test Suite is not installed.~nTo install it, " - "please follow the instructions on this page:~n~n " - "http://autobahn.ws/testsuite/installation.html"), - {skip, "Autobahn Test Suite not installed."}; - _ -> - {ok, _} = cowboy:start_clear(Name, [{port, 33080}], #{ - env => #{dispatch => init_dispatch()} - }), - Config - end; -init_per_group(Name = ws, Config) -> + [{ws, [parallel], ct_helper:all(?MODULE)}]. + +init_per_group(Name, Config) -> cowboy_test:init_http(Name, #{ env => #{dispatch => init_dispatch()} }, Config). @@ -90,34 +73,6 @@ init_dispatch() -> %% Tests. -autobahn_fuzzingclient(Config) -> - doc("Autobahn test suite for the Websocket protocol."), - Self = self(), - spawn_link(fun() -> do_start_port(Config, Self) end), - receive autobahn_exit -> ok end, - ct:log("

Full report

~n"), - Report = config(priv_dir, Config) ++ "reports/servers/index.html", - ct:print("Autobahn Test Suite report: file://~s~n", [Report]), - {ok, HTML} = file:read_file(Report), - case length(binary:matches(HTML, <<"case_failed">>)) > 2 of - true -> error(failed); - false -> ok - end. - -do_start_port(Config, Pid) -> - Port = open_port({spawn, "wstest -m fuzzingclient -s " ++ config(data_dir, Config) ++ "client.json"}, - [{line, 10000}, {cd, config(priv_dir, Config)}, binary, eof]), - do_receive_infinity(Port, Pid). - -do_receive_infinity(Port, Pid) -> - receive - {Port, {data, {eol, Line}}} -> - io:format(user, "~s~n", [Line]), - do_receive_infinity(Port, Pid); - {Port, eof} -> - Pid ! autobahn_exit - end. - unlimited_connections(Config) -> doc("Websocket connections are not limited. The connections " "are removed from the count after the handshake completes."), diff --git a/test/ws_SUITE_data/client.json b/test/ws_SUITE_data/client.json deleted file mode 100644 index 7899503..0000000 --- a/test/ws_SUITE_data/client.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "options": {"failByDrop": false}, - "enable-ssl": false, - - "servers": [{ - "agent": "Cowboy", - "url": "ws://localhost:33080/ws_echo", - "options": {"version": 18} - }], - - "cases": ["*"], - "exclude-cases": [], - "exclude-agent-cases": {} -} diff --git a/test/ws_autobahn_SUITE.erl b/test/ws_autobahn_SUITE.erl new file mode 100644 index 0000000..24d76e8 --- /dev/null +++ b/test/ws_autobahn_SUITE.erl @@ -0,0 +1,87 @@ +%% Copyright (c) 2011-2017, Loïc Hoguin +%% +%% Permission to use, copy, modify, and/or distribute this software for any +%% purpose with or without fee is hereby granted, provided that the above +%% copyright notice and this permission notice appear in all copies. +%% +%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +%% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +%% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +%% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +%% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +%% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +%% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +-module(ws_autobahn_SUITE). +-compile(export_all). +-compile(nowarn_export_all). + +-import(ct_helper, [config/2]). +-import(ct_helper, [doc/1]). + +%% ct. + +all() -> + [{group, autobahn}]. + +groups() -> + [{autobahn, [], [autobahn_fuzzingclient]}]. + +init_per_group(Name, Config) -> + %% Some systems have it named pip2. + Out = os:cmd("pip show autobahntestsuite ; pip2 show autobahntestsuite"), + case string:str(Out, "autobahntestsuite") of + 0 -> + ct:print("Skipping the autobahn group because the " + "Autobahn Test Suite is not installed.~nTo install it, " + "please follow the instructions on this page:~n~n " + "http://autobahn.ws/testsuite/installation.html"), + {skip, "Autobahn Test Suite not installed."}; + _ -> + {ok, _} = cowboy:start_clear(Name, [{port, 33080}], #{ + env => #{dispatch => init_dispatch()} + }), + Config + end. + +end_per_group(Listener, _Config) -> + cowboy:stop_listener(Listener). + +%% Dispatch configuration. + +init_dispatch() -> + cowboy_router:compile([ + {"localhost", [ + {"/ws_echo", ws_echo, []} + ]} + ]). + +%% Tests. + +autobahn_fuzzingclient(Config) -> + doc("Autobahn test suite for the Websocket protocol."), + Self = self(), + spawn_link(fun() -> do_start_port(Config, Self) end), + receive autobahn_exit -> ok end, + ct:log("

Full report

~n"), + Report = config(priv_dir, Config) ++ "reports/servers/index.html", + ct:print("Autobahn Test Suite report: file://~s~n", [Report]), + {ok, HTML} = file:read_file(Report), + case length(binary:matches(HTML, <<"case_failed">>)) > 2 of + true -> error(failed); + false -> ok + end. + +do_start_port(Config, Pid) -> + Port = open_port({spawn, "wstest -m fuzzingclient -s " ++ config(data_dir, Config) ++ "client.json"}, + [{line, 10000}, {cd, config(priv_dir, Config)}, binary, eof]), + do_receive_infinity(Port, Pid). + +do_receive_infinity(Port, Pid) -> + receive + {Port, {data, {eol, Line}}} -> + io:format(user, "~s~n", [Line]), + do_receive_infinity(Port, Pid); + {Port, eof} -> + Pid ! autobahn_exit + end. diff --git a/test/ws_autobahn_SUITE_data/client.json b/test/ws_autobahn_SUITE_data/client.json new file mode 100644 index 0000000..7899503 --- /dev/null +++ b/test/ws_autobahn_SUITE_data/client.json @@ -0,0 +1,14 @@ +{ + "options": {"failByDrop": false}, + "enable-ssl": false, + + "servers": [{ + "agent": "Cowboy", + "url": "ws://localhost:33080/ws_echo", + "options": {"version": 18} + }], + + "cases": ["*"], + "exclude-cases": [], + "exclude-agent-cases": {} +} -- cgit v1.2.3