diff options
Diffstat (limited to 'lib/test_server/doc/src')
20 files changed, 0 insertions, 5845 deletions
diff --git a/lib/test_server/doc/src/Makefile b/lib/test_server/doc/src/Makefile deleted file mode 100644 index 8c5418aee5..0000000000 --- a/lib/test_server/doc/src/Makefile +++ /dev/null @@ -1,140 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 2002-2012. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# - -include $(ERL_TOP)/make/target.mk -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# ---------------------------------------------------- -# Application version -# ---------------------------------------------------- -include ../../vsn.mk -VSN=$(TEST_SERVER_VSN) -APPLICATION=test_server - -DOC_EXTRA_FRONT_PAGE_INFO=Important note: \ -The Test Server application is obsolete and will be removed \ -in the next major OTP release - -# ---------------------------------------------------- -# Release directory specification -# ---------------------------------------------------- -RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) - -# ---------------------------------------------------- -# Target Specs -# ---------------------------------------------------- -XML_APPLICATION_FILES = ref_man.xml -XML_REF3_FILES = \ - test_server_ctrl.xml \ - test_server.xml -XML_REF3_INTERNAL = \ - ts.xml -XML_REF6_FILES = test_server_app.xml - -XML_PART_FILES = \ - part.xml \ - part_notes.xml \ - part_notes_history.xml - -XML_CHAPTER_FILES = \ - basics_chapter.xml \ - run_test_chapter.xml \ - write_test_chapter.xml \ - test_spec_chapter.xml \ - example_chapter.xml \ - write_framework_chapter.xml \ - notes.xml \ - notes_history.xml - -BOOK_FILES = book.xml - -XML_FILES = $(BOOK_FILES) $(XML_APPLICATION_FILES) $(XML_REF3_FILES) $(XML_REF6_FILES) \ - $(XML_PART_FILES) $(XML_CHAPTER_FILES) - -GIF_FILES = - -# ---------------------------------------------------- - -HTML_FILES = $(XML_APPLICATION_FILES:%.xml=$(HTMLDIR)/%.html) \ - $(XML_PART_FILES:%.xml=$(HTMLDIR)/%.html) - -HTML_INTERNAL = $(XML_REF3_INTERNAL:%.xml=$(HTMLDIR)/%.html) - -INFO_FILE = ../../info - -MAN3_FILES = $(XML_REF3_FILES:%.xml=$(MAN3DIR)/%.3) -MAN3_INTERNAL = $(XML_REF3_INTERNAL:%.xml=$(MAN3DIR)/%.3) -MAN6_FILES = $(XML_REF6_FILES:%_app.xml=$(MAN6DIR)/%.6) - -HTML_REF_MAN_FILE = $(HTMLDIR)/index.html - -TOP_PDF_FILE = $(PDFDIR)/test_server-$(VSN).pdf - -# ---------------------------------------------------- -# FLAGS -# ---------------------------------------------------- -XML_FLAGS += -DVIPS_FLAGS += - -# ---------------------------------------------------- -# Targets -# ---------------------------------------------------- -$(HTMLDIR)/%.gif: %.gif - $(INSTALL_DATA) $< $@ - -docs: pdf html man - -pdf: $(TOP_PDF_FILE) - -html: gifs $(HTML_REF_MAN_FILE) - -man: $(MAN3_FILES) $(MAN3_INTERNAL) $(MAN6_FILES) - -gifs: $(GIF_FILES:%=$(HTMLDIR)/%) - -debug opt: - -clean clean_docs: - rm -rf $(HTMLDIR)/* - rm -f $(MAN3DIR)/* - rm -f $(MAN6DIR)/* - rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) - rm -f errs core *~ - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_docs_spec: docs - $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" - $(INSTALL_DATA) $(HTMLDIR)/* \ - "$(RELSYSDIR)/doc/html" - $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" - $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" - $(INSTALL_DATA) $(MAN3_FILES) "$(RELEASE_PATH)/man/man3" - $(INSTALL_DIR) "$(RELEASE_PATH)/man/man6" - $(INSTALL_DATA) $(MAN6_FILES) "$(RELEASE_PATH)/man/man6" - -release_spec: - -release_tests_spec: diff --git a/lib/test_server/doc/src/basics_chapter.xml b/lib/test_server/doc/src/basics_chapter.xml deleted file mode 100644 index 9e9f38aab4..0000000000 --- a/lib/test_server/doc/src/basics_chapter.xml +++ /dev/null @@ -1,215 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Test Server Basics</title> - <prepared>Siri Hansen</prepared> - <docno></docno> - <date></date> - <rev></rev> - <file>basics_chapter.xml</file> - </header> - - <section> - <title>Introduction</title> - <p><em>Test Server</em> is a portable test tool for automated - testing of Erlang programs and OTP applications. It provides an - interface for running test programs directly with Test Server - as well as an interface for integrating Test Server - with a framework application. The latter makes it possible to use - Test Server as the engine of a higher level test tool - application.</p> - - <p>It is strongly recommended that Test Server be used from inside - a framework application, rather than interfaced directly for - running test programs. Test Server can be pretty difficult to use - since it's a very general and quite extensive and complex - application. Furthermore, the <c>test_server_ctrl</c> functions - are not meant to be used from within the actual test programs. The - framework should handle communication with Test Server and deal - with the more complex aspects of this interaction automatically so - that a higher level interface may be provided for the tester. For - test tool usage to be productive, a simpler, more intuitive and - (if required) more specific interface is required than what Test Server - can provide.</p> - - <p>OTP delivers a general purpose framework for Test Server, called - <em>Common Test</em>. This application is a tool well suited for - automated black box testing of target systems of <em>any kind</em> - (not necessarily implemented in Erlang). Common Test is also a very - useful tool for white box testing of Erlang programs and OTP - applications. Unless a more specific functionality and/or user - interface is required (in which case you might need to implement - your own framework), Common Test should do the job for - you. Please read the Common Test User's Guide and reference manual - for more information.</p> - - <p>Under normal circumstances, knowledge about the Test Server - application is not required for using the Common Test framework. - However, if you want to use Test Server without a framework, - or learn how to integrate it with your own framework, please read on... - </p> - </section> - <section> - <title>Getting started</title> - <p>Testing when using Test Server is done by running test - suites. A test suite is a number of test cases, where each test - case tests one or more things. The test case is the smallest unit - that the test server deals with. One or more test cases are - grouped together into one ordinary Erlang module, which is called - a test suite. Several test suite modules can be grouped together - in special test specification files representing whole application - and/or system test "jobs". - </p> - <p>The test suite Erlang module must follow a certain interface, - which is specified by Test Server. See the section on writing - test suites for details about this. - </p> - <p>Each test case is considered a success if it returns to the - caller, no matter what the returned value is. An exception to this - is the return value <c>{skip, Reason}</c> which indicates that the - test case is skipped. A failure is specified as a crash, no matter - what the crash reason is. - </p> - <p>As a test suite runs, all information (including output to - stdout) is recorded in several different log files. A minimum of - information is displayed to the user console. This only include - start and stop information, plus a note for each failed test case. - </p> - <p>The result from each test case is recorded in an HTML log file - which is created for each test run. Every test case gets one row - in a table presenting total time, whether the case was successful - or not, if it was skipped, and possibly also a comment. The HTML - file has links to each test case's logfile, which may be viewed - from e.g. Netscape or any other HTML capable browser. - </p> - <p>The Test Server consists of three parts: - </p> - <list type="bulleted"> - <item>The part that executes the test suites and - provides support for the test suite author is called - <c>test_server</c>. This is described in the chapter about - writing test cases in this user's guide, and in the reference - manual for the <c>test_server</c> module.</item> - <item>The controlling part, which provides the low level - operator interface, starts and stops slave nodes and writes - log files, is called - <c>test_server_ctrl</c>. The Test Server Controller should not - be used directly when running tests. Instead a framework built - on top of it should be used. More information - about how to write your own framework can be found - in this user's guide and in the reference manual for the - <c>test_server_ctrl</c> module.</item> - </list> - </section> - - <section> - <title>Definition of terms</title> - <taglist> - <tag><em>conf(iguration) case</em></tag> - <item>This is a group of test cases which need some specific - configuration. A conf case contains an initiation function which - sets up a specific configuration, one or more test cases using - this configuration, and a cleanup function which restores the - configuration. A conf case is specified in a test specification - either like this:<c>{conf,InitFunc,ListOfCases,CleanupFunc}</c>, - or this: <c>{conf,Properties,InitFunc,ListOfCases,CleanupFunc}</c> - </item> - <tag><em>datadir</em></tag> - <item>Data directory for a test suite. This directory contains - any files used by the test suite, e.g. additional erlang - modules, c code or data files. If the data directory contains - code which must be compiled before the test suite is run, it - should also contain a makefile source called Makefile.src - defining how to compile. - </item> - <tag><em>documentation clause</em></tag> - <item>One of the function clauses in a test case. This clause - shall return a list of strings describing what the test case - tests. - </item> - <tag><em>execution clause</em></tag> - <item>One of the function clauses in a test case. This clause - implements the actual test case, i.e. calls the functions that - shall be tested and checks results. The clause shall crash if it - fails. - </item> - <tag><em>major log file</em></tag> - <item>This is the test suites log file. - </item> - <tag><em>Makefile.src</em></tag> - <item>This file is used by the test server framework to generate - a makefile for a datadir. It contains some special characters - which are replaced according to the platform currently tested. - </item> - <tag><em>minor log file</em></tag> - <item>This is a separate log file for each test case. - </item> - <tag><em>privdir</em></tag> - <item>Private directory for a test suite. This directory should - be used when the test suite needs to write to files. - </item> - <tag><em>skip case</em></tag> - <item>A test case which shall be skipped. - </item> - <tag><em>specification clause</em></tag> - <item>One of the function clauses in a test case. This clause - shall return an empty list, a test specification or - <c>{skip,Reason}</c>. If an empty list is returned, it means - that the test case shall be executed, and so it must also have - an execution clause. - </item> - <tag><em>test case</em></tag> - <item>A single test included in a test suite. Typically it tests - one function in a module or application. A test case is - implemented as a function in a test suite module. The function - can have three clauses, the documentation-, specification- and - execution clause. - </item> - <tag><em>test specification</em></tag> - <item>A specification of which test suites and test cases to - run. There can be test specifications on three different levels - in a test. The top level is a test specification file which - roughly specifies what to test for a whole application. Then - there is a test specification for each test suite returned from - the <c>all(suite)</c> function in the suite. And there can also - be a test specification returned from the specification clause - of a test case. - </item> - <tag><em>test specification file</em></tag> - <item>This is a text file containing the test specification for - an application. The file has the extension ".spec" or - ".spec.Platform", where Platform is e.g. "vxworks". - </item> - <tag><em>test suite</em></tag> - <item>An erlang module containing a collection of test cases for - a specific application or module. - </item> - <tag><em>topcase</em></tag> - <item>The first "command" in a test specification file. This - command contains the test specification, like this: - <c>{topcase,TestSpecification}</c></item> - </taglist> - </section> -</chapter> - diff --git a/lib/test_server/doc/src/book.xml b/lib/test_server/doc/src/book.xml deleted file mode 100644 index 6eb7daae1a..0000000000 --- a/lib/test_server/doc/src/book.xml +++ /dev/null @@ -1,50 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE book SYSTEM "book.dtd"> - -<book xmlns:xi="http://www.w3.org/2001/XInclude"> - <header titlestyle="normal"> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Test Server</title> - <prepared>Siri Hansen</prepared> - <docno></docno> - <date>2002-07-11</date> - <rev></rev> - <file>book.xml</file> - </header> - <insidecover> - </insidecover> - <pagetext>Test Server</pagetext> - <preamble> - <contents level="2"></contents> - </preamble> - <parts lift="no"> - <xi:include href="part.xml"/> - </parts> - <applications> - <xi:include href="ref_man.xml"/> - </applications> - <releasenotes> - <xi:include href="notes.xml"/> - </releasenotes> - <listofterms></listofterms> - <index></index> -</book> - diff --git a/lib/test_server/doc/src/example_chapter.xml b/lib/test_server/doc/src/example_chapter.xml deleted file mode 100644 index ec152fdd6c..0000000000 --- a/lib/test_server/doc/src/example_chapter.xml +++ /dev/null @@ -1,151 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Examples</title> - <prepared>Siri Hansen</prepared> - <docno></docno> - <date></date> - <rev></rev> - <file>example_chapter.xml</file> - </header> - - <section> - <title>Test suite</title> - <code type="none"> --module(my_SUITE). - --export([all/1, - not_started/1, not_started_func1/1, not_started_func2/1, - start/1, stop/1, - func1/1, func2/1 - ]). - --export([init_per_testcase/2, end_per_testcase/2]). - --include("test_server.hrl"). - --define(default_timeout, ?t:minutes(1)). - -init_per_testcase(_Case, Config) -> - Dog=?t:timetrap(?default_timeout), - [{watchdog, Dog}|Config]. -end_per_testcase(_Case, Config) -> - Dog=?config(watchdog, Config), - ?t:timetrap_cancel(Dog), - ok. - -all(suite) -> - %% Test specification on test suite level - [not_started, - {conf, start, [func1, func2], stop}]. - -not_started(suite) -> - %% Test specification on test case level - [not_started_func1, not_started_func2]; -not_started(doc) -> - ["Testing all functions when application is not started"]. -%% No execution clause unless the specification clause returns []. - - -not_started_func1(suite) -> - []; -not_started_func1(doc) -> - ["Testing function 1 when application is not started"]. -not_started_func1(Config) when list(Config) -> - {error, not_started} = myapp:func1(dummy_ref,1), - {error, not_started} = myapp:func1(dummy_ref,2), - ok. - -not_started_func2(suite) -> - []; -not_started_func2(doc) -> - ["Testing function 2 when application is not started"]. -not_started_func2(Config) when list(Config) -> - {error, not_started} = myapp:func2(dummy_ref,1), - {error, not_started} = myapp:func2(dummy_ref,2), - ok. - - -%% No specification clause needed for an init function in a conf case!!! -start(doc) -> - ["Testing start of my application."]; -start(Config) when list(Config) -> - Ref = myapp:start(), - case erlang:whereis(my_main_process) of - Pid when pid(Pid) -> - [{myapp_ref,Ref}|Config]; - undefined -> - %% Since this is the init function in a conf case, the rest of the - %% cases in the conf case will be skipped if this case fails. - ?t:fail("my_main_process did not start") - end. - -func1(suite) -> - []; -func1(doc) -> - ["Test that func1 returns ok when argument is 1 and error if argument is 2"]; -func1(Config) when list(Config) -> - Ref = ?config(myapp_ref,Config), - ok = myapp:func1(Ref,1), - error = myapp:func1(Ref,2), - ok. - -func2(suite) -> - []; -func2(doc) -> - ["Test that func1 returns ok when argument is 3 and error if argument is 4"]; -func2(Config) when list(Config) -> - Ref = ?config(myapp_ref,Config), - ok = myapp:func2(Ref,3), - error = myapp:func2(Ref,4), - ok. - -%% No specification clause needed for a cleanup function in a conf case!!! -stop(doc) -> - ["Testing termination of my application"]; -stop(Config) when list(Config) -> - Ref = ?config(myapp_ref,Config), - ok = myapp:stop(Ref), - case erlang:whereis(my_main_process) of - undefined -> - lists:keydelete(myapp_ref,1,Config); - Pid when pid(Pid) -> - ?t:fail("my_main_process did not stop") - end. - </code> - </section> - - <section> - <title>Test specification file</title> - <p><em><c>myapp.spec:</c></em></p> - <code type="none"> -{topcase, {dir, "../myapp_test"}}. % Test specification on top level </code> - <p><em><c>myapp.spec.vxworks:</c></em></p> - <code type="none"> -{topcase, {dir, "../myapp_test"}}. % Test specification on top level -{skip,{my_SUITE,func2,"Not applicable on VxWorks"}}. </code> - </section> -</chapter> - - diff --git a/lib/test_server/doc/src/fascicules.xml b/lib/test_server/doc/src/fascicules.xml deleted file mode 100644 index 37feca543f..0000000000 --- a/lib/test_server/doc/src/fascicules.xml +++ /dev/null @@ -1,18 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE fascicules SYSTEM "fascicules.dtd"> - -<fascicules> - <fascicule file="part" href="part_frame.html" entry="no"> - User's Guide - </fascicule> - <fascicule file="ref_man" href="ref_man_frame.html" entry="yes"> - Reference Manual - </fascicule> - <fascicule file="part_notes" href="part_notes_frame.html" entry="no"> - Release Notes - </fascicule> - <fascicule file="" href="../../../../doc/print.html" entry="no"> - Off-Print - </fascicule> -</fascicules> - diff --git a/lib/test_server/doc/src/notes.xml b/lib/test_server/doc/src/notes.xml deleted file mode 100644 index b48bda94d0..0000000000 --- a/lib/test_server/doc/src/notes.xml +++ /dev/null @@ -1,1694 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2004</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>APPLICATION Release Notes</title> - <prepared>Peter Andersson</prepared> - <responsible>Peter Andersson</responsible> - <docno></docno> - <approved></approved> - <checked></checked> - <date>2007-11-30</date> - <rev>A</rev> - <file>notes.xml</file> - </header> - -<section><title>Test_Server 3.9.1</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - When generating Makefile from Makefile.src, - ts_lib:get_arg/4 earlier removed all spaces in the - extracted argument. The code was probably meant for - removing leading and trailing spaces only, and is now - corrected to do so.</p> - <p> - Own Id: OTP-13015</p> - </item> - <item> - <p> - With the Common Test 'create_priv_dir' start option set - to 'auto_per_tc', the name of the priv directory for a - configuration function could clash with the name of the - priv directory for a test case, which would cause Test - Server failure. This error has been corrected.</p> - <p> - Own Id: OTP-13181</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.9</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The status of an aborted test due to test suite - compilation error has changed from 'auto_skipped' to - 'failed'. This affects both the textual log file, event - handling and CT hook callbacks. The logging of - compilation failures has also been improved, especially - in the case of multiple test suites failing compilation.</p> - <p> - Own Id: OTP-10816</p> - </item> - <item> - <p> - The Test Server source code parser (erl2html2) failed to - handle the macro tuple in the syntax tree returned by - epp_dodger. This error has been corrected.</p> - <p> - Own Id: OTP-12740</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The Test Server application has been marked as obsolete - and will be removed from OTP in the next major release - (OTP 19.0).</p> - <p> - Own Id: OTP-10923 Aux Id: OTP-12705 </p> - </item> - <item> - <p> - When running OTP tests using the ts interface, it is now - possible to specify so called test categories per OTP - application. A test category is represented by a CT test - specification and defines an arbitrary subset of existing - test suites, groups and cases. Examples of test - categories are 'smoke' (smoke tests) and 'bench' - (benchmarks). (Call ts:help() for more info). Also, - functions for reading terms from the current test - specification during test, ct:get_testspec_terms/0 and - ct:get_testspec_terms/1, have been implemented.</p> - <p> - Own Id: OTP-11962</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.8.1</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - If the last expression in a test case causes a timetrap - timeout, the stack trace is ignored and not printed to - the test case log file. This happens because the - {Suite,TestCase,Line} info is not available in the stack - trace in this scenario, due to tail call elimination. - Common Test has been modified to handle this situation by - inserting a {Suite,TestCase,last_expr} tuple in the - correct place and printing the stack trace as expected.</p> - <p> - Own Id: OTP-12697 Aux Id: seq12848 </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.8</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - When installing test suites in a cross compilation - environment, ts_install was not able to read the values - of the environment variables specified in the - configuration file. This has been fixed.</p> - <p> - Own Id: OTP-11441</p> - </item> - <item> - <p> - Printouts by means of ct:log/2/3 or ct:pal/2/3 from the - hook functions on_tc_fail/2 and on_tc_skip/2 would (quite - unexpectedly) end up in the "unexpected i/o" log file - instead of in the test case log file. This behaviour has - been changed so that now, all printouts (including stdio - printouts) from these hook functions will be routed to - the test case log file.</p> - <p> - Own Id: OTP-12468</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The format of the information printed on top of the test - case (and configuration function) log file has been - slightly modified, mainly in order to make the start - configuration data easier to read and interpret.</p> - <p> - Own Id: OTP-12518 Aux Id: seq12808 </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.7.2</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The source code to html code generator in Test Server - (and Common Test) would fail to generate anchors in the - html code for functions with non-expandable macros, - resulting in bad html links to such functions. This - correction lets the code generator ignore macros that - can't be expanded (i.e. not pre-process them), so that - correct anchors will always be produced.</p> - <p> - Own Id: OTP-11766 Aux Id: seq12556 </p> - </item> - <item> - <p> - Make sure to install .hrl files when needed</p> - <p> - Own Id: OTP-12197</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Distribute <c>autoconf</c> helpers to applications at - build time instead of having multiple identical copies - committed in the repository.</p> - <p> - Own Id: OTP-12348</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.7.1</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The mechanism for running code cover analysis with - common_test has been improved. Earlier, if a test run - consisted of multiple tests, cover would be started and - stopped for each test. This would give "intermediate" - cover logs available from the "Coverage log" link on the - test suite result pages. To accumulate cover data over - all tests, the 'export' option had to be used in the - cover spec file. This was not well documented, and the - functionality was quite confusing.</p> - <p> - Using the 'nodes' option in the cover spec file would - fail when the test run consisted of multiple tests, since - the specified nodes would only be included in the cover - analysis of the first test.</p> - <p> - The repeated compilation and analysis of the same modules - was also very time consuming.</p> - <p> - To overcome these problems, ct will now only cover - compile and analyze modules once per test run, i.e. once - for each cover spec file. The log file is available via a - new button on the top level index page. The old "Coverage - log" links on the test suite result pages still exist, - but they all point to the same log containing the - accumulated result.</p> - <p> - Own Id: OTP-11971</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.7</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Application upgrade (appup) files are corrected for the - following applications: </p> - <p> - <c>asn1, common_test, compiler, crypto, debugger, - dialyzer, edoc, eldap, erl_docgen, et, eunit, gs, hipe, - inets, observer, odbc, os_mon, otp_mibs, parsetools, - percept, public_key, reltool, runtime_tools, ssh, - syntax_tools, test_server, tools, typer, webtool, wx, - xmerl</c></p> - <p> - A new test utility for testing appup files is added to - test_server. This is now used by most applications in - OTP.</p> - <p> - (Thanks to Tobias Schlager)</p> - <p> - Own Id: OTP-11744</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Calls to erlang:open_port/2 with 'spawn' are updated to - handle space in the command path.</p> - <p> - Own Id: OTP-10842</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.6.4</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p>The way Common Test handles skipping of test cases has - been updated. In previous versions, returning - <c>{skip,Reason}</c> from a configuration function (such - as init_per_suite or init_per_group), resulted in all - affected test cases getting skipped with status - <c>auto_skipped</c>. This was inappropriate, since this - status is supposed to be used to inform that Common Test - has taken the initiative to skip something (e.g. a test - case group if init_per_group failed). Therefore, in this - version of Common Test, whenever the user skips a suite, - group, or individual test case (by means of a - configuration function or test specification term), the - affected test cases get the status <c>user_skipped</c> - instead.</p> <p>This update has meant a few changes that - may affect Common Test users in various ways:</p> <list> - <item>The test results and statistics will be affected, - which is important to know when running regression tests - and comparing results to previous test runs.</item> - <item>Users that read or parse the textual log file - <c>suite.log</c> will notice that an auto skipped - function is now reported as <c>auto_skipped</c> rather - than <c>skipped</c> as before.</item> <item>When - <c>require</c> fails in an info function (such as suite/0 - or group/1), all affected configuration functions and - test cases are marked as <c>auto_skipped</c>.</item> - <item>If Common Test detects an error in the test suite - (such as e.g. an invalid all/0 function), all affected - configuration functions and test cases are marked as - <c>auto_skipped</c>.</item> <item>If a repeated test run - session reaches a deadline with <c>force_stop</c> - enabled, all remaining test cases are marked as - <c>auto_skipped</c> rather than <c>user_skipped</c> as - before.</item> <item>The event messages that Common Test - generates during test runs have been affected by this - update. For details see OTP-11524.</item> </list> - <p> - Own Id: OTP-11305 Aux Id: OTP-11524 </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.6.3</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Test Server installed an error handler (test_server_h) - only to be able to write the name of the current test - case to stdout whenever it received an error- or progress - report. This functionality was not useful and has been - removed. The built-in Common Test hook, cth_log_redirect, - has instead been improved to now also tag all error- and - progress reports in the log with suite-, group-, and/or - test case name.</p> - <p> - Own Id: OTP-11263 Aux Id: seq12251 </p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - A new log, the "Pre- and Post Test I/O Log", has been - introduced, which makes it possible to capture error- and - progress reports, as well as printouts made with ct:log/2 - and ct:pal/2, before and after a test run. (Some minor - improvements of the logging system have been made at the - same time). Links to the new log are found on the Common - Test Framework Log page. The Common Test User's Guide has - been updated with information about the new log and also - with a new section on how to synchronize external - applications with Common Test by means of the CT Hook - init and terminate functions.</p> - <p> - Own Id: OTP-11272</p> - </item> - </list> - </section> - - - <section><title>Known Bugs and Problems</title> - <list> - <item> - <p> - Test Server: Report auto_skipped in major log.</p> - <p> - Own Id: OTP-11297</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.6.2</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Some unused code related to remote targets is removed, - and documentation is updated.</p> - <p> - Own Id: OTP-10607 Aux Id: kunagi-338 [249] </p> - </item> - <item> - <p> - A bug in test_server_gl caused io requests containing - invalid data (i.e. not unicode:chardata()) to hang, since - no io reply was sent. This has been corrected.</p> - <p> - Own Id: OTP-10991</p> - </item> - <item> - <p> - Common Test would, in case of timetrap error, print a - warning in the log if end_per_testcase wasn't implemented - in the suite, even though it's an optional function. This - printout has been removed.</p> - <p> - Own Id: OTP-11052</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The '-force_stop' flag to use with time-limited repeats - of test runs can now be used with a new 'skip_rest' - option which causes the rest of the test cases in the - ongoing test job to be skipped when the time limit is - reached. E.g. 'ct_run -spec xxx -duration 010000 - -force_stop skip_rest'</p> - <p> - Own Id: OTP-10856 Aux Id: OTP-10832 </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.6.1</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The unicode update of test_server for R16A introduced a - few potential errors when logging to files. Sometimes ~tp - or ~ts was used for formatting also when writing to files - that were not opened with the {encoding,utf8} option. If - then the argument contained unicode characters above 255, - the file descriptor would crash. This has been corrected - by the following modifications:</p> <list> <item> Since the - 'unexpected_io' log file is used only when the test case - HTML file is not available (e.g. between test cases), - this file is now also a HTML file and as other - test_server HTML logs it is always UTF-8 encoded </item> - <item> Since it is possible to change which information - is going to which log file (with - test_server_ctrl:set_levels/3), we do not have full - control over which information is written to which file. - This means that any printout could be written to the - 'major' log file (suite.log), which was earlier encoded - as latin1. To avoid crashing this file descriptor due to - unicode strings, the 'major' log file is now also encoded - in UTF-8 (possible incopatibility). </item> <item> The - cross_cover.info file is no longer a text file which can - be read with file:consult/1, instead it is written as a - pure binary file using term_to_binary when writing and - binary_to_term when reading. </item> <item> The encoding - of the file named 'last_name', which only content is the - path to the last run.<timestamp> directory, is now - dependent on the file name mode of the VM. If file names - are expected to be unicode, then the 'last_name' file is - UTF-8 encoded, else it is latin1 encoded. </item> </list> - <p> - Also, ~tp has been changed back to ~p unless it is - somehow likely that the argument includes strings. It is - not obvious that this is the correct thing to do, but - some decission had to be taken...</p> - <p> - Own Id: OTP-10780</p> - </item> - <item> - <p> - Using the force_stop flag/option to interrupt a test run - caused a crash in Common Test. This problem has been - solved.</p> - <p> - Own Id: OTP-10832</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.6</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Line numbering of erlang files that were not correctly - indented could be wrong after coverting to html with - erl2html2:convert/[2,3] (the source code pointed to from - the test case). This has been corrected.</p> - <p> - Also, there are now link targets for each line and not - only for each 10th line, and link targets for functions - now include the arity and not only the function name - (e.g. func/1 has a link target "func-1").</p> - <p> - Own Id: OTP-9710 Aux Id: seq11945, kunagi-201 [112] </p> - </item> - <item> - <p> - Severe errors detected by <c>test_server</c> (e.g. if log - files directories cannot be created) will now be reported - to <c>common_test</c> and noted in the <c>common_test</c> - logs.</p> - <p> - Own Id: OTP-9769 Aux Id: kunagi-202 [113] </p> - </item> - <item> - <p> - The earlier undocumented cross cover feature for - accumulating cover data over multiple tests has now been - fixed and documented.</p> - <p> - Own Id: OTP-9870 Aux Id: kunagi-206 [117] </p> - </item> - <item> - <p> - If the test suite itself was included in code coverage - analysis, then the test_server would not manage to set - data_dir correctly for the test. This has been corrected.</p> - <p> - Own Id: OTP-9956 Aux Id: kunagi-207 [118] </p> - </item> - <item> - <p> - Any call to test_server:break/1 should cancel all active - timetramps. However, in some cases - Suite:end_per_testcase/2 is executed on a different - process than the test case itself, and if - test_server:break/1 was called from there, the timetraps - were not cancelled. This has been corrected.</p> - <p> - Own Id: OTP-10046 Aux Id: kunagi-174 [85] </p> - </item> - <item> - <p>When a test case failed because of a timetrap time - out, the <c>Config</c> data for the case was lost in the - following call to <c>end_per_testcase/2</c>, and also in - calls to the CT Hook function - <c>post_end_per_testcase/4</c>. This problem has been - solved and the <c>Config</c> data is now correctly passed - to the above functions after a timetrap timeout - failure.</p> - <p> - Own Id: OTP-10070 Aux Id: kunagi-175 [86] </p> - </item> - <item> - <p>In test_server, the same process would supervise the - currently running test case and be group leader (and IO - server) for the test case. Furthermore, when running - parallel test cases, new temporary supervisor/group - leader processes were spawned and the process that was - group leader for sequential test cases would not be - active. That would lead to several problems:</p> - <p>* Processes started by init_per_suite will inherit the - group leader of the init_per_suite process (and that - group leader would not process IO requests when parallel - test cases was running). If later a parallel test case - caused such a processto print using (for example) - io:format/2, the calling would hang.</p> - <p>* Similarly, if a process was spawned from a parallel - test case, it would inherit the temporary group leader - for that parallel test case. If that spawned process - later - when the group of parallel tests have finished - - attempted to print something, its group leader would be - dead and there would be <c>badarg</c> exception.</p> - <p>Those problems have been solved by having group - leaders separate from the processes that supervises the - test cases, and keeping temporary group leader process - for parallel test cases alive until no more process in - the system use them as group leaders.</p> - <p>Also, a new <c>unexpected_io.log</c> log file - (reachable from the summary page of each test suite) has - been introduced. All unexpected IO will be printed into - it(for example, IO to a group leader for a parallel test - case that has finished).</p> - <p> - Own Id: OTP-10101 Aux Id: OTP-10125 </p> - </item> - <item> - <p> - The stability of <c>common_test</c> and - <c>test_server</c> when running test cases in parallel - has been improved.</p> - <p> - Own Id: OTP-10480 Aux Id: kunagi-318 [229] </p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Added a general framework for executing benchmarks of - Erlang/OTP. Benchmarks for the Erlang VM and mnesia have - been incorporated in the framework. </p> - <p> - For details about how to add more benchmarks see - $ERL_TOP/HOWTO/BENCHMARKS.md in the source distribution.</p> - <p> - Own Id: OTP-10156</p> - </item> - <item> - <p> - Update common test modules to handle Unicode:</p> <list> - <item> Use UTF-8 encoding for all HTML files, except the - HTML version of the test suite generated with - erl2html2:convert, which will have the same encoding as - the original test suite (.erl) file. </item> <item> - Encode link targets in HTML files with - test_server_ctrl:uri_encode/1. </item> <item> Use unicode - modifier 't' with ~s when appropriate. </item> <item> Use - unicode:characters_to_list and - unicode:characters_to_binary for conversion between - binaries and strings instead of binary_to_list and - list_to_binary. </item> </list> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.5.3</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - test_server_h will now recognize info_reports written by - ct connection handlers (according to the description in - cth_conn_log) and ignore them as they will be completely - handled by by ct_conn_log_h.</p> - <p> - Earlier test_server_h would print a tag (testcase name) - before forwarding the report to error_logger_tty_h. This - would cause lots of tags in the log with no info report - following (since error_logger_tty_h did not handle them).</p> - <p> - Own Id: OTP-10571</p> - </item> - </list> - </section> - - - <section><title>Known Bugs and Problems</title> - <list> - <item> - <p> - Restore Config data if lost when test case fails.</p> - <p> - Own Id: OTP-10070 Aux Id: kunagi-175 [86] </p> - </item> - <item> - <p> - IO server error in test_server.</p> - <p> - Own Id: OTP-10125 Aux Id: OTP-10101, kunagi-177 [88] </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.5.2</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The documentation has been updated with the latest - changes for the test_server_ctrl:report/2 function.</p> - <p> - Own Id: OTP-10086 Aux Id: seq12066 </p> - </item> - <item> - <p> - The ct:get_status/0 function failed to report status if a - parallel test case group was running at the time of the - call. This has been fixed and the return value for the - function has been updated. Please see the ct reference - manual for details.</p> - <p> - Own Id: OTP-10172</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - It is now possible to sort the HTML tables by clicking on - the header elements. In order to reset a sorted table, - the browser window should simply be refreshed. This - feature requires that the browser supports javascript, - and has javascript execution enabled. If the 'ct_run - -basic_html' flag is used, no javascript code is included - in the generated HTML code.</p> - <p> - Own Id: OTP-9896 Aux Id: seq12034, OTP-9835 </p> - </item> - <item> - <p> - Verbosity levels for log printouts has been added. This - makes it possible to specify preferred verbosity for - different categories of log printouts, as well as general - printouts (such as standard IO), to allow control over - which strings get printed and which get ignored. New - versions of the Common Test logging functions, ct:log, - ct:pal and ct:print, have been introduced, with a new - Importance argument added. The Importance value is - compared to the verbosity level at runtime. More - information can be found in the chapter about Logging in - the Common Test User's Guide.</p> - <p> - Own Id: OTP-10067 Aux Id: seq12050 </p> - </item> - <item> - <p> - The Erlang/OTP test runner ts has been extended to allow - cross compilation of test suites. To cross compile the - test suites first follow the normal cross compilation - procedures and release the tests on the build host. Then - install ts using an xcomp specification file and compile - test suites using ts:compile_testcases/0. For more - details see $ERL_TOP/xcomp/README.md.</p> - <p> - Own Id: OTP-10074</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.5.1</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - After a test case timeout or abortion, the - end_per_testcase function executes on a new dedicated - process. The group leader for this process should be set - to the IO server for the test case, which was not done - properly. The result of this error was that no warnings - about end_per_testcase failing or timing out were ever - printed in the test case log. Also, help functions such - as e.g. test_server:stop_node/1, attempting to - synchronize with the IO server, would hang. The fault has - been corrected.</p> - <p> - Own Id: OTP-9666</p> - </item> - <item> - <p> - A deadlock situation could occur if Common Test is - forwarding error_handler printouts to Test Server at the - same time a new test case is starting. This error has - been fixed.</p> - <p> - Own Id: OTP-9894</p> - </item> - <item> - <p> - When a test case was killed because of a timetrap - timeout, the current location (suite, case and line) was - not printed correctly in the log files. This has been - corrected.</p> - <p> - Own Id: OTP-9930 Aux Id: seq12002 </p> - </item> - <item> - <p> - Test Server and Common Test would add new error handlers - with each test run and fail to remove previously added - ones. In the case of Test Server, this would only happen - if SASL was not running on the test node. This has been - fixed.</p> - <p> - Own Id: OTP-9941 Aux Id: seq12009 </p> - </item> - <item> - <p> - If a test case process was terminated due to an exit - signal from a linked process, Test Server failed to - report the correct name of the suite and case to the - framework. This has been corrected.</p> - <p> - Own Id: OTP-9958 Aux Id: OTP-9855 </p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - A new optional feature has been introduced that enables - Common Test to generate priv_dir directory names that are - unique for each test case or config function. The name of - the option/flag is 'create_priv_dir' and it can be set to - value 'auto_per_run' (which is the default, existing, - behaviour), or 'auto_per_tc' or 'manual_per_tc'. If - 'auto_per_tc' is used, Test Server creates a dedicated - priv_dir automatically for each test case (which can be - very expensive in case of many and/or repeated cases). If - 'manual_per_tc' is used, the user needs to create the - priv_dir explicitly by calling the new function - ct:make_priv_dir/0.</p> - <p> - Own Id: OTP-9659 Aux Id: seq11930 </p> - </item> - <item> - <p> - A column for test case group name has been added to the - suite overview HTML log file.</p> - <p> - Own Id: OTP-9730 Aux Id: seq11952 </p> - </item> - <item> - <p> - It is now possible to use the post_end_per_testcase CT - hook function to print a comment for a test case in the - overview log file, even if the test case gets killed by a - timetrap or unknown exit signal, or if the - end_per_testcase function times out.</p> - <p> - Own Id: OTP-9855 Aux Id: seq11979 </p> - </item> - <item> - <p> - Common Test will now print error information (with a time - stamp) in the test case log file immediately when a test - case fails. This makes it easier to see when, in time, - the fault actually occured, and aid the job of locating - relevant trace and debug printouts in the log.</p> - <p> - Own Id: OTP-9904 Aux Id: seq11985, OTP-9900 </p> - </item> - <item> - <p> - Test Server has been modified to check the SASL - errlog_type parameter when receiving an error logger - event, so that it doesn't print reports of type that the - user has disabled.</p> - <p> - Own Id: OTP-9955 Aux Id: seq12013 </p> - </item> - <item> - <p> - If an application cannot be found by ts it is - automatically skipped when testing.</p> - <p> - Own Id: OTP-9971</p> - </item> - <item> - <p> - By specifying a user defined function ({M,F,A} or fun) as - timetrap value, either by means of an info function or by - calling ct:timetrap/1, it is now possible to set a - timetrap that will be triggered when the user function - returns.</p> - <p> - Own Id: OTP-9988 Aux Id: OTP-9501, seq11894 </p> - </item> - <item> - <p> - If the optional configuration functions init_per_suite/1 - and end_per_suite/1 are not implemented in the test - suite, local Common Test versions of these functions are - called instead, and will be displayed in the overview log - file. Any printouts made by the pre- or - post_init_per_suite and pre- or post_end_per_suite hook - functions are saved in the log files for these functions.</p> - <p> - Own Id: OTP-9992</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.5</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The test case group info function has been implemented in - Common Test. Before execution of a test case group, a - call is now made to <c>TestSuite:group(GroupName)</c>. - The function returns a list of test properties, e.g. to - specify timetrap values, require configuration data, etc - (analogue to the test suite- and test case info - function). The scope of the properties set by - <c>group(GroupName)</c> is all test cases and sub-groups - of group <c>GroupName</c>.</p> - <p> - Own Id: OTP-9235</p> - </item> - <item> - <p> - The look of the HTML log files generated by Common Test - and Test Server has been improved (and made easier to - customize) by means of a CSS file.</p> - <p> - Own Id: OTP-9706</p> - </item> - </list> - </section> - - - <section><title>Known Bugs and Problems</title> - <list> - <item> - <p> - Fix problems in CT/TS due to line numbers in exceptions.</p> - <p> - Own Id: OTP-9203</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.4.5</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - An error in how comments are colored in the test suite - overview html log file has been corrected. As result, a - new framework callback function, format_comment/1, has - been introduced.</p> - <p> - Own Id: OTP-9237</p> - </item> - <item> - <p> - Test Server did not release SASL TTY handlers - (sasl_report_tty_h and error_logger_tty_h) properly after - each test run. This error has been fixed.</p> - <p> - Own Id: OTP-9311</p> - </item> - <item> - <p> - Automatically generated init- and end-configuration - functions for test case groups caused incorrect execution - order of test cases. This has been corrected.</p> - <p> - Own Id: OTP-9369</p> - </item> - <item> - <p> - If ct:log/2 was called with bad arguments, this could - cause the Common Test IO handling process to crash. This - fault has been corrected.</p> - <p> - Own Id: OTP-9371 Aux Id: OTP-8933 </p> - </item> - <item> - <p> - A bug has been fixed that made Test Server call the - end_tc/3 framework function with an incorrect module name - as first argument.</p> - <p> - Own Id: OTP-9379 Aux Id: seq11863 </p> - </item> - <item> - <p> - If end_per_testcase caused a timetrap timeout, the actual - test case status was discarded and the test case logged - as successful (even if the case had actually failed - before the call to end_per_testcase). This fault has been - fixed.</p> - <p> - Own Id: OTP-9397</p> - </item> - <item> - <p> - If a timetrap timeout occured during execution of of a - function in a lib module (i.e. a function called directly - or indirectly from a test case), the Suite argument in - the end_tc/3 framework callback function would not - correctly contain the name of the test suite, but the lib - module. (This would only happen if the lib module was - compiled with ct.hrl included). This error has been - solved.</p> - <p> - Own Id: OTP-9398</p> - </item> - <item> - <p> - Add a proplist() type</p> - <p> - Recently I was adding specs to an API and found that - there is no canonical proplist() type defined. (Thanks to - Ryan Zezeski)</p> - <p> - Own Id: OTP-9499</p> - </item> - <item> - <p> XML files have been corrected. </p> - <p> - Own Id: OTP-9550 Aux Id: OTP-9541 </p> - </item> - <item> - <p> - If a test suite would start with a test case group - defined without the init_per_group/2 and end_per_group/2 - function, init_per_suite/1 would not execute initially - and logging of the test run would fail. This error has - been fixed.</p> - <p> - Own Id: OTP-9584</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - A new option, 'logopts', has been introduced, to make it - possible to modify some aspects of the logging behaviour - in Common Test (or Test Server). For example, whenever an - io printout is made, test_server adds newline (\n) to the - end of the output string. This may not always be a - preferred action and can therefore be disabled by means - of "ct_run ... -logopts no_nl" (or ct:run_test([..., - {logopts,[no_nl]}])). A new framework callback function, - get_logopts/0, has been introduced (see the ct_framework - module for details).</p> - <p> - Own Id: OTP-9372 Aux Id: OTP-9396 </p> - </item> - <item> - <p> - A new option, 'logopts', has been introduced, to make it - possible to modify some aspects of the logging behaviour - in Common Test (or Test Server). For example, if the html - version of the test suite source code should not be - generated during the test run (and consequently be - unavailable in the log file system), the feature may be - disabled by means of "ct_run ... -logopts no_src" (or - ct:run_test([..., {logopts,[no_src]}])). A new framework - callback function, get_logopts/0, has been introduced - (see the ct_framework module for details).</p> - <p> - Own Id: OTP-9396 Aux Id: seq11869, OTP-9372 </p> - </item> - <item> - <p> - It is now possible to use a tuple {M,F,A}, or a fun, as - timetrap specification in the suite info function or test - case info functions. The function must return a valid - timeout value, as documented in the common_test man page - and in the User's Guide.</p> - <p> - Own Id: OTP-9501 Aux Id: seq11894 </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.4.4</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - It was previously not possible to use timetrap value - 'infinity' with ct:timetrap/1. This has been fixed.</p> - <p> - Own Id: OTP-9159</p> - </item> - <item> - <p> - A bug that made it impossible to cancel the previous - timetrap when calling ct:timetrap/1 has been corrected.</p> - <p> - Own Id: OTP-9233 Aux Id: OTP-9159 </p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - When running tests with auto-compilation disabled, Common - Test could only display the test suite source code on - html format in the test case log if the source file was - located in the same directory as the pre-compiled suite. - This has been modified so that Common Test now tries to - locate the source file by means of the test suite module - info (Suite:module_info/1). As a result, a suite may now - be compiled to a different output directory (e.g. - $MYTEST/bin) than the source code directory (e.g. - $MYTEST/src), without the source-code-to-html generation - being affected.</p> - <p> - Own Id: OTP-9138</p> - </item> - <item> - <p> - It is now possible to return a tuple {fail,Reason} from - init_per_testcase/2. The result is that the associated - test case gets logged as failed without ever executing.</p> - <p> - Own Id: OTP-9160 Aux Id: seq11502 </p> - </item> - <item> - <p> - Added DragonflyBSD check in test_server configure.</p> - <p> - Own Id: OTP-9249</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.4.3</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Updated the ts*.config files to contain information - relevant to testing Erlang/OTP in an open source - environment.</p> - <p> - Own Id: OTP-9017</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Alpha release of Common Test Hooks (CTH). CTHs allow the - users of common test to abtract out common behaviours - from test suites in a much more elegant and flexible way - than was possible before. Note that the addition of this - feature may introduce minor changes in the undocumented - behaviour of the interface inbetween common_test and - test_server.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-8851</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.4.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p>Miscellaneous updates</p> - <p> - Own Id: OTP-8976</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.4.1</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Returning {return_group_result,failed} from end_per_group - in a group that is part of a sequence, did not cause the - proceeding cases (or groups) to get skipped. This has - been fixed.</p> - <p> - Own Id: OTP-8753 Aux Id: seq11644 </p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Common Test has been updated to handle start options and - test specification terms for test case groups (and test - cases in groups). Also, an option named 'label', has been - added that associates the test run with a name that - Common Test prints in the overview HTML logs.</p> - <p> - Own Id: OTP-8725 Aux Id: OTP-8727 </p> - </item> - <item> - <p> - It is now possible to skip all tests in a suite, or a - group, by returning {fail,Reason} from the end_tc/5 - framework function for init_per_suite, or init_per_group.</p> - <p> - Own Id: OTP-8805 Aux Id: seq11664 </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.4</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Returning {fail,Reason} from the framework end_tc - function was not handled properly by Test Server for all - test suite functions.</p> - <p> - Own Id: OTP-8492 Aux Id: seq11502 </p> - </item> - <item> - <p> - If the framework end_tc function would hang and get - aborted by Test Server, there was no indication of - failure in the logs. This has been fixed.</p> - <p> - Own Id: OTP-8682 Aux Id: seq11504 </p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - It is now possible for the Test Server framework end_tc - function to change the status of the test case from ok or - auto-skipped to failed by returning {fail,Reason}.</p> - <p> - Own Id: OTP-8495 Aux Id: seq11502 </p> - </item> - <item> - <p> - Test Server will now call the end_per_testcase/2 function - even if the test case has been terminated explicitly - (with abort_current_testcase/1), or after a timetrap - timeout. Under these circumstances the return value of - end_per_testcase is completely ignored. Therefore the - function will not be able to change the reason for test - case termination by returning {fail,Reason}, nor will it - be able to save data with {save_config,Data}.</p> - <p> - Own Id: OTP-8500 Aux Id: seq11521 </p> - </item> - <item> - <p> - Previously, a repeat property of a test case group - specified the number of times the group should be - repeated after the main test run. I.e. {repeat,N} would - case the group to execute 1+N times. To be consistent - with the behaviour of the run_test repeat option, this - has been changed. N now specifies the absolute number of - executions instead.</p> - <p> - Own Id: OTP-8689 Aux Id: seq11502 </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.3.6</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The Test Server parse transform did not handle bit string - comprehensions. This has been fixed.</p> - <p> - Own Id: OTP-8458 Aux Id: OTP-8311 </p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The tc_status value in the Config list for a test case - that has failed because of a timetrap timeout, has - changed from {tc_status,timeout} to - {tc_status,timetrap_timeout}.</p> - <p> - Own Id: OTP-8302</p> - </item> - <item> - <p>The documentation is now possible to build in an open - source environment after a number of bugs are fixed and - some features are added in the documentation build - process. </p> - <p>- The arity calculation is updated.</p> - <p>- The module prefix used in the function names for - bif's are removed in the generated links so the links - will look like - "http://www.erlang.org/doc/man/erlang.html#append_element-2" - instead of - "http://www.erlang.org/doc/man/erlang.html#erlang:append_element-2".</p> - <p>- Enhanced the menu positioning in the html - documentation when a new page is loaded.</p> - <p>- A number of corrections in the generation of man - pages (thanks to Sergei Golovan)</p> - <p>- The legal notice is taken from the xml book file so - OTP's build process can be used for non OTP - applications.</p> - <p> - Own Id: OTP-8343</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.3.5</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - If the init_per_testcase/2 function fails, the test case - now gets marked and counted as auto skipped, not user - skipped (which would previously happen).</p> - <p> - Own Id: OTP-8289</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The documentation is now built with open source tools - (xsltproc and fop) that exists on most platforms. One - visible change is that the frames are removed.</p> - <p> - Own Id: OTP-8201</p> - </item> - <item> - <p> - It is now possible to fail a test case from the - end_per_testcase/2 function, by returning {fail,Reason}.</p> - <p> - Own Id: OTP-8284</p> - </item> - <item> - <p> - It is now possible to fail a test case by having the - end_tc/3 framework function return {fail,Reason} for the - test case.</p> - <p> - Own Id: OTP-8285</p> - </item> - <item> - <p> - The test_server framework API (e.g. the end_tc/3 - function) has been modified. See the test_server_ctrl - documentation for details.</p> - <p> - Own Id: OTP-8286 Aux Id: OTP-8285, OTP-8287 </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.3.4</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - When running a suite starting with a test case group, - Test Server crashed if init_per_suite/1 exited or - returned skip. This has been fixed.</p> - <p> - Own Id: OTP-8105 Aux Id: OTP-8089 </p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Various updates and fixes in Common Test and Test Server.</p> - <p> - Own Id: OTP-8045 Aux Id: OTP-8089,OTP-8105,OTP-8163 </p> - </item> - <item> - <p> - Errors in coverage data collection and analysis were - difficult to detect. The logging has been improved so - that more information about e.g. imported and missing - modules is printed to the html log files.</p> - <p> - Own Id: OTP-8163 Aux Id: seq11374 </p> - </item> - <item> - <p> - The Common Test HTML overview pages have been improved. - It is now possible to see if a test case has been skipped - explicitly or because a configuration function has - failed. Also, the history page (all_runs.html) now has - scrolling text displaying the test names. The old format - (showing names as a truncated string) can still be - generated by means of the flag/option 'basic_html'.</p> - <p> - Own Id: OTP-8177</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.3.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Various corrections and improvements of Common Test and - Test Server.</p> - <p> - Own Id: OTP-7981</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.3.1</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Minor updates and corrections.</p> - <p> - Own Id: OTP-7897</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.3</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The conf case in Test Server has been extended with - properties that make it possible to execute test cases in - parallel, in sequence and in shuffled order. It is now - also possible to repeat test cases according to different - criterias. The properties can be combined, making it - possible to e.g. repeat a conf case a certain number of - times and execute the test cases in different (random) - order every time. The properties are specified in a list - in the conf case definition: {conf, Properties, InitCase, - TestCases, EndCase}. The available properties are: - parallel, sequence, shuffle, repeat, repeat_until_all_ok, - repeat_until_any_ok, repeat_until_any_fail, - repeat_until_all_fail.</p> - <p> - Own Id: OTP-7511 Aux Id: OTP-7839 </p> - </item> - <item> - <p>The test server starts Cover on nodes of the same - version as the test server itself only.</p> - <p> - Own Id: OTP-7699</p> - </item> - <item> - <p> - The Erlang mode for Emacs has been updated with new and - modified skeletons for Common Test and TS. Syntax for - test case groups in Common Test (and conf cases with - properties in TS) has been added and a new minimal Common - Test suite skeleton has been introduced.</p> - <p> - Own Id: OTP-7856</p> - </item> - </list> - </section> - -</section> -<section><title>Test_Server 3.2.4.1</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The step functionality in Common Test (based on - interaction with Debugger) was broken. This has been - fixed, and some new step features have also been added. - Please see the Common Test User's Guide for details.</p> - <p> - Own Id: OTP-7800 Aux Id: seq11106 </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.2.4</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Miscellaneous updates.</p> - <p> - Own Id: OTP-7527</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.2.3</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - When a testcase terminated due to a timetrap, io sent to - the group leader from framework:end_tc/3 (using - ct:pal/2/3 or ct:log/2/3) would cause deadlock. This has - been fixed.</p> - <p> - Own Id: OTP-7447 Aux Id: seq11010 </p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Various updates and improvements, plus some minor bug - fixes, have been implemented in Common Test and Test - Server.</p> - <p> - Own Id: OTP-7112</p> - </item> - <item> - <p> - It is now possible, by means of the new function - ct:abort_current_testcase/1 or - test_server_ctrl:abort_current_testcase/1, to abort the - currently executing test case.</p> - <p> - Own Id: OTP-7518 Aux Id: OTP-7112 </p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.2.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p><c>erlang:system_info/1</c> now accepts the - <c>logical_processors</c>, and <c>debug_compiled</c> - arguments. For more info see the, <c>erlang(3)</c> - documentation.</p> <p>The scale factor returned by - <c>test_server:timetrap_scale_factor/0</c> is now also - effected if the emulator uses a larger amount of - scheduler threads than the amount of logical processors - on the system. </p> - <p> - Own Id: OTP-7175</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.2.1</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - When init_per_suite or end_per_suite terminated due to - runtime failure, test_server failed to format the line - number information properly and crashed. This error has - now been fixed.</p> - <p> - Own Id: OTP-7091</p> - </item> - </list> - </section> - -</section> - -<section><title>Test_Server 3.2.0</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Test Server is a portable test server for automated - application testing. The server can run test suites on - local or remote targets and log progress and results to - HTML pages. The main purpose of Test Server is to act as - engine inside customized test tools. A callback interface - for such framework applications is provided.</p> - <p> - Own Id: OTP-6989</p> - </item> - </list> - </section> - -</section> - -</chapter> - diff --git a/lib/test_server/doc/src/notes_history.xml b/lib/test_server/doc/src/notes_history.xml deleted file mode 100644 index ca7880d74f..0000000000 --- a/lib/test_server/doc/src/notes_history.xml +++ /dev/null @@ -1,113 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2006</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Test Server Release Notes History</title> - <prepared></prepared> - <docno></docno> - <date></date> - <rev></rev> - </header> - - <section> - <title>Test Server 3.1.1</title> - - <section> - <title>Improvements and new features</title> - <list type="bulleted"> - <item> - <p>Added functions <c>test_server:break/1</c> and - <c>test_server:continue/0</c> for semiautomatic testing.</p> - <p><c>test_server:timetrap/1</c> can now also take - <c>{hours,H} | {minutes,M | {seconds,S}</c>.</p> - <p>Added function - <c>test_server_ctrl:multiply_timetraps/1</c>, - <c>test_server_ctrl:add_case/3</c>, - <c>test_server_ctrl:add_cases/2/3</c>.</p> - <p>Added test suite functions <c>init_per_suite/1</c> and - <c>end_per_suite/1</c>.</p> - <p><c>fin_per_testcase/2</c> is changed to - <c>end_per_testcase/2</c>. <c>fin_per_testcase</c> is kept - for backwards compatibility.</p> - <p>Added support for writing own test server frameworks. - Callback functions <c>init_tc/1</c>, <c>end_tc/3</c>, - <c>get_suite/2</c>, <c>report/2</c>, <c>warn/1</c>.</p> - </item> - </list> - </section> - </section> - - <section> - <title>Test Server 3.1</title> - - <section> - <title>Improvements and New Features</title> - <list type="bulleted"> - <item> - <p>Added the options <c>cover</c> and <c>cover_details</c> - to <c>ts:run</c>. When one of these options is used, - the tested application will be cover compiled - before the test is run. The cover compiled code will also - be loaded on all slave or peer nodes started with - <c>test_server:start_node</c>. When the test is completed - coverage data from all nodes is collected and merged, and - presented in the coverage log to which there will be a link - from the test suite result page (i.e. the one with the - heading "Test suite ... results").</p> - <p>The <c>cover_details</c> option will do - <c>cover:analyse_to_file</c> for each cover compiled module, - while the <c>cover</c> option only will produce a list of - modules and the number of covered/uncovered lines in each - module.</p> - <p>To make it possible to run all test from a script (like in - the OTP daily builds), the following is added: - <c>ts:run([all_tests | Options])</c>.</p> - <p>This means that e.g. the following is possible: - <c>erl -s ts run all_tests batch cover</c>.</p> - <p>Note that it is also possible to run tests with cover even - if you don't use <c>ts</c>. - See <c>test_server_ctrl:cover/2/3</c>.</p> - <p>Own Id: OTP-4703</p> - </item> - <item> - <p>Removed module <c>ts_save.erl</c> and function - <c>ts:save/0/1</c><em>(incompatible)</em>.</p> - <p>Added config variable <c>ipv6_hosts</c> to - <c>ts:install/1</c> and test spec file.</p> - <p>No longer removing duplicates of test cases from test spec - <em>(incompatible)</em>.</p> - <p>Added function <c>test_server:run_on_shielded_node/2</c>.</p> - <p>Creation of html files for test suite source does no longer - crash if suite contains more than 9999 lines of code.</p> - <p>Added functionality for cross cover compilation, - i.e. collection of cover data from all tests.</p> - <p>Multiplying timetrap times with 10 when running with cover.</p> - <p>Added <c>ts:r/3</c> for running tests with cover.</p> - <p>*** POTENTIAL INCOMPATIBILITY ***</p> - <p>Own Id: OTP-5040</p> - </item> - </list> - </section> - </section> -</chapter> - diff --git a/lib/test_server/doc/src/part.xml b/lib/test_server/doc/src/part.xml deleted file mode 100644 index 685ed16a94..0000000000 --- a/lib/test_server/doc/src/part.xml +++ /dev/null @@ -1,46 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE part SYSTEM "part.dtd"> - -<part xmlns:xi="http://www.w3.org/2001/XInclude"> - <header> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Test Server User's Guide</title> - <prepared></prepared> - <docno></docno> - <date>2002-07-11</date> - <rev></rev> - </header> - <description> - <p><em>Test Server</em> is a portable test server for - automated application testing. The server can run test suites - and log progress and results to HTML - pages. The main purpose of Test Server is to act as engine - inside customized test tools. A callback interface for - such framework applications is provided.</p> - </description> - <xi:include href="basics_chapter.xml"/> - <xi:include href="test_spec_chapter.xml"/> - <xi:include href="write_test_chapter.xml"/> - <xi:include href="run_test_chapter.xml"/> - <xi:include href="write_framework_chapter.xml"/> - <xi:include href="example_chapter.xml"/> -</part> - diff --git a/lib/test_server/doc/src/part_notes.xml b/lib/test_server/doc/src/part_notes.xml deleted file mode 100644 index 8cb9b6c591..0000000000 --- a/lib/test_server/doc/src/part_notes.xml +++ /dev/null @@ -1,41 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE part SYSTEM "part.dtd"> - -<part xmlns:xi="http://www.w3.org/2001/XInclude"> - <header> - <copyright> - <year>2004</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Test Server Release Notes</title> - <prepared></prepared> - <docno></docno> - <date></date> - <rev></rev> - </header> - <description> - <p>The <em>Test Server</em> is a portable test server for - application testing. The test server can run automatic test suites - and log progress and results to HTML - pages. It also provides some support for test suite authors.</p> - <p>For information about older versions, see - <url href="part_notes_history_frame.html">Release Notes History</url>.</p> - </description> - <xi:include href="notes.xml"/> -</part> - diff --git a/lib/test_server/doc/src/part_notes_history.xml b/lib/test_server/doc/src/part_notes_history.xml deleted file mode 100644 index 468b5aa8ba..0000000000 --- a/lib/test_server/doc/src/part_notes_history.xml +++ /dev/null @@ -1,39 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE part SYSTEM "part.dtd"> - -<part> - <header> - <copyright> - <year>2006</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Test Server Release Notes History</title> - <prepared></prepared> - <docno></docno> - <date></date> - <rev></rev> - </header> - <description> - <p>The <em>Test Server</em> is a portable test server for - application testing. The test server can run automatic test suites - and log progress and results to HTML - pages. It also provides some support for test suite authors.</p> - </description> - <include file="notes_history"></include> -</part> - diff --git a/lib/test_server/doc/src/ref_man.xml b/lib/test_server/doc/src/ref_man.xml deleted file mode 100644 index 1b06d9750b..0000000000 --- a/lib/test_server/doc/src/ref_man.xml +++ /dev/null @@ -1,44 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE application SYSTEM "application.dtd"> - -<application xmlns:xi="http://www.w3.org/2001/XInclude"> - <header> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Test Server Reference Manual</title> - <prepared></prepared> - <docno></docno> - <date></date> - <rev></rev> - <file>ref_man.xml</file> - </header> - <description> - <p><em>Test Server</em> is a portable test server for - automated application testing. The server can run test suites - and log progress and results to HTML - pages. The main purpose of Test Server is to act as engine - inside customized test tools. A callback interface for - such framework applications is provided.</p> - </description> - <xi:include href="test_server_app.xml"/> - <xi:include href="test_server_ctrl.xml"/> - <xi:include href="test_server.xml"/> -</application> - diff --git a/lib/test_server/doc/src/run_test_chapter.xml b/lib/test_server/doc/src/run_test_chapter.xml deleted file mode 100644 index cb5b29c993..0000000000 --- a/lib/test_server/doc/src/run_test_chapter.xml +++ /dev/null @@ -1,50 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Running Test Suites</title> - <prepared>Siri Hansen</prepared> - <docno></docno> - <date></date> - <rev></rev> - <file>run_test_chapter.xml</file> - </header> - - <section> - <title>Using the test server controller</title> - <p>The test server controller provides a low level interface to - all the Test Server functionality. It is possible to use this - interface directly, but it is recommended to use a framework - such as <em>Common Test</em> instead. If no existing framework - suits your needs, you could of course build your own - on top of the test server controller. Some information about how - to do this can be found in the section named "Writing you own - test server framework" in the Test Server User's Guide. - </p> - <p>For information about using the controller directly, please see - all available functions in the reference manual for - <c>test_server_ctrl</c>. - </p> - </section> -</chapter> - diff --git a/lib/test_server/doc/src/test_server.xml b/lib/test_server/doc/src/test_server.xml deleted file mode 100644 index 96ff6de3ba..0000000000 --- a/lib/test_server/doc/src/test_server.xml +++ /dev/null @@ -1,853 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>2007</year> - <year>2013</year> - <holder>Ericsson AB, All Rights Reserved</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - The Initial Developer of the Original Code is Ericsson AB. - </legalnotice> - - <title>test_server</title> - <prepared>Siri Hansen</prepared> - <responsible></responsible> - <docno></docno> - <approved></approved> - <checked></checked> - <date></date> - <rev></rev> - <file>test_server_ref.sgml</file> - </header> - <module>test_server</module> - <modulesummary>This module provides support for test suite authors.</modulesummary> - <description> - <p>The <c>test_server</c> module aids the test suite author by providing - various support functions. The supported functionality includes: - </p> - <list type="bulleted"> - <item>Logging and timestamping - </item> - <item>Capturing output to stdout - </item> - <item>Retrieving and flushing the message queue of a process - </item> - <item>Watchdog timers, process sleep, time measurement and unit - conversion - </item> - <item>Private scratch directory for all test suites - </item> - <item>Start and stop of slave- or peer nodes</item> - </list> - <p>For more information on how to write test cases and for - examples, please see the Test Server User's Guide. - </p> - </description> - - <section> - <title>TEST SUITE SUPPORT FUNCTIONS</title> - <p>The following functions are supposed to be used inside a test - suite. - </p> - </section> - <funcs> - <func> - <name>os_type() -> OSType</name> - <fsummary>Returns the OS type of the target node</fsummary> - <type> - <v>OSType = term()</v> - <d>This is the same as returned from <c>os:type/0</c></d> - </type> - <desc> - <p>This function is equivalent to <c>os:type/0</c>. It is kept - for backwards compatibility.</p> - </desc> - </func> - <func> - <name>fail()</name> - <name>fail(Reason)</name> - <fsummary>Makes the test case fail.</fsummary> - <type> - <v>Reason = term()</v> - <d>The reason why the test case failed.</d> - </type> - <desc> - <p>This will make the test suite fail with a given reason, or - with <c>suite_failed</c> if no reason was given. Use this - function if you want to terminate a test case, as this will - make it easier to read the log- and HTML files. <c>Reason</c> - will appear in the comment field in the HTML log.</p> - </desc> - </func> - <func> - <name>timetrap(Timout) -> Handle</name> - <fsummary></fsummary> - <type> - <v>Timeout = integer() | {hours,H} | {minutes,M} | {seconds,S}</v> - <v>H = M = S = integer()</v> - <v>Pid = pid()</v> - <d>The process that is to be timetrapped (<c>self()</c>by default)</d> - </type> - <desc> - <p>Sets up a time trap for the current process. An expired - timetrap kills the process with reason - <c>timetrap_timeout</c>. The returned handle is to be given - as argument to <c>timetrap_cancel</c> before the timetrap - expires. If <c>Timeout</c> is an integer, it is expected to - be milliseconds.</p> - <note> - <p>If the current process is trapping exits, it will not be killed - by the exit signal with reason <c>timetrap_timeout</c>. - If this happens, the process will be sent an exit signal - with reason <c>kill</c> 10 seconds later which will kill the - process. Information about the timetrap timeout will in - this case not be found in the test logs. However, the - error_logger will be sent a warning.</p> - </note> - </desc> - </func> - <func> - <name>timetrap_cancel(Handle) -> ok</name> - <fsummary>Cancels a timetrap.</fsummary> - <type> - <v>Handle = term()</v> - <d>Handle returned from <c>timetrap</c></d> - </type> - <desc> - <p>This function cancels a timetrap. This must be done before - the timetrap expires.</p> - </desc> - </func> - <func> - <name>timetrap_scale_factor() -> ScaleFactor</name> - <fsummary>Returns the scale factor for timeouts.</fsummary> - <type> - <v>ScaleFactor = integer()</v> - </type> - <desc> - <p>This function returns the scale factor by which all timetraps - are scaled. It is normally 1, but can be greater than 1 if - the test_server is running <c>cover</c>, using a larger amount of - scheduler threads than the amount of logical processors on the - system, running under purify, valgrind or in a debug-compiled - emulator. The scale factor can be used if you need to scale you - own timeouts in test cases with same factor as the test_server - uses.</p> - </desc> - </func> - <func> - <name>sleep(MSecs) -> ok</name> - <fsummary>Suspens the calling task for a specified time.</fsummary> - <type> - <v>MSecs = integer() | float() | infinity</v> - <d>The number of milliseconds to sleep</d> - </type> - <desc> - <p>This function suspends the calling process for at least the - supplied number of milliseconds. There are two major reasons - why you should use this function instead of - <c>timer:sleep</c>, the first being that the module - <c>timer</c> may be unavailable at the time the test suite is - run, and the second that it also accepts floating point - numbers.</p> - </desc> - </func> - <func> - <name>adjusted_sleep(MSecs) -> ok</name> - <fsummary>Suspens the calling task for a specified time.</fsummary> - <type> - <v>MSecs = integer() | float() | infinity</v> - <d>The default number of milliseconds to sleep</d> - </type> - <desc> - <p>This function suspends the calling process for at least the - supplied number of milliseconds. The function behaves the same - way as <c>test_server:sleep/1</c>, only <c>MSecs</c> - will be multiplied by the 'multiply_timetraps' value, if set, - and also automatically scaled up if 'scale_timetraps' is set - to true (which it is by default).</p> - </desc> - </func> - <func> - <name>hours(N) -> MSecs</name> - <name>minutes(N) -> MSecs</name> - <name>seconds(N) -> MSecs</name> - <fsummary></fsummary> - <type> - <v>N = integer()</v> - <d>Value to convert to milliseconds.</d> - </type> - <desc> - <p>Theese functions convert <c>N</c> number of hours, minutes - or seconds into milliseconds. - </p> - <p>Use this function when you want to - <c>test_server:sleep/1</c> for a number of seconds, minutes or - hours(!).</p> - </desc> - </func> - <func> - <name>format(Format) -> ok</name> - <name>format(Format, Args)</name> - <name>format(Pri, Format)</name> - <name>format(Pri, Format, Args)</name> - <fsummary></fsummary> - <type> - <v>Format = string()</v> - <d>Format as described for <c>io_:format</c>.</d> - <v>Args = list()</v> - <d>List of arguments to format.</d> - </type> - <desc> - <p>Formats output just like <c>io:format</c> but sends the - formatted string to a logfile. If the urgency value, - <c>Pri</c>, is lower than some threshold value, it will also - be written to the test person's console. Default urgency is - 50, default threshold for display on the console is 1. - </p> - <p>Typically, the test person don't want to see everything a - test suite outputs, but is merely interested in if the test - cases succeeded or not, which the test server tells him. If he - would like to see more, he could manually change the threshold - values by using the <c>test_server_ctrl:set_levels/3</c> - function.</p> - </desc> - </func> - <func> - <name>capture_start() -> ok</name> - <name>capture_stop() -> ok</name> - <name>capture_get() -> list()</name> - <fsummary>Captures all output to stdout for a process.</fsummary> - <desc> - <p>These functions makes it possible to capture all output to - stdout from a process started by the test suite. The list of - characters captured can be purged by using <c>capture_get</c>.</p> - </desc> - </func> - <func> - <name>messages_get() -> list()</name> - <fsummary>Empty the message queue.</fsummary> - <desc> - <p>This function will empty and return all the messages - currently in the calling process' message queue.</p> - </desc> - </func> - <func> - <name>timecall(M, F, A) -> {Time, Value}</name> - <fsummary>Measures the time needed to call a function.</fsummary> - <type> - <v>M = atom()</v> - <d>The name of the module where the function resides.</d> - <v>F = atom()</v> - <d>The name of the function to call in the module.</d> - <v>A = list()</v> - <d>The arguments to supply the called function.</d> - <v>Time = integer()</v> - <d>The number of seconds it took to call the function.</d> - <v>Value = term()</v> - <d>Value returned from the called function.</d> - </type> - <desc> - <p>This function measures the time (in seconds) it takes to - call a certain function. The function call is <em>not</em> - caught within a catch.</p> - </desc> - </func> - <func> - <name>do_times(N, M, F, A) -> ok</name> - <name>do_times(N, Fun)</name> - <fsummary>Calls MFA or Fun N times.</fsummary> - <type> - <v>N = integer()</v> - <d>Number of times to call MFA.</d> - <v>M = atom()</v> - <d>Module name where the function resides.</d> - <v>F = atom()</v> - <d>Function name to call.</d> - <v>A = list()</v> - <d>Arguments to M:F.</d> - </type> - <desc> - <p>Calls MFA or Fun N times. Useful for extensive testing of a - sensitive function.</p> - </desc> - </func> - <func> - <name>m_out_of_n(M, N, Fun) -> ok | exit({m_out_of_n_failed, {R,left_to_do}}</name> - <fsummary>Fault tolerant <c>do_times</c>.</fsummary> - <type> - <v>N = integer()</v> - <d>Number of times to call the Fun.</d> - <v>M = integer()</v> - <d>Number of times to require a successful return.</d> - </type> - <desc> - <p>Repeatedly evaluates the given function until it succeeds - (doesn't crash) M times. If, after N times, M successful - attempts have not been accomplished, the process crashes with - reason {m_out_of_n_failed, {R,left_to_do}}, where R indicates - how many cases that was still to be successfully completed. - </p> - <p>For example: - </p> - <p><c>m_out_of_n(1,4,fun() -> tricky_test_case() end)</c> <br></br> -Tries to run tricky_test_case() up to 4 times, and is - happy if it succeeds once. - </p> - <p><c>m_out_of_n(7,8,fun() -> clock_sanity_check() end)</c> <br></br> -Tries running clock_sanity_check() up to 8 times,and - allows the function to fail once. This might be useful if - clock_sanity_check/0 is known to fail if the clock crosses an - hour boundary during the test (and the up to 8 test runs could - never cross 2 boundaries)</p> - </desc> - </func> - <func> - <name>call_crash(M, F, A) -> Result</name> - <name>call_crash(Time, M, F, A) -> Result</name> - <name>call_crash(Time, Crash, M, F, A) -> Result</name> - <fsummary>Calls MFA and succeeds if it crashes.</fsummary> - <type> - <v>Result = ok | exit(call_crash_timeout) | exit({wrong_crash_reason, Reason})</v> - <v>Crash = term()</v> - <d>Crash return from the function.</d> - <v>Time = integer()</v> - <d>Timeout in milliseconds.</d> - <v>M = atom()</v> - <d>Module name where the function resides.</d> - <v>F = atom()</v> - <d>Function name to call.</d> - <v>A = list()</v> - <d>Arguments to M:F.</d> - </type> - <desc> - <p>Spawns a new process that calls MFA. The call is considered - successful if the call crashes with the gives reason - (<c>Crash</c>) or any reason if not specified. The call must - terminate within the given time (default <c>infinity</c>), or - it is considered a failure.</p> - </desc> - </func> - <func> - <name>temp_name(Stem) -> Name</name> - <fsummary>Returns a unique filename.</fsummary> - <type> - <v>Stem = string()</v> - </type> - <desc> - <p>Returns a unique filename starting with <c>Stem</c> with - enough extra characters appended to make up a unique - filename. The filename returned is guaranteed not to exist in - the filesystem at the time of the call.</p> - </desc> - </func> - <func> - <name>break(Comment) -> ok</name> - <fsummary>Cancel all timetraps and wait for call to continue/0.</fsummary> - <type> - <v>Comment = string()</v> - </type> - <desc> - <p><c>Comment</c> is a string which will be written in - the shell, e.g. explaining what to do.</p> - <p>This function will cancel all timetraps and pause the - execution of the test case until the user executes the - <c>continue/0</c> function. It gives the user the opportunity - to interact with the erlang node running the tests, e.g. for - debugging purposes or for manually executing a part of the - test case.</p> - <p>When the <c>break/1</c> function is called, the shell will - look something like this:</p> - <code type="none"><![CDATA[ - --- SEMIAUTOMATIC TESTING --- - The test case executes on process <0.51.0> - - - "Here is a comment, it could e.g. instruct to pull out a card" - - - ----------------------------- - - Continue with --> test_server:continue(). ]]></code> - <p>The user can now interact with the erlang node, and when - ready call <c>test_server:continue().</c></p> - <p>Note that this function can not be used if the test is - executed with <c>ts:run/0/1/2/3/4</c> in <c>batch</c> mode.</p> - </desc> - </func> - <func> - <name>continue() -> ok</name> - <fsummary>Continue after break/1.</fsummary> - <desc> - <p>This function must be called in order to continue after a - test case has called <c>break/1</c>.</p> - </desc> - </func> - <func> - <name>run_on_shielded_node(Fun, CArgs) -> term()</name> - <fsummary>Execute a function a shielded node.</fsummary> - <type> - <v>Fun = function() (arity 0)</v> - <d>Function to execute on the shielded node.</d> - <v>CArg = string()</v> - <d>Extra command line arguments to use when starting the shielded node.</d> - </type> - <desc> - <p><c>Fun</c> is executed in a process on a temporarily created - hidden node with a proxy for communication with the test server - node. The node is called a shielded node (should have been called - a shield node). If <c>Fun</c> is successfully executed, the result - is returned. A peer node (see <c>start_node/3</c>) started from - the shielded node will be shielded from test server node, i.e. - they will not be aware of each other. This is useful when you want - to start nodes from earlier OTP releases than the OTP release of - the test server node.</p> - <p>Nodes from an earlier OTP release can normally not be started - if the test server hasn't been started in compatibility mode - (see the <c>+R</c> flag in the <c>erl(1)</c> documentation) of - an earlier release. If a shielded node is started in compatibility - mode of an earlier OTP release than the OTP release of the test - server node, the shielded node can start nodes of an earlier OTP - release.</p> - <note> - <p>You <em>must</em> make sure that nodes started by the shielded - node never communicate directly with the test server node.</p> - </note> - <note> - <p>Slave nodes always communicate with the test server node; - therefore, <em>never</em> start <em>slave nodes</em> from the - shielded node, <em>always</em> start <em>peer nodes</em>.</p> - </note> - </desc> - </func> - <func> - <name>start_node(Name, Type, Options) -> {ok, Node} | {error, Reason}</name> - <fsummary>Start a node.</fsummary> - <type> - <v>Name = atom() | string()</v> - <d>Name of the slavenode to start (as given to -sname or -name)</d> - <v>Type = slave | peer</v> - <d>The type of node to start.</d> - <v>Options = [{atom(), term()]</v> - <d>Tuplelist of options</d> - </type> - <desc> - <p>This functions starts a node, possibly on a remote machine, - and guarantees cross architecture transparency. Type is set to - either <c>slave</c> or <c>peer</c>. - </p> - <p><c>slave</c> means that the new node will have a master, - i.e. the slave node will terminate if the master terminates, - TTY output produced on the slave will be sent back to the - master node and file I/O is done via the master. The master is - normally the target node unless the target is itself a slave. - </p> - <p><c>peer</c> means that the new node is an independent node - with no master. - </p> - <p><c>Options</c> is a tuplelist which can contain one or more - of - </p> - <taglist> - <tag><c>{remote, true}</c></tag> - <item>Start the node on a remote host. If not specified, the - node will be started on the local host. Test cases that - require a remote host will fail with a reasonable comment if - no remote hosts are available at the time they are run. - </item> - <tag><c>{args, Arguments}</c></tag> - <item>Arguments passed directly to the node. This is - typically a string appended to the command line. - </item> - <tag><c>{wait, false}</c></tag> - <item>Don't wait until the node is up. By default, this - function does not return until the node is up and running, - but this option makes it return as soon as the node start - command is given.. - <br></br> -Only valid for peer nodes - </item> - <tag><c>{fail_on_error, false}</c></tag> - <item>Returns <c>{error, Reason}</c> rather than failing the - test case. - <br></br> -Only valid for peer nodes. Note that slave nodes always - act as if they had <c>fail_on_error=false</c></item> - <tag><c>{erl, ReleaseList}</c></tag> - <item>Use an Erlang emulator determined by ReleaseList when - starting nodes, instead of the same emulator as the test - server is running. ReleaseList is a list of specifiers, - where a specifier is either {release, Rel}, {prog, Prog}, or - 'this'. Rel is either the name of a release, e.g., "r12b_patched" - or 'latest'. 'this' means using the same emulator as the test - server. Prog is the name of an emulator executable. If the - list has more than one element, one of them is picked - randomly. (Only works on Solaris and Linux, and the test server - gives warnings when it notices that nodes are not of the same - version as itself.) - <br></br> - <br></br> - - When specifying this option to run a previous release, use - <c>is_release_available/1</c> function to test if the given - release is available and skip the test case if not. - <br></br> - <br></br> - - In order to avoid compatibility problems (may not appear right - away), use a shielded node (see <c>run_on_shielded_node/2</c>) - when starting nodes from different OTP releases than the test - server. - </item> - <tag><c>{cleanup, false}</c></tag> - <item>Tells the test server not to kill this node if it is - still alive after the test case is completed. This is useful - if the same node is to be used by a group of test cases. - </item> - <tag><c>{env, Env}</c></tag> - <item><c>Env</c> should be a list of tuples <c>{Name, Val}</c>, - where <c>Name</c> is the name of an environment variable, and - <c>Val</c> is the value it is to have in the started node. - Both <c>Name</c> and <c>Val</c> must be strings. The one - exception is <c>Val</c> being the atom <c>false</c> (in - analogy with <c>os:getenv/1</c>), which removes the - environment variable. Only valid for peer nodes. Not - available on VxWorks.</item> - <tag><c>{start_cover, false}</c></tag> - <item>By default the test server will start cover on all nodes - when the test is run with code coverage analysis. To make - sure cover is not started on a new node, set this option to - <c>false</c>. This can be necessary if the connection to - the node at some point will be broken but the node is - expected to stay alive. The reason is that a remote cover - node can not continue to run without its main node. Another - solution would be to explicitly stop cover on the node - before breaking the connection, but in some situations (if - old code resides in one or more processes) this is not - possible.</item> - </taglist> - </desc> - </func> - <func> - <name>stop_node(NodeName) -> bool()</name> - <fsummary>Stops a node</fsummary> - <type> - <v>NodeName = term()</v> - <d>Name of the node to stop</d> - </type> - <desc> - <p>This functions stops a node previously started with - <c>start_node/3</c>. Use this function to stop any node you - start, or the test server will produce a warning message in - the test logs, and kill the nodes automatically unless it was - started with the <c>{cleanup, false}</c> option.</p> - </desc> - </func> - <func> - <name>is_commercial() -> bool()</name> - <fsummary>Tests whether the emulator is commercially supported</fsummary> - <desc> - <p>This function test whether the emulator is commercially supported - emulator. The tests for a commercially supported emulator could be more - stringent (for instance, a commercial release should always contain - documentation for all applications).</p> - </desc> - </func> - - <func> - <name>is_release_available(Release) -> bool()</name> - <fsummary>Tests whether a release is available</fsummary> - <type> - <v>Release = string() | atom()</v> - <d>Release to test for</d> - </type> - <desc> - <p>This function test whether the release given by - <c>Release</c> (for instance, "r12b_patched") is available - on the computer that the test_server controller is running on. - Typically, you should skip the test case if not.</p> - <p>Caution: This function may not be called from the <c>suite</c> - clause of a test case, as the test_server will deadlock.</p> - </desc> - </func> - <func> - <name>is_native(Mod) -> bool()</name> - <fsummary>Checks whether the module is natively compiled or not</fsummary> - <type> - <v>Mod = atom()</v> - <d>A module name</d> - </type> - <desc> - <p>Checks whether the module is natively compiled or not</p> - </desc> - </func> - <func> - <name>app_test(App) -> ok | test_server:fail()</name> - <name>app_test(App,Mode)</name> - <fsummary>Checks an applications .app file for obvious errors</fsummary> - <type> - <v>App = term()</v> - <d>The name of the application to test</d> - <v>Mode = pedantic | tolerant</v> - <d>Default is pedantic</d> - </type> - <desc> - <p>Checks an applications .app file for obvious errors. - The following is checked: - </p> - <list type="bulleted"> - <item>required fields - </item> - <item>that all modules specified actually exists - </item> - <item>that all requires applications exists - </item> - <item>that no module included in the application has export_all - </item> - <item>that all modules in the ebin/ dir is included (If - <c>Mode==tolerant</c> this only produces a warning, as all - modules does not have to be included)</item> - </list> - </desc> - </func> - <func> - <name>appup_test(App) -> ok | test_server:fail()</name> - <fsummary>Checks an applications .appup file for obvious errors</fsummary> - <type> - <v>App = term()</v> - <d>The name of the application to test</d> - </type> - <desc> - <p>Checks an applications .appup file for obvious errors. - The following is checked: - </p> - <list type="bulleted"> - <item>syntax - </item> - <item>that .app file version and .appup file version match - </item> - <item>for non-library applications: validity of high-level upgrade - instructions, specifying no instructions is explicitly allowed - (in this case the application is not upgradeable)</item> - <item>for library applications: that there is exactly one wildcard - regexp clause restarting the application when upgrading or - downgrading from any version</item> - </list> - </desc> - </func> - <func> - <name>comment(Comment) -> ok</name> - <fsummary>Print a comment on the HTML result page</fsummary> - <type> - <v>Comment = string()</v> - </type> - <desc> - <p>The given String will occur in the comment field of the - table on the HTML result page. If called several times, only - the last comment is printed. comment/1 is also overwritten by - the return value {comment,Comment} from a test case or by - fail/1 (which prints Reason as a comment).</p> - </desc> - </func> - </funcs> - - <section> - <title>TEST SUITE EXPORTS</title> - <p>The following functions must be exported from a test suite - module. - </p> - </section> - <funcs> - <func> - <name>all(suite) -> TestSpec | {skip, Comment}</name> - <fsummary>Returns the module's test specification</fsummary> - <type> - <v>TestSpec = list()</v> - <v>Comment = string()</v> - <d>This comment will be printed on the HTML result page</d> - </type> - <desc> - <p>This function must return the test specification for the - test suite module. The syntax of a test specification is - described in the Test Server User's Guide.</p> - </desc> - </func> - <func> - <name>init_per_suite(Config0) -> Config1 | {skip, Comment}</name> - <fsummary>Test suite initiation</fsummary> - <type> - <v>Config0 = Config1 = [tuple()]</v> - <v>Comment = string()</v> - <d>Describes why the suite is skipped</d> - </type> - <desc> - <p>This function is called before all other test cases in the - suite. <c>Config</c> is the configuration which can be modified - here. Whatever is returned from this function is given as - <c>Config</c> to the test cases. - </p> - <p>If this function fails, all test cases in the suite will be - skipped.</p> - </desc> - </func> - <func> - <name>end_per_suite(Config) -> void()</name> - <fsummary>Test suite finalization</fsummary> - <type> - <v>Config = [tuple()]</v> - </type> - <desc> - <p>This function is called after the last test case in the - suite, and can be used to clean up whatever the test cases - have done. The return value is ignored.</p> - </desc> - </func> - <func> - <name>init_per_testcase(Case, Config0) -> Config1 | {skip, Comment}</name> - <fsummary>Test case initiation</fsummary> - <type> - <v>Case = atom()</v> - <v>Config0 = Config1 = [tuple()]</v> - <v>Comment = string()</v> - <d>Describes why the test case is skipped</d> - </type> - <desc> - <p>This function is called before each test case. The - <c>Case</c> argument is the name of the test case, and - <c>Config</c> is the configuration which can be modified - here. Whatever is returned from this function is given as - <c>Config</c> to the test case.</p> - </desc> - </func> - <func> - <name>end_per_testcase(Case, Config) -> void()</name> - <fsummary>Test case finalization</fsummary> - <type> - <v>Case = atom()</v> - <v>Config = [tuple()]</v> - </type> - <desc> - <p>This function is called after each test case, and can be - used to clean up whatever the test case has done. The return - value is ignored.</p> - </desc> - </func> - <func> - <name>Case(doc) -> [Decription]</name> - <name>Case(suite) -> [] | TestSpec | {skip, Comment}</name> - <name>Case(Config) -> {skip, Comment} | {comment, Comment} | Ok</name> - <fsummary>A test case</fsummary> - <type> - <v>Description = string()</v> - <d>Short description of the test case</d> - <v>TestSpec = list()</v> - <v>Comment = string()</v> - <d>This comment will be printed on the HTML result page</d> - <v>Ok = term()</v> - <v>Config = [tuple()]</v> - <d>Elements from the Config parameter can be read with the ?config macro, see section about test suite support macros</d> - </type> - <desc> - <p>The <em>documentation clause</em> (argument <c>doc</c>) can - be used for automatic generation of test documentation or test - descriptions. - </p> - <p>The <em>specification clause</em> (argument <c>spec</c>) - shall return an empty list, the test specification for the - test case or <c>{skip,Comment}</c>. The syntax of a test - specification is described in the Test Server User's Guide. - </p> - <p>The <em>execution clause</em> (argument <c>Config</c>) is - only called if the specification clause returns an empty list. - The execution clause is the real test case. Here you must call - the functions you want to test, and do whatever you need to - check the result. If something fails, make sure the process - crashes or call <c>test_server:fail/0/1</c> (which also will - cause the process to crash). - </p> - <p>You can return <c>{skip,Comment}</c> if you decide not to - run the test case after all, e.g. if it is not applicable on - this platform. - </p> - <p>You can return <c>{comment,Comment}</c> if you wish to - print some information in the 'Comment' field on the HTML - result page. - </p> - <p>If the execution clause returns anything else, it is - considered a success, unless it is <c>{'EXIT',Reason}</c> or - <c>{'EXIT',Pid,Reason}</c> which can't be distinguished from a - crash, and thus will be considered a failure. - </p> - <p>A <em>conf test case</em> is a group of test cases with an - init and a cleanup function. The init and cleanup functions - are also test cases, but they have special rules:</p> - <list type="bulleted"> - <item>They do not need a specification clause.</item> - <item>They must always have the execution clause.</item> - <item>They must return the <c>Config</c> parameter, a modified - version of it or <c>{skip,Comment}</c> from the execution clause.</item> - <item>The cleanup function may also return a tuple - <c>{return_group_result,Status}</c>, which is used to return the - status of the conf case to Test Server and/or to a conf case on a - higher level. (<c>Status = ok | skipped | failed</c>).</item> - <item><c>init_per_testcase</c> and <c>end_per_testcase</c> are - not called before and after these functions.</item> - </list> - </desc> - </func> - </funcs> - - - <section> - <title>TEST SUITE SUPPORT MACROS</title> - <p>There are some macros defined in the <c>test_server.hrl</c> - that are quite useful for test suite programmers: - </p> - <p>The <em>config</em> macro, is used to - retrieve information from the <c>Config</c> variable sent to all - test cases. It is used with two arguments, where the first is the - name of the configuration variable you wish to retrieve, and the - second is the <c>Config</c> variable supplied to the test case - from the test server. - </p> - <p>Possible configuration variables include:</p> - <list type="bulleted"> - <item><c>data_dir</c> - Data file directory.</item> - <item><c>priv_dir</c> - Scratch file directory.</item> - <item><c>nodes</c> - Nodes specified in the spec file</item> - <item><c>nodenames</c> - Generated nodenames.</item> - <item>Whatever added by conf test cases or - <c>init_per_testcase/2</c></item> - </list> - <p>Examples of the <c>config</c> macro can be seen in the Examples chapter - in the user's guide.</p> - <p>The <em>line</em> and <em>line_trace</em> macros are deprecated, see - below.</p> - </section> - - <section> - <title>TEST SUITE LINE NUMBERS</title> - <p>In the past, ERTS did not produce line numbers when generating - stacktraces, test_server was thus unable to provide them when reporting - test failures. It had instead two different mecanisms to do it: either by - using the <c>line</c> macro or by using the <c>test_server_line</c> parse - transform. Both are deprecated and should not be used in new tests - anymore.</p> - </section> -</erlref> - diff --git a/lib/test_server/doc/src/test_server_app.xml b/lib/test_server/doc/src/test_server_app.xml deleted file mode 100644 index 4830916561..0000000000 --- a/lib/test_server/doc/src/test_server_app.xml +++ /dev/null @@ -1,75 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE appref SYSTEM "appref.dtd"> - -<appref> - <header> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Test Server Application</title> - <prepared>Siri Hansen</prepared> - <responsible>Peter Andersson</responsible> - <docno></docno> - <approved></approved> - <checked></checked> - <date>2002-07-12</date> - <rev>PA1</rev> - <file>test_server_app.xml</file> - </header> - <app>test_server</app> - <appsummary>Test Server for manual or automatic testing of Erlang code</appsummary> - <description> - <p><em>Test Server</em> is a portable test server for - automated application testing. The server can run test suites - and log progress and results to HTML - pages. The main purpose of Test Server is to act as engine - inside customized test tools. A callback interface for - such framework applications is provided.</p> - <p>In brief the test server supports:</p> - <list type="bulleted"> - <item>Running multiple, concurrent test suites</item> - <item>Test suites may contain other test suites, in a tree fashion</item> - <item>Logging of the events in a test suite, on both suite and case levels</item> - <item>HTML presentation of test suite results</item> - <item>HTML presentation of test suite code</item> - <item>Support for test suite authors, e.g. start/stop slave nodes</item> - <item>Call trace on target and slave nodes</item> - </list> - <p>For information about how to write test cases and test suites, - please see the Test Server User's Guide and the reference - manual for the <c>test_server</c> module. - </p> - <p><em>Common Test</em> is an existing test tool application based on the - OTP Test Server. Please read the Common Test User's Guide for more information. - </p> - </description> - - <section> - <title>Configuration</title> - <p>There are currently no configuration parameters available for - this application. - </p> - </section> - - <section> - <title>SEE ALSO</title> - <p></p> - </section> -</appref> - diff --git a/lib/test_server/doc/src/test_server_ctrl.xml b/lib/test_server/doc/src/test_server_ctrl.xml deleted file mode 100644 index 2762997ece..0000000000 --- a/lib/test_server/doc/src/test_server_ctrl.xml +++ /dev/null @@ -1,844 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>2007</year> - <year>2013</year> - <holder>Ericsson AB, All Rights Reserved</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - The Initial Developer of the Original Code is Ericsson AB. - </legalnotice> - - <title>The Test Server Controller</title> - <prepared>Siri Hansen, Peter Andersson</prepared> - <responsible></responsible> - <docno></docno> - <approved></approved> - <checked></checked> - <date></date> - <rev></rev> - <file>test_server_ctrl_ref.sgml</file> - </header> - <module>test_server_ctrl</module> - <modulesummary>This module provides a low level interface to the Test Server.</modulesummary> - <description> - <p>The <c>test_server_ctrl</c> module provides a low level - interface to the Test Server. This interface is normally - not used directly by the tester, but through a framework built - on top of <c>test_server_ctrl</c>. - </p> - <p>Common Test is such a framework, well suited for automated - black box testing of target systems of any kind (not necessarily - implemented in Erlang). Common Test is also a very useful tool for - white box testing Erlang programs and OTP applications. - Please see the Common Test User's Guide and reference manual for - more information. - </p> - <p>If you want to write your own framework, some more information - can be found in the chapter "Writing your own test server - framework" in the Test Server User's Guide. Details about the - interface provided by <c>test_server_ctrl</c> follows below. - </p> - </description> - <funcs> - <func> - <name>start() -> Result</name> - <fsummary>Starts the test server.</fsummary> - <type> - <v>Result = ok | {error, {already_started, pid()}</v> - </type> - <desc> - <p>This function starts the test server.</p> - </desc> - </func> - <func> - <name>stop() -> ok</name> - <fsummary>Stops the test server immediately.</fsummary> - <desc> - <p>This stops the test server and - all its activity. The running test suite (if any) will be - halted.</p> - </desc> - </func> - <func> - <name>add_dir(Name, Dir) -> ok</name> - <name>add_dir(Name, Dir, Pattern) -> ok</name> - <name>add_dir(Name, [Dir|Dirs]) -> ok</name> - <name>add_dir(Name, [Dir|Dirs], Pattern) -> ok</name> - <fsummary>Add a directory to the job queue.</fsummary> - <type> - <v>Name = term()</v> - <d>The jobname for this directory.</d> - <v>Dir = term()</v> - <d>The directory to scan for test suites.</d> - <v>Dirs = [term()]</v> - <d>List of directories to scan for test suites.</d> - <v>Pattern = term()</v> - <d>Suite match pattern. Directories will be scanned for Pattern_SUITE.erl files.</d> - </type> - <desc> - <p>Puts a collection of suites matching (*_SUITE) in given - directories into the job queue. <c>Name</c> is an arbitrary - name for the job, it can be any erlang term. If <c>Pattern</c> - is given, only modules matching <c>Pattern*</c> will be added.</p> - </desc> - </func> - <func> - <name>add_module(Mod) -> ok</name> - <name>add_module(Name, [Mod|Mods]) -> ok</name> - <fsummary>Add a module to the job queue with or without a given name.</fsummary> - <type> - <v>Mod = atom()</v> - <v>Mods = [atom()]</v> - <d>The name(s) of the module(s) to add.</d> - <v>Name = term()</v> - <d>Name for the job.</d> - </type> - <desc> - <p>This function adds a module or a list of modules, to the - test servers job queue. <c>Name</c> may be any Erlang - term. When <c>Name</c> is not given, the job gets the name of - the module.</p> - </desc> - </func> - <func> - <name>add_case(Mod, Case) -> ok</name> - <fsummary>Adds one test case to the job queue.</fsummary> - <type> - <v>Mod = atom()</v> - <d>Name of the module the test case is in.</d> - <v>Case = atom() </v> - <d>Function name of the test case to add.</d> - </type> - <desc> - <p>This function will add one test case to the job queue. The - job will be given the module's name.</p> - </desc> - </func> - <func> - <name>add_case(Name, Mod, Case) -> ok</name> - <fsummary>Equivalent to add_case/2, but with specified name.</fsummary> - <type> - <v>Name = string()</v> - <d>Name to use for the test job.</d> - </type> - <desc> - <p>Equivalent to <c>add_case/2</c>, but the test job will get - the specified name.</p> - </desc> - </func> - <func> - <name>add_cases(Mod, Cases) -> ok</name> - <fsummary>Adds a list of test cases to the job queue.</fsummary> - <type> - <v>Mod = atom()</v> - <d>Name of the module the test case is in.</d> - <v>Cases = [Case] </v> - <v>Case = atom() </v> - <d>Function names of the test cases to add.</d> - </type> - <desc> - <p>This function will add one or more test cases to the job - queue. The job will be given the module's name.</p> - </desc> - </func> - <func> - <name>add_cases(Name, Mod, Cases) -> ok</name> - <fsummary>Equivalent to add_cases/2, but with specified name.</fsummary> - <type> - <v>Name = string()</v> - <d>Name to use for the test job.</d> - </type> - <desc> - <p>Equivalent to <c>add_cases/2</c>, but the test job will get - the specified name.</p> - </desc> - </func> - <func> - <name>add_spec(TestSpecFile) -> ok | {error, nofile}</name> - <fsummary>Adds a test specification file to the job queue.</fsummary> - <type> - <v>TestSpecFile = string()</v> - <d>Name of the test specification file</d> - </type> - <desc> - <p>This function will add the content of the given test - specification file to the job queue. The job will be given the - name of the test specification file, e.g. if the file is - called <c>test.spec</c>, the job will be called <c>test</c>. - </p> - <p>See the reference manual for the test server application - for details about the test specification file.</p> - </desc> - </func> - <func> - <name>add_dir_with_skip(Name, [Dir|Dirs], Skip) -> ok</name> - <name>add_dir_with_skip(Name, [Dir|Dirs], Pattern, Skip) -> ok</name> - <name>add_module_with_skip(Mod, Skip) -> ok</name> - <name>add_module_with_skip(Name, [Mod|Mods], Skip) -> ok</name> - <name>add_case_with_skip(Mod, Case, Skip) -> ok</name> - <name>add_case_with_skip(Name, Mod, Case, Skip) -> ok</name> - <name>add_cases_with_skip(Mod, Cases, Skip) -> ok</name> - <name>add_cases_with_skip(Name, Mod, Cases, Skip) -> ok</name> - <fsummary>Same purpose as functions listed above, but with extra Skip argument.</fsummary> - <type> - <v>Skip = [SkipItem]</v> - <d>List of items to be skipped from the test.</d> - <v>SkipItem = {Mod,Comment} | {Mod,Case,Comment} | {Mod,Cases,Comment}</v> - <v>Mod = atom()</v> - <d>Test suite name.</d> - <v>Comment = string()</v> - <d>Reason why suite or case is being skipped.</d> - <v>Cases = [Case]</v> - <v>Case = atom()</v> - <d>Name of test case function.</d> - </type> - <desc> - <p>These functions add test jobs just like the add_dir, add_module, - add_case and add_cases functions above, but carry an additional - argument, Skip. Skip is a list of items that should be skipped - in the current test run. Test job items that occur in the Skip - list will be logged as SKIPPED with the associated Comment.</p> - </desc> - </func> - <func> - <name>add_tests_with_skip(Name, Tests, Skip) -> ok</name> - <fsummary>Adds different types of jobs to the run queue.</fsummary> - <type> - <v>Name = term()</v> - <d>The jobname for this directory.</d> - <v>Tests = [TestItem]</v> - <d>List of jobs to add to the run queue.</d> - <v>TestItem = {Dir,all,all} | {Dir,Mods,all} | {Dir,Mod,Cases}</v> - <v>Dir = term()</v> - <d>The directory to scan for test suites.</d> - <v>Mods = [Mod]</v> - <v>Mod = atom()</v> - <d>Test suite name.</d> - <v>Cases = [Case]</v> - <v>Case = atom()</v> - <d>Name of test case function.</d> - <v>Skip = [SkipItem]</v> - <d>List of items to be skipped from the test.</d> - <v>SkipItem = {Mod,Comment} | {Mod,Case,Comment} | {Mod,Cases,Comment}</v> - <v>Comment = string()</v> - <d>Reason why suite or case is being skipped.</d> - </type> - <desc> - <p>This function adds various test jobs to the test_server_ctrl - job queue. These jobs can be of different type (all or specific suites - in one directory, all or specific cases in one suite, etc). It is also - possible to get particular items skipped by passing them along in the - Skip list (see the add_*_with_skip functions above).</p> - </desc> - </func> - <func> - <name>abort_current_testcase(Reason) -> ok | {error,no_testcase_running}</name> - <fsummary>Aborts the test case currently executing.</fsummary> - <type> - <v>Reason = term()</v> - <d>The reason for stopping the test case, which will be printed in the log.</d> - </type> - <desc> - <p>When calling this function, the currently executing test case will be aborted. - It is the user's responsibility to know for sure which test case is currently - executing. The function is therefore only safe to call from a function which - has been called (or synchronously invoked) by the test case.</p> - </desc> - </func> - <func> - <name>set_levels(Console, Major, Minor) -> ok</name> - <fsummary>Sets the levels of I/O.</fsummary> - <type> - <v>Console = integer()</v> - <d>Level for I/O to be sent to console.</d> - <v>Major = integer()</v> - <d>Level for I/O to be sent to the major logfile.</d> - <v>Minor = integer()</v> - <d>Level for I/O to be sent to the minor logfile.</d> - </type> - <desc> - <p>Determines where I/O from test suites/test server will - go. All text output from test suites and the test server is - tagged with a priority value which ranges from 0 to 100, 100 - being the most detailed. (see the section about log files in - the user's guide). Output from the test cases (using - <c>io:format/2</c>) has a detail level of 50. Depending on the - levels set by this function, this I/O may be sent to the - console, the major log file (for the whole test suite) or to - the minor logfile (separate for each test case). - </p> - <p>All output with detail level:</p> - <list type="bulleted"> - <item>Less than or equal to <c>Console</c> is displayed on - the screen (default 1) - </item> - <item>Less than or equal to <c>Major</c> is logged in the - major log file (default 19) - </item> - <item>Greater than or equal to <c>Minor</c> is logged in the - minor log files (default 10) - </item> - </list> - <p>To view the currently set thresholds, use the - <c>get_levels/0</c> function.</p> - </desc> - </func> - <func> - <name>get_levels() -> {Console, Major, Minor}</name> - <fsummary>Returns the current levels.</fsummary> - <desc> - <p>Returns the current levels. See <c>set_levels/3</c> for - types.</p> - </desc> - </func> - <func> - <name>jobs() -> JobQueue</name> - <fsummary>Returns the job queue.</fsummary> - <type> - <v>JobQueue = [{list(), pid()}]</v> - </type> - <desc> - <p>This function will return all the jobs currently in the job - queue.</p> - </desc> - </func> - <func> - <name>multiply_timetraps(N) -> ok</name> - <fsummary>All timetraps started after this will be multiplied by N.</fsummary> - <type> - <v>N = integer() | infinity</v> - </type> - <desc> - <p>This function should be called before a test is started - which requires extended timetraps, e.g. if extensive tracing - is used. All timetraps started after this call will be - multiplied by <c>N</c>.</p> - </desc> - </func> - <func> - <name>scale_timetraps(Bool) -> ok</name> - <fsummary>.</fsummary> - <type> - <v>Bool = true | false</v> - </type> - <desc> - <p>This function should be called before a test is started. - The parameter specifies if test_server should attempt - to automatically scale the timetrap value in order to compensate - for delays caused by e.g. the cover tool.</p> - </desc> - </func> - <func> - <name>get_timetrap_parameters() -> {N,Bool} </name> - <fsummary>Read the parameter values that affect timetraps.</fsummary> - <type> - <v>N = integer() | infinity</v> - <v>Bool = true | false</v> - </type> - <desc> - <p>This function may be called to read the values set by - <c>multiply_timetraps/1</c> and <c>scale_timetraps/1</c>.</p> - </desc> - </func> - <func> - <name>cover(Application,Analyse) -> ok</name> - <name>cover(CoverFile,Analyse) -> ok</name> - <name>cover(App,CoverFile,Analyse) -> ok</name> - <fsummary>Informs the test_server controller that next test shall run with code coverage analysis.</fsummary> - <type> - <v>Application = atom()</v> - <d>OTP application to cover compile</d> - <v>CoverFile = string()</v> - <d>Name of file listing modules to exclude from or include in cover compilation. The filename must include full path to the file.</d> - <v>Analyse = details | overview</v> - </type> - <desc> - <p>This function informs the test_server controller that next - test shall run with code coverage analysis. All timetraps will - automatically be multiplied by 10 when cover i run. - </p> - <p><c>Application</c> and <c>CoverFile</c> indicates what to - cover compile. If <c>Application</c> is given, the default is - that all modules in the <c>ebin</c> directory of the - application will be cover compiled. The <c>ebin</c> directory - is found by adding <c>ebin</c> to - <c>code:lib_dir(Application)</c>. - </p> - <p>A <c>CoverFile</c> can have the following entries:</p> - <code type="none"> -{exclude, all | ExcludeModuleList}. -{include, IncludeModuleList}. -{cross, CrossCoverInfo}.</code> - <p>Note that each line must end with a full - stop. <c>ExcludeModuleList</c> and <c>IncludeModuleList</c> - are lists of atoms, where each atom is a module name. - </p> - - <p><c>CrossCoverInfo</c> is used when collecting cover data - over multiple tests. Modules listed here are compiled, but - they will not be analysed when the test is finished. See - <seealso - marker="#cross_cover_analyse-2">cross_cover_analyse/2</seealso> - for more information about the cross cover mechanism and the - format of <c>CrossCoverInfo</c>. - </p> - <p>If both an <c>Application</c> and a <c>CoverFile</c> is - given, all modules in the application are cover compiled, - except for the modules listed in <c>ExcludeModuleList</c>. The - modules in <c>IncludeModuleList</c> are also cover compiled. - </p> - <p>If a <c>CoverFile</c> is given, but no <c>Application</c>, - only the modules in <c>IncludeModuleList</c> are cover - compiled. - </p> - <p><c>Analyse</c> indicates the detail level of the cover - analysis. If <c>Analyse = details</c>, each cover compiled - module will be analysed with - <c>cover:analyse_to_file/1</c>. If <c>Analyse = overview</c> - an overview of all cover compiled modules is created, listing - the number of covered and not covered lines for each module. - </p> - <p>If the test following this call starts any slave or peer - nodes with <c>test_server:start_node/3</c>, the same cover - compiled code will be loaded on all nodes. If the loading - fails, e.g. if the node runs an old version of OTP, the node - will simply not be a part of the coverage analysis. Note that - slave or peer nodes must be stopped with - <c>test_server:stop_node/1</c> for the node to be part of the - coverage analysis, else the test server will not be able to - fetch coverage data from the node. - </p> - <p>When the test is finished, the coverage analysis is - automatically completed, logs are created and the cover - compiled modules are unloaded. If another test is to be run - with coverage analysis, <c>test_server_ctrl:cover/2/3</c> must - be called again. - </p> - </desc> - </func> - <func> - <name>cross_cover_analyse(Level, Tests) -> ok</name> - <fsummary>Analyse cover data collected from multiple tests</fsummary> - <type> - <v>Level = details | overview</v> - <v>Tests = [{Tag,LogDir}]</v> - <v>Tag = atom()</v> - <d>Test identifier.</d> - <v>LogDir = string()</v> - <d>Log directory for the test identified by <c>Tag</c>. This - can either be the <c>run.<timestamp></c> directory or - the parent directory of this (in which case the latest - <c>run.<timestamp></c> directory is chosen.</d> - </type> - <desc> - <p>Analyse cover data collected from multiple tests. The modules - analysed are the ones listed in <c>cross</c> statements in - the cover files. These are modules that are heavily used by - other tests than the one where they belong or are explicitly - tested. They should then be listed as cross modules in the - cover file for the test where they are used but do not - belong. Se example below.</p> - <p>This function should be run after all tests are completed, - and the result will be stored in a file called - <c>cross_cover.html</c> in the <c>run.<timestamp></c> - directory of the test the modules belong to.</p> - <p>Note that the function can be executed on any node, and it - does not require <c>test_server_ctrl</c> to be started first.</p> - <p>The <c>cross</c> statement in the cover file must be like this:</p> - <code type="none"> -{cross,[{Tag,Modules}]}.</code> - <p>where <c>Tag</c> is the same as <c>Tag</c> in the - <c>Tests</c> parameter to this function and <c>Modules</c> is a - list of module names (atoms).</p> - <p><em>Example:</em></p> - <p>If the module <c>m1</c> belongs to system <c>s1</c> but is - heavily used also in the tests for another system <c>s2</c>, - then the cover files for the two systems' tests could be like - this:</p> -<code type="none"> -s1.cover: - {include,[m1]}. - -s2.cover: - {include,[....]}. % modules belonging to system s2 - {cross,[{s1,[m1]}]}.</code> - <p>When the tests for both <c>s1</c> and <c>s2</c> are completed, run</p> -<code type="none"> -test_server_ctrl:cross_cover_analyse(Level,[{s1,S1LogDir},{s2,S2LogDir}]) -</code> - - <p>and the accumulated cover data for <c>m1</c> will be written to - <c>S1LogDir/[run.<timestamp>/]cross_cover.html</c>.</p> - <p>Note that the <c>m1</c> module will also be presented in the - normal coverage log for <c>s1</c> (due to the include statement in - <c>s1.cover</c>), but that only includes the coverage achieved by the - <c>s1</c> test itself.</p> - <p>The Tag in the <c>cross</c> statement in the cover file has - no other purpose than mapping the list of modules - (<c>[m1]</c> in the example above) to the correct log - directory where it should be included in the - <c>cross_cover.html</c> file (<c>S1LogDir</c> in the example - above). I.e. the value of <c>Tag</c> has no meaning, it - could be <c>foo</c> as well as <c>s1</c> above, as long as - the same <c>Tag</c> is used in the cover file and in the - call to this function.</p> - </desc> - </func> - <func> - <name>trc(TraceInfoFile) -> ok | {error, Reason}</name> - <fsummary>Starts call trace on target and slave nodes</fsummary> - <type> - <v>TraceInfoFile = atom() | string()</v> - <d>Name of a file defining which functions to trace and how</d> - </type> - <desc> - <p>This function starts call trace on target and on slave or - peer nodes that are started or will be started by the test - suites. - </p> - <p>Timetraps are not extended automatically when tracing is - used. Use <c>multiply_timetraps/1</c> if necessary. - </p> - <p>Note that the trace support in the test server is in a very - early stage of the implementation, and thus not yet as - powerful as one might wish for. - </p> - <p>The trace information file specified by the - <c>TraceInfoFile</c> argument is a text file containing one or - more of the following elements: - </p> - <list type="bulleted"> - <item><c>{SetTP,Module,Pattern}.</c></item> - <item><c>{SetTP,Module,Function,Pattern}.</c></item> - <item><c>{SetTP,Module,Function,Arity,Pattern}.</c></item> - <item><c>ClearTP.</c></item> - <item><c>{ClearTP,Module}.</c></item> - <item><c>{ClearTP,Module,Function}.</c></item> - <item><c>{ClearTP,Module,Function,Arity}.</c></item> - </list> - <taglist> - <tag><c>SetTP = tp | tpl</c></tag> - <item>This is maps to the corresponding functions in the - <c>ttb</c> module in the <c>observer</c> - application. <c>tp</c> means set trace pattern on global - function calls. <c>tpl</c> means set trace pattern on local - and global function calls. - </item> - <tag><c>ClearTP = ctp | ctpl | ctpg</c></tag> - <item>This is maps to the corresponding functions in the - <c>ttb</c> module in the <c>observer</c> - application. <c>ctp</c> means clear trace pattern (i.e. turn - off) on global and local function calls. <c>ctpl</c> means - clear trace pattern on local function calls only and <c>ctpg</c> - means clear trace pattern on global function calls only. - </item> - <tag><c>Module = atom()</c></tag> - <item>The module to trace - </item> - <tag><c>Function = atom()</c></tag> - <item>The name of the function to trace - </item> - <tag><c>Arity = integer()</c></tag> - <item>The arity of the function to trace - </item> - <tag><c>Pattern = [] | match_spec()</c></tag> - <item>The trace pattern to set for the module or - function. For a description of the match_spec() syntax, - please turn to the User's guide for the runtime system - (erts). The chapter "Match Specification in Erlang" explains - the general match specification language. - </item> - </taglist> - <p>The trace result will be logged in a (binary) file called - <c>NodeName-test_server</c> in the current directory of the - test server controller node. The log must be formatted using - <c>ttb:format/1/2</c>. - </p> - </desc> - </func> - <func> - <name>stop_trace() -> ok | {error, not_tracing}</name> - <fsummary>Stops tracing on target and slave nodes.</fsummary> - <desc> - <p>This function stops tracing on target, and on slave or peer - nodes that are currently running. New slave or peer nodes will - no longer be traced after this.</p> - </desc> - </func> - </funcs> - - <section> - <title>FUNCTIONS INVOKED FROM COMMAND LINE</title> - <p>The following functions are supposed to be invoked from the - command line using the <c>-s</c> option when starting the erlang - node.</p> - </section> - <funcs> - <func> - <name>run_test(CommandLine) -> ok</name> - <fsummary>Runs the tests specified on the command line.</fsummary> - <type> - <v>CommandLine = FlagList</v> - </type> - <desc> - <p>This function is supposed to be invoked from the - commandline. It starts the test server, interprets the - argument supplied from the commandline, runs the tests - specified and when all tests are done, stops the test server - and returns to the Erlang prompt. - </p> - <p>The <c>CommandLine</c> argument is a list of command line - flags, typically <c>['KEY1', Value1, 'KEY2', Value2, ...]</c>. - The valid command line flags are listed below. - </p> - <p>Under a UNIX command prompt, this function can be invoked like this: - <br></br> -<c>erl -noshell -s test_server_ctrl run_test KEY1 Value1 KEY2 Value2 ... -s erlang halt</c></p> - <p>Or make an alias (this is for unix/tcsh) <br></br> -<c>alias erl_test 'erl -noshell -s test_server_ctrl run_test \!* -s erlang halt'</c></p> - <p>And then use it like this <br></br> -<c>erl_test KEY1 Value1 KEY2 Value2 ...</c> <br></br> -</p> - <p>The valid command line flags are</p> - <taglist> - <tag><c>DIR dir</c></tag> - <item>Adds all test modules in the directory <c>dir</c> to - the job queue. - </item> - <tag><c>MODULE mod</c></tag> - <item>Adds the module <c>mod</c> to the job queue. - </item> - <tag><c>CASE mod case</c></tag> - <item>Adds the case <c>case</c> in module <c>mod</c> to the - job queue. - </item> - <tag><c>SPEC spec</c></tag> - <item>Runs the test specification file <c>spec</c>. - </item> - <tag><c>SKIPMOD mod</c></tag> - <item>Skips all test cases in the module <c>mod</c></item> - <tag><c>SKIPCASE mod case</c></tag> - <item>Skips the test case <c>case</c> in module <c>mod</c>. - </item> - <tag><c>NAME name</c></tag> - <item>Names the test suite to something else than the - default name. This does not apply to <c>SPEC</c> which keeps - its names. - </item> - <tag><c>COVER app cover_file analyse</c></tag> - <item>Indicates that the test should be run with cover - analysis. <c>app</c>, <c>cover_file</c> and <c>analyse</c> - corresponds to the parameters to - <c>test_server_ctrl:cover/3</c>. If no cover file is used, - the atom <c>none</c> should be given. - </item> - <tag><c>TRACE traceinfofile</c></tag> - <item>Specifies a trace information file. When this option - is given, call tracing is started on the target node and all - slave or peer nodes that are started. The trace information - file specifies which modules and functions to trace. See the - function <c>trc/1</c> above for more information about the - syntax of this file. - </item> - </taglist> - </desc> - </func> - </funcs> - - <section> - <title>FRAMEWORK CALLBACK FUNCTIONS</title> - <p>A test server framework can be defined by setting the - environment variable <c>TEST_SERVER_FRAMEWORK</c> to a module - name. This module will then be framework callback module, and it - must export the following function:</p> - </section> - <funcs> - <func> - <name>get_suite(Mod,Func) -> TestCaseList</name> - <fsummary>Get subcases.</fsummary> - <type> - <v>Mod = atom()</v> - <d>Test suite name.</d> - <v>Func = atom()</v> - <d>Name of test case.</d> - <v>TestCaseList = [SubCase]</v> - <d>List of test cases.</d> - <v>SubCase = atom()</v> - <d>Name of a case.</d> - </type> - <desc> - <p>This function is called before a test case is started. The - purpose is to retrieve a list of subcases. The default - behaviour of this function should be to call - <c>Mod:Func(suite)</c> and return the result from this call.</p> - </desc> - </func> - <func> - <name>init_tc(Mod,Func,Args0) -> {ok,Args1} | {skip,ReasonToSkip} | {auto_skip,ReasonToSkip} | {fail,ReasonToFail}</name> - <fsummary>Preparation for a test case or configuration function.</fsummary> - <type> - <v>Mod = atom()</v> - <d>Test suite name.</d> - <v>Func = atom()</v> - <d>Name of test case or configuration function.</d> - <v>Args0 = Args1 = [tuple()]</v> - <d>Normally Args = [Config]</d> - <v>ReasonToSkip = term()</v> - <d>Reason to skip the test case or configuration function.</d> - <v>ReasonToFail = term()</v> - <d>Reason to fail the test case or configuration function.</d> - </type> - <desc> - <p>This function is called before a test case or configuration - function starts. It is called on the process executing the function - <c>Mod:Func</c>. Typical use of this function can be to alter - the input parameters to the test case function (<c>Args</c>) or - to set properties for the executing process.</p> - <p>By returning <c>{skip,Reason}</c>, <c>Func</c> gets skipped. - <c>Func</c> also gets skipped if <c>{auto_skip,Reason}</c> is returned, - but then gets an auto skipped status (rather than user skipped).</p> - <p>To fail <c>Func</c> immediately instead of executing it, return - <c>{fail,ReasonToFail}.</c></p> - </desc> - </func> - <func> - <name>end_tc(Mod,Func,Status) -> ok | {fail,ReasonToFail}</name> - <fsummary>Cleanup after a test case or configuration function.</fsummary> - <type> - <v>Mod = atom()</v> - <d>Test suite name.</d> - <v>Func = atom()</v> - <d>Name of test case or configuration function.</d> - <v>Status = {Result,Args} | {TCPid,Result,Args}</v> - <d>The status of the test case or configuration function.</d> - <v>ReasonToFail = term()</v> - <d>Reason to fail the test case or configuration function.</d> - <v>Result = ok | Skip | Fail</v> - <d>The final result of the test case or configuration function.</d> - <v>TCPid = pid()</v> - <d>Pid of the process executing Func</d> - <v>Skip = {skip,SkipReason}</v> - <v>SkipReason = term() | {failed,{Mod,init_per_testcase,term()}}</v> - <d>Reason why the function was skipped.</d> - <v>Fail = {error,term()} | {'EXIT',term()} | {timetrap_timeout,integer()} | - {testcase_aborted,term()} | testcase_aborted_or_killed | - {failed,term()} | {failed,{Mod,end_per_testcase,term()}}</v> - <d>Reason why the function failed.</d> - <v>Args = [tuple()]</v> - <d>Normally Args = [Config]</d> - </type> - <desc> - <p>This function is called when a test case, or a configuration function, - is finished. It is normally called on the process where the function - <c>Mod:Func</c> has been executing, but if not, the pid of the test - case process is passed with the <c>Status</c> argument.</p> - <p>Typical use of the <c>end_tc/3</c> function can be to clean up - after <c>init_tc/3</c>.</p> - <p>If <c>Func</c> is a test case, it is possible to analyse the value of - <c>Result</c> to verify that <c>init_per_testcase/2</c> and - <c>end_per_testcase/2</c> executed successfully.</p> - <p>It is possible with <c>end_tc/3</c> to fail an otherwise successful - test case, by returning <c>{fail,ReasonToFail}</c>. The test case <c>Func</c> - will be logged as failed with the provided term as reason.</p> - </desc> - </func> - <func> - <name>report(What,Data) -> ok</name> - <fsummary>Progress report for test.</fsummary> - <type> - <v>What = atom()</v> - <v>Data = term()</v> - </type> - <desc> - <p>This function is called in order to keep the framework up-to-date with - the progress of the test. This is useful e.g. if the - framework implements a GUI where the progress information is - constantly updated. The following can be reported: - </p> - <p><c>What = tests_start, Data = {Name,NumCases}</c><br></br> - <c>What = loginfo, Data = [{topdir,TestRootDir},{rundir,CurrLogDir}]</c><br></br> - <c>What = tests_done, Data = {Ok,Failed,{UserSkipped,AutoSkipped}}</c><br></br> - <c>What = tc_start, Data = {{Mod,{Func,GroupName}},TCLogFile}</c><br></br> - <c>What = tc_done, Data = {Mod,{Func,GroupName},Result}</c><br></br> - <c>What = tc_user_skip, Data = {Mod,{Func,GroupName},Comment}</c><br></br> - <c>What = tc_auto_skip, Data = {Mod,{Func,GroupName},Comment}</c><br></br> - <c>What = framework_error, Data = {{FWMod,FWFunc},Error}</c></p> - <p>Note that for a test case function that doesn't belong to a group, - <c>GroupName</c> has value <c>undefined</c>, otherwise the name of the test - case group.</p> - </desc> - </func> - <func> - <name>error_notification(Mod, Func, Args, Error) -> ok</name> - <fsummary>Inform framework of crashing testcase or configuration function.</fsummary> - <type> - <v>Mod = atom()</v> - <d>Test suite name.</d> - <v>Func = atom()</v> - <d>Name of test case or configuration function.</d> - <v>Args = [tuple()]</v> - <d>Normally Args = [Config]</d> - <v>Error = {Reason,Location}</v> - <v>Reason = term()</v> - <d>Reason for termination.</d> - <v>Location = unknown | [{Mod,Func,Line}]</v> - <d>Last known position in Mod before termination.</d> - <v>Line = integer()</v> - <d>Line number in file Mod.erl.</d> - </type> - <desc> - <p>This function is called as the result of function <c>Mod:Func</c> failing - with Reason at Location. The function is intended mainly to aid - specific logging or error handling in the framework application. Note - that for Location to have relevant values (i.e. other than unknown), - the <c>line</c> macro or <c>test_server_line</c> parse transform must - be used. For details, please see the section about test suite line numbers - in the <c>test_server</c> reference manual page.</p> - </desc> - </func> - <func> - <name>warn(What) -> boolean()</name> - <fsummary>Ask framework if test server should issue a warning for What.</fsummary> - <type> - <v>What = processes | nodes</v> - </type> - <desc> - <p>The test server checks the number of processes and nodes - before and after the test is executed. This function is a - question to the framework if the test server should warn when - the number of processes or nodes has changed during the test - execution. If <c>true</c> is returned, a warning will be written - in the test case minor log file.</p> - </desc> - </func> - <func> - <name>target_info() -> InfoStr</name> - <fsummary>Print info about the target system to the test case log.</fsummary> - <type> - <v>InfoStr = string() | ""</v> - </type> - <desc> - <p>The test server will ask the framework for information about - the test target system and print InfoStr in the test case - log file below the host information.</p> - </desc> - </func> - </funcs> -</erlref> - diff --git a/lib/test_server/doc/src/test_spec_chapter.xml b/lib/test_server/doc/src/test_spec_chapter.xml deleted file mode 100644 index 0a62010364..0000000000 --- a/lib/test_server/doc/src/test_spec_chapter.xml +++ /dev/null @@ -1,375 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Test Structure and Test Specifications</title> - <prepared>Siri Hansen</prepared> - <docno></docno> - <date></date> - <rev></rev> - <file>test_spec_chapter.xml</file> - </header> - - <section> - <title>Test structure</title> - <p>A test consists of a set of test cases. Each test case is - implemented as an erlang function. An erlang module implementing - one or more test cases is called a test suite. - </p> - </section> - - <section> - <title>Test specifications</title> - <p>A test specification is a specification of which test suites - and test cases to run and which to skip. A test specification can - also group several test cases into conf cases with init and - cleanup functions (see section about configuration cases - below). In a test there can be test specifications on three - different levels: - </p> - <p>The top level is a test specification file which roughly - specifies what to test for a whole application. The test - specification in such a file is encapsulated in a topcase - command. - </p> - <p>Then there is a test specification for each test suite, - specifying which test cases to run within the suite. The test - specification for a test suite is returned from the - <c>all(suite)</c> function in the test suite module. - </p> - <p>And finally there can be a test specification per test case, - specifying sub test cases to run. The test specification for a - test case is returned from the specification clause of the test - case. - </p> - <p>When a test starts, the total test specification is built in a - tree fashion, starting from the top level test specification. - </p> - <p>The following are the valid elements of a test - specification. The specification can be one of these elements or a - list with any combination of the elements: - </p> - <taglist> - <tag><c>{Mod, Case}</c></tag> - <item>This specifies the test case Mod:Case/1 - </item> - <tag><c>{dir, Dir}</c></tag> - <item>This specifies all modules <c>*_SUITE</c> in the directory - <c>Dir</c></item> - <tag><c>{dir, Dir, Pattern}</c></tag> - <item>This specifies all modules <c>Pattern*</c> in the - directory <c>Dir</c></item> - <tag><c>{conf, Init, TestSpec, Fin}</c></tag> - <item>This is a configuration case. In a test specification - file, <c>Init</c> and <c>Fin</c> must be - <c>{Mod,Func}</c>. Inside a module they can also be just - <c>Func</c>. See the section named Configuration Cases below for - more information about this. - </item> - <tag><c>{conf, Properties, Init, TestSpec, Fin}</c></tag> - <item>This is a configuration case as explained above, but - which also takes a list of execution properties for its group - of test cases and nested sub-groups. - </item> - <tag><c>{make, Init, TestSpec, Fin}</c></tag> - <item>This is a special version of a conf case which is only - used by the test server framework <c>ts</c>. <c>Init</c> and - <c>Fin</c> are make and unmake functions for a data - directory. <c>TestSpec</c> is the test specification for the - test suite owning the data directory in question. If the make - function fails, all tests in the test suite are skipped. The - difference between this "make case" and a normal conf case is - that for the make case, <c>Init</c> and <c>Fin</c> are given with - arguments (<c>{Mod,Func,Args}</c>). - </item> - <tag><c>Case</c></tag> - <item>This can only be used inside a module, i.e. not a test - specification file. It specifies the test case - <c>CurrentModule:Case</c>. - </item> - </taglist> - </section> - - <section> - <title>Test Specification Files</title> - <p>A test specification file is a text file containing the top - level test specification (a topcase command), and possibly one or - more additional commands. A "command" in a test specification file - means a key-value tuple ended by a dot-newline sequence. - </p> - <p>The following commands are valid: - </p> - <taglist> - <tag><c>{topcase, TestSpec}</c></tag> - <item>This command is mandatory in all test specification - files. <c>TestSpec</c> is the top level test specification of a - test. - </item> - <tag><c>{skip, {Mod, Comment}}</c></tag> - <item>This specifies that all cases in the module <c>Mod</c> - shall be skipped. <c>Comment</c> is a string. - </item> - <tag><c>{skip, {Mod, Case, Comment}}</c></tag> - <item>This specifies that the case <c>Mod:Case</c> shall be - skipped. - </item> - <tag><c>{skip, {Mod, CaseList, Comment}}</c></tag> - <item>This specifies that all cases <c>Mod:Case</c>, where - <c>Case</c> is in <c>CaseList</c>, shall be skipped. - </item> - <tag><c>{nodes, Nodes}</c></tag> - <item><c>Nodes</c> is a list of nodenames available to the test - suite. It will be added to the <c>Config</c> argument to all - test cases. <c>Nodes</c> is a list of atoms. - </item> - <tag><c>{require_nodenames, Num}</c></tag> - <item>Specifies how many nodenames the test suite will - need. Theese will be automatically generated and inserted into the - <c>Config</c> argument to all test cases. <c>Num</c> is an - integer. - </item> - <tag><c>{hosts, Hosts}</c></tag> - <item>This is a list of available hosts on which to start slave - nodes. It is used when the <c>{remote, true}</c> option is given - to the <c>test_server:start_node/3</c> function. Also, if - <c>{require_nodenames, Num}</c> is contained in a test - specification file, the generated nodenames will be spread over - all hosts given in this <c>Hosts</c> list. The hostnames are - atoms or strings. - </item> - <tag><c>{diskless, true}</c></tag> - <item>Adds <c>{diskless, true}</c> to the <c>Config</c> argument - to all test cases. This is kept for backwards compatibility and - should not be used. Use a configuration case instead. - </item> - <tag><c>{ipv6_hosts, Hosts}</c></tag> - <item>Adds <c>{ipv6_hosts, Hosts}</c> to the <c>Config</c> - argument to all test cases.</item> - </taglist> - <p>All test specification files shall have the extension - ".spec". If special test specification files are needed for - Windows or VxWorks platforms, additional files with the - extension ".spec.win" and ".spec.vxworks" shall be - used. This is useful e.g. if some test cases shall be skipped on - these platforms. - </p> - <p>Some examples for test specification files can be found in the - Examples section of this user's guide. - </p> - </section> - - <section> - <title>Configuration cases</title> - <p>If a group of test cases need the same initialization, a so called - <em>configuration</em> or <em>conf</em> case can be used. A conf - case consists of an initialization function, the group of test cases - needing this initialization and a cleanup or finalization function. - </p> - <p>If the init function in a conf case fails or returns - <c>{skip,Comment}</c>, the rest of the test cases in the conf case - (including the cleanup function) are skipped. If the init function - succeeds, the cleanup function will always be called, even if some - of the test cases in between failed. - </p> - <p>Both the init function and the cleanup function in a conf case - get the <c>Config</c> parameter as only argument. This parameter - can be modified or returned as is. Whatever is returned by the - init function is given as <c>Config</c> parameter to the rest of - the test cases in the conf case, including the cleanup function. - </p> - <p>If the <c>Config</c> parameter is changed by the init function, - it must be restored by the cleanup function. Whatever is returned - by the cleanup function will be given to the next test case called. - </p> - <p>The optional <c>Properties</c> list can be used to specify - execution properties for the test cases and possibly nested - sub-groups of the configuration case. The available properties are:</p> - <pre> - Properties = [parallel | sequence | Shuffle | {RepeatType,N}] - Shuffle = shuffle | {shuffle,Seed} - Seed = {integer(),integer(),integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | - repeat_until_any_ok | repeat_until_any_fail - N = integer() | forever</pre> - - <p>If the <c>parallel</c> property is specified, Test Server will execute - all test cases in the group in parallel. If <c>sequence</c> is specified, - the cases will be executed in a sequence, meaning if one case fails, all - following cases will be skipped. If <c>shuffle</c> is specified, the cases - in the group will be executed in random order. The <c>repeat</c> property - orders Test Server to repeat execution of the cases in the group a given - number of times, or until any, or all, cases fail or succeed.</p> - - <p>Properties may be combined so that e.g. if <c>shuffle</c>, - <c>repeat_until_any_fail</c> and <c>sequence</c> are all specified, the test - cases in the group will be executed repeatedly and in random order until - a test case fails, when execution is immediately stopped and the rest of - the cases skipped.</p> - - <p>The properties for a conf case is always printed on the top of the HTML log - for the group's init function. Also, the total execution time for a conf case - can be found at the bottom of the log for the group's end function.</p> - - <p>Configuration cases may be nested so that sets of grouped cases can be - configured with the same init- and end functions.</p> - </section> - - <section> - <title>The parallel property and nested configuration cases</title> - <p>If a conf case has a parallel property, its test cases will be spawned - simultaneously and get executed in parallel. A test case is not allowed - to execute in parallel with the end function however, which means - that the time it takes to execute a set of parallel cases is equal to the - execution time of the slowest test case in the group. A negative side - effect of running test cases in parallel is that the HTML summary pages - are not updated with links to the individual test case logs until the - end function for the conf case has finished.</p> - - <p>A conf case nested under a parallel conf case will start executing in - parallel with previous (parallel) test cases (no matter what properties the - nested conf case has). Since, however, test cases are never executed in - parallel with the init- or the end function of the same conf case, it's - only after a nested group of cases has finished that any remaining parallel - cases in the previous conf case get spawned.</p> - </section> - - <section> - <title>Repeated execution of test cases</title> - <marker id="repeated_cases"></marker> - <p>A conf case may be repeated a certain number of times - (specified by an integer) or indefinitely (specified by <c>forever</c>). - The repetition may also be stopped prematurely if any or all cases - fail or succeed, i.e. if the property <c>repeat_until_any_fail</c>, - <c>repeat_until_any_ok</c>, <c>repeat_until_all_fail</c>, or - <c>repeat_until_all_ok</c> is used. If the basic <c>repeat</c> - property is used, status of test cases is irrelevant for the repeat - operation.</p> - - <p>It is possible to return the status of a conf case (ok or - failed), to affect the execution of the conf case on the level above. - This is accomplished by, in the end function, looking up the value - of <c>tc_group_properties</c> in the <c>Config</c> list and checking the - result of the finished test cases. If status <c>failed</c> should be - returned from the conf case as a result, the end function should return - the value <c>{return_group_result,failed}</c>. The status of a nested conf - case is taken into account by Test Server when deciding if execution - should be repeated or not (unless the basic <c>repeat</c> property is used).</p> - - <p>The <c>tc_group_properties</c> value is a list of status tuples, - each with the key <c>ok</c>, <c>skipped</c> and <c>failed</c>. The - value of a status tuple is a list containing names of test cases - that have been executed with the corresponding status as result.</p> - - <p>Here's an example of how to return the status from a conf case:</p> - <pre> - conf_end_function(Config) -> - Status = ?config(tc_group_result, Config), - case proplists:get_value(failed, Status) of - [] -> % no failed cases - {return_group_result,ok}; - _Failed -> % one or more failed - {return_group_result,failed} - end.</pre> - - <p>It is also possible in the end function to check the status of - a nested conf case (maybe to determine what status the current conf case should - return). This is as simple as illustrated in the example above, only the - name of the end function of the nested conf case is stored in a tuple - <c>{group_result,EndFunc}</c>, which can be searched for in the status lists. - Example:</p> - <pre> - conf_end_function_X(Config) -> - Status = ?config(tc_group_result, Config), - Failed = proplists:get_value(failed, Status), - case lists:member({group_result,conf_end_function_Y}, Failed) of - true -> - {return_group_result,failed}; - false -> - {return_group_result,ok} - end; - ...</pre> - - <note><p>When a conf case is repeated, the init- and end functions - are also always called with each repetition.</p></note> - </section> - - <section> - <title>Shuffled test case order</title> - <p>The order that test cases in a conf case are executed, is under normal - circumstances the same as the order defined in the test specification. - With the <c>shuffle</c> property set, however, Test Server will instead - execute the test cases in random order.</p> - - <p>The user may provide a seed value (a tuple of three integers) with - the shuffle property: <c>{shuffle,Seed}</c>. This way, the same shuffling - order can be created every time the conf case is executed. If no seed value - is given, Test Server creates a "random" seed for the shuffling operation - (using the return value of <c>erlang:now()</c>). The seed value is always - printed to the log file of the init function so that it can be used to - recreate the same execution order in subsequent test runs.</p> - - <note><p>If execution of a conf case with shuffled test cases is repeated, - the seed will not be reset in between turns.</p></note> - - <p>If a nested conf case is specified in a conf case with a <c>shuffle</c> - property, the execution order of the nested cases in relation to the test cases - (and other conf cases) is also random. The order of the test cases in the nested - conf case is however not random (unless, of course, this one also has a - <c>shuffle</c> property).</p> - </section> - - <section> - <title>Skipping test cases</title> - <p>It is possible to skip certain test cases, for example if you - know beforehand that a specific test case fails. This might be - functionality which isn't yet implemented, a bug that is known but - not yet fixed or some functionality which doesn't work or isn't - applicable on a specific platform. - </p> - <p>There are several different ways to state that a test case - should be skipped:</p> - <list type="bulleted"> - <item>Using the <c>{skip,What}</c> command in a test - specification file - </item> - <item>Returning <c>{skip,Reason}</c> from the - <c>init_per_testcase/2</c> function - </item> - <item>Returning <c>{skip,Reason}</c> from the specification - clause of the test case - </item> - <item>Returning <c>{skip,Reason}</c> from the execution clause - of the test case - </item> - </list> - <p>The latter of course means that the execution clause is - actually called, so the author must make sure that the test case - is not run. For more information about the different clauses in a - test case, see the chapter about writing test cases. - </p> - <p>When a test case is skipped, it will be noted as <c>SKIPPED</c> - in the HTML log. - </p> - </section> -</chapter> - diff --git a/lib/test_server/doc/src/ts.xml b/lib/test_server/doc/src/ts.xml deleted file mode 100644 index 60dfdbc545..0000000000 --- a/lib/test_server/doc/src/ts.xml +++ /dev/null @@ -1,568 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>2007</year> - <year>2013</year> - <holder>Ericsson AB, All Rights Reserved</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - The Initial Developer of the Original Code is Ericsson AB. - </legalnotice> - - <title>The OTP Test Server Framework</title> - <prepared>Mattias Nilsson</prepared> - <responsible></responsible> - <docno></docno> - <approved></approved> - <checked></checked> - <date></date> - <rev></rev> - <file>ts.xml</file> - </header> - <module>ts</module> - <modulesummary>Test Server Framework for testing OTP</modulesummary> - <description> - <p>This is a framework for testing OTP. The <c>ts</c> module - implements the interface to all the functionality in the - framework. - </p> - <p>The framework is built on top of the Test Server Controller, - <c>test_server_ctrl</c>, and provides a high level operator - interface. The main features added by the framework are: - </p> - <list type="bulleted"> - <item>Automatic compilation of test suites and data directories - </item> - <item>Collection of files in central directories and creation of - additional HTML pages for better overview. - </item> - <item>Single command interface for running all available tests - </item> - <item>Spawns a new node with correct parameters before starting - the test server - </item> - <item>Atomatically creates the parameter file needed when - running tests on remote target - </item> - </list> - <p>More information about the Test Server Framework and how to run - test cases can be found in the Test Server User's Guide. - </p> - <p>For writing you own test server framework, please turn to the - reference manual for the Test Server Controller and chapter named - "Writing your own test server framework" in the Test Server User's - Guide. - </p> - <p>SETUP</p> - <p>To be able to run <c>ts</c>, you must first `install' - <c>ts</c> for the current environment. This is done by calling - <c>ts:install/0/1/2</c>. A file called `variables' is created - and used by <c>ts</c> when running test suites. It is not - recommended to edit this file, but it is possible to alter if - <c>ts</c> gets the wrong idea about your environment. - </p> - <p><c>ts:install/0</c> is used if the target platform is the - same as the controller host, i.e. if you run on "local target" - and no options are needed. Then running <c>ts:install/0</c> <c>ts</c> - will run an autoconf script for your current - environment and set up the necessary variables needed by the - test suites. - </p> - <p><c>ts:install/1</c> or <c>ts:install/2</c> is used if the - target platform is different from the controller host, i.e. if - you run on "remote target" or if special options are required - for your system. - </p> - <p>See the reference manual for detailed information about - <c>ts:install/0/1/2</c>. - </p> - <p>Some of the common variables in the 'variables' file are - described below. Do not make any assumptions as of what is found - in this file, as it may change at any time. - </p> - <list> - <item><c>longnames</c><br></br> - Set to true if the system is using fully qualified - nodenames. - </item> - <item><c>platform_id</c><br></br> - This is the currently installed platform identification - string. - </item> - <item><c>platform_filename</c><br></br> - This is the name used to create the final save directory - for test runs. - </item> - <item><c>platform_label</c><br></br> - This is the string presented in the generated test - results index page. - </item> - <item><c>rsh_name</c><br></br> - This is the rsh program to use when starting slave or - peer nodes on a remote host. - </item> - <item><c>erl_flags</c><br></br> - Compile time flags used when compiling test suites. - </item> - <item><c>erl_release</c><br></br> - The Erlang/OTP release being tested. - </item> - <item><c>'EMULATOR'</c><br></br> - The emulator being tested (e.g. beam) - </item> - <item><c>'CPU'</c><br></br> - The CPU in the machine running the tests, e.g. sparc. - </item> - <item><c>target_host</c><br></br> - The target host name - </item> - <item><c>os</c><br></br> - The target operating system, e.g. solaris2.8 - </item> - <item><c>target</c><br></br> - The current target platform, e.g. sparc-sun-solaris2.8 - </item> - </list> - <p>RUNNING TESTS</p> - <p>After installing <c>ts</c>, you can run your test with the - <c>ts:run/0/1/2/3/4</c> functions. These functions, however, - require a special directory structure to be able to find your - test suites. Both the test server and all tests must be located - under your $TESTROOT directory. The test server implementation - shall be located in the directory <c>$TESTROOT/test_server</c> - and for each application there must be a directory named - <c><![CDATA[$TESTROOT/<application>_test]]></c> containing the .spec file - and all test suites and data directories for the - application. Note that there shall only be one .spec file for - each application. - </p> - <p><c>$TESTROOT/test_server</c> must be the current directory - when calling the <c>ts:run/*</c> function. - </p> - <p>All available tests can be found with <c>ts:tests()</c>. This - will list all applications for which a test specification file - <c><![CDATA[../<application>_test/<application>.spec]]></c> can be found. - </p> - <p>To run all these tests, use <c>ts:run()</c>. - </p> - <p>To run one or some of the tests, use <c>ts:run(Tests)</c>, - where <c>Tests</c> is the name of the application you want to - test, or a list of such names. - </p> - <p>To run one test suite within a test, use - <c>ts:run(Test,Suite)</c>. - </p> - <p>To run one test case within a suite, use - <c>ts:run(Test,Suite,Case)</c></p> - <p>To all these functions, you can also add a list of - options. Please turn to the reference manual for the <c>ts</c> - module to see the valid options to use. - </p> - <p>The function <c>ts:help()</c> displays some simple help for - the functions in <c>ts</c>. Use this for quick reference. - </p> - <p>LOG FILES</p> - <p>As the execution of the test suites go on, events are logged in - four different ways: - </p> - <list type="bulleted"> - <item>Text to the operator's console.</item> - <item>Suite related information is sent to the major log file.</item> - <item>Case related information is sent to the minor log file.</item> - <item>The HTML log file gets updated with test results.</item> - </list> - <p>Typically the operator, who may run hundreds or thousands of - test cases, doesn't want to fill the screen with details - about/from the specific test cases. By default, the operator will - only see: - </p> - <list type="bulleted"> - <item>A confirmation that the test has started. - </item> - <item>A small note about each failed test case. - </item> - <item>A summary of all the run test cases. - </item> - <item>A confirmation that the test run is complete - </item> - <item>Some special information like error reports and progress - reports, printouts written with erlang:display/1 or io:format/3 - specifically addressed to somewhere other than - <c>standard_io</c>.</item> - </list> - <p>This is enough for the operator to know, and if he wants to dig - in deeper into a specific test case result, he can do so by - following the links in the HTML presentation to take a look in the - major or minor log files. - </p> - <p>A detailed report of the entire test suite is stored in the - major logfile, the exact reason for failure, time spent etc. - </p> - <p>The HTML log file is a summary of the major log file, but gives - a much better overview of the test run. It also has links to every - test case's log file for quick viewing with a HTML browser. - </p> - <p>The minor log file contain full details of every single test - case, each one in a separate file. This way the files should be - easy to compare with previous test runs, even if the set of test - cases change. - </p> - <p>Which information that goes where is user configurable via the - test server controller. Three threshold values determine what - comes out on screen, and in the major or minor log files. The - contents that goes to the HTML log file is fixed, and cannot be - altered. - </p> - - </description> - <funcs> - <func> - <name>install() -> ok | {error, Reason}</name> - <name>install(TargetSystem) -> ok | {error, Reason}</name> - <name>install(Opts) -> ok | {error, Reason}</name> - <name>install(TargetSystem,Opts) -> ok | {error, Reason}</name> - <fsummary>Installs the Test Server Framework</fsummary> - <type> - <v>TargetSystem = {Architecture, TargetHost}</v> - <v>Architecture = atom() or string()</v> - <d>e.g. "ose" or "vxworks_ppc603"</d> - <v>TargetHost = atom() or string()</v> - <d>The name of the target host</d> - <v>Opts = list()</v> - </type> - <desc> - <p>Installs and configures the Test Server Framework for - running test suites. If a remote host is to be used, the - <c>TargetSystem</c> argument must be given so that "cross - installation" can be done. Installation is required for - any of the functions in <c>ts</c> to work. - </p> - <p>Opts may be one or more of - </p> - <list> - <item><c>{longnames, Bool}</c><br></br> - Use fully qualified hostnames for test_server and - slave nodes. Bool is <c>true</c> or <c>false</c> (default). - </item> - <item><c>{verbose, Level}</c><br></br> - Verbosity level for test server output, set to 0, 1 or - 2, where 0 is quiet(default). - </item> - <item><c>{hosts, Hosts}</c><br></br> - This is a list of available hosts on which to start - slave nodes. It is used when the <c>{remote, true}</c> - option is given to the <c>test_server:start_node/3</c> - function. Also, if <c>{require_nodenames, Num}</c> is - contained in a test specification file, the generated - nodenames will be spread over all hosts given in this - <c>Hosts</c> list. The hostnames are given as atoms or - strings. - </item> - <item><c>{slavetargets, SlaveTarges}</c><br></br> - For VxWorks only. This is a list of - available hosts where slave nodes can be started. This is - necessary because only one node can run per host in the - VxWorks environment. This is not the same as - <c>{hosts, Hosts}</c> because it is used for all slave nodes - - not only the ones started with <c>{remote, true}</c>. The - hostnames are given as atoms or strings. - </item> - <item><c>{crossroot, TargetErlRoot}</c><br></br> - Erlang root directory on target host - <br></br> -This option is mandatory for remote targets - </item> - <item><c>{master, {MasterHost, MasterCookie}}</c><br></br> - If target is remote and the target node is started as - a slave node, this option - indicates which master and cookie to use. The given master - will also be used as master for slave nodes started with - <c>test_server:start_node/3</c>. It is expected that the - <c>erl_boot_server</c> is started on the master node before - the test is run. If this option is not given, the test - server controller node is used as master and the - <c>erl_boot_server</c> is automatically started. - </item> - <item><c>{erl_start_args, ArgString}</c><br></br> - Additional arguments to be used when starting the test - server controller node. <c>ArgString</c> will be appended to - the command line when starting the erlang node. Note that - this will only affect the startup of the <em>controller node</em>, - i.e. not the target node or any slave nodes - startet from a test case. - </item> - <item><c>{ipv6_hosts, HostList}</c><br></br> - This option will be inserted in the - <c>Config</c> parameter for each test case. <c>HostList</c> - is a list of hosts supporting IPv6. - </item> - </list> - </desc> - </func> - <func> - <name>help() -> ok</name> - <fsummary>Presents simple help on the functions in <c>ts</c></fsummary> - <desc> - <p>Presents simple help on the functions in <c>ts</c>. Useful - for quick reference.</p> - </desc> - </func> - <func> - <name>tests() -> Tests</name> - <fsummary>Returns the list of available tests</fsummary> - <desc> - <p>Returns the list of available tests. This is actually just - a list of all test specification files found by looking up - "../*_test/*.spec". - </p> - <p>In each ../Name_test/ directory there should be one test - specification file named Name.spec.</p> - </desc> - </func> - <func> - <name>run() -> ok | {error, Reason}</name> - <name>run([all_tests|Opts])</name> - <name>run(Specs)</name> - <name>run(Specs, Opts)</name> - <name>run(Spec, Module)</name> - <name>run(Spec, Module, Opts)</name> - <name>run(Spec, Module, Case)</name> - <name>run(Spec, Module, Case, Opts)</name> - <fsummary>Runs (specified) test suite(s)</fsummary> - <type> - <v>Specs = Spec | [Spec]</v> - <v>Spec = atom()</v> - <v>Module = atom()</v> - <v>Case = atom()</v> - <v>Opts = [Opt]</v> - <v>Opt = batch | verbose | {verbose, Level} | {vars, Vars} | keep_topcase | cover | cover_details |{cover,CoverFile} | {cover_details,CoverFile} | {trace, TraceSpec}</v> - <v>Level = integer(); 0 means silent</v> - <v>Vars = list() of key-value tuples</v> - <v>CoverFile = string(); name of file listing modules to exclude from or include in cover compilation. The name must include full path to the file.</v> - <v>Reason = term()</v> - </type> - <desc> - <p>This function runs test suite(s)/case(s). To be able to run - any tests, ts:install must first be called to create the - <c>variables</c> file needed. To run a whole test specification, - only specify the name of the test specification, and all test - suite modules belonging to that test spec will be run. To run - a single module in a test specification, use the Module - argument to specify the name of the module to run and all test - cases in that module will be run, and to run a specified test - case, specify the name of the test case using the Case - argument. If called with no argument, all test specifications - available will be run. Use ts:tests/0 to see the available - test specifications. - </p> - <p>If the <c>batch</c> option is not given, a new xterm is - started (unix) when <c>ts:run</c> is called. - </p> - <p>The <c>verbose</c> option sets the verbosity level for test - server output. This has the same effect as if given to - <c>ts:install/1/2</c></p> - <p>The <c>vars</c> option can be used for adding configuration - variables that are not in the <c>variables</c> file generated - during installation. Can be any of the <c>Opts</c> valid for - <c>ts:install/1/2</c>. - </p> - <p>The <c>keep_topcase</c> option forces <c>ts</c> to keep the - topcase in your test specification file as is. This option can - only be used if you don't give the <c>Module</c> or - <c>Case</c> parameters to <c>ts:run</c>. The - <c>keep_topcase</c> option is necessary if your topcase - contains anything other than <c><![CDATA[{dir,"../<Name>_test"}]]></c>. If - the option is not used, <c>ts</c> will modify your topcase. - </p> - <p>The <c>cover</c> and <c>cover_details</c> options indicates - that the test shall be run with code coverage - analysis. <c>cover_details</c> means that analysis shall be - done on the most detailed level. If the test is run with a - remote target, this option creates a list of uncovered lines - in each cover compiled module. If the test is run with a local - target, each cover compiled module will be analysed with - <c>cover:analyse_to_file/1</c>. The <c>cover</c> options will - only create an overview of all cover compiled modules with the - number of covered and not covered lines. - </p> - <p>The <c>CoverFile</c> which can be given with the - <c>cover</c> and <c>cover_details</c> options must be the - filename of a file listing modules to be excluded from or - included in the cover compilation. By default, <c>ts</c> - believes that <c>Spec</c> is the name of an OTP application - and that all modules in this application shall be cover - compiled. The <c>CoverFile</c> can exclude modules that belong - to the application and add modules that don't belong to the - application. The file can have the following entries:</p> - <code type="none"> -{exclude, all | ExcludeModuleList}. -{include, IncludeModuleList}. </code> - <p>Note that each line must end with a full - stop. <c>ExcludeModuleList</c> and <c>IncludeModuleList</c> - are lists of atoms, where each atom is a module name. - </p> - <p>If the <c>cover</c> or <c>cover_details</c> options are - given on their own, the directory <c><![CDATA[../<Spec>_test]]></c> is - searched for a <c>CoverFile</c> named <c><![CDATA[<Spec>.cover]]></c>. If - this file is not found, <c>Spec</c> is assumed to be the name - of an OTP application, and all modules in the <c>ebin</c> - directory for the application are cover compiled. The - <c>ebin</c> directory is found by adding <c>ebin</c> to - <c>code:lib_dir(Spec)</c>. - </p> - <p>The same cover compiled code will be loaded on all slave or - peer nodes started with <c>test_server:start_node/3</c>. The - exception is nodes that run an old version of OTP. If the loading - fails, the node will simply not be a part of the coverage - analysis. Note that slave and peer nodes must be stopped with - <c>test_server:stop_node/1</c> for the node to be part of the - coverage analysis, else the test server will not be able to - fetch coverage data from the node. - </p> - <p>The <c>trace</c> option is used to turn on call trace on - target and on slave or peer nodes started with - <c>test_server:start_node/3</c>. <c>TraceSpec</c> can be the - name of a trace information file, or a list of elements like - the ones in a trace information file. Please turn to the - reference manual for <c>test_server_ctrl:trc/1</c> for details - about the trace information file. - </p> - </desc> - </func> - <func> - <name>cross_cover_analyse(Level) -> ok</name> - <name>cross_cover_analyse([Level]) -> ok</name> - <fsummary>Analyse cover data collected from all tests</fsummary> - <desc> - <p>Analyse cover data collected from all tests. - </p> - <p>See test_server_ctrl:cross_cover_analyse/2 - </p> - </desc> - </func> - <func> - <name>r() -> ok</name> - <name>r(Opts) -> ok</name> - <name>r(SpecOrSuite) -> ok</name> - <name>r(SpecOrSuite,Opts) -> ok</name> - <name>r(Suite,Case) -> ok</name> - <name>r(Suite,Case,Opts) -> ok</name> - <fsummary>Run test suite or test case without <c>ts</c>installed</fsummary> - <type> - <v>SpecOrSuite = Spec | Suite</v> - <v>Spec = string()</v> - <d>"Name.spec" or "Name.spec.OsType", where OsType is vxworks</d> - <v>Suite = atom()</v> - <v>Case = atom()</v> - <v>Opts = [Opt]</v> - <v>Opt = {Cover,AppOrCoverFile} | {Cover,Application,CoverFile}</v> - <v>Cover = cover | cover_details</v> - <v>AppOrCoverFile = Application | CoverFile</v> - <v>Application = atom()</v> - <d>OTP application to cover compile</d> - <v>CoverFile = string()</v> - <d>Name of file listing modules to exclude from or include in cover compilation</d> - </type> - <desc> - <p>This function can be used to run a test suites or test - cases directly, without any of the additional features added - by the test server framework. It is simply a wrapper function - for the <c>add_dir</c>, <c>add_spec</c>, <c>add_module</c> and - <c>add_case</c> functions in <c>test_server_ctrl</c>: - </p> - <p><c>r() -> add_dir(".")</c> <br></br> -<c>r(Spec) -> add_spec(Spec)</c> <br></br> -<c>r(Suite) -> add_module(Suite)</c> <br></br> -<c>r(Suite,Case) -> add_case(Suite,Case)</c></p> - <p>To use this function, it is required that the test suite is - compiled and in the code path of the node where the function - is called. The function can be used without having <c>ts</c> - installed. - </p> - <p>For information about the <c>cover</c> and - <c>cover_details</c> options, see <c>test_server_ctrl:cover/2/3</c>.</p> - </desc> - </func> - <func> - <name>estone() -> ok | {error, Reason}</name> - <name>estone(Opts) -> ok</name> - <fsummary>Runs the EStone test</fsummary> - <desc> - <p>This function runs the EStone test. It is a shortcut for - running the test suite <c>estone_SUITE</c> in the - <c>kernel</c> application. - </p> - <p><c>Opts</c> is the same as the <c>Opts</c> argument for the - <c>ts:run</c> functions.</p> - </desc> - </func> - </funcs> - - <section> - <title>Makfile.src in Data Directory</title> - <p>If a data directory contains code which must be compiled before - the test suite is run, a makefile source called - <c>Makefile.src</c> can be placed in the data directory. This file - will be converted to a valid makefile by <c>ts:run/0/1/2/3/4</c>. - </p> - <p>The reason for generating the makefile is that you can use - variables from the <c>variables</c> file which was generated by - <c>ts:install/0/1/2</c>. All occurrences of <c>@Key@</c> in - <c>Makefile.src</c> is substituted by the <c>Value</c> from - <c>{Key,Value}</c> found in the <c>variables</c> file. Example: - </p> - <p>Cut from <c>variables</c>:</p> - <code type="none"> - ... - {'EMULATOR',"beam"}. - {'CFLAGS',"-g -O2"}. - {'LD',"$(CC) $(CFLAGS)"}. - {'CC',"gcc"}. - ... - </code> - <p><c>Makefile.src</c> for compiling erlang code could look - something like this:</p> - <code type="none"> - EFLAGS=+debug_info - - all: ordsets1.@EMULATOR@ - - ordsets1.@EMULATOR@: ordsets1.erl - erlc $(EFLAGS) ordsets1.erl - </code> - <p><c>Makefile.src</c> for compiling c code could look - something like this:</p> - <code type="none"> - CC = @CC@ - LD = @LD@ - CFLAGS = @CFLAGS@ -I@erl_include@ @DEFS@ - CROSSLDFLAGS = @CROSSLDFLAGS@ - - PROGS = nfs_check@exe@ - - all: $(PROGS) - - nfs_check@exe@: nfs_check@obj@ - $(LD) $(CROSSLDFLAGS) -o nfs_check nfs_check@obj@ @LIBS@ - - nfs_check@obj@: nfs_check.c - $(CC) -c -o nfs_check@obj@ $(CFLAGS) nfs_check.c - </code> - </section> -</erlref> - diff --git a/lib/test_server/doc/src/why_test_chapter.xml b/lib/test_server/doc/src/why_test_chapter.xml deleted file mode 100644 index 3d0e8271b1..0000000000 --- a/lib/test_server/doc/src/why_test_chapter.xml +++ /dev/null @@ -1,141 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Why Test</title> - <prepared>Siri Hansen</prepared> - <docno></docno> - <date></date> - <rev></rev> - </header> - - <section> - <title>Goals</title> - <p>It's not possible to prove that a program is correct by - testing. On the contrary, it has been formally proven that it is - impossible to prove programs in general by testing. Theoretical - program proofs or plain examination of code may be viable options - for those that wish to certify that a program is correct. The test - server, as it is based on testing, cannot be used for - certification. Its intended use is instead to (cost effectively) - <em>find bugs</em>. A successful test suite is one that reveals a - bug. If a test suite results in Ok, then we know very little that - we didn't know before. - </p> - </section> - - <section> - <title>What to test?</title> - <p>There are many kinds of test suites. Some concentrate on - calling every function in the interface to some module or - server. Some other do the same, but uses all kinds of illegal - parameters, and verifies that the server stays alive and rejects - the requests with reasonable error codes. Some test suites - simulate an application (typically consisting of a few modules of - an application), some try to do tricky requests in general, some - test suites even test internal functions. - </p> - <p>Another interesting category of test suites are the ones that - check that fixed bugs don't reoccur. When a bugfix is introduced, - a test case that checks for that specific bug should be written - and submitted to the affected test suite(s). - </p> - <p>Aim for finding bugs. Write whatever test that has the highest - probability of finding a bug, now or in the future. Concentrate - more on the critical parts. Bugs in critical subsystems are a lot - more expensive than others. - </p> - <p>Aim for functionality testing rather than implementation - details. Implementation details change quite often, and the test - suites should be long lived. Often implementation details differ - on different platforms and versions. If implementation details - have to be tested, try to factor them out into separate test - cases. Later on these test cases may be rewritten, or just - skipped. - </p> - <p>Also, aim for testing everything once, no less, no more. It's - not effective having every test case fail just because one - function in the interface changed. - </p> - </section> - - <section> - <title>How much to test</title> - <p>There is a unix shell script that counts the number of non - commented words (lines and characters too) of source code in each - application's test directory and divides with the number of such - source words in the src directory. This is a measure of how much - test code there is. - </p> - <p>There has been much debate over how much test code, compared to - production code, should be written in a project. More test code - finds more bugs, but test code needs to be maintained just like - the production code, and it's expensive to write it in the first - place. In several articles from relatively mature software - organizations that I have read, the amount of test code has been - about the same as the production code. </p> - <p>In OTP, at the time of - writing, few applications come even close to this, some have no - test code at all. - </p> - - <section> - <title>Full coverage</title> - <p>It is possible to cover compile the modules being tested - before running the test suites. Doing so displays which branches - of the code that are tested by the test suite, and which are - not. Many use this as a measure of a good test suite. When every - single line of source code is covered once by the test suite, - the test suite is finished. - </p> - <p>A coverage of 100% still proves nothing, though. It doesn't - mean that the code is error free, that everything is tested. For - instance, if a function contains a division, it has to be - executed at least twice. Once with parameters that cause - division by zero, and once with other parameters. - </p> - <p>High degree of coverage is good of course, it means that no - major parts of the code has been left untested. It's another - question whether it's cost effective. You're only likely to find - 50% more bugs when going from 67% to 100% coverage, but the work - (cost) is maybe 200% as large, or more, because reaching all of - those obscure branches is usually complicated. - </p> - <p>Again, the reason for testing with the test server is to find - bugs, not to create certificates of valid code. Maximizing the - number of found bugs per hour probably means not going for 100% - coverage. For some module the optimum may be 70%, for some other - maybe 250%. 100% shouldn't be a goal in itself.</p> - </section> - - <section> - <title>User interface testing</title> - <p>It is very difficult to do sensible testing of user - interfaces, especially the graphic ones. The test server has - some support for capturing the text I/O that goes to the user, - but none for graphics. There are several tools on the market - that help with this.</p> - </section> - </section> -</chapter> - diff --git a/lib/test_server/doc/src/write_framework_chapter.xml b/lib/test_server/doc/src/write_framework_chapter.xml deleted file mode 100644 index d10b580c34..0000000000 --- a/lib/test_server/doc/src/write_framework_chapter.xml +++ /dev/null @@ -1,160 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Write you own test server framework</title> - <prepared>Siri Hansen</prepared> - <docno></docno> - <date></date> - <rev></rev> - <file>write_framework_chapter.xml</file> - </header> - - <section> - <title>Introduction</title> - <p>The test server controller can be interfaced from the operating - system or from within Erlang. The nature of your new framework - will decide which interface to use. If you want your framework to - start a new node for each test, the operating system interface is - very convenient. If your node is already started, going from - within Erlang might be a more flexible solution. - </p> - <p>The two methods are described below. - </p> - </section> - - <section> - <title>Interfacing the test server controller from Erlang</title> - <p>Using the test server from Erlang means that you have to start - the test server and then add test jobs. Use - <c>test_server_ctrl:start/0</c> to start the test server, and - <c>test_server_ctrl:stop/0</c> to stop it. - </p> - - <section> - <title>Adding test jobs</title> - <p>There are many commands available for adding test cases to - the test server's job queue: <br></br> -</p> - <list type="bulleted"> - <item>Single test case <br></br> -<c>test_server_ctrl:add_case/2/3</c></item> - <item>Multiple test cases from same suite <br></br> -<c>test_server_ctrl:add_cases/2/3</c></item> - <item>Test suite module or modules <br></br> -<c>test_server_ctrl:add_module/1/2</c></item> - <item>Some or all test suite modules in a directory <br></br> -<c>test_server_ctrl:add_dir/2/3</c></item> - <item>Test cases specified in a test specification file <br></br> -<c>test_server_ctrl:add_spec/1</c></item> - </list> - <p>All test suites are given a unique name, which is usually - given when the test suite is added to the job queue. In some - cases, a default name is used, as in the case when a module is - added without a specified name. The test job name is used to - store logfiles, which are stored in the `name.logs' directory - under the current directory. - </p> - <p>See the reference manual for details about the functions for - adding test jobs. - </p> - </section> - </section> - - <section> - <title>Interfacing the test server controller from the operating system.</title> - <p>The function <c>run_test/1</c> is your interface in the test - server controller if you wish to use it from the operating - system. You simply start an erlang shell and invoke this function - with the <c>-s</c> option. <c>run_test/1</c> starts the test - server, runs the test specified by the command line and stops the - test server. The argument to <c>run_test/1</c> is a list of - command line flags, typically - <c>['KEY1', Value1, 'KEY2', Value2, ...]</c>. - The valid command line flags are listed in the reference manual - for <c>test_server_ctrl</c>. - </p> - <p>A typical command line may look like this <br></br> -<c>erl -noshell -s test_server_ctrl run_test KEY1 Value1 KEY2 Value2 ... -s erlang halt</c></p> - <p>Or make an alias (this is for unix/tcsh) <br></br> -<c>alias erl_test 'erl -noshell -s test_server_ctrl run_test \!* -s erlang halt'</c></p> - <p>And then use it like this <br></br> -<c>erl_test KEY1 Value1 KEY2 Value2 ...</c> <br></br> -</p> - - <section> - <title>An Example</title> - <p>An example of starting a test run from the command line <br></br> -</p> - <p><c>erl -name test_srv -noshell -rsh /home/super/otp/bin/ctrsh </c> <br></br> -<c>-pa /clearcase/otp/erts/lib/kernel/test </c> <br></br> -<c>-boot start_sasl -sasl errlog_type error </c> <br></br> -<c>-s test_server_ctrl run_test SPEC kernel.spec -s erlang halt</c> <br></br> -</p> - </section> - </section> - - <section> - <title>Framework callback functions</title> - <p>By defining the environment variable - <c>TEST_SERVER_FRAMEWORK</c> to a module name, the framework - callback functions can be used. The framework callback functions - are called by the test server in order let the framework interact - with the execution of the tests and to keep the framework upto - date with information about the test progress. - </p> - <p>The framework callback functions are described in the reference - manual for <c>test_server_ctrl</c>. - </p> - <p>Note that this topic is in an early stage of development, and - changes might occur. - </p> - </section> - - <section> - <title>Other concerns</title> - <p>Some things to think about when writing you own test server - framework: - </p> - <list type="bulleted"> - <item><c>emulator version</c> - Make sure that the intended - version of the emulator is started. - </item> - <item><c>operating system path</c> - If test cases use port - programs, make sure the paths are correct. - </item> - <item><c>recompilation</c> - Make sure all test suites are fresh - compiled. - </item> - <item><c>test_server.hrl</c> - Make sure the - <c>test_server.hrl</c> file is in the include path when - compiling test suites. - </item> - <item><c>running applications</c> - Some test suites require - some applications to be running (e.g. sasl). Make sure they are - started. - </item> - </list> - </section> -</chapter> - diff --git a/lib/test_server/doc/src/write_test_chapter.xml b/lib/test_server/doc/src/write_test_chapter.xml deleted file mode 100644 index c3e1881b8a..0000000000 --- a/lib/test_server/doc/src/write_test_chapter.xml +++ /dev/null @@ -1,228 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2002</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Writing Test Suites</title> - <prepared>Siri Hansen</prepared> - <docno></docno> - <date></date> - <rev></rev> - <file>write_test_chapter.xml</file> - </header> - - <section> - <title>Support for test suite authors</title> - <p>The <c>test_server</c> module provides some useful functions - to support the test suite author. This includes: - </p> - <list type="bulleted"> - <item>Starting and stopping slave or peer nodes</item> - <item>Capturing and checking stdout output</item> - <item>Retrieving and flushing process message queue</item> - <item>Watchdog timers</item> - <item>Checking that a function crashes</item> - <item>Checking that a function succeeds at least m out of n times</item> - <item>Checking .app files</item> - </list> - <p>Please turn to the reference manual for the <c>test_server</c> - module for details about these functions. - </p> - </section> - - <section> - <title>Test suites</title> - <p>A test suite is an ordinary Erlang module that contains test - cases. It's recommended that the module has a name on the form - *_SUITE.erl. Otherwise, the directory function will not find the - modules (by default). - </p> - <p>For some of the test server support, the test server include - file <c>test_server.hrl</c> must be included. Never include it - with the full path, for portability reasons. Use the compiler - include directive instead. - </p> - <p>The special function <c>all(suite)</c> in each module is called - to get the test specification for that module. The function - typically returns a list of test cases in that module, but any - test specification could be returned. Please see the chapter - about test specifications for details about this. - </p> - </section> - - <section> - <title>Init per test case</title> - <p>In each test suite module, the functions - <c>init_per_testcase/2</c> and <c>end_per_testcase/2</c> must be - implemented. - </p> - <p><c>init_per_testcase</c> is called before each test case in the - test suite, giving a (limited) possibility for initialization. - </p> - <p><c>end_per_testcase/2</c> is called after each test case is - completed, giving a possibility to clean up. - </p> - <p>The first argument to these functions is the name of the test - case. This can be used to do individual initialization and cleanup for - each test cases. - </p> - <p>The second argument is a list of tuples called - <c>Config</c>. The first element in a <c>Config</c> tuple - should be an atom - a key value to be used for searching. - <c>init_per_testcase/2</c> may modify the <c>Config</c> - parameter or just return it as is. Whatever is retuned by - <c>init_per_testcase/2</c> is given as <c>Config</c> parameter to - the test case itself. - </p> - <p>The return value of <c>end_per_testcase/2</c> is ignored by the - test server. - </p> - </section> - - <section> - <title>Test cases</title> - <p>The smallest unit that the test server is concerned with is a - test case. Each test case can in turn test many things, for - example make several calls to the same interface function with - different parameters. - </p> - <p>It is possible to put many or few tests into each test - case. How many things each test case tests is up to the author, - but here are some things to keep in mind. - </p> - <p>Very small test cases often leads to more code, since - initialization has to be duplicated. Larger code, especially with - a lot of duplication, increases maintenance and reduces - readability. - </p> - <p>Larger test cases make it harder to tell what went wrong if it - fails, and force us to skip larger portions of test code if a - specific part fails. These effects are accentuated when running on - multiple platforms because test cases often have to be skipped. - </p> - <p>A test case generally consists of three parts, the - documentation part, the specification part and the execution - part. These are implemented as three clauses of the same function. - </p> - <p>The documentation clause matches the argument '<c>doc</c>' and - returns a list for strings describing what the test case tests. - </p> - <p>The specification clause matches the argument '<c>suite</c>' - and returns the test specification for this particular test - case. If the test specification is an empty list, this indicates - that the test case is a leaf test case, i.e. one to be executed. - </p> - <p>The execution clause implements the actual test case. It takes - one argument, <c>Config</c>, which contain configuration - information like <c>data_dir</c> and <c>priv_dir</c>. See <seealso marker="#data_priv_dir">Data and Private Directories</seealso> for - more information about these. - </p> - <p>The <c>Config</c> variable can also contain the - <c>nodenames</c> key, if requested by the <c>require_nodenames</c> - command in the test suite specification file. All <c>Config</c> - items should be extracted using the <c>?config</c> macro. This is - to ensure future compatibility if the <c>Config</c> format - changes. See the reference manual for <c>test_server</c> for - details about this macro. - </p> - <p>If the execution clause crashes or exits, it is considered a - failure. If it returns <c>{skip,Reason}</c>, the test case is - considered skipped. If it returns <c>{comment,String}</c>, - the string will be added in the 'Comment' field on the HTML - result page. If the execution clause returns anything else, it is - considered a success, unless it is <c>{'EXIT',Reason}</c> or - <c>{'EXIT',Pid,Reason}</c> which can't be distinguished from a - crash, and thus will be considered a failure. - </p> - </section> - - <section> - <marker id="data_priv_dir"></marker> - <title>Data and Private Directories</title> - <p>The data directory (<c>data_dir</c>) is the directory where the test - module has its own files needed for the testing. A compiler test - case may have source files to feed into the compiler, a release - upgrade test case may have some old and new release of - something. A graphics test case may have some icons and a test - case doing a lot of math with bignums might store the correct - answers there. The name of the <c>data_dir</c> is the the name of - the test suite and then "_data". For example, - <c>"some_path/foo_SUITE.beam"</c> has the data directory - <c>"some_path/foo_SUITE_data/"</c>. - </p> - <p>The <c>priv_dir</c> is the test suite's private directory. This - directory should be used when a test case needs to write to - files. The name of the private directory is generated by the test - server, which also creates the directory. - </p> - <p><em>Warning:</em> Do not depend on current directory to be - writable, or to point to anything in particular. All scratch files - are to be written in the <c>priv_dir</c>, and all data files found - in <c>data_dir</c>. If the current directory has to be something - specific, it must be set with <c>file:set_cwd/1</c>. - </p> - </section> - - <section> - <title>Execution environment</title> - <p>Each time a test case is about to be executed, a new process is - created with <c>spawn_link</c>. This is so that the test case will - have no dependencies to earlier tests, with respect to process flags, - process links, messages in the queue, other processes having registered - the process, etc. As little as possible is done to change the initial - context of the process (what is created by plain spawn). Here is a - list of differences: - </p> - <list type="bulleted"> - <item>It has a link to the test server. If this link is removed, - the test server will not know when the test case is finished, - just wait infinitely. - </item> - <item>It often holds a few items in the process dictionary, all - with names starting with '<c>test_server_</c>'. This is to keep - track of if/where a test case fails. - </item> - <item>There is a top-level catch. All of the test case code is - catched, so that the location of a crash can be reported back to - the test server. If the test case process is killed by another - process (thus the catch code is never executed) the test server - is not able to tell where the test case was executing. - </item> - <item>It has a special group leader implemented by the test - server. This way the test server is able to capture the io that - the test case provokes. This is also used by some of the test - server support functions. - </item> - </list> - <p>There is no time limit for a test case, unless the test case - itself imposes such a limit, by calling - <c>test_server:timetrap/1</c> for example. The call can be made - in each test case, or in the <c>init_per_testcase/2</c> - function. Make sure to call the corresponding - <c>test_server:timetrap_cancel/1</c> function as well, e.g in the - <c>end_per_testcase/2</c> function, or else the test cases will - always fail. - </p> - </section> - -</chapter> - |