From 824a6db3ff111f8d2427ac5adad0362bf078630a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Thu, 2 Sep 2010 15:44:33 +0200 Subject: Add test suite for xmerl --- lib/xmerl/test/xmerl_xsd_lib.erl | 314 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 lib/xmerl/test/xmerl_xsd_lib.erl (limited to 'lib/xmerl/test/xmerl_xsd_lib.erl') diff --git a/lib/xmerl/test/xmerl_xsd_lib.erl b/lib/xmerl/test/xmerl_xsd_lib.erl new file mode 100644 index 0000000000..074e6dcd49 --- /dev/null +++ b/lib/xmerl/test/xmerl_xsd_lib.erl @@ -0,0 +1,314 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2006-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% +%%% Purpose : Sub routines for test suite for the xmerl application, +%%% xmerl_xsd module. +%%%------------------------------------------------------------------- +%%% @private +%%% File : xmerl_xsd_lib.erl +%%% Author : Bertil Karlsson +%%% Description : +%%% +%%% Created : 28 Apr 2006 by Bertil Karlsson +%%%------------------------------------------------------------------- +-module(xmerl_xsd_lib). + +-compile(export_all). + +-include("test_server.hrl"). +-include("xmerl.hrl"). +-include("xmerl_xsd.hrl"). +-include_lib("kernel/include/file.hrl"). + + +compare_test_results(Config,ST,IT) -> + ResST=compare_schema_test_results(ST), + ResIT=compare_instance_test_results(IT), + io:format("compare_test_results:~n ST = ~p~n IT = ~p~n ResST = ~p~n ResIT = ~p~n",[ST,IT,ResST,ResIT]), + case process_reference_results(Config,ResST,ResIT) of + error -> + error; + Diff -> + return_results(Diff,ResST,ResIT,length(ST)+length(IT)) + end. + +compare_schema_test_results(ST) -> + {[N||{N,false}<-ST],[N||{N,enoent}<-ST]}. +compare_instance_test_results(IT) -> + {[N||{N,false}<-IT],[N||{N,enoent}<-IT]}. + +return_results(Diff,{STErrs,_},{ITErrs,_},TotN) -> + NumErrs = length(STErrs++ITErrs), + case NumErrs == TotN of + true when TotN > 0 -> + ?line exit(all_tests_cases_failed); + _ -> + return_results2(Diff,TotN - NumErrs,TotN) + end. + +%% return_results2(Diff,{[],[]},{[],[]},TotN) -> +%% {comment,io_lib:format("~p successful test cases.~n"++Diff,[TotN])}; +%% return_results2(Diff,{STErrs,[]},{ITErrs,[]},TotN) -> +%% {comment,io_lib:format("Total number of test cases: ~p~nThe following ~p test cases failed: ~p~n"++Diff,[TotN,length(STErrs++ITErrs),STErrs++ITErrs])}; +%% return_results2(Diff,{STErrs,STOther},{ITErrs,ITOther},TotN) -> +%% {comment,io_lib:format("Total number of test cases: ~p~nThe following ~p test cases failed: ~p~nThe following ~p test cases was malicious ~p~~n"++Diff,[TotN,length(STErrs++ITErrs),STErrs++ITErrs,length(STOther++ITOther),STOther++ITOther])}. + +return_results2(_,0,0) -> + {comment,io_lib:format("This test case was empty.~n",[])}; +return_results2({[],[],[],[]},NumSucc,TotN) -> + {comment,io_lib:format("~p successful tests of totally ~p test cases.~n", + [NumSucc,TotN])}; +return_results2({NewFail,NewSuccess,NewMal,NewNotMal},NumSucc,TotN) -> + NFComm = + case NewFail of + [] -> ""; + _ -> io_lib:format("These ~p tests are new failures: ~p~n", + [length(NewFail),NewFail]) + end, + NSComm = + case NewSuccess of + [] -> ""; + _ -> io_lib:format("These ~p tests are new succeeding cases: ~p~n", + [length(NewSuccess),NewSuccess]) + end, + NMComm = + case NewMal of + [] -> ""; + _ -> io_lib:format("These ~p tests are now malicious: ~p~n", + [length(NewMal),NewMal]) + end, + NNMComm = + case NewNotMal of + [] -> ""; + _ -> io_lib:format("These ~p tests were malicious, but succeeds now: ~p~n",[length(NewNotMal),NewNotMal]) + end, + {comment,io_lib:format("~p successful tests of totally ~p test cases. ~n"++ + NFComm++NSComm++NMComm++NNMComm,[NumSucc,TotN])}. +%% return_results2(Diff,{STErrs,STOther},{ITErrs,ITOther},TotN) -> +%% {comment,io_lib:format("Total number of test cases: ~p~nThe following ~p test cases failed: ~p~nThe following ~p test cases was malicious ~p~~n",[TotN,length(STErrs++ITErrs),STErrs++ITErrs,length(STOther++ITOther),STOther++ITOther])}. + + +process_reference_results(Config,{ErrsST,MalST},{ErrsIT,MalIT}) -> + {RefFailed,RefMalicious} = xsd_reference_log(Config), + AllErrs = ErrsST ++ ErrsIT, + AllMals = MalST ++ MalIT, + %% test cases failed now but succeeded in reference results. + NewFailures = [X||X<-AllErrs, lists:member(X,RefFailed)==false], + %% test cases succeeded now but failed in reference results. + NewSucceeds = [X||X<-RefFailed,lists:member(X,AllErrs)==false], + %% test cases malicious now but succeeded in reference results. + NewMalicious = [X||X<-AllMals, lists:member(X,RefMalicious)==false], + %% test cases succeeded now but malicious in reference results. + NewNotMal = [X||X<-RefMalicious, lists:member(X,AllMals) == false], + write_in_log(Config,AllErrs,AllMals), +% io:format("process_reference_results:~n AllErrs = ~p~n NewFailures = ~p~n",[AllErrs,NewFailures]), + {NewFailures,NewSucceeds,NewMalicious,NewNotMal}. + +xsd_reference_log(Config) -> + DataDir = ?config(data_dir,Config), + Suite = ?config(suite,Config), + SuiteReferenceLog = + filename:join([DataDir,lists:concat([Suite,"_failed_cases.log"])]), + case file:consult(SuiteReferenceLog) of + {ok,List} when is_list(List) -> + case lists:keysearch(?config(testcase,Config),1,List) of + {value,{_,TCRefFails}} -> + TCRefFails; + _ -> + {[],[]} + end; + _ -> + {[],[]} + end. + +write_in_log(_Config,[],[]) -> + ok; +write_in_log(Config,AllErrs,AllMals) -> + ?line LogFileName = ?config(xmerl_error_log,Config), + {ok,IO}=file:open(LogFileName,[append]), + ?line TestCase = ?config(testcase,Config), + io:format(IO,"{~p,{~p,~p}}.~n",[TestCase,AllErrs,AllMals]), + file:close(IO), + ok. + +schema_test(Config,FileName,XsdBase,Validity) -> + ModuleName = filename:basename(FileName), + DataDir = ?config(data_dir,Config), + case xmerl_xsd:process_schema(filename:join([DataDir, FileName]),[{xsdbase,filename:join([DataDir,XsdBase])}]) of + {error,enoent} -> + {{ModuleName,enoent},#xsd_state{}}; + {Ok,S} -> + case Validity of + valid when Ok == ok -> +%% io:format("schema_test1: Validity=valid,Ok=ok,S=~p~n",[S]), + {{ModuleName,S#xsd_state.errors == []},S}; + invalid when Ok == error -> %% S is in this case an error reason + {{ModuleName,no_internal_error(S)},#xsd_state{}}; + notKnown -> + {{ModuleName,true},#xsd_state{}}; + valid -> + io:format("schema_test2: Validity=valid,Ok=~p,S=~p~n",[Ok,S]), +%% io:format("FileName: ~p~n",[FileName]), + {{ModuleName,false},#xsd_state{}}; + _ -> %% invalid Ok == ok + io:format("schema_test3: Validity=~p,Ok=~p,S=~p~n",[Validity,Ok,S]), + {{ModuleName,false},S} + end + end. +schema_test(Config,FileName,XsdBase,Validity,AccState) -> + ModuleName = filename:basename(FileName), + DataDir = ?config(data_dir,Config), + case xmerl_xsd:process_schema(filename:join([DataDir,FileName]), + [{xsdbase,filename:join([DataDir,XsdBase])}, AccState]) of + {error,enoent} -> + {{ModuleName,enoent},AccState}; + {Ok,S} -> + case Validity of + valid when Ok == ok -> + {{ModuleName,S#xsd_state.errors == []},S}; + invalid when Ok == error -> + {{ModuleName,no_internal_error(S)},AccState}; + notKnown -> + {{ModuleName,true},AccState}; + valid -> + {{ModuleName,false},AccState}; + _ -> + {{ModuleName,false},S} + end + end. +instance_test(Config,FileName,XMLBase,Validity,State) -> + ModuleName = filename:basename(FileName), + DataDir = ?config(data_dir,Config), + case xmerl_scan:file(filename:join([DataDir,FileName]), + [{xmlbase,filename:join([DataDir,XMLBase])}]) of + {error,enoent} -> + {ModuleName,enoent}; + {E,_} -> + {VE,S2} = xmerl_xsd:validate(E,State), + case Validity of + valid when is_record(VE,xmlElement) -> + case S2#xsd_state.errors of + [] -> ok; + _ -> io:format("test case ~p failed.~nValidity: ~p~nValidation result:~p~n",[FileName,Validity,VE]) + end, + {ModuleName,S2#xsd_state.errors == []}; + invalid when VE == error -> + {ModuleName,no_internal_error(S2)}; + notKnown -> + {ModuleName,true}; + _ -> + io:format("test case ~p failed.~nValidity: ~p~nValidation result:~p~n",[FileName,Validity,VE]), + {ModuleName,false} + end + end. + +no_internal_error(R) -> + case lists:keymember(internal_error,1,R) of + true -> + false; + _ -> + true + end. + +unpack(Config,Suite) -> + TarFile = suite_tar(Suite), + ?line file:set_cwd(?config(data_dir,Config)), + ?line ok=erl_tar:extract(TarFile,[compressed]), + change_mode(filename:rootname(TarFile,".tar.gz")). + +suite_tar(sun) -> + "suntest.tar.gz"; +suite_tar(msx) -> + "msxsdtest.tar.gz"; +suite_tar(nist) -> + "nisttest.tar.gz". + +change_mode(Files) -> + change_mode3(Files). +change_mode2(Dir)-> + ?line {ok,CWD} = file:get_cwd(), + ?line {ok,FileList} = file:list_dir(Dir), + ?line file:set_cwd(filename:join([CWD,Dir])), + change_mode3(FileList), + ?line file:set_cwd(CWD). +change_mode3([]) -> + ok; +change_mode3([F|Fs]) -> + case filelib:is_dir(F) of + true -> + chmod(F), + change_mode2(F); + _ -> + chmod(F) + end, + change_mode3(Fs). + +chmod(F) -> + case file:read_file_info(F) of + {ok,FileInfo} -> + Mode= FileInfo#file_info.mode, + file:write_file_info(F,FileInfo#file_info{mode=8#00777 bor Mode}); + _ -> + ok + end. + +rmdir(Config,Suite) -> + ?line file:set_cwd(?config(data_dir,Config)), + SuiteDir = filename:rootname(suite_tar(Suite),".tar.gz"), + ?line ok=rm_f_(SuiteDir). + +%% Dir is a directory +rm_f_(Dir) -> + ?line {ok,CWD} = file:get_cwd(), + ?line {ok,FileList} = file:list_dir(Dir), + ?line file:set_cwd(filename:join([CWD,Dir])), + rm_files(FileList), + ?line file:set_cwd(CWD), + ? line ok = file:del_dir(Dir). + +rm_files([])-> + ok; +rm_files([F|Fs]) -> + case filelib:is_dir(F) of + true -> + rm_f_(F); + _ -> + io:format("rm_files: ~p~n",[F]), + ?line ok = file:delete(F) + end, + rm_files(Fs). + +create_error_log_file(Config,Suite) -> + ?line {{Y,M,D},{H,Min,S}} = calendar:local_time(), + DTString=lists:concat([Y,"-",M,"-",D,"_",H,".",Min,".",S]), + FileName = lists:concat([Suite,"_",DTString,".errorlog"]), +%% ?line {ok,_IO} = file:open(filename:join([?config(priv_dir,Config), +%% FileName]),[append]). + +%% ?line {ok,_IO} = file:open(FileName,[append]). + io:format("error log file: ~p~n",[filename:join([?config(priv_dir,Config),FileName])]), + {ok,filename:join([?config(priv_dir,Config),FileName])}. + +close_error_log_file(Config) -> + case lists:keysearch(xmerl_error_log,1,Config) of + {value,{_,IO}} -> + file:close(IO); + _ -> + ok + end. -- cgit v1.2.3