%% %% %CopyrightBegin% %% %% Copyright Ericsson AB 2006-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. %% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. %% %% %CopyrightEnd% %% %% %%% Purpose : Sub routines for test suite for the xmerl application, %%% xmerl_xsd module. %%%------------------------------------------------------------------- %%% @private %%% File : xmerl_xsd_lib.erl %%% Author : Bertil Karlsson %%% Description : %%% %%% Created : 28 Apr 2006 by Bertil Karlsson %%%------------------------------------------------------------------- -module(xmerl_xsd_lib). -compile(export_all). -include("test_server.hrl"). -include("xmerl.hrl"). -include("xmerl_xsd.hrl"). -include_lib("kernel/include/file.hrl"). compare_test_results(Config,ST,IT) -> ResST=compare_schema_test_results(ST), ResIT=compare_instance_test_results(IT), io:format("compare_test_results:~n ST = ~p~n IT = ~p~n ResST = ~p~n ResIT = ~p~n",[ST,IT,ResST,ResIT]), case process_reference_results(Config,ResST,ResIT) of error -> error; Diff -> return_results(Diff,ResST,ResIT,length(ST)+length(IT)) end. compare_schema_test_results(ST) -> {[N||{N,false}<-ST],[N||{N,enoent}<-ST]}. compare_instance_test_results(IT) -> {[N||{N,false}<-IT],[N||{N,enoent}<-IT]}. return_results(Diff,{STErrs,_},{ITErrs,_},TotN) -> NumErrs = length(STErrs++ITErrs), case NumErrs == TotN of true when TotN > 0 -> ?line exit(all_tests_cases_failed); _ -> return_results2(Diff,TotN - NumErrs,TotN) end. %% return_results2(Diff,{[],[]},{[],[]},TotN) -> %% {comment,io_lib:format("~p successful test cases.~n"++Diff,[TotN])}; %% return_results2(Diff,{STErrs,[]},{ITErrs,[]},TotN) -> %% {comment,io_lib:format("Total number of test cases: ~p~nThe following ~p test cases failed: ~p~n"++Diff,[TotN,length(STErrs++ITErrs),STErrs++ITErrs])}; %% return_results2(Diff,{STErrs,STOther},{ITErrs,ITOther},TotN) -> %% {comment,io_lib:format("Total number of test cases: ~p~nThe following ~p test cases failed: ~p~nThe following ~p test cases was malicious ~p~~n"++Diff,[TotN,length(STErrs++ITErrs),STErrs++ITErrs,length(STOther++ITOther),STOther++ITOther])}. return_results2(_,0,0) -> {comment,io_lib:format("This test case was empty.~n",[])}; return_results2({[],[],[],[]},NumSucc,TotN) -> {comment,io_lib:format("~p successful tests of totally ~p test cases.~n", [NumSucc,TotN])}; return_results2({NewFail,NewSuccess,NewMal,NewNotMal},NumSucc,TotN) -> NFComm = case NewFail of [] -> ""; _ -> io_lib:format("These ~p tests are new failures: ~p~n", [length(NewFail),NewFail]) end, NSComm = case NewSuccess of [] -> ""; _ -> io_lib:format("These ~p tests are new succeeding cases: ~p~n", [length(NewSuccess),NewSuccess]) end, NMComm = case NewMal of [] -> ""; _ -> io_lib:format("These ~p tests are now malicious: ~p~n", [length(NewMal),NewMal]) end, NNMComm = case NewNotMal of [] -> ""; _ -> io_lib:format("These ~p tests were malicious, but succeeds now: ~p~n",[length(NewNotMal),NewNotMal]) end, {comment,io_lib:format("~p successful tests of totally ~p test cases. ~n"++ NFComm++NSComm++NMComm++NNMComm,[NumSucc,TotN])}. %% return_results2(Diff,{STErrs,STOther},{ITErrs,ITOther},TotN) -> %% {comment,io_lib:format("Total number of test cases: ~p~nThe following ~p test cases failed: ~p~nThe following ~p test cases was malicious ~p~~n",[TotN,length(STErrs++ITErrs),STErrs++ITErrs,length(STOther++ITOther),STOther++ITOther])}. process_reference_results(Config,{ErrsST,MalST},{ErrsIT,MalIT}) -> {RefFailed,RefMalicious} = xsd_reference_log(Config), AllErrs = ErrsST ++ ErrsIT, AllMals = MalST ++ MalIT, %% test cases failed now but succeeded in reference results. NewFailures = [X||X<-AllErrs, lists:member(X,RefFailed)==false], %% test cases succeeded now but failed in reference results. NewSucceeds = [X||X<-RefFailed,lists:member(X,AllErrs)==false], %% test cases malicious now but succeeded in reference results. NewMalicious = [X||X<-AllMals, lists:member(X,RefMalicious)==false], %% test cases succeeded now but malicious in reference results. NewNotMal = [X||X<-RefMalicious, lists:member(X,AllMals) == false], write_in_log(Config,AllErrs,AllMals), % io:format("process_reference_results:~n AllErrs = ~p~n NewFailures = ~p~n",[AllErrs,NewFailures]), {NewFailures,NewSucceeds,NewMalicious,NewNotMal}. xsd_reference_log(Config) -> DataDir = ?config(data_dir,Config), Suite = ?config(suite,Config), SuiteReferenceLog = filename:join([DataDir,lists:concat([Suite,"_failed_cases.log"])]), case file:consult(SuiteReferenceLog) of {ok,List} when is_list(List) -> case lists:keysearch(?config(testcase,Config),1,List) of {value,{_,TCRefFails}} -> TCRefFails; _ -> {[],[]} end; _ -> {[],[]} end. write_in_log(_Config,[],[]) -> ok; write_in_log(Config,AllErrs,AllMals) -> ?line LogFileName = ?config(xmerl_error_log,Config), {ok,IO}=file:open(LogFileName,[append]), ?line TestCase = ?config(testcase,Config), io:format(IO,"{~p,{~p,~p}}.~n",[TestCase,AllErrs,AllMals]), file:close(IO), ok. schema_test(Config,FileName,XsdBase,Validity) -> ModuleName = filename:basename(FileName), DataDir = ?config(data_dir,Config), case xmerl_xsd:process_schema(filename:join([DataDir, FileName]),[{xsdbase,filename:join([DataDir,XsdBase])}]) of {error,enoent} -> {{ModuleName,enoent},#xsd_state{}}; {Ok,S} -> case Validity of valid when Ok == ok -> %% io:format("schema_test1: Validity=valid,Ok=ok,S=~p~n",[S]), {{ModuleName,S#xsd_state.errors == []},S}; invalid when Ok == error -> %% S is in this case an error reason {{ModuleName,no_internal_error(S)},#xsd_state{}}; notKnown -> {{ModuleName,true},#xsd_state{}}; valid -> io:format("schema_test2: Validity=valid,Ok=~p,S=~p~n",[Ok,S]), %% io:format("FileName: ~p~n",[FileName]), {{ModuleName,false},#xsd_state{}}; _ -> %% invalid Ok == ok io:format("schema_test3: Validity=~p,Ok=~p,S=~p~n",[Validity,Ok,S]), {{ModuleName,false},S} end end. schema_test(Config,FileName,XsdBase,Validity,AccState) -> ModuleName = filename:basename(FileName), DataDir = ?config(data_dir,Config), case xmerl_xsd:process_schema(filename:join([DataDir,FileName]), [{xsdbase,filename:join([DataDir,XsdBase])}, AccState]) of {error,enoent} -> {{ModuleName,enoent},AccState}; {Ok,S} -> case Validity of valid when Ok == ok -> {{ModuleName,S#xsd_state.errors == []},S}; invalid when Ok == error -> {{ModuleName,no_internal_error(S)},AccState}; notKnown -> {{ModuleName,true},AccState}; valid -> {{ModuleName,false},AccState}; _ -> {{ModuleName,false},S} end end. instance_test(Config,FileName,XMLBase,Validity,State) -> ModuleName = filename:basename(FileName), DataDir = ?config(data_dir,Config), case xmerl_scan:file(filename:join([DataDir,FileName]), [{xmlbase,filename:join([DataDir,XMLBase])}]) of {error,enoent} -> {ModuleName,enoent}; {E,_} -> {VE,S2} = xmerl_xsd:validate(E,State), case Validity of valid when is_record(VE,xmlElement) -> case S2#xsd_state.errors of [] -> ok; _ -> io:format("test case ~p failed.~nValidity: ~p~nValidation result:~p~n",[FileName,Validity,VE]) end, {ModuleName,S2#xsd_state.errors == []}; invalid when VE == error -> {ModuleName,no_internal_error(S2)}; notKnown -> {ModuleName,true}; _ -> io:format("test case ~p failed.~nValidity: ~p~nValidation result:~p~n",[FileName,Validity,VE]), {ModuleName,false} end end. no_internal_error(R) -> case lists:keymember(internal_error,1,R) of true -> false; _ -> true end. unpack(Config,Suite) -> TarFile = suite_tar(Suite), ?line file:set_cwd(?config(data_dir,Config)), ?line ok=erl_tar:extract(TarFile,[compressed]), change_mode(filename:rootname(TarFile,".tar.gz")). suite_tar(sun) -> "suntest.tar.gz"; suite_tar(msx) -> "msxsdtest.tar.gz"; suite_tar(nist) -> "nisttest.tar.gz". change_mode(Files) -> change_mode3(Files). change_mode2(Dir)-> ?line {ok,CWD} = file:get_cwd(), ?line {ok,FileList} = file:list_dir(Dir), ?line file:set_cwd(filename:join([CWD,Dir])), change_mode3(FileList), ?line file:set_cwd(CWD). change_mode3([]) -> ok; change_mode3([F|Fs]) -> case filelib:is_dir(F) of true -> chmod(F), change_mode2(F); _ -> chmod(F) end, change_mode3(Fs). chmod(F) -> case file:read_file_info(F) of {ok,FileInfo} -> Mode= FileInfo#file_info.mode, file:write_file_info(F,FileInfo#file_info{mode=8#00777 bor Mode}); _ -> ok end. rmdir(Config,Suite) -> ?line file:set_cwd(?config(data_dir,Config)), SuiteDir = filename:rootname(suite_tar(Suite),".tar.gz"), ?line ok=rm_f_(SuiteDir). %% Dir is a directory rm_f_(Dir) -> ?line {ok,CWD} = file:get_cwd(), ?line {ok,FileList} = file:list_dir(Dir), ?line file:set_cwd(filename:join([CWD,Dir])), rm_files(FileList), ?line file:set_cwd(CWD), ? line ok = file:del_dir(Dir). rm_files([])-> ok; rm_files([F|Fs]) -> case filelib:is_dir(F) of true -> rm_f_(F); _ -> io:format("rm_files: ~p~n",[F]), ?line ok = file:delete(F) end, rm_files(Fs). create_error_log_file(Config,Suite) -> ?line {{Y,M,D},{H,Min,S}} = calendar:local_time(), DTString=lists:concat([Y,"-",M,"-",D,"_",H,".",Min,".",S]), FileName = lists:concat([Suite,"_",DTString,".errorlog"]), %% ?line {ok,_IO} = file:open(filename:join([?config(priv_dir,Config), %% FileName]),[append]). %% ?line {ok,_IO} = file:open(FileName,[append]). io:format("error log file: ~p~n",[filename:join([?config(priv_dir,Config),FileName])]), {ok,filename:join([?config(priv_dir,Config),FileName])}. close_error_log_file(Config) -> case lists:keysearch(xmerl_error_log,1,Config) of {value,{_,IO}} -> file:close(IO); _ -> ok end.