From 985d201454d0cb43d5ed3230d6afeaeea0a1fe2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Tue, 23 Mar 2010 04:31:14 +0000 Subject: OTP-8533 Update observer test suite for R14 --- lib/observer/test/crashdump_viewer_SUITE.erl | 723 +++++++++++++++++++++++++++ 1 file changed, 723 insertions(+) create mode 100644 lib/observer/test/crashdump_viewer_SUITE.erl (limited to 'lib/observer/test/crashdump_viewer_SUITE.erl') diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl new file mode 100644 index 0000000000..fcf383dc2e --- /dev/null +++ b/lib/observer/test/crashdump_viewer_SUITE.erl @@ -0,0 +1,723 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2003-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(crashdump_viewer_SUITE). + +%% Test functions +-export([all/1,translate/1,start/1,fini/1,load_file/1, + non_existing/1,not_a_crashdump/1,old_crashdump/1]). +-export([init_per_suite/1, end_per_suite/1]). +-export([init_per_testcase/2, end_per_testcase/2]). + +-include("test_server.hrl"). +-include("test_server_line.hrl"). +-include_lib("kernel/include/file.hrl"). + +-include_lib("stdlib/include/ms_transform.hrl"). + +-define(default_timeout, ?t:minutes(30)). +-define(sl_alloc_vsns,[r9b]). + +init_per_testcase(_Case, Config) -> + DataDir = ?config(data_dir,Config), + Fs = filelib:wildcard(filename:join(DataDir,"*translated*")), + lists:foreach(fun(F) -> file:delete(F) end,Fs), + catch crashdump_viewer:stop(), + Dog = ?t:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. +end_per_testcase(_Case, Config) -> + Dog=?config(watchdog, Config), + ?t:timetrap_cancel(Dog), + ok. + +all(suite) -> + [translate,{conf,start,[load_file,non_existing,not_a_crashdump, + old_crashdump],fini}]. + +init_per_suite(doc) -> + ["Create a lot of crashdumps which can be used in the testcases below"]; +init_per_suite(Config) when is_list(Config) -> + Dog = ?t:timetrap(?default_timeout), + application:start(inets), % will be using the http client later + http:set_options([{ipv6,disabled}]), + DataDir = ?config(data_dir,Config), + Rels = [R || R <- [r12b,r13b], ?t:is_release_available(R)] ++ [current], + io:format("Creating crash dumps for the following releases: ~p", [Rels]), + AllDumps = create_dumps(DataDir,Rels), + ?t:timetrap_cancel(Dog), + [{dumps,AllDumps}|Config]. + +translate(suite) -> + []; +translate(doc) -> + ["Test that crash dumps from OTP R9B can be translated"]; +translate(Config) when is_list(Config) -> + DataDir = ?config(data_dir,Config), + OutFile = filename:join(DataDir,"translated"), + + R9BFiles = filelib:wildcard(filename:join(DataDir,"r9b_dump.*")), + AllFiles = R9BFiles, + lists:foreach( + fun(File) -> + io:format("Translating file: ~s~n",[File]), + ok = crashdump_translate:old2new(File,OutFile), + check_result(File,OutFile) + end, + AllFiles), + ok. + +start(suite) -> + []; +start(doc) -> + ["Test start and stop of the Crashdump Viewer"]; +start(Config) when is_list(Config) -> + %% Set a much shorter timeout here... We don't have all the time in world. + AngryDog = ?t:timetrap(?t:seconds(30)), + Port = start_cdv(), + true = is_pid(whereis(crashdump_viewer_server)), + true = is_pid(whereis(web_tool)), + Html = contents(Port,"start_page"), + "Welcome to the Web BasedErlang Crash Dump Analyser" = strip(Html), + ok = crashdump_viewer:stop(), + timer:sleep(10), % give some time to stop + undefined = whereis(crashdump_viewer_server), + undefined = whereis(web_tool), + Url = cdv_url(Port,"start_page"), + {error,_} = http:request(get,{Url,[]},[],[]), +% exit(whereis(httpc_manager),kill), + ?t:timetrap_cancel(AngryDog), + ok. + +fini(Config) when is_list(Config) -> + ok. + +load_file(suite) -> + []; +load_file(doc) -> + ["Load files into the tool and view all pages"]; +load_file(Config) when is_list(Config) -> + case ?t:is_debug() of + true -> + {skip,"Debug-compiled emulator -- far too slow"}; + false -> + load_file_1(Config) + end. + + +load_file_1(Config) -> + DataDir = ?config(data_dir,Config), + Port = start_cdv(), + + AllFiles = filelib:wildcard(filename:join(DataDir,"r*_dump.*")), + lists:foreach( + fun(File) -> + browse_file(Port,File), + special(Port,File) + end, + AllFiles), + ok = crashdump_viewer:stop(). + +non_existing(suite) -> + []; +non_existing(doc) -> + ["Try to load nonexisting file"]; +non_existing(Config) when is_list(Config) -> + Port = start_cdv(), + Url = "http://localhost:"++Port++"/cdv_erl/crashdump_viewer/read_file", + Html = request_sync(post,{Url,[],[],"path=nonexistingfile"}), + "Please wait..." = title(Html), + "An error occured:nonexistingfile is not an Erlang crash dump" = + strip(wait(10,Port,"redirect")), + ok = crashdump_viewer:stop(). + +old_crashdump(doc) -> + ["Try to load nonexisting file"]; +old_crashdump(Config) when is_list(Config) -> + Port = start_cdv(), + DataDir = ?config(data_dir, Config), + OldCrashDump = filename:join(DataDir, "old_format.dump"), + Url = "http://localhost:"++Port++"/cdv_erl/crashdump_viewer/read_file", + Html = request_sync(post,{Url,[],[],"path="++OldCrashDump}), + "Please wait..." = title(Html), + Str = "An error occured:The crashdump "++OldCrashDump++ + " is in the pre-R10B format, which is no longer supported.", + Str = strip(wait(10,Port,"redirect")), + ok = crashdump_viewer:stop(). + + +not_a_crashdump(suite) -> + []; +not_a_crashdump(doc) -> + ["Try to load a file which is not an erlang crashdump"]; +not_a_crashdump(Config) when is_list(Config) -> + Port = start_cdv(), + NoCrashdump = code:which(?MODULE), + Url = "http://localhost:"++Port++"/cdv_erl/crashdump_viewer/read_file", + Html = request_sync(post,{Url,[],[],"path="++NoCrashdump}), + "Please wait..." = title(Html), + Str = "An error occured:"++NoCrashdump++" is not an Erlang crash dump", + Str = strip(wait(10,Port,"redirect")), + ok = crashdump_viewer:stop(), +% exit(whereis(httpc_manager),kill), + ok. + + + +end_per_suite(doc) -> + ["Remove generated crashdumps"]; +end_per_suite(Config) when is_list(Config) -> + Dumps = ?config(dumps,Config), + lists:foreach(fun(CD) -> ok = file:delete(CD) end,Dumps), + lists:keydelete(dumps,1,Config). + + +%%%----------------------------------------------------------------- +%%% Internal +start_cdv() -> + ?t:capture_start(), + ok = crashdump_viewer:start(), + "WebTool is available at http://localhost:" ++ Where = + lists:flatten(?t:capture_get()), + ?t:capture_stop(), + [Port|_] = string:tokens(Where,"/"), + Port. + + +check_result(File,OutFile) -> + {ok,#file_info{size=FS}} = file:read_file_info(File), + {ok,#file_info{size=OFS}} = file:read_file_info(OutFile), + Rel = + if OFS > 0 -> FS/OFS; + true -> 1.25 + end, + if Rel>0.75, Rel<1.25 -> ok; + true -> ?t:fail({unreasonable_size,File,FS,OFS}) + end, + {ok,Fd} = file:open(OutFile,[read]), + "=erl_crash_dump:0.0\n" = io:get_line(Fd,''), + case is_truncated(File) of + true -> + ok; + false -> + {ok,_} = file:position(Fd,{eof,-5}), + case io:get_line(Fd,'') of + "=end\n" -> ok; + Other -> ?t:fail({truncated,File,Other}) + end + end, + ok = file:close(Fd). + + +%% Read a page and check that the page title matches Title +contents(Port,Link) -> + Url = cdv_url(Port,Link), + request_sync(get,{Url,[]}). + +cdv_url(Port,Link) -> + "http://localhost:" ++ Port ++ "/cdv_erl/crashdump_viewer/" ++ Link. + +request_sync(Method,HTTPReqCont) -> + case http:request(Method, + HTTPReqCont, + [{timeout,30000}], + [{full_result, false}]) of + {ok,{200,Html}} -> + Html; + {ok,{Code,Html}} -> + io:format("~s\n", [Html]), + io:format("Received ~w from http:request(...) with\nMethod=~w\n" + "HTTPReqCont=~p\n", + [Code,Method,HTTPReqCont]), + ?t:fail(); + Other -> + io:format( + "Received ~w from http:request(...) with\nMethod=~w\n" + "HTTPReqCont=~p\n", + [Other,Method,HTTPReqCont]), + ?t:fail() + end. + + + + +strip([$<|Html]) -> + strip(drop_tag(Html)); +strip([$\n|Html]) -> + strip(Html); +strip([X|Html]) -> + [X|strip(Html)]; +strip([]) -> + []. +drop_tag([$>|Html]) -> + Html; +drop_tag([_|Html]) -> + drop_tag(Html). + +title(Port,Link,Title) -> + Html = contents(Port,Link), + Title = title(Html). + +wait(0,_Port,Link) -> + ?t:fail({wait,Link,timeout}); +wait(Time,Port,Link) -> + Html = contents(Port,Link), + case title(Html) of + "Please wait..." -> + timer:sleep(1000), + wait(Time-1,Port,Link); + _Title -> + Html + end. + +title([$<,$T,$I,$T,$L,$E,$>|Html]) -> + title_end(Html); +title([_|Html]) -> + title(Html); +title([]) -> + []. + +title_end([$<,$/,$T,$I,$T,$L,$E,$>|_]) -> + []; +title_end([X|Html]) -> + [X|title_end(Html)]. + + +%%%----------------------------------------------------------------- +%%% General check of what is displayed for a dump +browse_file(Port,File) -> + io:format("Browsing file: ~s~n",[File]), + + %% The page where a filename can be entered + title(Port,"read_file_frame","Read File"), + + %% Load a file + Url = "http://localhost:"++Port++"/cdv_erl/crashdump_viewer/read_file", + Html = request_sync(post,{Url,[],[],"path="++File}), + "Please wait..." = title(Html), + "Crashdump Viewer Start Page" = title(wait(10,Port,"start_page")), + + %% The frame with the initial information for a dump + title(Port,"initial_info_frame","General Information"), + + %% Topmost frame of the page + FilenameFrame = contents(Port,"filename_frame"), + Match = "FilenameCrashdump currently viewed:" ++ File, + true = lists:prefix(Match,strip(FilenameFrame)), + + %% Toggle a menu item and check that it explodes/collapses + title(Port,"menu_frame","Menu"), + exploded = toggle_menu(Port), + collapsed = toggle_menu(Port), + + %% Open each page in menu and check that correct title is shown + title(Port,"general_info","General Information"), + title(Port,"processes","Process Information"), + title(Port,"sort_procs?sort=state","Process Information"), + title(Port,"sort_procs?sort=state","Process Information"), + title(Port,"sort_procs?sort=pid","Process Information"), + title(Port,"sort_procs?sort=pid","Process Information"), + title(Port,"sort_procs?sort=msg_q_len","Process Information"), + title(Port,"sort_procs?sort=msg_q_len","Process Information"), + title(Port,"sort_procs?sort=reds","Process Information"), + title(Port,"sort_procs?sort=reds","Process Information"), + title(Port,"sort_procs?sort=mem","Process Information"), + title(Port,"sort_procs?sort=mem","Process Information"), + title(Port,"sort_procs?sort=name","Process Information"), + title(Port,"sort_procs?sort=name","Process Information"), + title(Port,"sort_procs?sort=init_func","Process Information"), + title(Port,"sort_procs?sort=init_func","Process Information"), + title(Port,"ports","Port Information"), + title(Port,"ets_tables","ETS Table Information"), + title(Port,"timers","Timer Information"), + title(Port,"fun_table","Fun Information"), + title(Port,"atoms","Atoms"), + title(Port,"dist_info","Distribution Information"), + title(Port,"loaded_modules","Loaded Modules Information"), + title(Port,"hash_tables","Hash Table Information"), + title(Port,"index_tables","Index Table Information"), + title(Port,"memory","Memory Information"), + title(Port,"allocated_areas","Information about allocated areas"), + title(Port,"allocator_info","Allocator Information"), + + case is_truncated(File) of + true -> + ok; + _ -> + proc_details(Port), + port_details(Port), + title(Port,"loaded_mod_details?mod=kernel","kernel") + end, + + ok. + + +special(Port,File) -> + case filename:extension(File) of + ".full_dist" -> + contents(Port,"processes"), + AllProcs = contents(Port,"sort_procs?sort=name"), + + %% I registered a process as aaaaaaaa in the full_dist dumps + %% to make sure it will be the first in the list when sorted + %% on names. There are some special data here, så I'll thoroughly + %% read the process details for this process. Other processes + %% are just briefly traversed. + {Pid,Rest1} = get_first_process(AllProcs), + + ProcDetails = contents(Port,"proc_details?pid=" ++ Pid), + ProcTitle = "Process " ++ Pid, + ProcTitle = title(ProcDetails), + title(Port,"ets_tables?pid="++Pid,"ETS Tables for Process "++Pid), + title(Port,"timers?pid="++Pid,"Timers for Process "++Pid), + + case filename:basename(File) of + "r10b_dump.full_dist" -> + [MsgQueueLink,DictLink,StackDumpLink] = + expand_memory_links(ProcDetails), + MsgQueue = contents(Port,MsgQueueLink), + "MsgQueue" = title(MsgQueue), + title(Port,DictLink,"Dictionary"), + title(Port,StackDumpLink,"StackDump"), + + ExpandBinaryLink = expand_binary_link(MsgQueue), + title(Port,ExpandBinaryLink,"Expanded binary"), + lookat_all_pids(Port,Rest1); + _ -> + ok + end; + ".250atoms" -> + Html1 = contents(Port,"atoms"), + NextLink1 = next_link(Html1), + "Atoms" = title(Html1), + Html2 = contents(Port,NextLink1), + NextLink2 = next_link(Html2), + "Atoms" = title(Html2), + Html3 = contents(Port,NextLink2), + "" = next_link(Html3), + "Atoms" = title(Html3); + _ -> + ok + end, + case filename:basename(File) of + "r10b_dump." ++ _ -> + lookat_all_pids(Port,contents(Port,"processes")); + "r11b_dump." ++ _ -> + lookat_all_pids(Port,contents(Port,"processes")); + _ -> + ok + end, + ok. + + +lookat_all_pids(Port,Pids) -> + case get_first_process(Pids) of + {Pid,Rest} -> + ProcDetails = contents(Port,"proc_details?pid=" ++ Pid), + ProcTitle = "Process " ++ Pid, + ProcTitle = title(ProcDetails), + title(Port,"ets_tables?pid="++Pid,"ETS Tables for Process "++Pid), + title(Port,"timers?pid="++Pid,"Timers for Process "++Pid), + + MemoryLinks = expand_memory_links(ProcDetails), + lists:foreach( + fun(Link) -> + Cont = contents(Port,Link), + true = lists:member(title(Cont), + ["MsgQueue", + "Dictionary", + "StackDump"]) + end, + MemoryLinks), + lookat_all_pids(Port,Rest); + false -> + ok + end. + + +get_first_process([]) -> + false; +get_first_process(Html) -> + case Html of + " + {string:sub_word(Rest,1,$"),Rest}; + [_H|T] -> + get_first_process(T) + end. + +expand_memory_links(Html) -> + case Html of + "MsgQueue + [string:sub_word(Rest,1,$")|expand_memory_links(Rest)]; + "Dictionary + [string:sub_word(Rest,1,$")|expand_memory_links(Rest)]; + "StackDump + [string:sub_word(Rest,1,$")]; + [_H|T] -> + expand_memory_links(T); + [] -> + [] + end. + +expand_binary_link(Html) -> + case Html of + " + "expand_binary?pos=" ++ string:sub_word(Rest,1,$"); + [_H|T] -> + expand_binary_link(T) + end. + + +next_link(Html) -> + case Html of + " + "next?pos=" ++ string:sub_word(Rest,1,$"); + [_H|T] -> + next_link(T); + [] -> + [] + end. + + + +toggle_menu(Port) -> + Html = contents(Port,"toggle?index=10"), + check_toggle(Html). + +check_toggle(Html) -> + case Html of + " + collapsed; + " + exploded; + [_H|T] -> + check_toggle(T) + end. + + +proc_details(Port) -> + ProcDetails = contents(Port,"proc_details?pid=<0.0.0>"), + "Process <0.0.0>" = title(ProcDetails), + + ExpandLink = expand_link(ProcDetails), + title(Port,ExpandLink,"StackDump"), + + Unknown = contents(Port,"proc_details?pid=<0.9999.0>"), + "Could not find process: <0.9999.0>" = title(Unknown). + +expand_link(Html) -> + case Html of + "StackDump + string:sub_word(Rest,1,$"); + [_H|T] -> + expand_link(T) + end. + + +port_details(Port) -> + Port1 = contents(Port,"ports?port=Port<0.1>"), + "#Port<0.1>" = title(Port1), + + Port0 = contents(Port,"ports?port=Port<0.0>"), + "Could not find port: #Port<0.0>" = title(Port0). + +is_truncated(File) -> + case filename:extension(filename:rootname(File)) of + ".trunc" -> true; + _ -> false + end. + + +%%%----------------------------------------------------------------- +%%% +create_dumps(DataDir,Rels) -> + create_dumps(DataDir,Rels,[]). +create_dumps(DataDir,[Rel|Rels],Acc) -> + Fun = fun() -> do_create_dumps(DataDir,Rel) end, + Pa = filename:dirname(code:which(?MODULE)), + {SlAllocDumps,Dumps,DosDump} = + ?t:run_on_shielded_node(Fun, compat_rel(Rel) ++ "-pa " ++ Pa), + create_dumps(DataDir,Rels,SlAllocDumps ++ Dumps ++ Acc ++ DosDump); +create_dumps(_DataDir,[],Acc) -> + Acc. + +do_create_dumps(DataDir,Rel) -> + SlAllocDumps = + case lists:member(Rel,?sl_alloc_vsns) of + true -> + [dump_with_args(DataDir,Rel,"no_sl_alloc","+Se false"), + dump_with_args(DataDir,Rel,"sl_alloc_1","+Se true +Sr 1"), + dump_with_args(DataDir,Rel,"sl_alloc_2","+Se true +Sr 2")]; + false -> + [] + end, + CD1 = full_dist_dump(DataDir,Rel), + CD2 = dump_with_args(DataDir,Rel,"port_is_unix_fd","-oldshell"), + DosDump = + case os:type() of + {unix,sunos} -> dos_dump(DataDir,Rel,CD1); + _ -> [] + end, + case Rel of + current -> + CD3 = dump_with_args(DataDir,Rel,"instr","+Mim true"), + {SlAllocDumps, [CD1,CD2,CD3], DosDump}; + _ -> + {SlAllocDumps, [CD1,CD2], DosDump} + end. + + +%% Create a dump which has two visible nodes, one hidden and one +%% not connected node, and with monitors and links between nodes. +full_dist_dump(DataDir,Rel) -> + Opt = rel_opt(Rel), + Pz = "-pz " ++ filename:dirname(code:which(?MODULE)), + PzOpt = [{args,Pz}], + {ok,N1} = ?t:start_node(n1,peer,Opt ++ PzOpt), + {ok,N2} = ?t:start_node(n2,peer,Opt ++ PzOpt), + {ok,N3} = ?t:start_node(n3,peer,Opt ++ PzOpt), + {ok,N4} = ?t:start_node(n4,peer,Opt ++ [{args,"-hidden " ++ Pz}]), + Creator = self(), + + HelperMod = crashdump_helper, + + P1 = rpc:call(N1,HelperMod,n1_proc,[N2,Creator]), + P2 = rpc:call(N2,HelperMod,remote_proc,[P1,Creator]), + P3 = rpc:call(N3,HelperMod,remote_proc,[P1,Creator]), + P4 = rpc:call(N4,HelperMod,remote_proc,[P1,Creator]), + + get_response(P2), + get_response(P3), + get_response(P4), + get_response(P1), + + L = lists:seq(0,255), + BigMsg = {message,list_to_binary(L),L}, + Port = hd(erlang:ports()), + {aaaaaaaa,N1} ! {short,message,1,2.5,"hello world",Port,{}}, + {aaaaaaaa,N1} ! BigMsg, + + ?t:stop_node(N3), + DumpName = "full_dist", + CD = dump(N1,DataDir,Rel,DumpName), + + ?t:stop_node(N2), + ?t:stop_node(N4), + CD. + +get_response(P) -> + receive {P,done} -> ok + after 3000 -> ?t:fail({get_response_timeout,P,node(P)}) + end. + + +dump_with_args(DataDir,Rel,DumpName,Args) -> + RelOpt = rel_opt(Rel), + Opt = RelOpt ++ [{args,Args}], + {ok,N1} = ?t:start_node(n1,peer,Opt), + CD = dump(N1,DataDir,Rel,DumpName), + ?t:stop_node(n1), + CD. + + + +dump(Node,DataDir,Rel,DumpName) -> + rpc:call(Node,erlang,halt,[DumpName]), + Crashdump0 = filename:join(filename:dirname(code:which(?t)), + "erl_crash_dump.n1"), + Crashdump1 = filename:join(DataDir, dump_prefix(Rel)++DumpName), + ok = rename(Crashdump0,Crashdump1), + Crashdump1. + +rename(From,To) -> + ok = check_complete(From), + case file:rename(From,To) of + {error,exdev} -> + {ok,_} = file:copy(From,To), + ok = file:delete(From); + ok -> + ok + end. + +check_complete(File) -> + check_complete1(File,5). + +check_complete1(_File,0) -> + {error,enoent}; +check_complete1(File,N) -> + case file:read_file_info(File) of + {error,enoent} -> + timer:sleep(500), + check_complete1(File,N-1); + {ok,#file_info{size=Size}} -> + check_complete2(File,Size) + end. + +check_complete2(File,Size) -> + timer:sleep(500), + case file:read_file_info(File) of + {ok,#file_info{size=Size}} -> + ok; + {ok,#file_info{size=OtherSize}} -> + check_complete2(File,OtherSize) + end. + +dos_dump(DataDir,Rel,Dump) -> + DosDumpName = filename:join(DataDir,dump_prefix(Rel)++"dos"), + Cmd = "unix2dos " ++ Dump ++ " > " ++ DosDumpName, + Port = open_port({spawn,Cmd},[exit_status]), + receive + {Port,{exit_status,0}} -> + [DosDumpName]; + {Port,{exit_status,_Error}} -> + ?t:comment("Couldn't run \'unix2dos\'"), + [] + end. + +rel_opt(Rel) -> + case Rel of + r9b -> [{erl,[{release,"r9b_patched"}]}]; + r9c -> [{erl,[{release,"r9c_patched"}]}]; + r10b -> [{erl,[{release,"r10b_patched"}]}]; + r11b -> [{erl,[{release,"r11b_patched"}]}]; + r12b -> [{erl,[{release,"r12b_patched"}]}]; + r13b -> [{erl,[{release,"r13b_patched"}]}]; + current -> [] + end. + +dump_prefix(Rel) -> + case Rel of + r9b -> "r9b_dump."; + r9c -> "r9c_dump."; + r10b -> "r10b_dump."; + r11b -> "r11b_dump."; + r12b -> "r12b_dump."; + r13b -> "r13b_dump."; + current -> "r14b_dump." + end. + +compat_rel(Rel) -> + case Rel of + r9b -> "+R9 "; + r9c -> "+R9 "; + r10b -> "+R10 "; + r11b -> "+R11 "; + r12b -> "+R12 "; + r13b -> "+R13 "; + current -> "" + end. -- cgit v1.2.3