Merge branch 'pan/update_pcre_8.33'

* pan/update_pcre_8.33:
  Workaround TR gnu/181328, GCC 4.2.1 20070831 on FreeBSD 9.1
  Clarify relation between erts_iolist_{size|to_buf}
  Fix backslash in titles of manpages
  Correct UTF-8 in stdlib's notes.xml
  Add more tests for corner error cases in erl_bif_re.c
  Add documentation of report_errors and match_limit(_recursion)
  Add match_limit and match_limit_recursion options
  Add return_errors option to re:run/3
  Add README for updating PCRE
  Add documentation of extensions to re module
  Add new options to Erlang re interface and mend dupnames
  Update PCRE doc part of re.xml to PCRE 8.33 state
  Integrate new PCRE test suites
  Integrate patch for PCRE bug id 1370
  Handle CRLF correctly in global regexp
  Add erts_prefix to pcre_library and update erl_bif_re
  Update to PCRE 8.33, w/o the erts_ prefix added

OTP-11204
OTP-11205
OTP-10285

1 files changed, 161 insertions, 128 deletions

diff --git a/lib/stdlib/test/run_pcre_tests.erl b/lib/stdlib/test/run_pcre_tests.erl
index 78b4803fc8..c4a8afc092 100644
--- a/lib/stdlib/test/run_pcre_tests.erl
+++ b/lib/stdlib/test/run_pcre_tests.erl
@@ -468,25 +468,26 @@ stru([{Line,<<Ch,Re0/binary>>}|T0]) ->
     {NewRe,<< Ch, Options/binary >>} = end_of_re(Ch,Re),
     case interpret_options_x(backstrip(frontstrip(Options)),NewRe) of
 	{Olist,<<>>} -> 
+	    U = lists:member(unicode,Olist),
 	    case T of
 		[{_,<<$-,_/binary>>}|Con] ->
 		    %Debug output, we skip those 
 		    %io:format("Skipping debug (~w)~n",[Line]),
 		    TmpT = skip_debug(Con),
-		    {NewT,Matches} = stru2(TmpT),
+		    {NewT,Matches} = stru2(TmpT,U),
 		    [{NewRe,Line,Olist,Matches}|stru(NewT)];
 		[{_,<<$C,$a,$p,$t,$u,$r,$i,$n,$g,_/binary>>}|_] ->
 		    NewT0 = skip_extra_info(T),
-		    {NewT,Matches} = stru2(NewT0),
+		    {NewT,Matches} = stru2(NewT0,U),
 		    [{NewRe,Line,Olist,Matches}|stru(NewT)];
 		[{_,<<Bla,_/binary>>}|_] when Bla =/= $  ->
 		    %io:format("Skipping blabla (~w)~n",[Line]),
 		    NewT = skip_until_empty(T),
 		    stru(NewT);
 		_ ->
-		    {NewT,Matches} = stru2(T),
+		    {NewT,Matches} = stru2(T,U),
 		    %erlang:display({NewRe,Line,Olist,Matches}),
-		    Matches1 = case lists:member(unicode,Olist) of
+		    Matches1 = case U of
 				   true ->
 				       Matches ++
 					   [ {unicode:characters_to_list(E1,unicode),E2,E3,E4} ||
@@ -498,14 +499,7 @@ stru([{Line,<<Ch,Re0/binary>>}|T0]) ->
 		    [{NewRe,Line,Olist,Matches1}|stru(NewT)]
 	    end;
 	{_,Rest} ->
-%% 	    case T of
-%% 		[{_,<<Bla,_/binary>>}|_] when Bla =/= $  ->
-%% 		    io:format("Skipping blabla (~w)~n",[Line]);
-%% 		_ ->
-%% 		    ok
-%% 	    end,
 	    NewT = skip_until_empty(T),
-	    %{NewT,_Matches} = stru2(T),
 	    info("Skipping options ~s for now (~w)~n",[binary_to_list(Rest),Line]),
 	    case NewT of
 		[{Li,_}|_] ->
@@ -568,10 +562,16 @@ tr_option($g) ->
 tr_option(_) ->
     false.
 
+
 interpret_options(<<$<,Rest0/binary>>) ->
     {Option,Rest} = pinch_cr(Rest0),
-    {Olist,NRest} = interpret_options(Rest),
-    {[Option | Olist], NRest};
+    case Option of
+	{not_supported,{newline,_Offender}} ->
+	    {[],<<$<,Rest0/binary>>};
+	_ ->
+	    {Olist,NRest} = interpret_options(Rest),
+	    {[Option | Olist], NRest}
+    end;
 interpret_options(<<$L,$f,$r,$_,$F,$R,Rest/binary>>) ->
     info("Accepting (and ignoring) french locale~n",[]),
     {Olist,NRest} = interpret_options(Rest),
@@ -603,13 +603,11 @@ backslash_end(<<_>>) ->
 backslash_end(<<_,R/binary>>) ->
     backslash_end(R).
 
-%stru2([<<$ ,$ ,$ ,$ , $*,$*,$*,$ ,_/binary>> | T]) ->
-%    stru2(T);
-stru2([{Line,<<$ ,Rest/binary>>} | T]) ->
+stru2([{Line,<<$ ,Rest/binary>>} | T],U) ->
     % A challenge
-    case  (catch responses(T)) of
+    case  (catch responses(T,U)) of
 	{NewT,Rlist} ->
-	    {NewNewT,StrList} = stru2(NewT),
+	    {NewNewT,StrList} = stru2(NewT,U),
 	    %% Hack...
 	    FS = case backstrip(frontstrip(Rest)) of
 		     <<"\\">> -> 
@@ -624,7 +622,7 @@ stru2([{Line,<<$ ,Rest/binary>>} | T]) ->
 				 OFS
 			 end
 		 end,
-	    {ExecOpts,NFS} = escape(FS),
+	    {ExecOpts,NFS} = escape(FS,U),
 	    case find_unsupported(ExecOpts) of
 		[] ->
 		    {NewNewT,[{NFS,Line,ExecOpts,
@@ -641,30 +639,29 @@ stru2([{Line,<<$ ,Rest/binary>>} | T]) ->
 	    {NewT,[]}
     end;
 	
-stru2(X) ->
+stru2(X,_) ->
     {X,[]}.
-%responses([<< $ ,$ ,$ ,$ ,$*,$*,$*,$ ,_/binary>>|T]) ->
-%    responses(T);
-responses([{_Line,<< X:2/binary,$:,$ ,Resp/binary>>}|T]) ->
-    {NT,R2} = responses(T),
+
+responses([{_Line,<< X:2/binary,$:,$ ,Resp/binary>>}|T],U) ->
+    {NT,R2} = responses(T,U),
     NX=list_to_integer(binary_to_list(frontstrip(X))),
-    {NT,[{NX,escape2(Resp)} | R2]};
-responses([{_Line,<< X:3/binary,$:,$ ,Resp/binary>>}|T]) ->
-    {NT,R2} = responses(T),
+    {NT,[{NX,escape2(Resp,U)} | R2]};
+responses([{_Line,<< X:3/binary,$:,$ ,Resp/binary>>}|T],U) ->
+    {NT,R2} = responses(T,U),
     NX=list_to_integer(binary_to_list(frontstrip(X))),
-    {NT,[{NX,escape2(Resp)} | R2]};
-responses([{_Line,<<$N,$o,$ ,$m,$a,$t,$c,$h,_/binary>>}|T]) ->
+    {NT,[{NX,escape2(Resp,U)} | R2]};
+responses([{_Line,<<$N,$o,$ ,$m,$a,$t,$c,$h,_/binary>>}|T],_) ->
     {T,nomatch};
-responses([{Line,<<$ ,No,Ch,_/binary>>}|T]) when No >= $0, No =< $9, Ch >= $A, Ch =< $Z ->
+responses([{Line,<<$ ,No,Ch,_/binary>>}|T],U) when No >= $0, No =< $9, Ch >= $A, Ch =< $Z ->
     info("Skipping strange debug response at line ~p~n",[Line]),
-    responses(T);
-responses([{Line,<<$ ,$ ,Ch,_/binary>>}|T]) when Ch =:= $G; Ch =:= $C ->
+    responses(T,U);
+responses([{Line,<<$ ,$ ,Ch,_/binary>>}|T],U) when Ch =:= $G; Ch =:= $C ->
     info("Skipping stranger debug response at line ~p~n",[Line]),
-    responses(T);
-responses([{Line,<<C,_/binary>>=X}|_]) when C =/= $  ->
+    responses(T,U);
+responses([{Line,<<C,_/binary>>=X}|_],_) when C =/= $  ->
     info("Offending response line(~w)! ~p~n",[Line,X]),
     throw(fail);
-responses(X) ->
+responses(X,_) ->
     {X,[]}.
     
 
@@ -753,24 +750,38 @@ splitby(Ch,<<Ch,Rest/binary>>,Acc) ->
     {Acc,Rest};
 splitby(Ch,<<OCh,Rest/binary>>,Acc) ->
     splitby(Ch,Rest,<<Acc/binary,OCh>>).
+
+pick_number(N,<<Ch:8,Rest/binary>>) when Ch >= $0, Ch =< $9 ->
+    pick_number(N*10+(Ch - $0),Rest);
+pick_number(N,Rest) ->
+    {N,Rest}.
+
+pick_offset(Rest) ->
+    {Int,NRest} = pick_number(0,Rest),
+    {{offset,Int},NRest}.
 	    
 
-escape(<<>>) ->
+escape(<<>>,_) ->
     {[],<<>>};
-escape(<<$\\, Ch, Rest/binary>>) when Ch >= $A, Ch =< $Z; Ch =:= $? ->
+escape(<<$\\, Ch, Rest/binary>>,U) when Ch >= $A, Ch =< $Z; Ch =:= $? ->
     %Options in the string...
     NewOpts = eopt(Ch),
-    {MoreOpts,Tail} = escape(Rest),
+    {MoreOpts,Tail} = escape(Rest,U),
     {NewOpts ++ MoreOpts,Tail};
-escape(<<$\\, $<, Rest/binary>>) ->
+escape(<<$\\, $>, Rest/binary>>,U) ->
+    %Offset Options in the string...
+    {NewOpt,NewRest} = pick_offset(Rest),
+    {MoreOpts,Tail} = escape(NewRest,U),
+    {[NewOpt|MoreOpts],Tail};
+escape(<<$\\, $<, Rest/binary>>,U) ->
     %CR Options in the string...
     {NewOpt,NewRest} = pinch_cr(Rest),
-    {MoreOpts,Tail} = escape(NewRest),
+    {MoreOpts,Tail} = escape(NewRest,U),
     {[NewOpt|MoreOpts],Tail};
-escape(<<$\\, Ch, Rest/binary>>) ->
+escape(<<$\\, Ch, Rest/binary>>,U) ->
     {C,NR} = case single_esc(Ch) of
 		 no ->
-		     case multi_esc(<<Ch,Rest/binary>>) of
+		     case multi_esc(<<Ch,Rest/binary>>,U) of
 			 {CharBin,NewRest} ->
 			     {CharBin,NewRest};
 			 no ->
@@ -780,30 +791,28 @@ escape(<<$\\, Ch, Rest/binary>>) ->
 		     %erlang:display({escape,CCC}),
 		     {<<CCC>>,Rest}
 	     end,
-    {MoreOpts,Tail} = escape(NR),
+    {MoreOpts,Tail} = escape(NR,U),
     {MoreOpts,<<C/binary,Tail/binary>>};
-%escape(<<$\\,Rest/binary>>) ->
-%    escape(<<Rest/binary>>);
-escape(<<Ch,Rest/binary>>) ->
-    {X,RR} = escape(<<Rest/binary>>),
+escape(<<Ch,Rest/binary>>,U) ->
+    {X,RR} = escape(<<Rest/binary>>,U),
     {X,<<Ch,RR/binary>>};
-escape(Any) ->
+escape(Any,_) ->
     {[],Any}.
-escape2(<<>>) ->
+escape2(<<>>,_) ->
     <<>>;
-escape2(<<$\\, Ch, Rest/binary>>) ->
-    {C,NR} = case multi_esc(<<Ch,Rest/binary>>) of
+escape2(<<$\\, Ch, Rest/binary>>,U) ->
+    {C,NR} = case multi_esc(<<Ch,Rest/binary>>,U) of
 		 {CharBin,NewRest} ->
 		     {CharBin,NewRest};
 		 no ->
 		     {<<$\\>>,<<Ch,Rest/binary>>}
 	     end,
-    Tail = escape2(NR),
+    Tail = escape2(NR,U),
     <<C/binary,Tail/binary>>;
-escape2(<<Ch,Rest/binary>>) ->
-    RR = escape2(<<Rest/binary>>),
+escape2(<<Ch,Rest/binary>>,U) ->
+    RR = escape2(<<Rest/binary>>,U),
     <<Ch,RR/binary>>;
-escape2(Any) ->
+escape2(Any,_) ->
     Any.
 
 
@@ -848,28 +857,34 @@ list_to_utf8(L) when is_list(L); is_binary(L) ->
 list_to_utf8({Tag,_,_}) when Tag =:= incomplete ; Tag =:= error ->
     throw(skip).
 
-multi_esc(<<M,N,O,Rest/binary>>) 
+multi_esc(<<M,N,O,Rest/binary>>,_) 
    when M >= $0, M =< $7, N >= $0, N =< $7, O  >= $0, O =< $7 ->
     Cha = ((M - $0) bsl 6) bor ((N - $0) bsl 3) bor (O - $0),
     {<<Cha>>,Rest};
-multi_esc(<<N,O,Rest/binary>>) 
+multi_esc(<<N,O,Rest/binary>>,_) 
    when N >= $0, N =< $7, O  >= $0, O =< $7 ->
     Cha = ((N - $0) bsl 3) bor (O - $0),
     {<<Cha>>,Rest};
-multi_esc(<<O,Rest/binary>>) 
+multi_esc(<<O,Rest/binary>>,_) 
    when O  >= $0, O =< $7 ->
     Cha = (O - $0),
     {<<Cha>>,Rest};
 
 
-multi_esc(<<$x,${,N,O,$},Rest/binary>>) 
+multi_esc(<<$x,${,N,O,$},Rest/binary>>,Unicode) 
     when ((((N >= $0) and (N =< $9)) or ((N >= $A) and (N =< $F)) or 
 	 ((N >= $a) and (N =< $f))) and 
 	(((O >= $0) and (O =< $9)) or ((O >= $A) and (O =< $F)) or 
 	 ((O >= $a) and (O =< $f)))) -> 
+    %io:format("~p(~p)~n",[<<$x,${,N,O,$}>>,get(unicode)]),
     Cha = (trx(N) bsl 4) bor trx(O),
-    {int_to_utf8(Cha),Rest};
-multi_esc(<<$x,${,N,O,P,$},Rest/binary>>) 
+    case Unicode of
+	false ->
+	    {<<Cha:8>>,Rest};
+	_ ->
+	    {int_to_utf8(Cha),Rest}
+    end;
+multi_esc(<<$x,${,N,O,P,$},Rest/binary>>,_) 
     when ((((N >= $0) and (N =< $9)) or ((N >= $A) and (N =< $F)) or 
 	 ((N >= $a) and (N =< $f))) and 
 	(((O >= $0) and (O =< $9)) or ((O >= $A) and (O =< $F)) or 
@@ -878,7 +893,7 @@ multi_esc(<<$x,${,N,O,P,$},Rest/binary>>)
 	 ((P >= $a) and (P =< $f)))) -> 
     Cha = (trx(N) bsl 8) bor (trx(O) bsl 4) bor trx(P),
     {int_to_utf8(Cha),Rest};
-multi_esc(<<$x,${,N,O,P,Q,$},Rest/binary>>) 
+multi_esc(<<$x,${,N,O,P,Q,$},Rest/binary>>,_) 
     when ((((N >= $0) and (N =< $9)) or ((N >= $A) and (N =< $F)) or 
 	 ((N >= $a) and (N =< $f))) and 
 	(((O >= $0) and (O =< $9)) or ((O >= $A) and (O =< $F)) or 
@@ -889,7 +904,7 @@ multi_esc(<<$x,${,N,O,P,Q,$},Rest/binary>>)
 	 ((Q >= $a) and (Q =< $f)))) -> 
     Cha = (trx(N) bsl 12) bor (trx(O) bsl 8) bor (trx(P) bsl 4) bor trx(Q),
     {int_to_utf8(Cha),Rest};
-multi_esc(<<$x,${,N,O,P,Q,R,$},Rest/binary>>) 
+multi_esc(<<$x,${,N,O,P,Q,R,$},Rest/binary>>,_) 
     when ((((N >= $0) and (N =< $9)) or ((N >= $A) and (N =< $F)) or 
 	 ((N >= $a) and (N =< $f))) and 
 	(((O >= $0) and (O =< $9)) or ((O >= $A) and (O =< $F)) or 
@@ -902,7 +917,7 @@ multi_esc(<<$x,${,N,O,P,Q,R,$},Rest/binary>>)
 	 ((R >= $a) and (R =< $f)))) -> 
     Cha = (trx(N) bsl 16) bor (trx(O) bsl 12) bor (trx(P) bsl 8) bor (trx(Q) bsl 4) bor trx(R),
     {int_to_utf8(Cha),Rest};
-multi_esc(<<$x,${,N,O,P,Q,R,S,$},Rest/binary>>) 
+multi_esc(<<$x,${,N,O,P,Q,R,S,$},Rest/binary>>,_) 
     when ((((N >= $0) and (N =< $9)) or ((N >= $A) and (N =< $F)) or 
 	 ((N >= $a) and (N =< $f))) and 
 	(((O >= $0) and (O =< $9)) or ((O >= $A) and (O =< $F)) or 
@@ -917,19 +932,19 @@ multi_esc(<<$x,${,N,O,P,Q,R,S,$},Rest/binary>>)
 	 ((S >= $a) and (S =< $f)))) -> 
     Cha = (trx(N) bsl 20) bor (trx(O) bsl 16) bor (trx(P) bsl 12) bor (trx(Q) bsl 8) bor (trx(R) bsl 4) bor trx(S),
     {int_to_utf8(Cha),Rest};
-multi_esc(<<$x,N,O,Rest/binary>>) 
+multi_esc(<<$x,N,O,Rest/binary>>,_) 
   when ((((N >= $0) and (N =< $9)) or ((N >= $A) and (N =< $F)) or 
 	 ((N >= $a) and (N =< $f))) and 
 	(((O >= $0) and (O =< $9)) or ((O >= $A) and (O =< $F)) or 
 	 ((O >= $a) and (O =< $f)))) -> 
     Cha = (trx(N) bsl 4) bor trx(O),
     {<<Cha>>,Rest};
-multi_esc(<<$x,N,Rest/binary>>) 
+multi_esc(<<$x,N,Rest/binary>>,_) 
   when (((N >= $0) and (N =< $9)) or ((N >= $A) and (N =< $F)) or 
 	 ((N >= $a) and (N =< $f)))  -> 
     Cha = trx(N),
     {<<Cha>>,Rest};
-multi_esc(_) ->
+multi_esc(_,_) ->
     no.
 
 single_esc($") ->
@@ -980,15 +995,14 @@ gen_split_test(OneFile) ->
     Lines = splitfile(0,Bin,1),
     Structured = stru(Lines),
     PerlShellScript = OneFile++"_split_test_gen.sh",
-    dumpsplit(Structured,PerlShellScript),
-    PerlShellScript,
+    FunList = dumpsplit(Structured,PerlShellScript),
     ErlModule = "re_"++filename:basename(OneFile)++"_split_test",
     ErlFileName = ErlModule++".erl",
     {ok,F}= file:open(ErlFileName,[write]),
     io:format(F,"-module(~s).~n",[ErlModule]),
     io:format(F,"-compile(export_all).~n",[]),
     io:format(F,"-compile(no_native).~n",[]),
-    io:format(F,"-include(\"test_server.hrl\").~n",[]),
+    %io:format(F,"-include(\"test_server.hrl\").~n",[]),
     %io:format(F,"-define(line,erlang:display(?LINE),).~n",[]),
     io:format(F,"%% This file is generated by running ~w:gen_split_test(~p)~n",
 	      [?MODULE,OneFile]),
@@ -996,62 +1010,72 @@ gen_split_test(OneFile) ->
     io:format(F,"join([A]) -> [A];~n",[]),
     io:format(F,"join([H|T]) -> [H,<<\":\">>|join(T)].~n",[]),
     io:format(F,"run() ->~n",[]),
+    [ io:format(F,"    ~s(),~n",[FunName]) || FunName <- FunList ],
     file:close(F),
     os:cmd("sh "++ PerlShellScript++" 2>/dev/null >> "++ErlFileName),
-    {ok,F2}= file:open(ErlFileName,[append]),
-    io:format(F2,"ok.~n",[]),
-    file:close(F2),
     io:format("~s~n",[os:cmd("wc -l "++ErlFileName)]),
     ok.
 
 dumpsplit(S,Fname) ->
     {ok,F}= file:open(Fname,[write]),
-    dodumpsplit(F,S),
-    file:close(F).
-
-dodumpsplit(_,[]) ->
-    ok;
-dodumpsplit(F,[H|T]) ->
+    Res = dodumpsplit(F,S,0,[],0),
+    file:close(F),
+    Res.
+
+dodumpsplit(F,[],_,Acc,_) ->
+    io:format(F,"echo \"    ok.\"~n",[]),
+    lists:reverse(Acc);
+dodumpsplit(F,L,0,Acc,FunNum) ->
+    NewFun = "run"++integer_to_list(FunNum),
+    io:format(F,"echo \"    ok.\"~n",[]),
+    io:format(F,"echo \"~s() ->\"~n",[NewFun]),
+    dodumpsplit(F,L,20,[NewFun|Acc],FunNum+1);
+dodumpsplit(F,[H|T],N,Acc,FunNum) ->
     dumponesplit(F,H),
-    dodumpsplit(F,T).
+    dodumpsplit(F,T,N-1,Acc,FunNum).
 
-dumponesplit(F,{RE,_,O,TS}) ->
+dumponesplit(F,{RE,Line,O,TS}) ->
     [begin
 	 {NO,_} = pick_exec_options(O++Op),
 	 SSS = opt_to_string(NO),
-	 io:format(F,"perl -e '$x = join(\":\",split(/~s/~s,\"~s\")); "
-		   "$x =~~ s/\\\\/\\\\\\\\/g; $x =~~ s/\\\"/\\\\\"/g; "
-		   "print \"?line <<\\\"$x\\\">> = "
-		   "iolist_to_binary(join(re:split(\\\"~s\\\","
-		   "\\\"~s\\\",~p))), \\n\";'~n",
-		   [zsafe(safe(RE)),
-		    SSS,
-		    ysafe(safe(Str)),
-		    dsafe(safe(Str)),
-		    dsafe2(safe(RE)),
-		    NO++[trim]]),
-	 io:format(F,"perl -e '$x = join(\":\",split(/~s/~s,\"~s\",2)); "
-		   "$x =~~ s/\\\\/\\\\\\\\/g; $x =~~ s/\\\"/\\\\\"/g; "
-		   "print \"?line <<\\\"$x\\\">> = "
-		   "iolist_to_binary(join(re:split(\\\"~s\\\","
-		   "\\\"~s\\\",~p))), \\n\";'~n",
-		   [zsafe(safe(RE)),
-		    SSS,
-		    ysafe(safe(Str)),
-		    dsafe(safe(Str)),
-		    dsafe2(safe(RE)),
-		    NO++[{parts,2}]]),
-	 io:format(F,"perl -e '$x = join(\":\",split(/~s/~s,\"~s\",-1)); "
-		   "$x =~~ s/\\\\/\\\\\\\\/g; $x =~~ s/\\\"/\\\\\"/g; "
-		   "print \"?line <<\\\"$x\\\">> = "
-		   "iolist_to_binary(join(re:split(\\\"~s\\\","
-		   "\\\"~s\\\",~p))), \\n\";'~n",
-		   [zsafe(safe(RE)),
-		    SSS,
-		    ysafe(safe(Str)),
-		    dsafe(safe(Str)),
-		    dsafe2(safe(RE)),
-		    NO])
+	 LLL = unicode:characters_to_list(RE),
+	 case (catch iolist_to_binary(LLL)) of
+	     X when is_binary(X) -> 
+		 io:format(F,"perl -e '$x = join(\":\",split(/~s/~s,\"~s\")); "
+			   "$x =~~ s/\\\\/\\\\\\\\/g; $x =~~ s/\\\"/\\\\\"/g; "
+			   "print \"    <<\\\"$x\\\">> = "
+			   "iolist_to_binary(join(re:split(\\\"~s\\\","
+			   "\\\"~s\\\",~p))), \\n\";'~n",
+			   [zsafe(safe(RE)),
+			    SSS,
+			    ysafe(safe(Str)),
+			    dsafe(safe(Str)),
+			    dsafe2(safe(RE)),
+			    NO++[trim]]),
+		 io:format(F,"perl -e '$x = join(\":\",split(/~s/~s,\"~s\",2)); "
+			   "$x =~~ s/\\\\/\\\\\\\\/g; $x =~~ s/\\\"/\\\\\"/g; "
+			   "print \"    <<\\\"$x\\\">> = "
+			   "iolist_to_binary(join(re:split(\\\"~s\\\","
+			   "\\\"~s\\\",~p))), \\n\";'~n",
+			   [zsafe(safe(RE)),
+			    SSS,
+			    ysafe(safe(Str)),
+			    dsafe(safe(Str)),
+			    dsafe2(safe(RE)),
+			    NO++[{parts,2}]]),
+		 io:format(F,"perl -e '$x = join(\":\",split(/~s/~s,\"~s\",-1)); "
+			   "$x =~~ s/\\\\/\\\\\\\\/g; $x =~~ s/\\\"/\\\\\"/g; "
+			   "print \"    <<\\\"$x\\\">> = "
+			   "iolist_to_binary(join(re:split(\\\"~s\\\","
+			   "\\\"~s\\\",~p))), \\n\";'~n",
+			   [zsafe(safe(RE)),
+			    SSS,
+			    ysafe(safe(Str)),
+			    dsafe(safe(Str)),
+			    dsafe2(safe(RE)),
+			    NO]);
+	     _ -> io:format("Found fishy character at line ~w~n",[Line])
+	 end
      end ||
 	{Str,_,Op,_} <- TS].
 
@@ -1063,42 +1087,51 @@ gen_repl_test(OneFile) ->
     Lines = splitfile(0,Bin,1),
     Structured = stru(Lines),
     PerlShellScript = OneFile++"_replacement_test_gen.sh",
-    dump(Structured,PerlShellScript),
+    FunList = dump(Structured,PerlShellScript),
     ErlModule = "re_"++filename:basename(OneFile)++"_replacement_test",
     ErlFileName = ErlModule++".erl",
     {ok,F}= file:open(ErlFileName,[write]),
     io:format(F,"-module(~s).~n",[ErlModule]),
     io:format(F,"-compile(export_all).~n",[]),
     io:format(F,"-compile(no_native).~n",[]),
-    io:format(F,"-include(\"test_server.hrl\").~n",[]),
+    %io:format(F,"-include(\"test_server.hrl\").~n",[]),
     io:format(F,"%% This file is generated by running ~w:gen_repl_test(~p)~n",
 	      [?MODULE,OneFile]),
     io:format(F,"run() ->~n",[]),
+    [ io:format(F,"    ~s(),~n",[FunName]) || FunName <- FunList ],
     file:close(F),
     os:cmd("sh "++ PerlShellScript++" 2>/dev/null >> "++ErlFileName),
-    {ok,F2}= file:open(ErlFileName,[append]),
-    io:format(F2,"ok.~n",[]),
-    file:close(F2),
     io:format("~s~n",[os:cmd("wc -l "++ErlFileName)]),
     ok.
 dump(S,Fname) ->
     {ok,F}= file:open(Fname,[write]),
-    dodump(F,S),
-    file:close(F).
+    Res = dodump(F,S,0,[],0),
+    file:close(F),
+    Res.
     
-dodump(_,[]) ->
-    ok;
-dodump(F,[H|T]) ->
+dodump(F,[],_,Acc,_) ->
+    io:format(F,"echo \"    ok.\"~n",[]),
+    lists:reverse(Acc);
+dodump(F,L,0,Acc,FunNum) ->
+    NewFun = "run"++integer_to_list(FunNum),
+    io:format(F,"echo \"    ok.\"~n",[]),
+    io:format(F,"echo \"~s() ->\"~n",[NewFun]),
+    dodump(F,L,20,[NewFun|Acc],FunNum+1);
+dodump(F,[H|T],N,Acc,FunNum) ->
     dumpone(F,H),
-    dodump(F,T).
+    dodump(F,T,N-1,Acc,FunNum).
 
-dumpone(F,{RE,_,O,TS}) ->
+dumpone(F,{RE,Line,O,TS}) ->
     [begin
 	 {NO,_} = pick_exec_options(O++Op),
 	 SSS = opt_to_string(NO),
 	 RS = ranstring(),
-	 io:format(F,"perl -e '$x = \"~s\"; $x =~~ s/~s/~s/~s; $x =~~ s/\\\\/\\\\\\\\/g; $x =~~ s/\\\"/\\\\\"/g; print \"?line <<\\\"$x\\\">> = iolist_to_binary(re:replace(\\\"~s\\\",\\\"~s\\\",\\\"~s\\\",~p)), \\n\";'~n",[ysafe(safe(Str)),zsafe(safe(RE)),perlify(binary_to_list(RS)),SSS,dsafe(safe(Str)),dsafe(safe(RE)),xsafe(RS),NO]),
-	 io:format(F,"perl -e '$x = \"~s\"; $x =~~ s/~s/~s/g~s; $x =~~ s/\\\\/\\\\\\\\/g; $x =~~ s/\\\"/\\\\\"/g; print \"?line <<\\\"$x\\\">> = iolist_to_binary(re:replace(\\\"~s\\\",\\\"~s\\\",\\\"~s\\\",~p)), \\n\";'~n",[ysafe(safe(Str)),zsafe(safe(RE)),perlify(binary_to_list(RS)),SSS,dsafe(safe(Str)),dsafe(safe(RE)),xsafe(RS),NO++[global]])
+	 LLL = unicode:characters_to_list(RE),
+	 case (catch iolist_to_binary(LLL)) of
+	     X when is_binary(X) -> io:format(F,"perl -e '$x = \"~s\"; $x =~~ s/~s/~s/~s; $x =~~ s/\\\\/\\\\\\\\/g; $x =~~ s/\\\"/\\\\\"/g; print \"    <<\\\"$x\\\">> = iolist_to_binary(re:replace(\\\"~s\\\",\\\"~s\\\",\\\"~s\\\",~p)), \\n\";'~n",[ysafe(safe(Str)),zsafe(safe(RE)),perlify(binary_to_list(RS)),SSS,dsafe(safe(Str)),dsafe(safe(RE)),xsafe(RS),NO]),
+	 io:format(F,"perl -e '$x = \"~s\"; $x =~~ s/~s/~s/g~s; $x =~~ s/\\\\/\\\\\\\\/g; $x =~~ s/\\\"/\\\\\"/g; print \"    <<\\\"$x\\\">> = iolist_to_binary(re:replace(\\\"~s\\\",\\\"~s\\\",\\\"~s\\\",~p)), \\n\";'~n",[ysafe(safe(Str)),zsafe(safe(RE)),perlify(binary_to_list(RS)),SSS,dsafe(safe(Str)),dsafe(safe(RE)),xsafe(RS),NO++[global]]);
+	     _ -> io:format("Found fishy character at line ~w~n",[Line])
+	 end
      end ||
 	{Str,_,Op,_} <- TS].