Convert command string to encoded binary in heart:set_cmd/1

OTP-10843

In order to allow unicode filenames in heart commands, the command
string is now encoded according to the file name translation mode of
the emulator (file:native_name_encoding()) before it is sent to the
heart process.

2 files changed, 21 insertions, 12 deletions

diff --git a/lib/kernel/doc/src/heart.xml b/lib/kernel/doc/src/heart.xml
index 2856d84dcf..3464832360 100644
--- a/lib/kernel/doc/src/heart.xml
+++ b/lib/kernel/doc/src/heart.xml
@@ -4,7 +4,7 @@
 <erlref>
   <header>
     <copyright>
-      <year>1996</year><year>2012</year>
+      <year>1996</year><year>2013</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -118,8 +118,13 @@
           the system. The new Erlang runtime system will (if it
           misbehaves) use the environment variable
           <c>HEART_COMMAND</c> to reboot.</p>
-        <p>Limitations: The length of the <c><anno>Cmd</anno></c> command string
-          must be less than 2047 characters.</p>
+
+        <p>Limitations: The <c><anno>Cmd</anno></c> command string
+        will be sent to the heart program as a ISO-latin-1 or UTF-8
+        encoded binary depending on the file name encoding mode of the
+        emulator (see
+	<seealso marker="kernel:file#native_name_encoding/0"><c>file:native_name_encoding/0</c></seealso>).
+	The size of the encoded binary must be less than 2047 bytes.</p>
       </desc>
     </func>
     <func>
diff --git a/lib/kernel/src/heart.erl b/lib/kernel/src/heart.erl
index 87cb9d7f51..a3790e90cf 100644
--- a/lib/kernel/src/heart.erl
+++ b/lib/kernel/src/heart.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2012. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2013. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -184,14 +184,18 @@ wait_ack(Port) ->
 loop(Parent, Port, Cmd) ->
     send_heart_beat(Port),
     receive
-	{From, set_cmd, NewCmd} when length(NewCmd) < 2047 ->
-	    send_heart_cmd(Port, NewCmd),
-	    wait_ack(Port),
-	    From ! {heart, ok},
-	    loop(Parent, Port, NewCmd);
-	{From, set_cmd, NewCmd} ->
-	    From ! {heart, {error, {bad_cmd, NewCmd}}},
-	    loop(Parent, Port, Cmd);
+	{From, set_cmd, NewCmd0} ->
+	    Enc = file:native_name_encoding(),
+	    case catch unicode:characters_to_binary(NewCmd0,Enc,Enc) of
+		NewCmd when is_binary(NewCmd), byte_size(NewCmd) < 2047 ->
+		    send_heart_cmd(Port, NewCmd),
+		    wait_ack(Port),
+		    From ! {heart, ok},
+		    loop(Parent, Port, NewCmd);
+		_ ->
+		    From ! {heart, {error, {bad_cmd, NewCmd0}}},
+		    loop(Parent, Port, Cmd)
+	    end;
 	{From, clear_cmd} ->
 	    From ! {heart, ok},
 	    send_heart_cmd(Port, ""),