From ba8f70f0e9279e84350be840529c5e9b66f7f55b Mon Sep 17 00:00:00 2001
From: Dmitry Belyaev <be.dmitry@gmail.com>
Date: Thu, 13 Nov 2014 16:11:20 +1100
Subject: Fix signedness bug

The regression was introduced by making byte type signed.

Bitwise OR (|) applied to signed char (byte) and any longer type reqires the former to expand to the same size.
When the char is negative it is expanded with sign bit filling the extra bytes causing the unexpected result.
---
 lib/odbc/c_src/odbcserver.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

(limited to 'lib/odbc')

diff --git a/lib/odbc/c_src/odbcserver.c b/lib/odbc/c_src/odbcserver.c
index be4e1bbeec..f4b0a5d8d0 100644
--- a/lib/odbc/c_src/odbcserver.c
+++ b/lib/odbc/c_src/odbcserver.c
@@ -1802,10 +1802,23 @@ static int read_exact(byte *buffer, int len) {
 #endif
 
 
+static size_t length_buffer_to_size(byte length_buffer[LENGTH_INDICATOR_SIZE])
+{
+  size_t size = 0, i;
+
+  for (i = 0; i < LENGTH_INDICATOR_SIZE; ++i) {
+    size <<= 8;
+    size |= (unsigned char)length_buffer[i];
+  }
+
+  return size;
+}
+
+
 /* Recieive (read) data from erlang on stdin */
 static byte * receive_erlang_port_msg(void)
 {
-    int i, len = 0;
+    size_t len;
     byte *buffer;
     byte lengthstr[LENGTH_INDICATOR_SIZE];
 
@@ -1814,10 +1827,8 @@ static byte * receive_erlang_port_msg(void)
     {
 	DO_EXIT(EXIT_STDIN_HEADER);
     }
-    for(i=0; i < LENGTH_INDICATOR_SIZE; i++) {
-	len <<= 8;
-	len |= lengthstr[i];
-    }
+
+    len = length_buffer_to_size(lengthstr);
     
     if (len <= 0 || len > 1024) {
 	DO_EXIT(EXIT_STDIN_HEADER);
@@ -1918,8 +1929,7 @@ static byte * receive_msg(int socket)
 #endif
 {
     byte lengthstr[LENGTH_INDICATOR_SIZE];
-    size_t msg_len = 0;
-    int i;
+    size_t msg_len;
     byte *buffer = NULL;
     
     if(!receive_msg_part(socket, lengthstr, LENGTH_INDICATOR_SIZE)) {
@@ -1927,10 +1937,7 @@ static byte * receive_msg(int socket)
 	DO_EXIT(EXIT_SOCKET_RECV_HEADER);
     }
     
-    for(i = 0; i < LENGTH_INDICATOR_SIZE; i++) {
-	msg_len <<= 8;
-	msg_len |= lengthstr[i];
-    }
+    msg_len = length_buffer_to_size(lengthstr);
     
     buffer = (byte *)safe_malloc(msg_len);
 
-- 
cgit v1.2.3