From 693df762a0621dd5c55f3556eea7bedcb64b1016 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 25 Jan 2013 13:51:30 +0100 Subject: jinterface: don't need another FilterOutputStream wrapper DeflaterOutputStream is already an FilterOutputStream --- lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java index 22ebb4688a..dafc758b65 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java @@ -773,10 +773,8 @@ public class OtpOutputStream extends ByteArrayOutputStream { final OtpOutputStream oos = new OtpOutputStream(o); write1(OtpExternal.compressedTag); write4BE(oos.size()); - final java.io.FilterOutputStream fos = new java.io.FilterOutputStream( - this); final java.util.zip.DeflaterOutputStream dos = new java.util.zip.DeflaterOutputStream( - fos); + this); try { oos.writeTo(dos); dos.close(); -- cgit v1.2.3 From 770bd5cd6dbda2783efb6d786983c4652840a446 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 25 Jan 2013 13:53:46 +0100 Subject: jinterface: fix typo in error message if encoding fails --- lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java index dafc758b65..b448c29435 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java @@ -780,7 +780,7 @@ public class OtpOutputStream extends ByteArrayOutputStream { dos.close(); } catch (final IOException e) { throw new java.lang.IllegalArgumentException( - "Intremediate stream failed for Erlang object " + o); + "Intermediate stream failed for Erlang object " + o); } } -- cgit v1.2.3 From 9061aa49cf578c3b79b79ed8564cc061847d2a73 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 25 Jan 2013 15:06:43 +0100 Subject: jinterface, OtpOutputStream: properly override the three write() methods to ensure our growth strategy Previously, if code called e.g. write(byte[] b, int off, int len), the growth strategy of the parent class ByteArrayOutputStream was used. --- .../com/ericsson/otp/erlang/OtpOutputStream.java | 38 ++++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java index b448c29435..85100c1911 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java @@ -142,19 +142,37 @@ public class OtpOutputStream extends ByteArrayOutputStream { super.buf[super.count++] = b; } - /** - * Write an array of bytes to the stream. - * - * @param buf - * the array of bytes to write. - * + /* (non-Javadoc) + * @see java.io.ByteArrayOutputStream#write(byte[]) */ - @Override public void write(final byte[] buf) { - ensureCapacity(super.count + buf.length); - System.arraycopy(buf, 0, super.buf, super.count, buf.length); - super.count += buf.length; + // don't assume that super.write(byte[]) calls write(buf, 0, buf.length) + write(buf, 0, buf.length); + } + + /* (non-Javadoc) + * @see java.io.ByteArrayOutputStream#write(int) + */ + @Override + public synchronized void write(int b) { + ensureCapacity(super.count + 1); + super.buf[super.count] = (byte) b; + count += 1; + } + + /* (non-Javadoc) + * @see java.io.ByteArrayOutputStream#write(byte[], int, int) + */ + @Override + public synchronized void write(byte[] b, int off, int len) { + if ((off < 0) || (off > b.length) || (len < 0) + || ((off + len) - b.length > 0)) { + throw new IndexOutOfBoundsException(); + } + ensureCapacity(super.count + len); + System.arraycopy(b, off, super.buf, super.count, len); + super.count += len; } /** -- cgit v1.2.3 From 8e5e2a11f069935644ff5404ffd8758773834f29 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 25 Jan 2013 15:16:52 +0100 Subject: jinterface: don't compress small erlang terms < 5 bytes Compression always has at least 5 bytes (the compressed tag + original size) so we can't get a smaller external term if the original term is already smaller than 5 bytes. --- .../com/ericsson/otp/erlang/OtpOutputStream.java | 38 ++++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java index 85100c1911..8c3f48968e 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java @@ -789,16 +789,34 @@ public class OtpOutputStream extends ByteArrayOutputStream { */ public void write_compressed(final OtpErlangObject o) { final OtpOutputStream oos = new OtpOutputStream(o); - write1(OtpExternal.compressedTag); - write4BE(oos.size()); - final java.util.zip.DeflaterOutputStream dos = new java.util.zip.DeflaterOutputStream( - this); - try { - oos.writeTo(dos); - dos.close(); - } catch (final IOException e) { - throw new java.lang.IllegalArgumentException( - "Intermediate stream failed for Erlang object " + o); + /* + * similar to erts_term_to_binary() in external.c: + * We don't want to compress if compression actually increases the size. + * Since compression uses 5 extra bytes (COMPRESSED tag + size), don't + * compress if the original term is smaller. + */ + if (oos.size() < 5) { + try { + oos.writeTo(this); + // if the term is written as a compressed term, the output + // stream is closed, so we do this here, too + this.close(); + } catch (IOException e) { + throw new java.lang.IllegalArgumentException( + "Intermediate stream failed for Erlang object " + o); + } + } else { + write1(OtpExternal.compressedTag); + write4BE(oos.size()); + final java.util.zip.DeflaterOutputStream dos = new java.util.zip.DeflaterOutputStream( + this); + try { + oos.writeTo(dos); + dos.close(); // note: closes this, too! + } catch (final IOException e) { + throw new java.lang.IllegalArgumentException( + "Intermediate stream failed for Erlang object " + o); + } } } -- cgit v1.2.3 From 9239bca48cad7653b43f89f71c49b3d0be682c51 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 25 Jan 2013 16:12:41 +0100 Subject: jinterface: don't return compressed external term if bigger than uncompressed Now, OtpOutputStream#write_compressed() uses the same mechanism as erts_term_to_binary() in external.c: it tries to compress the given term into a buffer of the size of the uncompressed term and if this is not possible, i.e. the compression plus headers is bigger, it uses the uncompressed external term format instead. --- .../com/ericsson/otp/erlang/OtpOutputStream.java | 43 +++++++++++++++++++--- lib/jinterface/test/nc_SUITE.erl | 10 ++++- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java index 8c3f48968e..541df20369 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java @@ -48,6 +48,8 @@ public class OtpOutputStream extends ByteArrayOutputStream { private static final BigDecimal ten = new BigDecimal(10.0); private static final BigDecimal one = new BigDecimal(1.0); + private boolean fixedSize = false; + /** * Create a stream with the default initial size (2048 bytes). */ @@ -101,10 +103,16 @@ public class OtpOutputStream extends ByteArrayOutputStream { * the storage of an OtpOutputStream instance. */ public void trimToSize() { - if (super.count < super.buf.length) { - final byte[] tmp = new byte[super.count]; - System.arraycopy(super.buf, 0, tmp, 0, super.count); + resize(super.count); + } + + private void resize(int size) { + if (size < super.buf.length) { + final byte[] tmp = new byte[size]; + System.arraycopy(super.buf, 0, tmp, 0, size); super.buf = tmp; + } else if (size > super.buf.length) { + ensureCapacity(size); } } @@ -118,6 +126,9 @@ public class OtpOutputStream extends ByteArrayOutputStream { public void ensureCapacity(int minCapacity) { int oldCapacity = super.buf.length; if (minCapacity > oldCapacity) { + if (fixedSize) { + throw new IllegalArgumentException("Trying to increase fixed-size buffer"); + } int newCapacity = (oldCapacity * 3)/2 + 1; if (newCapacity < oldCapacity + defaultIncrement) newCapacity = oldCapacity + defaultIncrement; @@ -796,6 +807,7 @@ public class OtpOutputStream extends ByteArrayOutputStream { * compress if the original term is smaller. */ if (oos.size() < 5) { + // fast path for small terms try { oos.writeTo(this); // if the term is written as a compressed term, the output @@ -806,16 +818,37 @@ public class OtpOutputStream extends ByteArrayOutputStream { "Intermediate stream failed for Erlang object " + o); } } else { - write1(OtpExternal.compressedTag); - write4BE(oos.size()); + int startCount = super.count; + // we need destCount bytes for an uncompressed term + // -> if compression uses more, use the uncompressed term! + int destCount = startCount + oos.size(); + this.resize(destCount); + this.fixedSize = true; final java.util.zip.DeflaterOutputStream dos = new java.util.zip.DeflaterOutputStream( this); try { + write1(OtpExternal.compressedTag); + write4BE(oos.size()); oos.writeTo(dos); dos.close(); // note: closes this, too! + } catch (final IllegalArgumentException e) { + // could not make the value smaller than originally + // -> reset to starting count, write uncompressed + super.count = startCount; + try { + oos.writeTo(this); + // if the term is written as a compressed term, the output + // stream is closed, so we do this here, too + this.close(); + } catch (IOException e2) { + throw new java.lang.IllegalArgumentException( + "Intermediate stream failed for Erlang object " + o); + } } catch (final IOException e) { throw new java.lang.IllegalArgumentException( "Intermediate stream failed for Erlang object " + o); + } finally { + this.fixedSize = false; } } } diff --git a/lib/jinterface/test/nc_SUITE.erl b/lib/jinterface/test/nc_SUITE.erl index 9c88400c2a..ba96ecf8a6 100644 --- a/lib/jinterface/test/nc_SUITE.erl +++ b/lib/jinterface/test/nc_SUITE.erl @@ -188,7 +188,10 @@ decompress_roundtrip(doc) -> []; decompress_roundtrip(suite) -> []; decompress_roundtrip(Config) when is_list(Config) -> Terms = - [0.0, + [{}, + {a,b,c}, + [], + 0.0, math:sqrt(2), <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,31:5>>, make_ref()], @@ -206,7 +209,10 @@ compress_roundtrip(doc) -> []; compress_roundtrip(suite) -> []; compress_roundtrip(Config) when is_list(Config) -> Terms = - [0.0, + [{}, + {a,b,c}, + [], + 0.0, math:sqrt(2), <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,31:5>>, make_ref()], -- cgit v1.2.3 From bae8d6a4f357f8c9be0661e80a9d163fba56ee7c Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Mon, 28 Jan 2013 19:43:01 +0100 Subject: jinterface: new limited OutputStream implementation without the need to resize (saves memory re-allocations) --- .../java_src/com/ericsson/otp/erlang/OtpOutputStream.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java index 541df20369..1512f9f2aa 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java @@ -48,7 +48,7 @@ public class OtpOutputStream extends ByteArrayOutputStream { private static final BigDecimal ten = new BigDecimal(10.0); private static final BigDecimal one = new BigDecimal(1.0); - private boolean fixedSize = false; + private int fixedSize = Integer.MAX_VALUE; /** * Create a stream with the default initial size (2048 bytes). @@ -124,16 +124,17 @@ public class OtpOutputStream extends ByteArrayOutputStream { * @param minCapacity the desired minimum capacity */ public void ensureCapacity(int minCapacity) { + if (minCapacity > fixedSize) { + throw new IllegalArgumentException("Trying to increase fixed-size buffer"); + } int oldCapacity = super.buf.length; if (minCapacity > oldCapacity) { - if (fixedSize) { - throw new IllegalArgumentException("Trying to increase fixed-size buffer"); - } int newCapacity = (oldCapacity * 3)/2 + 1; if (newCapacity < oldCapacity + defaultIncrement) newCapacity = oldCapacity + defaultIncrement; if (newCapacity < minCapacity) newCapacity = minCapacity; + newCapacity = Math.min(fixedSize, newCapacity); // minCapacity is usually close to size, so this is a win: final byte[] tmp = new byte[newCapacity]; System.arraycopy(super.buf, 0, tmp, 0, super.count); @@ -822,8 +823,7 @@ public class OtpOutputStream extends ByteArrayOutputStream { // we need destCount bytes for an uncompressed term // -> if compression uses more, use the uncompressed term! int destCount = startCount + oos.size(); - this.resize(destCount); - this.fixedSize = true; + this.fixedSize = destCount; final java.util.zip.DeflaterOutputStream dos = new java.util.zip.DeflaterOutputStream( this); try { @@ -848,7 +848,7 @@ public class OtpOutputStream extends ByteArrayOutputStream { throw new java.lang.IllegalArgumentException( "Intermediate stream failed for Erlang object " + o); } finally { - this.fixedSize = false; + this.fixedSize = Integer.MAX_VALUE; } } } -- cgit v1.2.3 From d78f2ee3f2bd9ce38d77d70c0893e16b9d0f8df4 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Mon, 28 Jan 2013 20:01:14 +0100 Subject: jinterface: fix a memory leak after the first try to compress the value with a fixed buffer size, the deflater must be closed so that memory can be (instantly) reused --- .../java_src/com/ericsson/otp/erlang/OtpOutputStream.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java index 1512f9f2aa..884a0cd21c 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java @@ -26,6 +26,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.text.DecimalFormat; import java.util.Arrays; +import java.util.zip.Deflater; /** * Provides a stream for encoding Erlang terms to external format, for @@ -824,14 +825,18 @@ public class OtpOutputStream extends ByteArrayOutputStream { // -> if compression uses more, use the uncompressed term! int destCount = startCount + oos.size(); this.fixedSize = destCount; + Deflater def = new Deflater(); final java.util.zip.DeflaterOutputStream dos = new java.util.zip.DeflaterOutputStream( - this); + this, def); try { write1(OtpExternal.compressedTag); write4BE(oos.size()); oos.writeTo(dos); dos.close(); // note: closes this, too! } catch (final IllegalArgumentException e) { + // discard further un-compressed data + // -> if not called, there may be memory leaks! + def.end(); // could not make the value smaller than originally // -> reset to starting count, write uncompressed super.count = startCount; -- cgit v1.2.3 From 283569b8ede00003c24776961e77051f37dc5594 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Mon, 28 Jan 2013 20:05:24 +0100 Subject: jinterface, OtpOutputStream: add a write_compressed(object, level) method Now that we use an own deflater, we can also allow the user to specify the compression level as in Erlang's term_to_binary/2. --- .../com/ericsson/otp/erlang/OtpOutputStream.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java index 884a0cd21c..f7d5891a27 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java @@ -796,11 +796,23 @@ public class OtpOutputStream extends ByteArrayOutputStream { /** * Write an arbitrary Erlang term to the stream in compressed format. - * + * * @param o - * the Erlang tem to write. + * the Erlang term to write. */ public void write_compressed(final OtpErlangObject o) { + write_compressed(o, Deflater.DEFAULT_COMPRESSION); + } + + /** + * Write an arbitrary Erlang term to the stream in compressed format. + * + * @param o + * the Erlang term to write. + * @param level + * the compression level (0..9) + */ + public void write_compressed(final OtpErlangObject o, int level) { final OtpOutputStream oos = new OtpOutputStream(o); /* * similar to erts_term_to_binary() in external.c: @@ -825,7 +837,7 @@ public class OtpOutputStream extends ByteArrayOutputStream { // -> if compression uses more, use the uncompressed term! int destCount = startCount + oos.size(); this.fixedSize = destCount; - Deflater def = new Deflater(); + Deflater def = new Deflater(level); final java.util.zip.DeflaterOutputStream dos = new java.util.zip.DeflaterOutputStream( this, def); try { -- cgit v1.2.3