diff --git a/README.md b/README.md index 0d72aca8..e847a6cf 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ # QuickBuffers - Fast Protocol Buffers without Allocations [![Build](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/maven.yml/badge.svg?branch=main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/maven.yml) -[![Protobuf Conformance Tests](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml) -[![Protobuf Conformance Tests](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=develop)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml) -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/us.hebi.quickbuf/quickbuf-runtime/badge.svg)](https://maven-badges.herokuapp.com/maven-central/us.hebi.quickbuf/quickbuf-runtime) +[![Conformance (main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml) +[![Conformance (develop)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=develop)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml) +[![Maven Central](https://img.shields.io/maven-central/v/us.hebi.quickbuf/quickbuf-runtime)](https://central.sonatype.com/artifact/us.hebi.quickbuf/quickbuf-runtime) QuickBuffers is a Java implementation of [Google's Protocol Buffers](https://developers.google.com/protocol-buffers/) that has been developed for low latency use cases in zero-allocation environments. It has no external dependencies, and the API follows Protobuf-Java where feasible to simplify migration. @@ -52,13 +52,13 @@ In order to use QuickBuffers you need to generate messages and add the correspon The message generator `protoc-gen-quickbuf` is set up as a plugin for the protocol buffers compiler `protoc`. You can install one of the [pre-built packages](https://hebirobotics.github.io/QuickBuffers/download.html) and run: ```sh -protoc-quickbuf --quickbuf_out=${options>: +protoc-quickbuf --quickbuf_out=: ``` or use a [protoc-gen-quickbuf-${version}-${arch}.exe](https://repo1.maven.org/maven2/us/hebi/quickbuf/protoc-gen-quickbuf/) plugin binary with an absolute `pluginPath`: ```sh -protoc --plugin-protoc-gen-quickbuf=${exePath} --quickbuf_out=${options>: +protoc --plugin=protoc-gen-quickbuf=${exePath} --quickbuf_out=: ``` or build messages in Maven using the [protoc-jar-maven-plugin](https://github.com/os72/protoc-jar-maven-plugin): @@ -150,7 +150,7 @@ for (int i = 0; i < 4; i++) { } ``` -Messages can be read from a `ProtoSource` and written to a `ProtoSink`. `newInstance` instantiates optimized implementations for accessing contiguous blocks of memory such as `byte[]` and `ByteBuffer`. Reads and writes do not modify the `ByteBuffer` state, so positions and limits need to be manually if needed. +Messages can be read from a `ProtoSource` and written to a `ProtoSink`. `newInstance` instantiates optimized implementations for accessing contiguous blocks of memory such as `byte[]` and `ByteBuffer`. Reads and writes do not modify the `ByteBuffer` state, so positions and limits need to be updated manually if needed. ```Java // Convenience wrappers @@ -182,6 +182,23 @@ ProtoSink.newInstance(new ByteArrayOutputStream()); ProtoSource.newInstance(new ByteArrayInputStream(bytes)); ``` +A `RepeatedByte` output grows as needed, so the size does not need to be known up front. Together with the length delimited form it can hold a stream of multiple messages. + +```Java +// Write multiple messages to a growing buffer +RepeatedByte bytes = RepeatedByte.newEmptyInstance(); +ProtoSink sink = ProtoSink.newInstance(bytes); +for (RootMessage msg : messages) { + msg.writeDelimitedTo(sink); +} + +// Read them back one at a time +ProtoSource source = ProtoSource.newInstance(bytes); +while (!source.isAtEnd()) { + result.clearQuick().mergeDelimitedFrom(source); +} +``` + Keep in mind that mutability comes at the cost of thread-safety, so contents should be cloned with `ProtoMessage::clone` or copied with `ProtoMessage::copyFrom` before being passed to another thread. **Direct Source/Sink** @@ -416,7 +433,7 @@ Be aware that this prevents the definition of cycles in the message definitions. ## Proguard configuration -There are no reflections, so none of the fields need to be preserved or special cased. However, Proguard may warn about missing methods when obfuscating against an older runtime. This is related to an intentional workaround, so the warnings can just be disabled by adding the line below to the `proguard.conf` file. R8 should automatically pick it up from the bundled [config file](./runtime/src/main/resources/META-INF/proguard/quickbuf-runtime.pro). +There are no reflections, so none of the fields need to be preserved or special cased. However, Proguard may warn about missing methods when obfuscating against an older runtime. This is related to an intentional workaround, so the warnings can just be disabled by adding the line below to the `proguard.conf` file. R8 should automatically pick it up from the bundled [config file](./quickbuf-runtime/src/main/resources/META-INF/proguard/quickbuf-runtime.pro). ```text -dontwarn us.hebi.quickbuf.JdkMethods diff --git a/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/encoding/DecodingBenchmark.java b/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/encoding/DecodingBenchmark.java index b121cd18..9519517e 100644 --- a/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/encoding/DecodingBenchmark.java +++ b/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/encoding/DecodingBenchmark.java @@ -53,7 +53,7 @@ * DecodingBenchmark.skipRawVarint thrpt 10 260,887 ± 3,749 ops/us * * @author Florian Enner - * @since 07 Mär 2022 + * @since 07 Mar 2022 */ @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.MICROSECONDS) diff --git a/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/json/JsonSourceBenchmark.java b/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/json/JsonSourceBenchmark.java index 68261349..9651921d 100644 --- a/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/json/JsonSourceBenchmark.java +++ b/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/json/JsonSourceBenchmark.java @@ -64,7 +64,7 @@ * JsonSourceBenchmark.testJsonMarketDecode thrpt 10 484,539 ± 49,410 ops/ms * * @author Florian Enner - * @since 01 Mär 2022 + * @since 01 Mar 2022 */ @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.MILLISECONDS) diff --git a/protoc-gen-quickbuf/src/main/java/us/hebi/quickbuf/generator/Javadoc.java b/protoc-gen-quickbuf/src/main/java/us/hebi/quickbuf/generator/Javadoc.java index d76a45e7..0a194920 100644 --- a/protoc-gen-quickbuf/src/main/java/us/hebi/quickbuf/generator/Javadoc.java +++ b/protoc-gen-quickbuf/src/main/java/us/hebi/quickbuf/generator/Javadoc.java @@ -38,7 +38,6 @@ class Javadoc { public static String inherit() { // Note: JavaDoc automatically adds the superclass doc, // so we don't need to manually call it out. -// return "{@inheritDoc}"; return ""; } @@ -72,7 +71,7 @@ public static CodeBlock forEnumValue(RequestInfo.EnumValueInfo info) { public static CodeBlock.Builder withComments(SourceCodeInfo.Location location) { // Protobuf-java seems to prefer leading comments and only use trailing as a fallback. - // They also remove the first space as well as empty lines, but that' + // They also remove the first space as well as empty lines, which is not done here. CodeBlock.Builder builder = CodeBlock.builder(); String format = "
\n$L
\n\n"; if (location.hasLeadingComments()) { diff --git a/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/GsonSource.java b/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/GsonSource.java index 2849a578..27e1fa29 100644 --- a/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/GsonSource.java +++ b/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/GsonSource.java @@ -97,16 +97,24 @@ public void readString(Utf8String store) throws IOException { @Override public void readBytes(RepeatedByte store) throws IOException { - decodeBase64(reader.nextString(), store.clear()); + readBase64(reader.nextString(), store.clear()); } @Override public ProtoSource readBytesAsSource() throws IOException { RepeatedByte store = RepeatedByte.newEmptyInstance(); - decodeBase64(reader.nextString(), store); + readBase64(reader.nextString(), store); return ProtoSource.newInstance(store); } + private static void readBase64(String input, RepeatedByte output) throws InvalidJsonException { + try { + decodeBase64(input, output); + } catch (IllegalArgumentException invalid) { + throw new InvalidJsonException(invalid.getMessage()); + } + } + @Override public void skipValue() throws IOException { reader.skipValue(); diff --git a/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/JacksonSource.java b/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/JacksonSource.java index 0c60500a..d065f99d 100644 --- a/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/JacksonSource.java +++ b/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/JacksonSource.java @@ -20,6 +20,8 @@ package us.hebi.quickbuf.compat; +import com.fasterxml.jackson.core.Base64Variant; +import com.fasterxml.jackson.core.Base64Variants; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; @@ -113,7 +115,7 @@ public void readString(Utf8String store) throws IOException { @Override public void readBytes(RepeatedByte store) throws IOException { next(); - store.copyFrom(reader.getBinaryValue()); + store.copyFrom(reader.getBinaryValue(BASE64)); } @Override @@ -121,7 +123,7 @@ public ProtoSource readBytesAsSource() throws IOException { if (next() == JsonToken.VALUE_NULL) { return ProtoSource.newArraySource(); } - return ProtoSource.newInstance(reader.getBinaryValue()); + return ProtoSource.newInstance(reader.getBinaryValue(BASE64)); } @Override @@ -238,4 +240,8 @@ private IOException unexpectedTokenError(JsonToken expectedToken) { JsonToken peekToken = null; + // proto3 json accepts base64 with and without padding + private static final Base64Variant BASE64 = Base64Variants.MIME_NO_LINEFEEDS + .withReadPadding(Base64Variant.PaddingReadBehaviour.PADDING_ALLOWED); + } diff --git a/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java b/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java index ece29e57..fd49e536 100644 --- a/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java +++ b/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java @@ -48,6 +48,16 @@ public void testBadInputs() { testError(CompatibilityTest.JSON_UNKNOWN_FIELD_NULL, "Encountered unknown field: 'unknownField'"); } + @Override + public void testInvalidUnquotedValues() { + // not run: Gson throws unchecked NumberFormatExceptions for invalid numbers + } + + @Override + public void testIntegersOutOfRange() { + // not run: Gson throws unchecked NumberFormatExceptions for invalid numbers + } + @Override protected JsonSource newJsonSource(String json) { return new GsonSource(json); diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySink.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySink.java index d8581ca0..887f42dd 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySink.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySink.java @@ -218,7 +218,7 @@ public ProtoSink setOutput(byte[] buffer, long offset, int length) { return super.setOutput(buffer, offset, length); } else { // Wrapping direct memory - if (offset <= 0) { + if (offset == 0) { // some Android phones use the sign bit as a flag, so negative addresses can be valid throw new NullPointerException("null reference with invalid address offset"); } this.buffer = null; diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java index a3431b99..b3251227 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java @@ -31,9 +31,9 @@ * Source that reads from a flat array * * @author Florian Enner - * @since 06 Mär 2022 + * @since 06 Mar 2022 */ -class ArraySource extends ProtoSource{ +class ArraySource extends ProtoSource { private int bufferSizeAfterLimit; protected int offset; @@ -53,10 +53,10 @@ public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { if (byteLimit < 0) { throw InvalidProtocolBufferException.negativeSize(); } - byteLimit += position; - if (byteLimit > limit) { // limit is always Math.min(limit, currentLimit) + if (byteLimit > limit - position) { // limit is always Math.min(limit, currentLimit) throw InvalidProtocolBufferException.truncatedMessage(); } + byteLimit += position; final int oldLimit = currentLimit; currentLimit = byteLimit; recomputeBufferSizeAfterLimit(); @@ -95,6 +95,7 @@ public void rewindTo(int position) { this.position = offset + position; } + @Override protected void requireRemaining(final int numBytes) throws IOException { if (numBytes < 0) { throw InvalidProtocolBufferException.negativeSize(); @@ -306,7 +307,7 @@ public ProtoSource setInput(byte[] buffer, long offset, int length) { // Wrapping heap buffer baseOffset = BYTE_ARRAY_OFFSET; return super.setInput(buffer, offset, length); - } else if (offset <= 0) { + } else if (offset == 0) { // some Android phones use the sign bit as a flag, so negative addresses can be valid throw new NullPointerException("null reference with invalid base address"); } else if (length < 0) { throw new ArrayIndexOutOfBoundsException("negative length"); diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/Base64.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/Base64.java index 933e0766..2bec2707 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/Base64.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/Base64.java @@ -369,7 +369,7 @@ public static byte[] decode(byte[] sArr) * @param sOff The starting position in the source array. * @param sLen The number of bytes to decode from the source array. Length 0 will return an empty array. * @return The decoded array of bytes. May be of length 0. Will be null if the legal characters - * (including '=') isn't divideable by 4. (I.e. definitely corrupted). + * (including '=') leave a remainder of 1 when divided by 4. (I.e. definitely corrupted). */ public static byte[] decode(byte[] sArr, int sOff, int sLen) { @@ -380,8 +380,8 @@ public static byte[] decode(byte[] sArr, int sOff, int sLen) if (IA[sArr[sOff + i] & 0xff] < 0) sepCnt++; - // Check so that legal chars (including '=') are evenly divideable by 4 as specified in RFC 2045. - if ((sLen - sepCnt) % 4 != 0) + // MODIFIED (ennerf): accept missing padding as allowed by proto3 json. A single remaining char can't encode a byte. + if ((sLen - sepCnt) % 4 == 1) return null; int pad = 0; @@ -390,13 +390,16 @@ public static byte[] decode(byte[] sArr, int sOff, int sLen) pad++; int len = ((sLen - sepCnt) * 6 >> 3) - pad; + if (len < 0) // MODIFIED (ennerf): more padding than content + return null; byte[] dArr = new byte[len]; // Preallocate byte[] of exact length for (int s = 0, d = 0; d < len;) { // Assemble three bytes into an int from four "valid" characters. int i = 0; - for (int j = 0; j < 4; j++) { // j only increased if a valid char was found. + // MODIFIED (ennerf): unpadded input ends within the last group + for (int j = 0; j < 4 && s < sLen; j++) { // j only increased if a valid char was found. int c = IA[sArr[sOff + s++] & 0xff]; if (c >= 0) i |= c << (18 - j * 6); diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java index 494d7a46..098c6881 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java @@ -22,7 +22,7 @@ /** * @author Florian Enner - * @since 18 Mär 2022 + * @since 18 Mar 2022 */ public class InvalidJsonException extends InvalidProtocolBufferException { @@ -39,6 +39,10 @@ static InvalidJsonException truncatedMessage() { "input has been truncated or that the input is invalid."); } + static InvalidJsonException invalidBase64() { + return new InvalidJsonException("Invalid base64 value."); + } + static InvalidJsonException illegalNumberFormat() { return new InvalidJsonException("Illegal number format."); } diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidProtocolBufferException.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidProtocolBufferException.java index c59522ef..94a90f5d 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidProtocolBufferException.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidProtocolBufferException.java @@ -106,7 +106,7 @@ static InvalidProtocolBufferException invalidWireType() { static InvalidProtocolBufferException recursionLimitExceeded() { return new InvalidProtocolBufferException( "Protocol message had too many levels of nesting. May be malicious. " + - "Use ProtoSink.setRecursionLimit() to increase the depth limit."); + "Use ProtoSource.setRecursionLimit() to increase the depth limit."); } static InvalidProtocolBufferException sizeLimitExceeded() { diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java index 31bde442..cab53307 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java @@ -35,7 +35,7 @@ * but a basic version is working. * * @author Florian Enner - * @since 17 Mär 2022 + * @since 17 Mar 2022 */ class JsonDecoding { @@ -113,10 +113,10 @@ public static boolean isWhitespace(int ch) { } public static boolean isBreak(int ch) { - return breaks[ch]; + return ch < 0 || breaks[ch]; // EOF also ends a value } - private static final boolean[] breaks = new boolean[127]; + private static final boolean[] breaks = new boolean[256]; final static int[] sHexValues = new int[256]; final static int[] TRUE_VALUE = {INT_t, INT_r, INT_u, INT_e}; final static int[] FALSE_VALUE = {INT_f, INT_a, INT_l, INT_s, INT_e}; @@ -271,6 +271,8 @@ static char escapedToRawChar(int escapedChar) throws InvalidJsonException { return '\\'; case '"': return '"'; + case '/': + return '/'; case 'b': return '\b'; case 't': @@ -359,7 +361,7 @@ static long readLong(byte[] buffer, int tokenStart, int tokenEnd) throws Invalid int digit; for (result = 0L; i < len; result -= (long) digit) { digit = (buffer[i++ + tokenStart] & 0xFF) - INT_0; - if (digit < 0 || result < multmin) { + if (digit < 0 || digit > 9 || result < multmin) { throw InvalidJsonException.illegalNumberFormat(); } diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSink.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSink.java index a4f8d22d..a418ab4f 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSink.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSink.java @@ -682,7 +682,7 @@ private void writeChar(char c) { @Override public void close() throws IOException { - output.extendCapacityTo(0); + // the output may be owned by the caller and is still needed after closing } @Override diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java index 18f16920..bd3ed32d 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java @@ -203,7 +203,7 @@ public long readFixed64() throws IOException { * Read a {@code fixed32} field value from the source. */ public int readFixed32() throws IOException { - return this.readInt32(); // TODO: read unsigned 32 + return this.readUInt32(); } /** @@ -214,10 +214,15 @@ public > ProtoMsg readGroup(final ProtoM } /** - * Read a {@code uint32} field value from the source. + * Read a {@code uint32} field value from the source. Negative values + * written by older versions represent the same bits. */ public int readUInt32() throws IOException { - return this.readInt32(); // TODO: read unsigned 32 + final long value = this.readInt64(); + if (value < Integer.MIN_VALUE || value > 0xFFFFFFFFL) { + throw outOfRange("uint32", value); + } + return (int) value; } /** @@ -265,7 +270,7 @@ public void readRepeatedSInt64(final RepeatedLong value) throws IOException { } public void readRepeatedFixed32(final RepeatedInt value) throws IOException { - readRepeatedInt32(value); // TODO: read unsigned 32 + readRepeatedUInt32(value); } public void readRepeatedSFixed32(final RepeatedInt value) throws IOException { @@ -273,7 +278,11 @@ public void readRepeatedSFixed32(final RepeatedInt value) throws IOException { } public void readRepeatedUInt32(final RepeatedInt value) throws IOException { - readRepeatedInt32(value); // TODO: read unsigned 32 + beginArray(); + while (!isAtEnd()) { + value.add(readUInt32()); + } + endArray(); } public void readRepeatedSInt32(final RepeatedInt value) throws IOException { @@ -419,12 +428,35 @@ public boolean isAtField(FieldName fieldName) { // ==================== Implementation ==================== + InvalidJsonException outOfRange(String type, Object value) { + String atField = (currentField == null ? "" : " for field '" + currentField + "'"); + return new InvalidJsonException("Value out of range for " + type + ": " + value + atField); + } + + // Note: proto3 json accepts base64 with and without padding protected static void decodeBase64(String input, RepeatedByte output) { - output.addAll(Base64.decodeFast(input)); + final byte[] bytes = input.getBytes(ISO_8859_1); // other chars become an ignored '?' + output.addAll(decodeBase64Unchecked(bytes, bytes.length)); } protected static void decodeBase64(RepeatedByte input, RepeatedByte output) { - output.addAll(Base64.decode(input.array, 0, input.length)); + output.addAll(decodeBase64Unchecked(input.array, input.length)); + } + + static byte[] decodeBase64(byte[] input, int length) throws InvalidJsonException { + final byte[] bytes = Base64.decode(input, 0, length); + if (bytes == null) { + throw InvalidJsonException.invalidBase64(); + } + return bytes; + } + + private static byte[] decodeBase64Unchecked(byte[] input, int length) { + final byte[] bytes = Base64.decode(input, 0, length); + if (bytes == null) { + throw new IllegalArgumentException("Invalid base64 value."); + } + return bytes; } @@ -486,7 +518,11 @@ public double readDouble() throws IOException { @Override public int readInt32() throws IOException { - return (int) this.readInt64(); + final long value = this.readInt64(); + if ((int) value != value) { + throw outOfRange("int32", value); + } + return (int) value; } @Override @@ -531,15 +567,22 @@ public void readString(Utf8String store) throws IOException { @Override public void readBytes(RepeatedByte store) throws IOException { - getValueAsBytes(buffer); + readUnescapedString(buffer); store.clear(); - decodeBase64(buffer, store); + store.addAll(decodeBase64(buffer.array, buffer.length)); } @Override public ProtoSource readBytesAsSource() throws IOException { - getValueAsBytes(buffer); - return ProtoSource.newInstance(Base64.decode(buffer.array, 0, buffer.length)); + readUnescapedString(buffer); + return ProtoSource.newInstance(decodeBase64(buffer.array, buffer.length)); + } + + private void readUnescapedString(RepeatedByte buffer) throws IOException { + checkNotAtNull(); + checkCurrentToken(INT_QUOTE); + JsonDecoding.StringDecoding.readQuotedUtf8(this, buffer); + token = readNextToken(); } @Override @@ -581,6 +624,8 @@ private void skipArray() throws IOException { case INT_RBRACKET: level--; break; + case INT_EOF: + throw InvalidJsonException.truncatedMessage(); } } } @@ -599,6 +644,8 @@ private void skipObject() throws IOException { case INT_RCURLY: level--; break; + case INT_EOF: + throw InvalidJsonException.truncatedMessage(); } } } diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoMessage.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoMessage.java index d95a7753..e3e6d6be 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoMessage.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoMessage.java @@ -127,6 +127,10 @@ public int getCachedSize() { * Computes the number of bytes required to encode this message. * The size is cached and the cached result can be retrieved * using getCachedSize(). + *

+ * Computing the size also encodes string fields to utf8 and caches + * the result, so calling this ahead of time removes the encoding + * cost from subsequent writes of unmodified strings. * * @return the size of the serialized proto form */ diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java index 1ab29f29..e7359767 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java @@ -294,6 +294,7 @@ public boolean skipField(final int tag, final RepeatedByte unknownBytes) throws } case WireFormat.WIRETYPE_LENGTH_DELIMITED:{ int length = readLength(); + requireRemaining(length); ByteUtil.writeUInt32(unknownBytes, tag); ByteUtil.writeUInt32(unknownBytes, length); ByteUtil.writeBytes(unknownBytes, length, this); @@ -358,6 +359,7 @@ protected void skipMessage(RepeatedByte unknownBytes) throws IOException { /** Read a repeated (packed) {@code double} field value from the source. */ public void readPackedDouble(RepeatedDouble store) throws IOException { final int length = readLength(); + requireRemaining(length); final int limit = pushLimit(length); final int count = roundedCount64(length); final int offset = store.addLength(count); @@ -390,6 +392,7 @@ public double readDouble() throws IOException { /** Read a repeated (packed) {@code float} field value from the source. */ public void readPackedFloat(RepeatedFloat store) throws IOException { final int length = readLength(); + requireRemaining(length); final int limit = pushLimit(length); final int count = roundedCount32(length); final int offset = store.addLength(count); @@ -427,6 +430,7 @@ public void readPackedSFixed64(RepeatedLong store) throws IOException { /** Read a repeated (packed) {@code fixed64} field value from the source. */ public void readPackedFixed64(RepeatedLong store) throws IOException { final int length = readLength(); + requireRemaining(length); final int limit = pushLimit(length); final int count = roundedCount64(length); final int offset = store.addLength(count); @@ -474,6 +478,7 @@ public void readPackedSFixed32(RepeatedInt store) throws IOException { /** Read a repeated (packed) {@code fixed32} field value from the source. */ public void readPackedFixed32(RepeatedInt store) throws IOException { final int length = readLength(); + requireRemaining(length); final int limit = pushLimit(length); final int count = roundedCount32(length); final int offset = store.addLength(count); @@ -652,6 +657,7 @@ public int readSInt32() throws IOException { /** Read a repeated (packed) {@code bool} field value from the source. */ public void readPackedBool(RepeatedBoolean store) throws IOException { final int length = readLength(); + requireRemaining(length); final int limit = pushLimit(length); store.reserve(length / MIN_BOOL_SIZE); while (!isAtEnd()) { @@ -706,7 +712,11 @@ public int readEnum() throws IOException { /** Read a length delimiter from the source. */ public int readLength() throws IOException { - return readRawVarint32(); + final int length = readRawVarint32(); + if (length < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + return length; } // ------------------------------ DELIMITED TYPES ------------------------------ @@ -724,6 +734,7 @@ public int readRepeatedString(final RepeatedString store, final int tag) throws /** Read a {@code string} field value from the source. */ public void readString(final Utf8String store) throws IOException { final int length = readLength(); + requireRemaining(length); store.setSize(length); readRawBytes(store.bytes(), 0, length); } @@ -787,6 +798,7 @@ public int readRepeatedBytes(final RepeatedBytes store, final int tag) throws IO public void readBytes(RepeatedByte store) throws IOException { // note: bytes type gets replaced rather than merged final int length = readLength(); + requireRemaining(length); store.setLength(length); readRawBytes(store.array, 0, length); } @@ -997,6 +1009,9 @@ public int setRecursionLimit(final int limit) { * without harming your app's functionality. Note that size limits only apply when reading from an * {@code InputStream}, not when constructed around a raw byte array. * + *

A stream can't know how much input remains, so length prefixes are only checked against + * this limit before allocating storage. Untrusted streams should always set a limit. + * *

If you want to read several messages from a single CodedInputStream, you could call {@link * #resetSizeCounter()} after each one to avoid hitting the size limit. * @@ -1054,10 +1069,11 @@ public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { if (byteLimit < 0) { throw InvalidProtocolBufferException.negativeSize(); } - byteLimit += getTotalBytesRead(); - if (byteLimit > currentLimit) { + final int totalBytesRead = getTotalBytesRead(); + if (byteLimit > currentLimit - totalBytesRead) { throw InvalidProtocolBufferException.truncatedMessage(); } + byteLimit += totalBytesRead; final int oldLimit = currentLimit; currentLimit = byteLimit; return oldLimit; @@ -1132,6 +1148,17 @@ protected void reserveRepeatedFieldCapacity(RepeatedField store, int tag) t // implement in child classes that support looking ahead } + /** + * Checks that the source has enough remaining bytes before space for them gets + * allocated, so that corrupt lengths fail without allocating large arrays. Note + * that this can only be enforced if the source knows the remaining size. + * + * @param numBytes number of bytes that are about to be read + */ + protected void requireRemaining(final int numBytes) throws IOException { + // implement in child classes that know the remaining size + } + /** * Reserves space in the repeated field store to hold the number of remaining varints until * the limit or end of stream is reached. Note that this is an optimization that may do @@ -1302,6 +1329,16 @@ public void readRawBytes(byte[] buffer, int offset, int length) throws IOExcepti } } + @Override + protected void requireRemaining(int numBytes) throws IOException { + final int newPosition = position + numBytes; + if (newPosition - sizeLimit > 0) { // int overflow conscious + throw InvalidProtocolBufferException.sizeLimitExceeded(); + } else if (newPosition - currentLimit > 0) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + } + private void require(int size) throws IOException { final int newPosition = position + size; if (size < 0) { @@ -1356,6 +1393,9 @@ public void rewindTo(int totalBytesRead) { @Override public byte readRawByte() throws IOException { + if (buffer.position() == currentLimit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } try { return buffer.get(); } catch (BufferUnderflowException truncated) { @@ -1364,15 +1404,23 @@ public byte readRawByte() throws IOException { } @Override - public void skipRawBytes(int size) throws IOException { - if (buffer.remaining() < size) { + protected void requireRemaining(int numBytes) throws IOException { + if (numBytes < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } else if (numBytes > buffer.remaining() || numBytes > currentLimit - buffer.position()) { throw InvalidProtocolBufferException.truncatedMessage(); } + } + + @Override + public void skipRawBytes(int size) throws IOException { + requireRemaining(size); ((Buffer) buffer).position(buffer.position() + size); } @Override public void readRawBytes(byte[] values, int offset, int length) throws IOException { + requireRemaining(length); try { buffer.get(values, offset, length); } catch (BufferUnderflowException truncated) { diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java index 347935ca..fb8aff4c 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java @@ -64,7 +64,7 @@ public RepeatedEnum addAll(final E... values) { public RepeatedEnum addAll(final E[] buffer, final int offset, final int length) { final int pos = addLength(length); for (int i = 0; i < length; i++) { - array[pos + i] = buffer[i].getNumber(); + array[pos + i] = buffer[offset + i].getNumber(); } return this; } @@ -91,8 +91,9 @@ public void copyFrom(RepeatedEnum other) { @Override public void addAll(RepeatedEnum values) { - int pos = addLength(values.length); - System.arraycopy(values.array, 0, array, pos, values.length); + final int count = values.length; // values may be this + final int pos = addLength(count); + System.arraycopy(values.array, 0, array, pos, count); } public int getValue(int index) { diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java index 05babe18..74948ad3 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java @@ -57,7 +57,7 @@ public final void addAll(IN[] values) { public final void addAll(IN[] buffer, int offset, int length) { reserve(length); - for (int i = offset; i < length; i++) { + for (int i = offset; i < offset + length; i++) { add(buffer[i]); } } @@ -72,13 +72,19 @@ public final void copyFrom(IN[] buffer) { } public final void copyFrom(IN[] buffer, int offset, int length) { + if (length < this.length) { + clearElementsFrom(length); + } this.length = 0; addAll(buffer, offset, length); } @Override public final void copyFrom(SubType other) { - if (other.length > length) { + if (other.length < length) { + clearElementsFrom(other.length); + } + if (other.length > capacity()) { extendCapacityTo(other.length); } length = other.length; @@ -87,10 +93,17 @@ public final void copyFrom(SubType other) { } } + /** Elements past the length get reused by next(), so they need to be cleared when the length shrinks */ + private void clearElementsFrom(int newLength) { + for (int i = newLength; i < length; i++) { + clearIndex0(i); + } + } + @Override public void addAll(SubType values) { final int newLength = length + values.length; - extendCapacityTo(newLength); + reserve(values.length); for (int i = 0; i < values.length; i++) { copyFrom0(array[length + i], values.array[i]); } diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/UnsafeAccess.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/UnsafeAccess.java index ee6e01ad..66f7491a 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/UnsafeAccess.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/UnsafeAccess.java @@ -1,8 +1,8 @@ /*- * #%L - * MAT File Library + * quickbuf-runtime * %% - * Copyright (C) 2018 HEBI Robotics + * Copyright (C) 2019 HEBI Robotics * %% * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/Base64Test.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/Base64Test.java index 1a1dbe16..8238aeec 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/Base64Test.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/Base64Test.java @@ -67,8 +67,19 @@ public void testDecodeBase64_url_noPadding() throws Exception { } } - private byte[] decodeBase64(CharSequence input) { - return us.hebi.quickbuf.Base64.decodeFast(input.toString()); + @Test + public void testDecodeBase64_keepsInput() throws Exception { + RepeatedByte input = RepeatedByte.newEmptyInstance().addAll(new byte[]{'A', 'Q', 'I'}); + RepeatedByte output = RepeatedByte.newEmptyInstance(); + JsonSource.decodeBase64(input, output); + assertArrayEquals(new byte[]{1, 2}, output.toArray()); + assertArrayEquals(new byte[]{'A', 'Q', 'I'}, input.toArray()); + } + + private byte[] decodeBase64(CharSequence input) throws InvalidJsonException { + RepeatedByte output = RepeatedByte.newEmptyInstance(); + JsonSource.decodeBase64(input.toString(), output); + return output.toArray(); } diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSinkTest.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSinkTest.java index 2e4b1f61..facee0ea 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSinkTest.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSinkTest.java @@ -38,6 +38,15 @@ */ public class JsonSinkTest { + @Test + public void testCloseKeepsOutput() throws IOException { + RepeatedByte output = RepeatedByte.newEmptyInstance(); + try (JsonSink sink = JsonSink.newInstance().setOutput(output)) { + sink.writeMessage(TestAllTypes.newInstance().setOptionalInt32(1)); + } + assertEquals("{\"optionalInt32\":1}", new String(output.toArray(), "UTF-8")); + } + @Test public void testEnumNumbers() throws IOException { TestAllTypes msg = TestAllTypes.newInstance(); diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java index 78bf3ddf..a8e8bae2 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java @@ -72,6 +72,83 @@ public void testSkipping() throws Exception { } + @Test(timeout = 5000) + public void testSkippingTruncatedInput() throws Exception { + for (String json : new String[]{"{\"unknown\":[1", "{\"unknown\":{\"a\":1"}) { + try { + ForeignMessage.newInstance().mergeFrom(newJsonSource(json).setIgnoreUnknownFields(true)); + fail("expected to fail on truncated input: " + json); + } catch (IOException expected) { + } + } + } + + @Test + public void testUnpaddedBase64() throws Exception { + assertArrayEquals(new byte[]{1}, parseJson("{\"optionalBytes\":\"AQ\"}").getOptionalBytes().toArray()); + assertArrayEquals(new byte[]{1, 2}, parseJson("{\"optionalBytes\":\"AQI\"}").getOptionalBytes().toArray()); + assertArrayEquals(new byte[]{1, 2, 3}, parseJson("{\"optionalBytes\":\"AQID\"}").getOptionalBytes().toArray()); + assertArrayEquals(new byte[]{1, 2}, parseJson("{\"optionalBytes\":\"AQI=\"}").getOptionalBytes().toArray()); + + // escapes get decoded first, so escaped line breaks get ignored like other illegal chars + assertArrayEquals(new byte[]{1, 2, 3, 4, 5, 6}, parseJson("{\"optionalBytes\":\"AQID\\r\\nBAUG\"}").getOptionalBytes().toArray()); + assertArrayEquals(new byte[]{1, 2}, parseJson("{\"optionalBytes\":\"\\u0041QI\"}").getOptionalBytes().toArray()); + + for (String json : new String[]{"{\"optionalBytes\":\"A\"}", "{\"optionalBytes\":\"AQIDB\"}"}) { + try { + parseJson(json); + fail("expected to fail on invalid base64 length: " + json); + } catch (IOException expected) { + } + } + } + + @Test + public void testEscapedSolidus() throws Exception { + assertEquals("a/b", parseJson("{\"optionalString\":\"a\\/b\"}").getOptionalString()); + + // PHP escapes forward slashes in base64 values + assertArrayEquals(new byte[]{(byte) 0xAB, (byte) 0xFF}, parseJson("{\"optionalBytes\":\"q\\/8=\"}").getOptionalBytes().toArray()); + assertArrayEquals(new byte[]{(byte) 0xAB, (byte) 0xFF}, parseJson("{\"optionalBytes\":\"q\\/8\"}").getOptionalBytes().toArray()); + } + + @Test + public void testInvalidUnquotedValues() throws Exception { + for (String json : new String[]{"{\"optionalInt32\":1" + (char) 0xE9 + "}", "{\"optionalInt32\":12a}", "{\"optionalInt32\":"}) { + try { + parseJson(json); + fail("expected to fail on invalid input: " + json); + } catch (IOException expected) { + } + } + } + + @Test + public void testIntegersOutOfRange() throws Exception { + for (String json : new String[]{ + "{\"optionalInt32\":2147483648}", + "{\"optionalInt32\":4294967297}", + "{\"optionalUint32\":4294967296}", + "{\"optionalUint32\":-2147483649}"}) { + try { + parseJson(json); + fail("expected to fail on out of range value: " + json); + } catch (IOException expected) { + } + } + } + + @Test + public void testUnsignedInt32() throws Exception { + TestAllTypes msg = parseJson("{\"optionalUint32\":4294967295,\"optionalFixed32\":2147483648," + + "\"repeatedUint32\":[4294967295,-1],\"repeatedFixed32\":[2147483648]}"); + assertEquals(-1, msg.getOptionalUint32()); + assertEquals(Integer.MIN_VALUE, msg.getOptionalFixed32()); + assertEquals(-1, msg.getRepeatedUint32().get(0)); + assertEquals(-1, msg.getRepeatedUint32().get(1)); + assertEquals(Integer.MIN_VALUE, msg.getRepeatedFixed32().get(0)); + } + @Test public void testManualInput() throws Exception { TestAllTypes msg = parseJson(CompatibilityTest.JSON_MANUAL_INPUT); diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java index d5814b8b..c5c6ba46 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java @@ -496,4 +496,72 @@ public void testStreamLimitsExceeded() throws IOException { } + @Test + public void testLimitOverflow() throws IOException { + // nested message with a length of Integer.MAX_VALUE after the tag + assertInvalidForAllSources(new byte[]{(byte) 0x92, 0x01, -1, -1, -1, -1, 0x07}); + } + + @Test(timeout = 5000) + public void testNegativeLength() throws IOException { + // bytes field 15 and unknown field 1000 with a length of -11 + assertInvalidForAllSources(new byte[]{0x7A, -11, -1, -1, -1, -1, -1, -1, -1, -1, 0x01}); + assertInvalidForAllSources(new byte[]{(byte) 0xC2, 0x3E, -11, -1, -1, -1, -1, -1, -1, -1, -1, 0x01}); + } + + @Test + public void testLengthBeyondInput() throws IOException { + // string field 14, bytes field 15, and unknown field 1000 with a length of Integer.MAX_VALUE + assertInvalidForAllSources(new byte[]{0x72, -1, -1, -1, -1, 0x07}); + assertInvalidForAllSources(new byte[]{0x7A, -1, -1, -1, -1, 0x07}); + assertInvalidForAllSources(new byte[]{(byte) 0xC2, 0x3E, -1, -1, -1, -1, 0x07}); + } + + @Test + public void testPackedLengthBeyondInput() throws IOException { + // packed fixed32, fixed64, float, double, and bool fields 37, 38, 41, 42, 43 with a length of 0x7FFFFFF0 + for (byte tag : new byte[]{(byte) 0xAA, (byte) 0xB2, (byte) 0xCA, (byte) 0xD2, (byte) 0xDA}) { + byte[] bytes = Arrays.copyOf(new byte[]{tag, 0x02, (byte) 0xF0, -1, -1, -1, 0x07}, 20); + for (ProtoSource source : newSources(bytes)) { + TestAllTypes msg = TestAllTypes.newInstance(); + try { + msg.mergeFrom(source); + fail(source.getClass().getSimpleName() + " accepted invalid input"); + } catch (InvalidProtocolBufferException expected) { + } + assertEquals(0, msg.getRepeatedFixed32().capacity() + msg.getRepeatedFixed64().capacity() + + msg.getRepeatedFloat().capacity() + msg.getRepeatedDouble().capacity() + + msg.getRepeatedBool().capacity()); + } + } + } + + @Test + public void testPackedLengthNotMultipleOfElementSize() throws IOException { + // packed repeated double field 42 with a length of 5 followed by a string field 14 + assertInvalidForAllSources(new byte[]{(byte) 0xD2, 0x02, 0x05, 1, 2, 3, 4, 5, 0x72, 0x01, 0x61}); + } + + private static void assertInvalidForAllSources(byte[] bytes) throws IOException { + for (ProtoSource source : newSources(bytes)) { + try { + TestAllTypes.newInstance().mergeFrom(source); + fail(source.getClass().getSimpleName() + " accepted invalid input"); + } catch (InvalidProtocolBufferException expected) { + } + } + } + + private static ProtoSource[] newSources(byte[] bytes) { + // streams don't know the remaining size, so they need a size limit to protect against large lengths + ProtoSource stream = ProtoSource.newStreamSource().setInput(new ByteArrayInputStream(bytes)); + stream.setSizeLimit(bytes.length); + return new ProtoSource[]{ + ProtoSource.newArraySource().setInput(bytes), + ProtoSource.newDirectSource().setInput(bytes), + stream, + ProtoSource.newBufferSource().setInput(ByteBuffer.wrap(bytes)) + }; + } + } diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java index 7af0b951..124fd44a 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java @@ -456,6 +456,105 @@ public void testRepeatedStrings() throws IOException { assertEquals(msg, actual); } + @Test + public void testRepeatedObjectsWithOffset() { + RepeatedString strings = RepeatedString.newEmptyInstance(); + strings.addAll(new String[]{"a", "b", "c", "d"}, 1, 2); + assertEquals("[b, c]", strings.toString()); + strings.copyFrom(new String[]{"a", "b", "c", "d"}, 2, 2); + assertEquals("[c, d]", strings.toString()); + + RepeatedBytes bytes = RepeatedBytes.newEmptyInstance(); + bytes.addAll(new byte[][]{{1}, {2}, {3}, {4}}, 1, 2); + assertEquals(2, bytes.length()); + assertArrayEquals(new byte[]{2}, bytes.get(0).toArray()); + assertArrayEquals(new byte[]{3}, bytes.get(1).toArray()); + bytes.copyFrom(new byte[][]{{1}, {2}, {3}, {4}}, 2, 2); + assertEquals(2, bytes.length()); + assertArrayEquals(new byte[]{3}, bytes.get(0).toArray()); + assertArrayEquals(new byte[]{4}, bytes.get(1).toArray()); + + TestAllTypes[] values = new TestAllTypes[4]; + for (int i = 0; i < values.length; i++) { + values[i] = TestAllTypes.newInstance().setOptionalInt32(i); + } + RepeatedMessage messages = RepeatedMessage.newEmptyInstance(TestAllTypes.getFactory()); + messages.addAll(values, 1, 2); + assertEquals(2, messages.length()); + assertEquals(values[1], messages.get(0)); + assertEquals(values[2], messages.get(1)); + messages.copyFrom(values, 2, 2); + assertEquals(2, messages.length()); + assertEquals(values[2], messages.get(0)); + assertEquals(values[3], messages.get(1)); + } + + @Test + public void testRepeatedEnumsWithOffset() { + NestedEnum[] values = new NestedEnum[]{NestedEnum.FOO, NestedEnum.BAR, NestedEnum.BAZ}; + RepeatedEnum enums = RepeatedEnum.newEmptyInstance(NestedEnum.converter()); + enums.addAll(values, 1, 2); + assertEquals("[2, 3]", enums.toString()); + enums.copyFrom(values, 2, 1); + assertEquals("[3]", enums.toString()); + } + + @Test + public void testRepeatedEnumsAddSelf() { + RepeatedEnum enums = RepeatedEnum.newEmptyInstance(NestedEnum.converter()); + enums.addAll(NestedEnum.FOO, NestedEnum.BAR); + enums.addAll(enums); + assertEquals("[1, 2, 1, 2]", enums.toString()); + } + + @Test + public void testRepeatedMessageShrinkingCopy() { + TestAllTypes[] values = new TestAllTypes[]{ + TestAllTypes.newInstance().setOptionalInt32(1).setOptionalInt64(2), + TestAllTypes.newInstance().setOptionalInt32(3).setOptionalInt64(4) + }; + RepeatedMessage shorter = RepeatedMessage.newEmptyInstance(TestAllTypes.getFactory()); + shorter.add(values[0]); + + // reused elements past the new length must not keep old values + RepeatedMessage messages = RepeatedMessage.newEmptyInstance(TestAllTypes.getFactory()); + messages.addAll(values); + messages.copyFrom(shorter); + messages.clear(); + assertEquals(TestAllTypes.newInstance(), messages.next()); + assertEquals(TestAllTypes.newInstance(), messages.next()); + + messages.clear(); + messages.addAll(values); + messages.copyFrom(new TestAllTypes[]{values[0]}); + messages.clear(); + assertEquals(TestAllTypes.newInstance(), messages.next()); + assertEquals(TestAllTypes.newInstance(), messages.next()); + } + + @Test + public void testRepeatedMessageAddAllKeepsCapacity() { + RepeatedMessage other = RepeatedMessage.newEmptyInstance(TestAllTypes.getFactory()); + other.next().setOptionalInt32(1); + other.next().setOptionalInt32(2); + + RepeatedMessage messages = RepeatedMessage.newEmptyInstance(TestAllTypes.getFactory()); + messages.reserve(8); + TestAllTypes first = messages.next().setOptionalInt32(0); + messages.addAll(other); + assertEquals(8, messages.capacity()); + assertSame(first, messages.get(0)); + assertEquals(3, messages.length()); + assertEquals(2, messages.get(2).getOptionalInt32()); + + messages.clear(); + messages.copyFrom(other); + assertEquals(8, messages.capacity()); + assertSame(first, messages.get(0)); + assertEquals(2, messages.length()); + assertEquals(1, messages.get(0).getOptionalInt32()); + } + @Test public void testBytes() throws IOException { byte[] utf8Bytes = "optionalByteString\uD83D\uDCA9".getBytes(UTF_8); @@ -757,6 +856,26 @@ public void testDelimitedStream() throws IOException { assertArrayEquals(outData, inData); } + @Test + public void testDelimitedStreamInGrowingBuffer() throws IOException { + RepeatedByte bytes = RepeatedByte.newEmptyInstance(); + ProtoSink sink = ProtoSink.newInstance(bytes); + for (int i = 0; i < 100; i++) { + TestAllTypes.newInstance().setOptionalInt32(i).setOptionalString("value " + i).writeDelimitedTo(sink); + } + + ProtoSource source = ProtoSource.newInstance(bytes); + TestAllTypes result = TestAllTypes.newInstance(); + int count = 0; + while (!source.isAtEnd()) { + result.clearQuick().mergeDelimitedFrom(source); + assertEquals(count, result.getOptionalInt32()); + assertEquals("value " + count, result.getOptionalString()); + count++; + } + assertEquals(100, count); + } + @Test public void testCollidingFieldNames() throws IOException { TestAllTypes expected = TestAllTypes.newInstance() diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/UnsafeTest.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/UnsafeTest.java index 006cc6f5..68720c03 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/UnsafeTest.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/UnsafeTest.java @@ -83,6 +83,26 @@ public void testUnsafeDirectMemory() throws IOException { assertEquals(message, TestAllTypes.newInstance().mergeFrom(source)); } + @Test + public void testTaggedDirectAddress() { + // e.g. Android with ARM top byte ignore puts a tag into the upper bits + long address = -16L << 56 | 0x1000; + ProtoSink.newDirectSink().setOutput(null, address, 8); + ProtoSource.newDirectSource().setInput(null, address, 8); + + try { + ProtoSink.newDirectSink().setOutput(null, 0, 8); + fail("null address sink"); + } catch (NullPointerException npe) { + } + + try { + ProtoSource.newDirectSource().setInput(null, 0, 8); + fail("null address source"); + } catch (NullPointerException npe) { + } + } + @Test public void testRepeatedPacked() throws IOException { RepeatedPackables.Packed msg = RepeatedPackables.Packed.parseFrom(CompatibilityTest.repeatedPackablesPacked());