survey

README.md

@@ -7,9 +7,9 @@
 # QuickBuffers - Fast Protocol Buffers without Allocations
 
 [![Build](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/maven.yml/badge.svg?branch=main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/maven.yml)
-[![Protobuf Conformance Tests](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml)
-[![Protobuf Conformance Tests](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=develop)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml)
-[![Maven Central](https://maven-badges.herokuapp.com/maven-central/us.hebi.quickbuf/quickbuf-runtime/badge.svg)](https://maven-badges.herokuapp.com/maven-central/us.hebi.quickbuf/quickbuf-runtime)
+[![Conformance (main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml)
+[![Conformance (develop)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=develop)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml)
+[![Maven Central](https://img.shields.io/maven-central/v/us.hebi.quickbuf/quickbuf-runtime)](https://central.sonatype.com/artifact/us.hebi.quickbuf/quickbuf-runtime)
 
 QuickBuffers is a Java implementation of [Google's Protocol Buffers](https://developers.google.com/protocol-buffers/) that has been developed for low latency use cases in zero-allocation environments. It has no external dependencies, and the API follows Protobuf-Java where feasible to simplify migration.
 
@@ -52,13 +52,13 @@ In order to use QuickBuffers you need to generate messages and add the correspon
 The message generator `protoc-gen-quickbuf` is set up as a plugin for the protocol buffers compiler `protoc`. You can install one of the [pre-built packages](https://hebirobotics.github.io/QuickBuffers/download.html) and run:
 
 ```sh
-protoc-quickbuf --quickbuf_out=${options>:<outputDir> <protoFiles>
+protoc-quickbuf --quickbuf_out=<options>:<outputDir> <protoFiles>
 ```
 
 or use a [protoc-gen-quickbuf-${version}-${arch}.exe](https://repo1.maven.org/maven2/us/hebi/quickbuf/protoc-gen-quickbuf/) plugin binary with an absolute `pluginPath`:
 
 ```sh
-protoc --plugin-protoc-gen-quickbuf=${exePath} --quickbuf_out=${options>:<outputDir> <protoFiles>
+protoc --plugin=protoc-gen-quickbuf=${exePath} --quickbuf_out=<options>:<outputDir> <protoFiles>
 ```
 
 or build messages in Maven using the [protoc-jar-maven-plugin](https://github.com/os72/protoc-jar-maven-plugin):
@@ -150,7 +150,7 @@ for (int i = 0; i < 4; i++) {
 }
 ```
 
-Messages can be read from a `ProtoSource` and written to a `ProtoSink`. `newInstance` instantiates optimized implementations for accessing contiguous blocks of memory such as `byte[]` and `ByteBuffer`. Reads and writes do not modify the `ByteBuffer` state, so positions and limits need to be manually if needed.
+Messages can be read from a `ProtoSource` and written to a `ProtoSink`. `newInstance` instantiates optimized implementations for accessing contiguous blocks of memory such as `byte[]` and `ByteBuffer`. Reads and writes do not modify the `ByteBuffer` state, so positions and limits need to be updated manually if needed.
 
 ```Java
 // Convenience wrappers
@@ -182,6 +182,23 @@ ProtoSink.newInstance(new ByteArrayOutputStream());
 ProtoSource.newInstance(new ByteArrayInputStream(bytes));
 ```
 
+A `RepeatedByte` output grows as needed, so the size does not need to be known up front. Together with the length delimited form it can hold a stream of multiple messages.
+
+```Java
+// Write multiple messages to a growing buffer
+RepeatedByte bytes = RepeatedByte.newEmptyInstance();
+ProtoSink sink = ProtoSink.newInstance(bytes);
+for (RootMessage msg : messages) {
+    msg.writeDelimitedTo(sink);
+}
+
+// Read them back one at a time
+ProtoSource source = ProtoSource.newInstance(bytes);
+while (!source.isAtEnd()) {
+    result.clearQuick().mergeDelimitedFrom(source);
+}
+```
+
 Keep in mind that mutability comes at the cost of thread-safety, so contents should be cloned with `ProtoMessage::clone` or copied with `ProtoMessage::copyFrom` before being passed to another thread.
 
 **Direct Source/Sink**
@@ -416,7 +433,7 @@ Be aware that this prevents the definition of cycles in the message definitions.
 
 ## Proguard configuration
 
-There are no reflections, so none of the fields need to be preserved or special cased. However, Proguard may warn about missing methods when obfuscating against an older runtime. This is related to an intentional workaround, so the warnings can just be disabled by adding the line below to the `proguard.conf` file. R8 should automatically pick it up from the bundled [config file](./runtime/src/main/resources/META-INF/proguard/quickbuf-runtime.pro).
+There are no reflections, so none of the fields need to be preserved or special cased. However, Proguard may warn about missing methods when obfuscating against an older runtime. This is related to an intentional workaround, so the warnings can just be disabled by adding the line below to the `proguard.conf` file. R8 should automatically pick it up from the bundled [config file](./quickbuf-runtime/src/main/resources/META-INF/proguard/quickbuf-runtime.pro).
 
 ```text
 -dontwarn us.hebi.quickbuf.JdkMethods

benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/encoding/DecodingBenchmark.java

@@ -53,7 +53,7 @@
  * DecodingBenchmark.skipRawVarint    thrpt   10  260,887 ± 3,749  ops/us
  *
  * @author Florian Enner
- * @since 07 Mär 2022
+ * @since 07 Mar 2022
  */
 @BenchmarkMode(Mode.Throughput)
 @OutputTimeUnit(TimeUnit.MICROSECONDS)

benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/json/JsonSourceBenchmark.java

@@ -64,7 +64,7 @@
  * JsonSourceBenchmark.testJsonMarketDecode     thrpt   10  484,539 ± 49,410  ops/ms
  *
  * @author Florian Enner
- * @since 01 Mär 2022
+ * @since 01 Mar 2022
  */
 @BenchmarkMode(Mode.Throughput)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)

protoc-gen-quickbuf/src/main/java/us/hebi/quickbuf/generator/Javadoc.java

@@ -38,7 +38,6 @@ class Javadoc {
     public static String inherit() {
         // Note: JavaDoc automatically adds the superclass doc,
         // so we don't need to manually call it out.
-//        return "{@inheritDoc}";
         return "";
     }
 
@@ -72,7 +71,7 @@ public static CodeBlock forEnumValue(RequestInfo.EnumValueInfo info) {
 
     public static CodeBlock.Builder withComments(SourceCodeInfo.Location location) {
         // Protobuf-java seems to prefer leading comments and only use trailing as a fallback.
-        // They also remove the first space as well as empty lines, but that'
+        // They also remove the first space as well as empty lines, which is not done here.
         CodeBlock.Builder builder = CodeBlock.builder();
         String format = "<pre>\n$L</pre>\n\n";
         if (location.hasLeadingComments()) {

quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/GsonSource.java

@@ -97,16 +97,24 @@ public void readString(Utf8String store) throws IOException {
 
     @Override
     public void readBytes(RepeatedByte store) throws IOException {
-        decodeBase64(reader.nextString(), store.clear());
+        readBase64(reader.nextString(), store.clear());
     }
 
     @Override
     public ProtoSource readBytesAsSource() throws IOException {
         RepeatedByte store = RepeatedByte.newEmptyInstance();
-        decodeBase64(reader.nextString(), store);
+        readBase64(reader.nextString(), store);
         return ProtoSource.newInstance(store);
     }
 
+    private static void readBase64(String input, RepeatedByte output) throws InvalidJsonException {
+        try {
+            decodeBase64(input, output);
+        } catch (IllegalArgumentException invalid) {
+            throw new InvalidJsonException(invalid.getMessage());
+        }
+    }
+
     @Override
     public void skipValue() throws IOException {
         reader.skipValue();

quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/JacksonSource.java

@@ -20,6 +20,8 @@
 
 package us.hebi.quickbuf.compat;
 
+import com.fasterxml.jackson.core.Base64Variant;
+import com.fasterxml.jackson.core.Base64Variants;
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.JsonParser;
@@ -113,15 +115,15 @@ public void readString(Utf8String store) throws IOException {
     @Override
     public void readBytes(RepeatedByte store) throws IOException {
         next();
-        store.copyFrom(reader.getBinaryValue());
+        store.copyFrom(reader.getBinaryValue(BASE64));
     }
 
     @Override
     public ProtoSource readBytesAsSource() throws IOException {
         if (next() == JsonToken.VALUE_NULL) {
             return ProtoSource.newArraySource();
         }
-        return ProtoSource.newInstance(reader.getBinaryValue());
+        return ProtoSource.newInstance(reader.getBinaryValue(BASE64));
     }
 
     @Override
@@ -238,4 +240,8 @@ private IOException unexpectedTokenError(JsonToken expectedToken) {
 
     JsonToken peekToken = null;
 
+    // proto3 json accepts base64 with and without padding
+    private static final Base64Variant BASE64 = Base64Variants.MIME_NO_LINEFEEDS
+            .withReadPadding(Base64Variant.PaddingReadBehaviour.PADDING_ALLOWED);
+
 }

quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java

@@ -48,6 +48,16 @@ public void testBadInputs() {
         testError(CompatibilityTest.JSON_UNKNOWN_FIELD_NULL, "Encountered unknown field: 'unknownField'");
     }
 
+    @Override
+    public void testInvalidUnquotedValues() {
+        // not run: Gson throws unchecked NumberFormatExceptions for invalid numbers
+    }
+
+    @Override
+    public void testIntegersOutOfRange() {
+        // not run: Gson throws unchecked NumberFormatExceptions for invalid numbers
+    }
+
     @Override
     protected JsonSource newJsonSource(String json) {
         return new GsonSource(json);

quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySink.java

@@ -218,7 +218,7 @@ public ProtoSink setOutput(byte[] buffer, long offset, int length) {
                 return super.setOutput(buffer, offset, length);
             } else {
                 // Wrapping direct memory
-                if (offset <= 0) {
+                if (offset == 0) { // some Android phones use the sign bit as a flag, so negative addresses can be valid
                     throw new NullPointerException("null reference with invalid address offset");
                 }
                 this.buffer = null;

quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java

@@ -31,9 +31,9 @@
  * Source that reads from a flat array
  *
  * @author Florian Enner
- * @since 06 Mär 2022
+ * @since 06 Mar 2022
  */
-class ArraySource extends ProtoSource{
+class ArraySource extends ProtoSource {
 
     private int bufferSizeAfterLimit;
     protected int offset;
@@ -53,10 +53,10 @@ public int pushLimit(int byteLimit) throws InvalidProtocolBufferException {
         if (byteLimit < 0) {
             throw InvalidProtocolBufferException.negativeSize();
         }
-        byteLimit += position;
-        if (byteLimit > limit) { // limit is always Math.min(limit, currentLimit)
+        if (byteLimit > limit - position) { // limit is always Math.min(limit, currentLimit)
             throw InvalidProtocolBufferException.truncatedMessage();
         }
+        byteLimit += position;
         final int oldLimit = currentLimit;
         currentLimit = byteLimit;
         recomputeBufferSizeAfterLimit();
@@ -95,6 +95,7 @@ public void rewindTo(int position) {
         this.position = offset + position;
     }
 
+    @Override
     protected void requireRemaining(final int numBytes) throws IOException {
         if (numBytes < 0) {
             throw InvalidProtocolBufferException.negativeSize();
@@ -306,7 +307,7 @@ public ProtoSource setInput(byte[] buffer, long offset, int length) {
                 // Wrapping heap buffer
                 baseOffset = BYTE_ARRAY_OFFSET;
                 return super.setInput(buffer, offset, length);
-            } else if (offset <= 0) {
+            } else if (offset == 0) { // some Android phones use the sign bit as a flag, so negative addresses can be valid
                 throw new NullPointerException("null reference with invalid base address");
             } else if (length < 0) {
                 throw new ArrayIndexOutOfBoundsException("negative length");

quickbuf-runtime/src/main/java/us/hebi/quickbuf/Base64.java

@@ -369,7 +369,7 @@ public static byte[] decode(byte[] sArr)
      * @param sOff The starting position in the source array.
      * @param sLen The number of bytes to decode from the source array. Length 0 will return an empty array.
      * @return The decoded array of bytes. May be of length 0. Will be <code>null</code> if the legal characters
-     * (including '=') isn't divideable by 4. (I.e. definitely corrupted).
+     * (including '=') leave a remainder of 1 when divided by 4. (I.e. definitely corrupted).
      */
     public static byte[] decode(byte[] sArr, int sOff, int sLen)
     {
@@ -380,8 +380,8 @@ public static byte[] decode(byte[] sArr, int sOff, int sLen)
             if (IA[sArr[sOff + i] & 0xff] < 0)
                 sepCnt++;
 
-        // Check so that legal chars (including '=') are evenly divideable by 4 as specified in RFC 2045.
-        if ((sLen - sepCnt) % 4 != 0)
+        // MODIFIED (ennerf): accept missing padding as allowed by proto3 json. A single remaining char can't encode a byte.
+        if ((sLen - sepCnt) % 4 == 1)
             return null;
 
         int pad = 0;
@@ -390,13 +390,16 @@ public static byte[] decode(byte[] sArr, int sOff, int sLen)
                 pad++;
 
         int len = ((sLen - sepCnt) * 6 >> 3) - pad;
+        if (len < 0) // MODIFIED (ennerf): more padding than content
+            return null;
 
         byte[] dArr = new byte[len];       // Preallocate byte[] of exact length
 
         for (int s = 0, d = 0; d < len;) {
             // Assemble three bytes into an int from four "valid" characters.
             int i = 0;
-            for (int j = 0; j < 4; j++) {   // j only increased if a valid char was found.
+            // MODIFIED (ennerf): unpadded input ends within the last group
+            for (int j = 0; j < 4 && s < sLen; j++) {   // j only increased if a valid char was found.
                 int c = IA[sArr[sOff + s++] & 0xff];
                 if (c >= 0)
                     i |= c << (18 - j * 6);

quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java

@@ -22,7 +22,7 @@
 
 /**
  * @author Florian Enner
- * @since 18 Mär 2022
+ * @since 18 Mar 2022
  */
 public class InvalidJsonException extends InvalidProtocolBufferException {
 
@@ -39,6 +39,10 @@ static InvalidJsonException truncatedMessage() {
                         "input has been truncated or that the input is invalid.");
     }
 
+    static InvalidJsonException invalidBase64() {
+        return new InvalidJsonException("Invalid base64 value.");
+    }
+
     static InvalidJsonException illegalNumberFormat() {
         return new InvalidJsonException("Illegal number format.");
     }

quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidProtocolBufferException.java

@@ -106,7 +106,7 @@ static InvalidProtocolBufferException invalidWireType() {
     static InvalidProtocolBufferException recursionLimitExceeded() {
         return new InvalidProtocolBufferException(
                 "Protocol message had too many levels of nesting.  May be malicious.  " +
-                        "Use ProtoSink.setRecursionLimit() to increase the depth limit.");
+                        "Use ProtoSource.setRecursionLimit() to increase the depth limit.");
     }
 
     static InvalidProtocolBufferException sizeLimitExceeded() {

quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java

@@ -35,7 +35,7 @@
  * but a basic version is working.
  *
  * @author Florian Enner
- * @since 17 Mär 2022
+ * @since 17 Mar 2022
  */
 class JsonDecoding {
 
@@ -113,10 +113,10 @@ public static boolean isWhitespace(int ch) {
         }
 
         public static boolean isBreak(int ch) {
-            return breaks[ch];
+            return ch < 0 || breaks[ch]; // EOF also ends a value
         }
 
-        private static final boolean[] breaks = new boolean[127];
+        private static final boolean[] breaks = new boolean[256];
         final static int[] sHexValues = new int[256];
         final static int[] TRUE_VALUE = {INT_t, INT_r, INT_u, INT_e};
         final static int[] FALSE_VALUE = {INT_f, INT_a, INT_l, INT_s, INT_e};
@@ -271,6 +271,8 @@ static char escapedToRawChar(int escapedChar) throws InvalidJsonException {
                     return '\\';
                 case '"':
                     return '"';
+                case '/':
+                    return '/';
                 case 'b':
                     return '\b';
                 case 't':
@@ -359,7 +361,7 @@ static long readLong(byte[] buffer, int tokenStart, int tokenEnd) throws Invalid
                 int digit;
                 for (result = 0L; i < len; result -= (long) digit) {
                     digit = (buffer[i++ + tokenStart] & 0xFF) - INT_0;
-                    if (digit < 0 || result < multmin) {
+                    if (digit < 0 || digit > 9 || result < multmin) {
                         throw InvalidJsonException.illegalNumberFormat();
                     }
 

quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSink.java

@@ -682,7 +682,7 @@ private void writeChar(char c) {
 
         @Override
         public void close() throws IOException {
-            output.extendCapacityTo(0);
+            // the output may be owned by the caller and is still needed after closing
         }
 
         @Override