From ce701c7dd996225d8903c400a74348b312395434 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Wed, 21 Oct 2026 22:10:00 +0200 Subject: [PATCH 01/20] fixed surveyed typos, links and comments --- README.md | 14 +++++++------- .../benchmarks/encoding/DecodingBenchmark.java | 2 +- .../benchmarks/json/JsonSourceBenchmark.java | 2 +- .../java/us/hebi/quickbuf/generator/Javadoc.java | 3 +-- .../main/java/us/hebi/quickbuf/ArraySource.java | 4 ++-- .../us/hebi/quickbuf/InvalidJsonException.java | 2 +- .../main/java/us/hebi/quickbuf/JsonDecoding.java | 2 +- .../main/java/us/hebi/quickbuf/UnsafeAccess.java | 4 ++-- 8 files changed, 16 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 0d72aca8..5ba29ff9 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ # QuickBuffers - Fast Protocol Buffers without Allocations [![Build](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/maven.yml/badge.svg?branch=main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/maven.yml) -[![Protobuf Conformance Tests](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml) -[![Protobuf Conformance Tests](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=develop)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml) -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/us.hebi.quickbuf/quickbuf-runtime/badge.svg)](https://maven-badges.herokuapp.com/maven-central/us.hebi.quickbuf/quickbuf-runtime) +[![Conformance (main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=main)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml) +[![Conformance (develop)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml/badge.svg?branch=develop)](https://github.com/HebiRobotics/QuickBuffers/actions/workflows/conformance.yml) +[![Maven Central](https://img.shields.io/maven-central/v/us.hebi.quickbuf/quickbuf-runtime)](https://central.sonatype.com/artifact/us.hebi.quickbuf/quickbuf-runtime) QuickBuffers is a Java implementation of [Google's Protocol Buffers](https://developers.google.com/protocol-buffers/) that has been developed for low latency use cases in zero-allocation environments. It has no external dependencies, and the API follows Protobuf-Java where feasible to simplify migration. @@ -52,13 +52,13 @@ In order to use QuickBuffers you need to generate messages and add the correspon The message generator `protoc-gen-quickbuf` is set up as a plugin for the protocol buffers compiler `protoc`. You can install one of the [pre-built packages](https://hebirobotics.github.io/QuickBuffers/download.html) and run: ```sh -protoc-quickbuf --quickbuf_out=${options>: +protoc-quickbuf --quickbuf_out=: ``` or use a [protoc-gen-quickbuf-${version}-${arch}.exe](https://repo1.maven.org/maven2/us/hebi/quickbuf/protoc-gen-quickbuf/) plugin binary with an absolute `pluginPath`: ```sh -protoc --plugin-protoc-gen-quickbuf=${exePath} --quickbuf_out=${options>: +protoc --plugin=protoc-gen-quickbuf=${exePath} --quickbuf_out=: ``` or build messages in Maven using the [protoc-jar-maven-plugin](https://github.com/os72/protoc-jar-maven-plugin): @@ -150,7 +150,7 @@ for (int i = 0; i < 4; i++) { } ``` -Messages can be read from a `ProtoSource` and written to a `ProtoSink`. `newInstance` instantiates optimized implementations for accessing contiguous blocks of memory such as `byte[]` and `ByteBuffer`. Reads and writes do not modify the `ByteBuffer` state, so positions and limits need to be manually if needed. +Messages can be read from a `ProtoSource` and written to a `ProtoSink`. `newInstance` instantiates optimized implementations for accessing contiguous blocks of memory such as `byte[]` and `ByteBuffer`. Reads and writes do not modify the `ByteBuffer` state, so positions and limits need to be updated manually if needed. ```Java // Convenience wrappers @@ -416,7 +416,7 @@ Be aware that this prevents the definition of cycles in the message definitions. ## Proguard configuration -There are no reflections, so none of the fields need to be preserved or special cased. However, Proguard may warn about missing methods when obfuscating against an older runtime. This is related to an intentional workaround, so the warnings can just be disabled by adding the line below to the `proguard.conf` file. R8 should automatically pick it up from the bundled [config file](./runtime/src/main/resources/META-INF/proguard/quickbuf-runtime.pro). +There are no reflections, so none of the fields need to be preserved or special cased. However, Proguard may warn about missing methods when obfuscating against an older runtime. This is related to an intentional workaround, so the warnings can just be disabled by adding the line below to the `proguard.conf` file. R8 should automatically pick it up from the bundled [config file](./quickbuf-runtime/src/main/resources/META-INF/proguard/quickbuf-runtime.pro). ```text -dontwarn us.hebi.quickbuf.JdkMethods diff --git a/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/encoding/DecodingBenchmark.java b/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/encoding/DecodingBenchmark.java index b121cd18..9519517e 100644 --- a/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/encoding/DecodingBenchmark.java +++ b/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/encoding/DecodingBenchmark.java @@ -53,7 +53,7 @@ * DecodingBenchmark.skipRawVarint thrpt 10 260,887 ± 3,749 ops/us * * @author Florian Enner - * @since 07 Mär 2022 + * @since 07 Mar 2022 */ @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.MICROSECONDS) diff --git a/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/json/JsonSourceBenchmark.java b/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/json/JsonSourceBenchmark.java index 68261349..9651921d 100644 --- a/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/json/JsonSourceBenchmark.java +++ b/benchmarks/src/main/java/us/hebi/quickbuf/benchmarks/json/JsonSourceBenchmark.java @@ -64,7 +64,7 @@ * JsonSourceBenchmark.testJsonMarketDecode thrpt 10 484,539 ± 49,410 ops/ms * * @author Florian Enner - * @since 01 Mär 2022 + * @since 01 Mar 2022 */ @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.MILLISECONDS) diff --git a/protoc-gen-quickbuf/src/main/java/us/hebi/quickbuf/generator/Javadoc.java b/protoc-gen-quickbuf/src/main/java/us/hebi/quickbuf/generator/Javadoc.java index d76a45e7..0a194920 100644 --- a/protoc-gen-quickbuf/src/main/java/us/hebi/quickbuf/generator/Javadoc.java +++ b/protoc-gen-quickbuf/src/main/java/us/hebi/quickbuf/generator/Javadoc.java @@ -38,7 +38,6 @@ class Javadoc { public static String inherit() { // Note: JavaDoc automatically adds the superclass doc, // so we don't need to manually call it out. -// return "{@inheritDoc}"; return ""; } @@ -72,7 +71,7 @@ public static CodeBlock forEnumValue(RequestInfo.EnumValueInfo info) { public static CodeBlock.Builder withComments(SourceCodeInfo.Location location) { // Protobuf-java seems to prefer leading comments and only use trailing as a fallback. - // They also remove the first space as well as empty lines, but that' + // They also remove the first space as well as empty lines, which is not done here. CodeBlock.Builder builder = CodeBlock.builder(); String format = "
\n$L
\n\n"; if (location.hasLeadingComments()) { diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java index a3431b99..e58c4cb8 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java @@ -31,9 +31,9 @@ * Source that reads from a flat array * * @author Florian Enner - * @since 06 Mär 2022 + * @since 06 Mar 2022 */ -class ArraySource extends ProtoSource{ +class ArraySource extends ProtoSource { private int bufferSizeAfterLimit; protected int offset; diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java index 494d7a46..98fd885f 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java @@ -22,7 +22,7 @@ /** * @author Florian Enner - * @since 18 Mär 2022 + * @since 18 Mar 2022 */ public class InvalidJsonException extends InvalidProtocolBufferException { diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java index 31bde442..5bd10c48 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java @@ -35,7 +35,7 @@ * but a basic version is working. * * @author Florian Enner - * @since 17 Mär 2022 + * @since 17 Mar 2022 */ class JsonDecoding { diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/UnsafeAccess.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/UnsafeAccess.java index ee6e01ad..66f7491a 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/UnsafeAccess.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/UnsafeAccess.java @@ -1,8 +1,8 @@ /*- * #%L - * MAT File Library + * quickbuf-runtime * %% - * Copyright (C) 2018 HEBI Robotics + * Copyright (C) 2019 HEBI Robotics * %% * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From d0824c87c37c617be9c2aa2d14e0a09b00cd66ba Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 13:24:39 +0200 Subject: [PATCH 02/20] documented the utf8 caching side effect of computing the serialized size --- .../src/main/java/us/hebi/quickbuf/ProtoMessage.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoMessage.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoMessage.java index d95a7753..e3e6d6be 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoMessage.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoMessage.java @@ -127,6 +127,10 @@ public int getCachedSize() { * Computes the number of bytes required to encode this message. * The size is cached and the cached result can be retrieved * using getCachedSize(). + *

+ * Computing the size also encodes string fields to utf8 and caches + * the result, so calling this ahead of time removes the encoding + * cost from subsequent writes of unmodified strings. * * @return the size of the serialized proto form */ From e5f496ee2a8b93613a290985615297e523a84b31 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 13:25:37 +0200 Subject: [PATCH 03/20] documented growing outputs and length delimited streams --- README.md | 17 ++++++++++++++++ .../java/us/hebi/quickbuf/ProtoTests.java | 20 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/README.md b/README.md index 5ba29ff9..e847a6cf 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,23 @@ ProtoSink.newInstance(new ByteArrayOutputStream()); ProtoSource.newInstance(new ByteArrayInputStream(bytes)); ``` +A `RepeatedByte` output grows as needed, so the size does not need to be known up front. Together with the length delimited form it can hold a stream of multiple messages. + +```Java +// Write multiple messages to a growing buffer +RepeatedByte bytes = RepeatedByte.newEmptyInstance(); +ProtoSink sink = ProtoSink.newInstance(bytes); +for (RootMessage msg : messages) { + msg.writeDelimitedTo(sink); +} + +// Read them back one at a time +ProtoSource source = ProtoSource.newInstance(bytes); +while (!source.isAtEnd()) { + result.clearQuick().mergeDelimitedFrom(source); +} +``` + Keep in mind that mutability comes at the cost of thread-safety, so contents should be cloned with `ProtoMessage::clone` or copied with `ProtoMessage::copyFrom` before being passed to another thread. **Direct Source/Sink** diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java index 7af0b951..3bc06e7a 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java @@ -757,6 +757,26 @@ public void testDelimitedStream() throws IOException { assertArrayEquals(outData, inData); } + @Test + public void testDelimitedStreamInGrowingBuffer() throws IOException { + RepeatedByte bytes = RepeatedByte.newEmptyInstance(); + ProtoSink sink = ProtoSink.newInstance(bytes); + for (int i = 0; i < 100; i++) { + TestAllTypes.newInstance().setOptionalInt32(i).setOptionalString("value " + i).writeDelimitedTo(sink); + } + + ProtoSource source = ProtoSource.newInstance(bytes); + TestAllTypes result = TestAllTypes.newInstance(); + int count = 0; + while (!source.isAtEnd()) { + result.clearQuick().mergeDelimitedFrom(source); + assertEquals(count, result.getOptionalInt32()); + assertEquals("value " + count, result.getOptionalString()); + count++; + } + assertEquals(100, count); + } + @Test public void testCollidingFieldNames() throws IOException { TestAllTypes expected = TestAllTypes.newInstance() From ca0a443cc1ac2903169d9f6b139475f2d1d0874b Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:21:08 +0200 Subject: [PATCH 04/20] fixed repeated objects ignoring the array offset in addAll --- .../java/us/hebi/quickbuf/RepeatedObject.java | 2 +- .../java/us/hebi/quickbuf/ProtoTests.java | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java index 05babe18..3ebb5b94 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java @@ -57,7 +57,7 @@ public final void addAll(IN[] values) { public final void addAll(IN[] buffer, int offset, int length) { reserve(length); - for (int i = offset; i < length; i++) { + for (int i = offset; i < offset + length; i++) { add(buffer[i]); } } diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java index 3bc06e7a..e4f26088 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java @@ -456,6 +456,39 @@ public void testRepeatedStrings() throws IOException { assertEquals(msg, actual); } + @Test + public void testRepeatedObjectsWithOffset() { + RepeatedString strings = RepeatedString.newEmptyInstance(); + strings.addAll(new String[]{"a", "b", "c", "d"}, 1, 2); + assertEquals("[b, c]", strings.toString()); + strings.copyFrom(new String[]{"a", "b", "c", "d"}, 2, 2); + assertEquals("[c, d]", strings.toString()); + + RepeatedBytes bytes = RepeatedBytes.newEmptyInstance(); + bytes.addAll(new byte[][]{{1}, {2}, {3}, {4}}, 1, 2); + assertEquals(2, bytes.length()); + assertArrayEquals(new byte[]{2}, bytes.get(0).toArray()); + assertArrayEquals(new byte[]{3}, bytes.get(1).toArray()); + bytes.copyFrom(new byte[][]{{1}, {2}, {3}, {4}}, 2, 2); + assertEquals(2, bytes.length()); + assertArrayEquals(new byte[]{3}, bytes.get(0).toArray()); + assertArrayEquals(new byte[]{4}, bytes.get(1).toArray()); + + TestAllTypes[] values = new TestAllTypes[4]; + for (int i = 0; i < values.length; i++) { + values[i] = TestAllTypes.newInstance().setOptionalInt32(i); + } + RepeatedMessage messages = RepeatedMessage.newEmptyInstance(TestAllTypes.getFactory()); + messages.addAll(values, 1, 2); + assertEquals(2, messages.length()); + assertEquals(values[1], messages.get(0)); + assertEquals(values[2], messages.get(1)); + messages.copyFrom(values, 2, 2); + assertEquals(2, messages.length()); + assertEquals(values[2], messages.get(0)); + assertEquals(values[3], messages.get(1)); + } + @Test public void testBytes() throws IOException { byte[] utf8Bytes = "optionalByteString\uD83D\uDCA9".getBytes(UTF_8); From 687a3eb08a0d41cc9b1d7c159bf27bf97bd88a7b Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:22:49 +0200 Subject: [PATCH 05/20] fixed reused repeated elements keeping old values after a shrinking copy --- .../java/us/hebi/quickbuf/RepeatedObject.java | 13 ++++++++++ .../java/us/hebi/quickbuf/ProtoTests.java | 25 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java index 3ebb5b94..337e3f35 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java @@ -72,12 +72,18 @@ public final void copyFrom(IN[] buffer) { } public final void copyFrom(IN[] buffer, int offset, int length) { + if (length < this.length) { + clearElementsFrom(length); + } this.length = 0; addAll(buffer, offset, length); } @Override public final void copyFrom(SubType other) { + if (other.length < length) { + clearElementsFrom(other.length); + } if (other.length > length) { extendCapacityTo(other.length); } @@ -87,6 +93,13 @@ public final void copyFrom(SubType other) { } } + /** Elements past the length get reused by next(), so they need to be cleared when the length shrinks */ + private void clearElementsFrom(int newLength) { + for (int i = newLength; i < length; i++) { + clearIndex0(i); + } + } + @Override public void addAll(SubType values) { final int newLength = length + values.length; diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java index e4f26088..2dee486a 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java @@ -489,6 +489,31 @@ public void testRepeatedObjectsWithOffset() { assertEquals(values[3], messages.get(1)); } + @Test + public void testRepeatedMessageShrinkingCopy() { + TestAllTypes[] values = new TestAllTypes[]{ + TestAllTypes.newInstance().setOptionalInt32(1).setOptionalInt64(2), + TestAllTypes.newInstance().setOptionalInt32(3).setOptionalInt64(4) + }; + RepeatedMessage shorter = RepeatedMessage.newEmptyInstance(TestAllTypes.getFactory()); + shorter.add(values[0]); + + // reused elements past the new length must not keep old values + RepeatedMessage messages = RepeatedMessage.newEmptyInstance(TestAllTypes.getFactory()); + messages.addAll(values); + messages.copyFrom(shorter); + messages.clear(); + assertEquals(TestAllTypes.newInstance(), messages.next()); + assertEquals(TestAllTypes.newInstance(), messages.next()); + + messages.clear(); + messages.addAll(values); + messages.copyFrom(new TestAllTypes[]{values[0]}); + messages.clear(); + assertEquals(TestAllTypes.newInstance(), messages.next()); + assertEquals(TestAllTypes.newInstance(), messages.next()); + } + @Test public void testBytes() throws IOException { byte[] utf8Bytes = "optionalByteString\uD83D\uDCA9".getBytes(UTF_8); From 94283bd41e94d0681d71b4b6841e8cf9d4aca728 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:24:05 +0200 Subject: [PATCH 06/20] fixed repeated enums ignoring the array offset in addAll --- .../src/main/java/us/hebi/quickbuf/RepeatedEnum.java | 2 +- .../src/test/java/us/hebi/quickbuf/ProtoTests.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java index 347935ca..968b0dc8 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java @@ -64,7 +64,7 @@ public RepeatedEnum addAll(final E... values) { public RepeatedEnum addAll(final E[] buffer, final int offset, final int length) { final int pos = addLength(length); for (int i = 0; i < length; i++) { - array[pos + i] = buffer[i].getNumber(); + array[pos + i] = buffer[offset + i].getNumber(); } return this; } diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java index 2dee486a..45bec59f 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java @@ -489,6 +489,16 @@ public void testRepeatedObjectsWithOffset() { assertEquals(values[3], messages.get(1)); } + @Test + public void testRepeatedEnumsWithOffset() { + NestedEnum[] values = new NestedEnum[]{NestedEnum.FOO, NestedEnum.BAR, NestedEnum.BAZ}; + RepeatedEnum enums = RepeatedEnum.newEmptyInstance(NestedEnum.converter()); + enums.addAll(values, 1, 2); + assertEquals("[2, 3]", enums.toString()); + enums.copyFrom(values, 2, 1); + assertEquals("[3]", enums.toString()); + } + @Test public void testRepeatedMessageShrinkingCopy() { TestAllTypes[] values = new TestAllTypes[]{ From c33b0de17a2c2fc15419bffd008a20ba40675b08 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:24:24 +0200 Subject: [PATCH 07/20] fixed an exception when adding repeated enums to themselves --- .../src/main/java/us/hebi/quickbuf/RepeatedEnum.java | 5 +++-- .../src/test/java/us/hebi/quickbuf/ProtoTests.java | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java index 968b0dc8..fb8aff4c 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedEnum.java @@ -91,8 +91,9 @@ public void copyFrom(RepeatedEnum other) { @Override public void addAll(RepeatedEnum values) { - int pos = addLength(values.length); - System.arraycopy(values.array, 0, array, pos, values.length); + final int count = values.length; // values may be this + final int pos = addLength(count); + System.arraycopy(values.array, 0, array, pos, count); } public int getValue(int index) { diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java index 45bec59f..0c845546 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java @@ -499,6 +499,14 @@ public void testRepeatedEnumsWithOffset() { assertEquals("[3]", enums.toString()); } + @Test + public void testRepeatedEnumsAddSelf() { + RepeatedEnum enums = RepeatedEnum.newEmptyInstance(NestedEnum.converter()); + enums.addAll(NestedEnum.FOO, NestedEnum.BAR); + enums.addAll(enums); + assertEquals("[1, 2, 1, 2]", enums.toString()); + } + @Test public void testRepeatedMessageShrinkingCopy() { TestAllTypes[] values = new TestAllTypes[]{ From 9d04e84a067407c9c7cb4b1766d34aa871364e46 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:26:06 +0200 Subject: [PATCH 08/20] fixed json sources hanging when skipping truncated values --- .../src/main/java/us/hebi/quickbuf/JsonSource.java | 4 ++++ .../test/java/us/hebi/quickbuf/JsonSourceTest.java | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java index 18f16920..08fb422c 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java @@ -581,6 +581,8 @@ private void skipArray() throws IOException { case INT_RBRACKET: level--; break; + case INT_EOF: + throw InvalidJsonException.truncatedMessage(); } } } @@ -599,6 +601,8 @@ private void skipObject() throws IOException { case INT_RCURLY: level--; break; + case INT_EOF: + throw InvalidJsonException.truncatedMessage(); } } } diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java index 78bf3ddf..409af620 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java @@ -72,6 +72,17 @@ public void testSkipping() throws Exception { } + @Test(timeout = 5000) + public void testSkippingTruncatedInput() throws Exception { + for (String json : new String[]{"{\"unknown\":[1", "{\"unknown\":{\"a\":1"}) { + try { + ForeignMessage.newInstance().mergeFrom(newJsonSource(json).setIgnoreUnknownFields(true)); + fail("expected to fail on truncated input: " + json); + } catch (IOException expected) { + } + } + } + @Test public void testManualInput() throws Exception { TestAllTypes msg = parseJson(CompatibilityTest.JSON_MANUAL_INPUT); From c53ce47a1cdc0c3a83cc7bcaa45fa48dd1df78eb Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:27:30 +0200 Subject: [PATCH 09/20] fixed an integer overflow when pushing very large limits --- .../java/us/hebi/quickbuf/ArraySource.java | 4 ++-- .../java/us/hebi/quickbuf/ProtoSource.java | 5 +++-- .../java/us/hebi/quickbuf/ProtoFailTests.java | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java index e58c4cb8..25531692 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java @@ -53,10 +53,10 @@ public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { if (byteLimit < 0) { throw InvalidProtocolBufferException.negativeSize(); } - byteLimit += position; - if (byteLimit > limit) { // limit is always Math.min(limit, currentLimit) + if (byteLimit > limit - position) { // limit is always Math.min(limit, currentLimit) throw InvalidProtocolBufferException.truncatedMessage(); } + byteLimit += position; final int oldLimit = currentLimit; currentLimit = byteLimit; recomputeBufferSizeAfterLimit(); diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java index 1ab29f29..dd80a3e6 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java @@ -1054,10 +1054,11 @@ public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { if (byteLimit < 0) { throw InvalidProtocolBufferException.negativeSize(); } - byteLimit += getTotalBytesRead(); - if (byteLimit > currentLimit) { + final int totalBytesRead = getTotalBytesRead(); + if (byteLimit > currentLimit - totalBytesRead) { throw InvalidProtocolBufferException.truncatedMessage(); } + byteLimit += totalBytesRead; final int oldLimit = currentLimit; currentLimit = byteLimit; return oldLimit; diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java index d5814b8b..d1be0f61 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java @@ -496,4 +496,26 @@ public void testStreamLimitsExceeded() throws IOException { } + @Test + public void testLimitOverflow() throws IOException { + // nested message with a length of Integer.MAX_VALUE after the tag + assertInvalidForAllSources(new byte[]{(byte) 0x92, 0x01, -1, -1, -1, -1, 0x07}); + } + + private static void assertInvalidForAllSources(byte[] bytes) throws IOException { + ProtoSource[] sources = new ProtoSource[]{ + ProtoSource.newArraySource().setInput(bytes), + ProtoSource.newDirectSource().setInput(bytes), + ProtoSource.newStreamSource().setInput(new ByteArrayInputStream(bytes)), + ProtoSource.newBufferSource().setInput(ByteBuffer.wrap(bytes)) + }; + for (ProtoSource source : sources) { + try { + TestAllTypes.newInstance().mergeFrom(source); + fail(source.getClass().getSimpleName() + " accepted invalid input"); + } catch (InvalidProtocolBufferException expected) { + } + } + } + } From 77ff4508532874cdbed5122b5ba0982870d5b6f5 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:29:19 +0200 Subject: [PATCH 10/20] fixed negative lengths causing exceptions or endless loops instead of parse errors --- .../src/main/java/us/hebi/quickbuf/ProtoSource.java | 6 +++++- .../src/test/java/us/hebi/quickbuf/ProtoFailTests.java | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java index dd80a3e6..8c4f5508 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java @@ -706,7 +706,11 @@ public int readEnum() throws IOException { /** Read a length delimiter from the source. */ public int readLength() throws IOException { - return readRawVarint32(); + final int length = readRawVarint32(); + if (length < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + return length; } // ------------------------------ DELIMITED TYPES ------------------------------ diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java index d1be0f61..8e459776 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java @@ -502,6 +502,13 @@ public void testLimitOverflow() throws IOException { assertInvalidForAllSources(new byte[]{(byte) 0x92, 0x01, -1, -1, -1, -1, 0x07}); } + @Test(timeout = 5000) + public void testNegativeLength() throws IOException { + // bytes field 15 and unknown field 1000 with a length of -11 + assertInvalidForAllSources(new byte[]{0x7A, -11, -1, -1, -1, -1, -1, -1, -1, -1, 0x01}); + assertInvalidForAllSources(new byte[]{(byte) 0xC2, 0x3E, -11, -1, -1, -1, -1, -1, -1, -1, -1, 0x01}); + } + private static void assertInvalidForAllSources(byte[] bytes) throws IOException { ProtoSource[] sources = new ProtoSource[]{ ProtoSource.newArraySource().setInput(bytes), From fa6bca06d86ee8d56ff98c6375c3483243c87475 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:31:14 +0200 Subject: [PATCH 11/20] fixed large lengths allocating memory before checking the remaining input --- .../java/us/hebi/quickbuf/ArraySource.java | 1 + .../java/us/hebi/quickbuf/ProtoSource.java | 39 +++++++++++++++ .../java/us/hebi/quickbuf/ProtoFailTests.java | 47 ++++++++++++++++--- 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java index 25531692..4ae9d1a1 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java @@ -95,6 +95,7 @@ public void rewindTo(int position) { this.position = offset + position; } + @Override protected void requireRemaining(final int numBytes) throws IOException { if (numBytes < 0) { throw InvalidProtocolBufferException.negativeSize(); diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java index 8c4f5508..2d5e3c5c 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java @@ -294,6 +294,7 @@ public boolean skipField(final int tag, final RepeatedByte unknownBytes) throws } case WireFormat.WIRETYPE_LENGTH_DELIMITED:{ int length = readLength(); + requireRemaining(length); ByteUtil.writeUInt32(unknownBytes, tag); ByteUtil.writeUInt32(unknownBytes, length); ByteUtil.writeBytes(unknownBytes, length, this); @@ -358,6 +359,7 @@ protected void skipMessage(RepeatedByte unknownBytes) throws IOException { /** Read a repeated (packed) {@code double} field value from the source. */ public void readPackedDouble(RepeatedDouble store) throws IOException { final int length = readLength(); + requireRemaining(length); final int limit = pushLimit(length); final int count = roundedCount64(length); final int offset = store.addLength(count); @@ -390,6 +392,7 @@ public double readDouble() throws IOException { /** Read a repeated (packed) {@code float} field value from the source. */ public void readPackedFloat(RepeatedFloat store) throws IOException { final int length = readLength(); + requireRemaining(length); final int limit = pushLimit(length); final int count = roundedCount32(length); final int offset = store.addLength(count); @@ -427,6 +430,7 @@ public void readPackedSFixed64(RepeatedLong store) throws IOException { /** Read a repeated (packed) {@code fixed64} field value from the source. */ public void readPackedFixed64(RepeatedLong store) throws IOException { final int length = readLength(); + requireRemaining(length); final int limit = pushLimit(length); final int count = roundedCount64(length); final int offset = store.addLength(count); @@ -474,6 +478,7 @@ public void readPackedSFixed32(RepeatedInt store) throws IOException { /** Read a repeated (packed) {@code fixed32} field value from the source. */ public void readPackedFixed32(RepeatedInt store) throws IOException { final int length = readLength(); + requireRemaining(length); final int limit = pushLimit(length); final int count = roundedCount32(length); final int offset = store.addLength(count); @@ -652,6 +657,7 @@ public int readSInt32() throws IOException { /** Read a repeated (packed) {@code bool} field value from the source. */ public void readPackedBool(RepeatedBoolean store) throws IOException { final int length = readLength(); + requireRemaining(length); final int limit = pushLimit(length); store.reserve(length / MIN_BOOL_SIZE); while (!isAtEnd()) { @@ -728,6 +734,7 @@ public int readRepeatedString(final RepeatedString store, final int tag) throws /** Read a {@code string} field value from the source. */ public void readString(final Utf8String store) throws IOException { final int length = readLength(); + requireRemaining(length); store.setSize(length); readRawBytes(store.bytes(), 0, length); } @@ -791,6 +798,7 @@ public int readRepeatedBytes(final RepeatedBytes store, final int tag) throws IO public void readBytes(RepeatedByte store) throws IOException { // note: bytes type gets replaced rather than merged final int length = readLength(); + requireRemaining(length); store.setLength(length); readRawBytes(store.array, 0, length); } @@ -1001,6 +1009,9 @@ public int setRecursionLimit(final int limit) { * without harming your app's functionality. Note that size limits only apply when reading from an * {@code InputStream}, not when constructed around a raw byte array. * + *

A stream can't know how much input remains, so length prefixes are only checked against + * this limit before allocating storage. Untrusted streams should always set a limit. + * *

If you want to read several messages from a single CodedInputStream, you could call {@link * #resetSizeCounter()} after each one to avoid hitting the size limit. * @@ -1137,6 +1148,17 @@ protected void reserveRepeatedFieldCapacity(RepeatedField store, int tag) t // implement in child classes that support looking ahead } + /** + * Checks that the source has enough remaining bytes before space for them gets + * allocated, so that corrupt lengths fail without allocating large arrays. Note + * that this can only be enforced if the source knows the remaining size. + * + * @param numBytes number of bytes that are about to be read + */ + protected void requireRemaining(final int numBytes) throws IOException { + // implement in child classes that know the remaining size + } + /** * Reserves space in the repeated field store to hold the number of remaining varints until * the limit or end of stream is reached. Note that this is an optimization that may do @@ -1307,6 +1329,16 @@ public void readRawBytes(byte[] buffer, int offset, int length) throws IOExcepti } } + @Override + protected void requireRemaining(int numBytes) throws IOException { + final int newPosition = position + numBytes; + if (newPosition - sizeLimit > 0) { // int overflow conscious + throw InvalidProtocolBufferException.sizeLimitExceeded(); + } else if (newPosition - currentLimit > 0) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + } + private void require(int size) throws IOException { final int newPosition = position + size; if (size < 0) { @@ -1368,6 +1400,13 @@ public byte readRawByte() throws IOException { } } + @Override + protected void requireRemaining(int numBytes) throws IOException { + if (numBytes > buffer.remaining()) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + } + @Override public void skipRawBytes(int size) throws IOException { if (buffer.remaining() < size) { diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java index 8e459776..876d6030 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java @@ -509,14 +509,35 @@ public void testNegativeLength() throws IOException { assertInvalidForAllSources(new byte[]{(byte) 0xC2, 0x3E, -11, -1, -1, -1, -1, -1, -1, -1, -1, 0x01}); } + @Test + public void testLengthBeyondInput() throws IOException { + // string field 14, bytes field 15, and unknown field 1000 with a length of Integer.MAX_VALUE + assertInvalidForAllSources(new byte[]{0x72, -1, -1, -1, -1, 0x07}); + assertInvalidForAllSources(new byte[]{0x7A, -1, -1, -1, -1, 0x07}); + assertInvalidForAllSources(new byte[]{(byte) 0xC2, 0x3E, -1, -1, -1, -1, 0x07}); + } + + @Test + public void testPackedLengthBeyondInput() throws IOException { + // packed fixed32, fixed64, float, double, and bool fields 37, 38, 41, 42, 43 with a length of 0x7FFFFFF0 + for (byte tag : new byte[]{(byte) 0xAA, (byte) 0xB2, (byte) 0xCA, (byte) 0xD2, (byte) 0xDA}) { + byte[] bytes = Arrays.copyOf(new byte[]{tag, 0x02, (byte) 0xF0, -1, -1, -1, 0x07}, 20); + for (ProtoSource source : newSources(bytes)) { + TestAllTypes msg = TestAllTypes.newInstance(); + try { + msg.mergeFrom(source); + fail(source.getClass().getSimpleName() + " accepted invalid input"); + } catch (InvalidProtocolBufferException expected) { + } + assertEquals(0, msg.getRepeatedFixed32().capacity() + msg.getRepeatedFixed64().capacity() + + msg.getRepeatedFloat().capacity() + msg.getRepeatedDouble().capacity() + + msg.getRepeatedBool().capacity()); + } + } + } + private static void assertInvalidForAllSources(byte[] bytes) throws IOException { - ProtoSource[] sources = new ProtoSource[]{ - ProtoSource.newArraySource().setInput(bytes), - ProtoSource.newDirectSource().setInput(bytes), - ProtoSource.newStreamSource().setInput(new ByteArrayInputStream(bytes)), - ProtoSource.newBufferSource().setInput(ByteBuffer.wrap(bytes)) - }; - for (ProtoSource source : sources) { + for (ProtoSource source : newSources(bytes)) { try { TestAllTypes.newInstance().mergeFrom(source); fail(source.getClass().getSimpleName() + " accepted invalid input"); @@ -525,4 +546,16 @@ private static void assertInvalidForAllSources(byte[] bytes) throws IOException } } + private static ProtoSource[] newSources(byte[] bytes) { + // streams don't know the remaining size, so they need a size limit to protect against large lengths + ProtoSource stream = ProtoSource.newStreamSource().setInput(new ByteArrayInputStream(bytes)); + stream.setSizeLimit(bytes.length); + return new ProtoSource[]{ + ProtoSource.newArraySource().setInput(bytes), + ProtoSource.newDirectSource().setInput(bytes), + stream, + ProtoSource.newBufferSource().setInput(ByteBuffer.wrap(bytes)) + }; + } + } From e29c7f6731d0b44236dceb3b627c877cc2c23950 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:38:26 +0200 Subject: [PATCH 12/20] fixed json sources rejecting base64 without padding --- .../us/hebi/quickbuf/compat/GsonSource.java | 12 +++++- .../hebi/quickbuf/compat/JacksonSource.java | 10 ++++- .../main/java/us/hebi/quickbuf/Base64.java | 11 ++++-- .../hebi/quickbuf/InvalidJsonException.java | 4 ++ .../java/us/hebi/quickbuf/JsonSource.java | 37 ++++++++++++++++--- .../java/us/hebi/quickbuf/Base64Test.java | 15 +++++++- .../java/us/hebi/quickbuf/JsonSourceTest.java | 20 ++++++++++ 7 files changed, 93 insertions(+), 16 deletions(-) diff --git a/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/GsonSource.java b/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/GsonSource.java index 2849a578..27e1fa29 100644 --- a/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/GsonSource.java +++ b/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/GsonSource.java @@ -97,16 +97,24 @@ public void readString(Utf8String store) throws IOException { @Override public void readBytes(RepeatedByte store) throws IOException { - decodeBase64(reader.nextString(), store.clear()); + readBase64(reader.nextString(), store.clear()); } @Override public ProtoSource readBytesAsSource() throws IOException { RepeatedByte store = RepeatedByte.newEmptyInstance(); - decodeBase64(reader.nextString(), store); + readBase64(reader.nextString(), store); return ProtoSource.newInstance(store); } + private static void readBase64(String input, RepeatedByte output) throws InvalidJsonException { + try { + decodeBase64(input, output); + } catch (IllegalArgumentException invalid) { + throw new InvalidJsonException(invalid.getMessage()); + } + } + @Override public void skipValue() throws IOException { reader.skipValue(); diff --git a/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/JacksonSource.java b/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/JacksonSource.java index 0c60500a..d065f99d 100644 --- a/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/JacksonSource.java +++ b/quickbuf-compat/src/main/java/us/hebi/quickbuf/compat/JacksonSource.java @@ -20,6 +20,8 @@ package us.hebi.quickbuf.compat; +import com.fasterxml.jackson.core.Base64Variant; +import com.fasterxml.jackson.core.Base64Variants; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; @@ -113,7 +115,7 @@ public void readString(Utf8String store) throws IOException { @Override public void readBytes(RepeatedByte store) throws IOException { next(); - store.copyFrom(reader.getBinaryValue()); + store.copyFrom(reader.getBinaryValue(BASE64)); } @Override @@ -121,7 +123,7 @@ public ProtoSource readBytesAsSource() throws IOException { if (next() == JsonToken.VALUE_NULL) { return ProtoSource.newArraySource(); } - return ProtoSource.newInstance(reader.getBinaryValue()); + return ProtoSource.newInstance(reader.getBinaryValue(BASE64)); } @Override @@ -238,4 +240,8 @@ private IOException unexpectedTokenError(JsonToken expectedToken) { JsonToken peekToken = null; + // proto3 json accepts base64 with and without padding + private static final Base64Variant BASE64 = Base64Variants.MIME_NO_LINEFEEDS + .withReadPadding(Base64Variant.PaddingReadBehaviour.PADDING_ALLOWED); + } diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/Base64.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/Base64.java index 933e0766..2bec2707 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/Base64.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/Base64.java @@ -369,7 +369,7 @@ public static byte[] decode(byte[] sArr) * @param sOff The starting position in the source array. * @param sLen The number of bytes to decode from the source array. Length 0 will return an empty array. * @return The decoded array of bytes. May be of length 0. Will be null if the legal characters - * (including '=') isn't divideable by 4. (I.e. definitely corrupted). + * (including '=') leave a remainder of 1 when divided by 4. (I.e. definitely corrupted). */ public static byte[] decode(byte[] sArr, int sOff, int sLen) { @@ -380,8 +380,8 @@ public static byte[] decode(byte[] sArr, int sOff, int sLen) if (IA[sArr[sOff + i] & 0xff] < 0) sepCnt++; - // Check so that legal chars (including '=') are evenly divideable by 4 as specified in RFC 2045. - if ((sLen - sepCnt) % 4 != 0) + // MODIFIED (ennerf): accept missing padding as allowed by proto3 json. A single remaining char can't encode a byte. + if ((sLen - sepCnt) % 4 == 1) return null; int pad = 0; @@ -390,13 +390,16 @@ public static byte[] decode(byte[] sArr, int sOff, int sLen) pad++; int len = ((sLen - sepCnt) * 6 >> 3) - pad; + if (len < 0) // MODIFIED (ennerf): more padding than content + return null; byte[] dArr = new byte[len]; // Preallocate byte[] of exact length for (int s = 0, d = 0; d < len;) { // Assemble three bytes into an int from four "valid" characters. int i = 0; - for (int j = 0; j < 4; j++) { // j only increased if a valid char was found. + // MODIFIED (ennerf): unpadded input ends within the last group + for (int j = 0; j < 4 && s < sLen; j++) { // j only increased if a valid char was found. int c = IA[sArr[sOff + s++] & 0xff]; if (c >= 0) i |= c << (18 - j * 6); diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java index 98fd885f..098c6881 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidJsonException.java @@ -39,6 +39,10 @@ static InvalidJsonException truncatedMessage() { "input has been truncated or that the input is invalid."); } + static InvalidJsonException invalidBase64() { + return new InvalidJsonException("Invalid base64 value."); + } + static InvalidJsonException illegalNumberFormat() { return new InvalidJsonException("Illegal number format."); } diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java index 08fb422c..28d2c0aa 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java @@ -419,12 +419,30 @@ public boolean isAtField(FieldName fieldName) { // ==================== Implementation ==================== + // Note: proto3 json accepts base64 with and without padding protected static void decodeBase64(String input, RepeatedByte output) { - output.addAll(Base64.decodeFast(input)); + final byte[] bytes = input.getBytes(ISO_8859_1); // other chars become an ignored '?' + output.addAll(decodeBase64Unchecked(bytes, bytes.length)); } protected static void decodeBase64(RepeatedByte input, RepeatedByte output) { - output.addAll(Base64.decode(input.array, 0, input.length)); + output.addAll(decodeBase64Unchecked(input.array, input.length)); + } + + static byte[] decodeBase64(byte[] input, int length) throws InvalidJsonException { + final byte[] bytes = Base64.decode(input, 0, length); + if (bytes == null) { + throw InvalidJsonException.invalidBase64(); + } + return bytes; + } + + private static byte[] decodeBase64Unchecked(byte[] input, int length) { + final byte[] bytes = Base64.decode(input, 0, length); + if (bytes == null) { + throw new IllegalArgumentException("Invalid base64 value."); + } + return bytes; } @@ -531,15 +549,22 @@ public void readString(Utf8String store) throws IOException { @Override public void readBytes(RepeatedByte store) throws IOException { - getValueAsBytes(buffer); + readUnescapedString(buffer); store.clear(); - decodeBase64(buffer, store); + store.addAll(decodeBase64(buffer.array, buffer.length)); } @Override public ProtoSource readBytesAsSource() throws IOException { - getValueAsBytes(buffer); - return ProtoSource.newInstance(Base64.decode(buffer.array, 0, buffer.length)); + readUnescapedString(buffer); + return ProtoSource.newInstance(decodeBase64(buffer.array, buffer.length)); + } + + private void readUnescapedString(RepeatedByte buffer) throws IOException { + checkNotAtNull(); + checkCurrentToken(INT_QUOTE); + JsonDecoding.StringDecoding.readQuotedUtf8(this, buffer); + token = readNextToken(); } @Override diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/Base64Test.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/Base64Test.java index 1a1dbe16..8238aeec 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/Base64Test.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/Base64Test.java @@ -67,8 +67,19 @@ public void testDecodeBase64_url_noPadding() throws Exception { } } - private byte[] decodeBase64(CharSequence input) { - return us.hebi.quickbuf.Base64.decodeFast(input.toString()); + @Test + public void testDecodeBase64_keepsInput() throws Exception { + RepeatedByte input = RepeatedByte.newEmptyInstance().addAll(new byte[]{'A', 'Q', 'I'}); + RepeatedByte output = RepeatedByte.newEmptyInstance(); + JsonSource.decodeBase64(input, output); + assertArrayEquals(new byte[]{1, 2}, output.toArray()); + assertArrayEquals(new byte[]{'A', 'Q', 'I'}, input.toArray()); + } + + private byte[] decodeBase64(CharSequence input) throws InvalidJsonException { + RepeatedByte output = RepeatedByte.newEmptyInstance(); + JsonSource.decodeBase64(input.toString(), output); + return output.toArray(); } diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java index 409af620..96a79f13 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java @@ -83,6 +83,26 @@ public void testSkippingTruncatedInput() throws Exception { } } + @Test + public void testUnpaddedBase64() throws Exception { + assertArrayEquals(new byte[]{1}, parseJson("{\"optionalBytes\":\"AQ\"}").getOptionalBytes().toArray()); + assertArrayEquals(new byte[]{1, 2}, parseJson("{\"optionalBytes\":\"AQI\"}").getOptionalBytes().toArray()); + assertArrayEquals(new byte[]{1, 2, 3}, parseJson("{\"optionalBytes\":\"AQID\"}").getOptionalBytes().toArray()); + assertArrayEquals(new byte[]{1, 2}, parseJson("{\"optionalBytes\":\"AQI=\"}").getOptionalBytes().toArray()); + + // escapes get decoded first, so escaped line breaks get ignored like other illegal chars + assertArrayEquals(new byte[]{1, 2, 3, 4, 5, 6}, parseJson("{\"optionalBytes\":\"AQID\\r\\nBAUG\"}").getOptionalBytes().toArray()); + assertArrayEquals(new byte[]{1, 2}, parseJson("{\"optionalBytes\":\"\\u0041QI\"}").getOptionalBytes().toArray()); + + for (String json : new String[]{"{\"optionalBytes\":\"A\"}", "{\"optionalBytes\":\"AQIDB\"}"}) { + try { + parseJson(json); + fail("expected to fail on invalid base64 length: " + json); + } catch (IOException expected) { + } + } + } + @Test public void testManualInput() throws Exception { TestAllTypes msg = parseJson(CompatibilityTest.JSON_MANUAL_INPUT); From 4c82f95527f72e01ab10f163c525abf8bc42db11 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:39:07 +0200 Subject: [PATCH 13/20] fixed json sources rejecting escaped forward slashes --- .../src/main/java/us/hebi/quickbuf/JsonDecoding.java | 2 ++ .../src/test/java/us/hebi/quickbuf/JsonSourceTest.java | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java index 5bd10c48..0df53768 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java @@ -271,6 +271,8 @@ static char escapedToRawChar(int escapedChar) throws InvalidJsonException { return '\\'; case '"': return '"'; + case '/': + return '/'; case 'b': return '\b'; case 't': diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java index 96a79f13..6135463a 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java @@ -103,6 +103,15 @@ public void testUnpaddedBase64() throws Exception { } } + @Test + public void testEscapedSolidus() throws Exception { + assertEquals("a/b", parseJson("{\"optionalString\":\"a\\/b\"}").getOptionalString()); + + // PHP escapes forward slashes in base64 values + assertArrayEquals(new byte[]{(byte) 0xAB, (byte) 0xFF}, parseJson("{\"optionalBytes\":\"q\\/8=\"}").getOptionalBytes().toArray()); + assertArrayEquals(new byte[]{(byte) 0xAB, (byte) 0xFF}, parseJson("{\"optionalBytes\":\"q\\/8\"}").getOptionalBytes().toArray()); + } + @Test public void testManualInput() throws Exception { TestAllTypes msg = parseJson(CompatibilityTest.JSON_MANUAL_INPUT); From 4453d30f512f761d379ec23285b95f9f3df7cd5b Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:40:18 +0200 Subject: [PATCH 14/20] fixed closing a json sink clearing the output buffer --- .../src/main/java/us/hebi/quickbuf/JsonSink.java | 2 +- .../src/test/java/us/hebi/quickbuf/JsonSinkTest.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSink.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSink.java index a4f8d22d..a418ab4f 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSink.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSink.java @@ -682,7 +682,7 @@ private void writeChar(char c) { @Override public void close() throws IOException { - output.extendCapacityTo(0); + // the output may be owned by the caller and is still needed after closing } @Override diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSinkTest.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSinkTest.java index 2e4b1f61..facee0ea 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSinkTest.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSinkTest.java @@ -38,6 +38,15 @@ */ public class JsonSinkTest { + @Test + public void testCloseKeepsOutput() throws IOException { + RepeatedByte output = RepeatedByte.newEmptyInstance(); + try (JsonSink sink = JsonSink.newInstance().setOutput(output)) { + sink.writeMessage(TestAllTypes.newInstance().setOptionalInt32(1)); + } + assertEquals("{\"optionalInt32\":1}", new String(output.toArray(), "UTF-8")); + } + @Test public void testEnumNumbers() throws IOException { TestAllTypes msg = TestAllTypes.newInstance(); From 5df618d293fba7ccc4a7a63d8d7ec62312364534 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:43:50 +0200 Subject: [PATCH 15/20] fixed buffer sources reading past the limits of nested fields --- .../src/main/java/us/hebi/quickbuf/ProtoSource.java | 12 ++++++++---- .../test/java/us/hebi/quickbuf/ProtoFailTests.java | 6 ++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java index 2d5e3c5c..e7359767 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ProtoSource.java @@ -1393,6 +1393,9 @@ public void rewindTo(int totalBytesRead) { @Override public byte readRawByte() throws IOException { + if (buffer.position() == currentLimit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } try { return buffer.get(); } catch (BufferUnderflowException truncated) { @@ -1402,21 +1405,22 @@ public byte readRawByte() throws IOException { @Override protected void requireRemaining(int numBytes) throws IOException { - if (numBytes > buffer.remaining()) { + if (numBytes < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } else if (numBytes > buffer.remaining() || numBytes > currentLimit - buffer.position()) { throw InvalidProtocolBufferException.truncatedMessage(); } } @Override public void skipRawBytes(int size) throws IOException { - if (buffer.remaining() < size) { - throw InvalidProtocolBufferException.truncatedMessage(); - } + requireRemaining(size); ((Buffer) buffer).position(buffer.position() + size); } @Override public void readRawBytes(byte[] values, int offset, int length) throws IOException { + requireRemaining(length); try { buffer.get(values, offset, length); } catch (BufferUnderflowException truncated) { diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java index 876d6030..c5c6ba46 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoFailTests.java @@ -536,6 +536,12 @@ public void testPackedLengthBeyondInput() throws IOException { } } + @Test + public void testPackedLengthNotMultipleOfElementSize() throws IOException { + // packed repeated double field 42 with a length of 5 followed by a string field 14 + assertInvalidForAllSources(new byte[]{(byte) 0xD2, 0x02, 0x05, 1, 2, 3, 4, 5, 0x72, 0x01, 0x61}); + } + private static void assertInvalidForAllSources(byte[] bytes) throws IOException { for (ProtoSource source : newSources(bytes)) { try { From 9cae9e14586fd5717133dffa5b73ff7f6a45fb5f Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 12:47:29 +0200 Subject: [PATCH 16/20] fixed invalid characters in json numbers causing index exceptions or wrong values --- .../java/us/hebi/quickbuf/compat/GsonSourceTest.java | 5 +++++ .../src/main/java/us/hebi/quickbuf/JsonDecoding.java | 6 +++--- .../test/java/us/hebi/quickbuf/JsonSourceTest.java | 11 +++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java b/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java index ece29e57..385e2e6b 100644 --- a/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java +++ b/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java @@ -48,6 +48,11 @@ public void testBadInputs() { testError(CompatibilityTest.JSON_UNKNOWN_FIELD_NULL, "Encountered unknown field: 'unknownField'"); } + @Override + public void testInvalidUnquotedValues() { + // not run: Gson throws unchecked NumberFormatExceptions for invalid numbers + } + @Override protected JsonSource newJsonSource(String json) { return new GsonSource(json); diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java index 0df53768..cab53307 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonDecoding.java @@ -113,10 +113,10 @@ public static boolean isWhitespace(int ch) { } public static boolean isBreak(int ch) { - return breaks[ch]; + return ch < 0 || breaks[ch]; // EOF also ends a value } - private static final boolean[] breaks = new boolean[127]; + private static final boolean[] breaks = new boolean[256]; final static int[] sHexValues = new int[256]; final static int[] TRUE_VALUE = {INT_t, INT_r, INT_u, INT_e}; final static int[] FALSE_VALUE = {INT_f, INT_a, INT_l, INT_s, INT_e}; @@ -361,7 +361,7 @@ static long readLong(byte[] buffer, int tokenStart, int tokenEnd) throws Invalid int digit; for (result = 0L; i < len; result -= (long) digit) { digit = (buffer[i++ + tokenStart] & 0xFF) - INT_0; - if (digit < 0 || result < multmin) { + if (digit < 0 || digit > 9 || result < multmin) { throw InvalidJsonException.illegalNumberFormat(); } diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java index 6135463a..f86dc47e 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java @@ -112,6 +112,17 @@ public void testEscapedSolidus() throws Exception { assertArrayEquals(new byte[]{(byte) 0xAB, (byte) 0xFF}, parseJson("{\"optionalBytes\":\"q\\/8\"}").getOptionalBytes().toArray()); } + @Test + public void testInvalidUnquotedValues() throws Exception { + for (String json : new String[]{"{\"optionalInt32\":1" + (char) 0xE9 + "}", "{\"optionalInt32\":12a}", "{\"optionalInt32\":"}) { + try { + parseJson(json); + fail("expected to fail on invalid input: " + json); + } catch (IOException expected) { + } + } + } + @Test public void testManualInput() throws Exception { TestAllTypes msg = parseJson(CompatibilityTest.JSON_MANUAL_INPUT); From 73a57dac3b5d109d9283322115e8573ddf9d7acd Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Wed, 30 Sep 2026 02:18:49 +0200 Subject: [PATCH 17/20] fixed the recursion limit error naming ProtoSink instead of ProtoSource --- .../java/us/hebi/quickbuf/InvalidProtocolBufferException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidProtocolBufferException.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidProtocolBufferException.java index c59522ef..94a90f5d 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidProtocolBufferException.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/InvalidProtocolBufferException.java @@ -106,7 +106,7 @@ static InvalidProtocolBufferException invalidWireType() { static InvalidProtocolBufferException recursionLimitExceeded() { return new InvalidProtocolBufferException( "Protocol message had too many levels of nesting. May be malicious. " + - "Use ProtoSink.setRecursionLimit() to increase the depth limit."); + "Use ProtoSource.setRecursionLimit() to increase the depth limit."); } static InvalidProtocolBufferException sizeLimitExceeded() { From 0fe5de6f957166d71eec42b5903605f96eef1222 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Tue, 29 Sep 2026 18:19:54 +0200 Subject: [PATCH 18/20] fixed out of range json values for int32 fields getting truncated --- .../hebi/quickbuf/compat/GsonSourceTest.java | 5 ++++ .../java/us/hebi/quickbuf/JsonSource.java | 30 +++++++++++++++---- .../java/us/hebi/quickbuf/JsonSourceTest.java | 26 ++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java b/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java index 385e2e6b..fd49e536 100644 --- a/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java +++ b/quickbuf-compat/src/test/java/us/hebi/quickbuf/compat/GsonSourceTest.java @@ -53,6 +53,11 @@ public void testInvalidUnquotedValues() { // not run: Gson throws unchecked NumberFormatExceptions for invalid numbers } + @Override + public void testIntegersOutOfRange() { + // not run: Gson throws unchecked NumberFormatExceptions for invalid numbers + } + @Override protected JsonSource newJsonSource(String json) { return new GsonSource(json); diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java index 28d2c0aa..bd3ed32d 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/JsonSource.java @@ -203,7 +203,7 @@ public long readFixed64() throws IOException { * Read a {@code fixed32} field value from the source. */ public int readFixed32() throws IOException { - return this.readInt32(); // TODO: read unsigned 32 + return this.readUInt32(); } /** @@ -214,10 +214,15 @@ public > ProtoMsg readGroup(final ProtoM } /** - * Read a {@code uint32} field value from the source. + * Read a {@code uint32} field value from the source. Negative values + * written by older versions represent the same bits. */ public int readUInt32() throws IOException { - return this.readInt32(); // TODO: read unsigned 32 + final long value = this.readInt64(); + if (value < Integer.MIN_VALUE || value > 0xFFFFFFFFL) { + throw outOfRange("uint32", value); + } + return (int) value; } /** @@ -265,7 +270,7 @@ public void readRepeatedSInt64(final RepeatedLong value) throws IOException { } public void readRepeatedFixed32(final RepeatedInt value) throws IOException { - readRepeatedInt32(value); // TODO: read unsigned 32 + readRepeatedUInt32(value); } public void readRepeatedSFixed32(final RepeatedInt value) throws IOException { @@ -273,7 +278,11 @@ public void readRepeatedSFixed32(final RepeatedInt value) throws IOException { } public void readRepeatedUInt32(final RepeatedInt value) throws IOException { - readRepeatedInt32(value); // TODO: read unsigned 32 + beginArray(); + while (!isAtEnd()) { + value.add(readUInt32()); + } + endArray(); } public void readRepeatedSInt32(final RepeatedInt value) throws IOException { @@ -419,6 +428,11 @@ public boolean isAtField(FieldName fieldName) { // ==================== Implementation ==================== + InvalidJsonException outOfRange(String type, Object value) { + String atField = (currentField == null ? "" : " for field '" + currentField + "'"); + return new InvalidJsonException("Value out of range for " + type + ": " + value + atField); + } + // Note: proto3 json accepts base64 with and without padding protected static void decodeBase64(String input, RepeatedByte output) { final byte[] bytes = input.getBytes(ISO_8859_1); // other chars become an ignored '?' @@ -504,7 +518,11 @@ public double readDouble() throws IOException { @Override public int readInt32() throws IOException { - return (int) this.readInt64(); + final long value = this.readInt64(); + if ((int) value != value) { + throw outOfRange("int32", value); + } + return (int) value; } @Override diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java index f86dc47e..a8e8bae2 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/JsonSourceTest.java @@ -123,6 +123,32 @@ public void testInvalidUnquotedValues() throws Exception { } } + @Test + public void testIntegersOutOfRange() throws Exception { + for (String json : new String[]{ + "{\"optionalInt32\":2147483648}", + "{\"optionalInt32\":4294967297}", + "{\"optionalUint32\":4294967296}", + "{\"optionalUint32\":-2147483649}"}) { + try { + parseJson(json); + fail("expected to fail on out of range value: " + json); + } catch (IOException expected) { + } + } + } + + @Test + public void testUnsignedInt32() throws Exception { + TestAllTypes msg = parseJson("{\"optionalUint32\":4294967295,\"optionalFixed32\":2147483648," + + "\"repeatedUint32\":[4294967295,-1],\"repeatedFixed32\":[2147483648]}"); + assertEquals(-1, msg.getOptionalUint32()); + assertEquals(Integer.MIN_VALUE, msg.getOptionalFixed32()); + assertEquals(-1, msg.getRepeatedUint32().get(0)); + assertEquals(-1, msg.getRepeatedUint32().get(1)); + assertEquals(Integer.MIN_VALUE, msg.getRepeatedFixed32().get(0)); + } + @Test public void testManualInput() throws Exception { TestAllTypes msg = parseJson(CompatibilityTest.JSON_MANUAL_INPUT); From 8115da7810a926feeaec03cae4ad9573978ca165 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Sun, 18 Oct 2026 16:55:57 +0200 Subject: [PATCH 19/20] accepted tagged native addresses in direct sinks and sources --- .../main/java/us/hebi/quickbuf/ArraySink.java | 2 +- .../java/us/hebi/quickbuf/ArraySource.java | 2 +- .../java/us/hebi/quickbuf/UnsafeTest.java | 20 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySink.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySink.java index d8581ca0..887f42dd 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySink.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySink.java @@ -218,7 +218,7 @@ public ProtoSink setOutput(byte[] buffer, long offset, int length) { return super.setOutput(buffer, offset, length); } else { // Wrapping direct memory - if (offset <= 0) { + if (offset == 0) { // some Android phones use the sign bit as a flag, so negative addresses can be valid throw new NullPointerException("null reference with invalid address offset"); } this.buffer = null; diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java index 4ae9d1a1..b3251227 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/ArraySource.java @@ -307,7 +307,7 @@ public ProtoSource setInput(byte[] buffer, long offset, int length) { // Wrapping heap buffer baseOffset = BYTE_ARRAY_OFFSET; return super.setInput(buffer, offset, length); - } else if (offset <= 0) { + } else if (offset == 0) { // some Android phones use the sign bit as a flag, so negative addresses can be valid throw new NullPointerException("null reference with invalid base address"); } else if (length < 0) { throw new ArrayIndexOutOfBoundsException("negative length"); diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/UnsafeTest.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/UnsafeTest.java index 006cc6f5..68720c03 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/UnsafeTest.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/UnsafeTest.java @@ -83,6 +83,26 @@ public void testUnsafeDirectMemory() throws IOException { assertEquals(message, TestAllTypes.newInstance().mergeFrom(source)); } + @Test + public void testTaggedDirectAddress() { + // e.g. Android with ARM top byte ignore puts a tag into the upper bits + long address = -16L << 56 | 0x1000; + ProtoSink.newDirectSink().setOutput(null, address, 8); + ProtoSource.newDirectSource().setInput(null, address, 8); + + try { + ProtoSink.newDirectSink().setOutput(null, 0, 8); + fail("null address sink"); + } catch (NullPointerException npe) { + } + + try { + ProtoSource.newDirectSource().setInput(null, 0, 8); + fail("null address source"); + } catch (NullPointerException npe) { + } + } + @Test public void testRepeatedPacked() throws IOException { RepeatedPackables.Packed msg = RepeatedPackables.Packed.parseFrom(CompatibilityTest.repeatedPackablesPacked()); From 90d289d3dcd01ab5ca2b6c57fb222449a39424d1 Mon Sep 17 00:00:00 2001 From: Florian Enner Date: Wed, 21 Oct 2026 22:40:00 +0200 Subject: [PATCH 20/20] fixed repeated objects reallocating in addAll and copyFrom despite enough capacity --- .../java/us/hebi/quickbuf/RepeatedObject.java | 4 ++-- .../java/us/hebi/quickbuf/ProtoTests.java | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java index 337e3f35..74948ad3 100644 --- a/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java +++ b/quickbuf-runtime/src/main/java/us/hebi/quickbuf/RepeatedObject.java @@ -84,7 +84,7 @@ public final void copyFrom(SubType other) { if (other.length < length) { clearElementsFrom(other.length); } - if (other.length > length) { + if (other.length > capacity()) { extendCapacityTo(other.length); } length = other.length; @@ -103,7 +103,7 @@ private void clearElementsFrom(int newLength) { @Override public void addAll(SubType values) { final int newLength = length + values.length; - extendCapacityTo(newLength); + reserve(values.length); for (int i = 0; i < values.length; i++) { copyFrom0(array[length + i], values.array[i]); } diff --git a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java index 0c845546..124fd44a 100644 --- a/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java +++ b/quickbuf-runtime/src/test/java/us/hebi/quickbuf/ProtoTests.java @@ -532,6 +532,29 @@ public void testRepeatedMessageShrinkingCopy() { assertEquals(TestAllTypes.newInstance(), messages.next()); } + @Test + public void testRepeatedMessageAddAllKeepsCapacity() { + RepeatedMessage other = RepeatedMessage.newEmptyInstance(TestAllTypes.getFactory()); + other.next().setOptionalInt32(1); + other.next().setOptionalInt32(2); + + RepeatedMessage messages = RepeatedMessage.newEmptyInstance(TestAllTypes.getFactory()); + messages.reserve(8); + TestAllTypes first = messages.next().setOptionalInt32(0); + messages.addAll(other); + assertEquals(8, messages.capacity()); + assertSame(first, messages.get(0)); + assertEquals(3, messages.length()); + assertEquals(2, messages.get(2).getOptionalInt32()); + + messages.clear(); + messages.copyFrom(other); + assertEquals(8, messages.capacity()); + assertSame(first, messages.get(0)); + assertEquals(2, messages.length()); + assertEquals(1, messages.get(0).getOptionalInt32()); + } + @Test public void testBytes() throws IOException { byte[] utf8Bytes = "optionalByteString\uD83D\uDCA9".getBytes(UTF_8);