Add opt-in incremental bounded HEIC decoding

.github/workflows/tests.yml

@@ -38,9 +38,13 @@ jobs:
 
             - run: cargo test --all-features --locked
 
-            - run: cargo test --locked bounded::
+            - run: "cargo test --locked bounded::"
 
-            - run: cargo test --locked --no-default-features bounded::
+            - run: "cargo test --locked --no-default-features bounded::"
+
+            - run: cargo test --release --locked --features incremental-experiment --lib --bins
+
+            - run: cargo test --release --locked --no-default-features --features incremental-experiment --lib
 
     # Full TESTING.md correctness pass: pixel-for-pixel and ICC-profile
     # comparison against a pinned heif-dec validator over the libheif corpus,
@@ -72,7 +76,7 @@ jobs:
             - name: Install system dependencies
               run: |
                   sudo apt-get update
-                  sudo apt-get install -y cmake pkg-config ffmpeg \
+                  sudo apt-get install -y cmake pkg-config ffmpeg ruby \
                       libde265-dev libx265-dev libaom-dev libdav1d-dev \
                       libopenjp2-7-dev libjpeg-dev libbrotli-dev zlib1g-dev
 
@@ -120,10 +124,45 @@ jobs:
             - name: Full correctness pass
               run: scripts/heic_tests.sh verify --full --require-exts heic,avif
 
+            - name: Incremental decoder oracle comparisons
+              run: |
+                  prefix="$PWD/.heic-test-assets/libpng-install"
+                  cmake -S scripts/incremental -B .heic-test-runs/incremental-oracle \
+                      -DHEIF_SOURCE_DIR="$PWD/.heic-test-assets/libheif" \
+                      -DHEIF_BUILD_DIR="$PWD/.heic-test-runs/validator-build" \
+                      -DPNG_PNG_INCLUDE_DIR="$prefix/include" \
+                      -DPNG_LIBRARY="$prefix/lib/libpng16.a"
+                  cmake --build .heic-test-runs/incremental-oracle --parallel
+                  ruby scripts/incremental/generate.rb \
+                      .heic-test-runs/incremental-oracle/primary-oracle \
+                      .heic-test-assets/libheif/fuzzing/data/corpus/colors-no-alpha.heic
+                  cargo test --release --locked --features incremental-experiment --test incremental-memory
+                  cargo test --release --locked --no-default-features --features incremental-experiment --test incremental-memory
+                  cargo build --release --locked --features incremental-experiment --bins
+                  ruby scripts/incremental/verify.rb \
+                      .heic-test-runs/incremental-oracle/primary-oracle \
+                      target/release/incremental-allocation target/release/incremental-compare \
+                      .heic-test-assets/incremental-corpus/*.heic \
+                      .heic-test-assets/libheif/fuzzing/data/corpus/colors-no-alpha.heic \
+                      .heic-test-assets/libheif/fuzzing/data/corpus/colors-no-alpha-thumbnail.heic \
+                      .heic-test-assets/libheif/fuzzing/data/corpus/hevc32.heif \
+                      .heic-test-assets/stress-corpus/nowpp_photo_small.heic \
+                      .heic-test-assets/stress-corpus/wpp_narrow.heic \
+                      .heic-test-assets/stress-corpus/wpp_narrow2.heic
+                  for name in direct tall pipeline; do
+                      prefix="$PWD/.heic-test-runs/incremental/$name"
+                      ANNEX_B_OUTPUT="$prefix.hevc" target/release/incremental-check ".heic-test-assets/incremental-corpus/$name.heic"
+                      ffmpeg -v error -y -i "$prefix.hevc" -frames:v 1 -pix_fmt yuv420p -f rawvideo "$prefix.yuv"
+                      REFERENCE_YUV="$prefix.yuv" target/release/incremental-check ".heic-test-assets/incremental-corpus/$name.heic"
+                  done
+
             - name: Upload verify report
               if: failure()
               uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
               with:
                   name: verify-report
-                  path: .heic-test-runs/verify/run/report.txt
+                  path: |
+                      .heic-test-runs/verify/run/report.txt
+                      .heic-test-runs/incremental/*.json*
+                      .heic-test-assets/incremental-corpus/
                   if-no-files-found: ignore

API.md

@@ -356,3 +356,40 @@ fn decode(path: &Path) -> Result<(), heic_decoder::DecodeError> {
     Ok(())
 }
 ```
+
+## Experimental incremental bounded decoding
+
+Enable the `incremental-experiment` Cargo feature to use
+`decode_incremental_experiment(BoundedInput, BoundedDecodeOptions)`. It returns
+`(BoundedRgbImage, [u64; 3])`; the diagnostic counters are reconstructed rows,
+SAO edge components, and SAO band components across all coded items. This
+feature and entry point are experimental. Existing decode functions retain
+their behavior and do not select this path automatically.
+
+The incremental path supports opaque 8-bit 4:2:0 HEVC still images with one
+IDR slice per coded item, including non-grid images and uniform grids whose
+individual tiles exceed the output cap. WPP, HEVC tiles, multiple slices,
+higher bit depths, alpha, other chroma formats, and unsupported color profiles
+return errors. The output is display-oriented sRGB RGB8, using area averaging
+when reduced, with a maximum side from 1 through 6000. Known depth, semantic
+matte, and gain-map auxiliaries may accompany the SDR base image; this does
+not apply their effects or provide HDR output.
+
+Reconstruction, deblocking, and SAO finish in rolling bands before conversion
+and reduction. Large coded images use a bounded two-thread pipeline. The
+working state grows with coded width and CTU size, while output storage is
+capped by `max_side`; it does not retain a full source raster or use temporary
+files or repeated reconstruction. A caller's borrowed compressed input is
+separate from decoder-owned memory. Admission is conservative and may reject
+budgets smaller than its reserved workspace, including a worker stack.
+Allocator overhead and process RSS are not the same as requested allocation
+payloads. Use Path input when avoiding a caller-owned compressed input buffer
+is also important.
+
+This first subset retains limits of 16384 per coded side, 256 million display
+pixels, bounded metadata, and a 64 KiB slice-header prefix. It supports one
+clean-aperture crop before orientation, with chroma interpolation for odd
+origins on a single coded image or one-tile grid. Odd-origin crops across
+multiple grid items and repeated crops or crops after orientation are
+unsupported. Exif orientation applies when no effective container orientation
+is present. `decoder-tracing` disables bounded decoding.

Cargo.toml

@@ -8,6 +8,7 @@ license-file = "LICENSE"
 repository = "https://github.com/ente-io/heic-decoder"
 
 [features]
+incremental-experiment = []
 default = ["std", "parallel-grid"]
 decoder-tracing = ["std"]
 image-integration = ["dep:image"]
@@ -40,3 +41,34 @@ scuffle-h265 = "0.2.2"
 
 [dev-dependencies]
 qcms = "0.3.0"
+
+[[bin]]
+name = "incremental-allocation"
+path = "src/bin/incremental-allocation.rs"
+required-features = ["incremental-experiment"]
+
+[[bin]]
+name = "incremental-check"
+path = "src/bin/incremental-check.rs"
+required-features = ["incremental-experiment"]
+
+[[bin]]
+name = "capability-inspect"
+path = "src/bin/capability-inspect.rs"
+required-features = ["incremental-experiment"]
+
+[[bin]]
+name = "incremental-compare"
+path = "src/bin/incremental-compare.rs"
+required-features = ["incremental-experiment"]
+
+[[bin]]
+name = "incremental-bench"
+path = "src/bin/incremental-bench.rs"
+required-features = ["incremental-experiment"]
+
+[[test]]
+name = "incremental-memory"
+path = "tests/incremental-memory.rs"
+harness = false
+required-features = ["incremental-experiment"]

TESTING.md

@@ -161,3 +161,121 @@ libheif corpus and the ente fixtures.
 
 Generated reports and PNG artifacts are under `.heic-test-runs/`. Use
 `--keep-artifacts` with `verify` when debugging a pixel mismatch.
+
+## Incremental bounded decoder
+
+The opt-in `incremental-experiment` feature is tested separately because
+`--all-features` also enables decoder tracing, which disables bounded decode.
+The standalone allocation test imposes both live-heap and individual-request
+ceilings across decoding and conversion threads. It covers Path/Bytes parity,
+source-height growth, odd crops on both axes, grid clipping, tile color-profile
+inheritance, threaded grid tiles, and early budget rejection. The malformed
+reference fixture uses `.bin` and is tested under a 256 KiB ceiling separately
+from the positive `*.heic` oracle comparisons.
+
+The external primary-image oracle is only a test executable; it adds no native
+dependency to the Rust decoder. After the existing harness builds libheif,
+build and run it with CMake, libpng development files, Ruby, and FFmpeg with
+libx265. No incremental image assets are committed. The generator starts
+from the existing external libheif corpus and creates the geometry, metadata,
+and height-growth cases missing from that corpus in the ignored
+`.heic-test-assets/incremental-corpus` directory. All reference PNGs come from
+live strict libheif decoding. Its manifest records source, oracle, FFmpeg,
+and input identities; encoded bytes can differ across libx265 versions.
+
+```bash
+cmake -S scripts/incremental -B .heic-test-runs/incremental-oracle \
+  -DHEIF_SOURCE_DIR="$PWD/.heic-test-assets/libheif" \
+  -DHEIF_BUILD_DIR="$PWD/.heic-test-runs/validator-build"
+cmake --build .heic-test-runs/incremental-oracle --parallel
+ruby scripts/incremental/generate.rb \
+  .heic-test-runs/incremental-oracle/primary-oracle \
+  .heic-test-assets/libheif/fuzzing/data/corpus/colors-no-alpha.heic
+cargo test --release --locked --features incremental-experiment --lib --bins --test incremental-memory
+cargo test --release --locked --no-default-features --features incremental-experiment --lib --test incremental-memory
+cargo build --release --locked --features incremental-experiment --bins
+ruby scripts/incremental/verify.rb \
+  .heic-test-runs/incremental-oracle/primary-oracle \
+  target/release/incremental-allocation target/release/incremental-compare \
+  .heic-test-assets/incremental-corpus/*.heic
+```
+
+Supply `PNG_PNG_INCLUDE_DIR` and `PNG_LIBRARY` when using a custom libpng,
+as the CI workflow does. The runner compares original-size (up to side 6000)
+and side-65 outputs, writes identities and metrics under
+`.heic-test-runs/incremental`, and fails on any decoder, oracle, geometry, or
+pixel-comparison failure. `INCREMENTAL_TEST_ROOT` overrides the output folder.
+The generator accepts an optional output directory; set
+`INCREMENTAL_FIXTURES_DIR` to that directory for allocation tests. Missing
+generated inputs fail the test with setup instructions rather than skipping it.
+Every supplied input is required to decode; unsupported inputs do not count
+as successful comparisons. CI also exercises the six currently supported
+files in the libheif/stress corpus. The separate normal suite keeps its exact
+RGB/ICC comparisons and existing expected-failure accounting.
+
+The oracle enables libheif strict decoding and rejects warnings. RGB display
+comparisons normalize embedded ICC to sRGB and apply an independent floating
+point area average. The `rgb8-rounding` profile allows at most one value per
+RGB channel per pixel; alpha remains exact. The `exact` profile allows no
+sample changes. Dimensions and raster lengths are always checked. Average
+error, PSNR, signed bias and local error are diagnostics, never substitutes
+for the per-sample gate. These profiles do not authorize different tone
+mapping, resampling filters, or high-bit-depth rounding. Native reconstruction
+must retain exact sample checks against an independent decoder. ICC
+normalization currently shares moxcms with the implementation; separate qcms
+unit tests cover that boundary.
+
+The pinned libheif has a bilinear chroma-border indexing defect. Odd-grid
+regressions retain an interior aperture so they still exercise the final
+chroma neighbor without depending on its erroneous outer-border conversion.
+Every supplied input uses the unmodified live strict oracle; there are no
+stored corrected PNGs or input-specific comparison exceptions. This leaves
+the affected outer-border RGB conversion outside independent libheif coverage.
+Do not raise the global tolerance or silently classify discrepancies as
+oracle defects.
+
+A separate high-frequency odd-crop probe remains outside the passing corpus:
+at side 6000, one green sample is 72 versus the strict oracle's 70, despite
+exact native YUV agreement with FFmpeg. The sample lies inside a band, and
+the display-conversion cause remains unresolved. The one-value RGB gate is
+unchanged; this probe is not counted as a passing input. Preserve it for
+the decoder compatibility follow-up.
+
+For a supplied large supported image, measure allocator requests separately
+from uninstrumented decode timing:
+
+```bash
+target/release/incremental-allocation bounded image.heic 6000 128
+target/release/incremental-allocation bytes image.heic 6000 128
+target/release/incremental-bench bounded image.heic
+target/release/incremental-bench normal image.heic
+```
+
+Run timing trials serially, after warm-up, in alternating order. Normal mode
+returns the full raster; bounded mode includes capped output and reduction.
+No claim of equal work or universal speedup follows from those timings.
+Use a process memory tool separately for RSS. The allocator probe reports
+requested heap bytes and excludes the caller's borrowed input allocation.
+
+`incremental-check` is a deliberately unbounded verification tool comparing
+native reconstructed samples against the full Rust decoder. For an 8-bit
+4:2:0 fixture with no conformance-window crop, provide independently decoded
+planar YUV to require exact independent agreement too:
+
+```bash
+ANNEX_B_OUTPUT=reference.hevc target/release/incremental-check image.heic
+ffmpeg -v error -y -i reference.hevc -frames:v 1 -pix_fmt yuv420p -f rawvideo reference.yuv
+REFERENCE_YUV=reference.yuv target/release/incremental-check image.heic
+```
+
+This verification tool is not part of the bounded path or its memory evidence.
+`capability-inspect` prints coded-item SPS/PPS features. `scripts/incremental/wrap.rb`
+packages a supplied one-IDR Annex-B 8-bit 4:2:0 stream into a direct HEIC or
+repeated-tile grid; its dimensions must match the stream. `CROP`, `ROTATION`
+and `MIRROR` environment variables add fixture transforms without re-encoding.
+`GRID=1` creates a single-tile grid, `CANVAS` clips its visible dimensions,
+`TILE_NCLX` and `PRIMARY_NCLX` set color metadata, and `REFERENCE_COUNT`
+constructs malformed reference-count regression inputs.
+The 200 MP photographic fixture used during development was externally
+sourced and re-encoded as Main Still Picture Level 8.5, with WPP disabled;
+it is not committed and is not a native camera HEIC compatibility claim.

scripts/incremental/CMakeLists.txt

@@ -0,0 +1,11 @@
+cmake_minimum_required(VERSION 3.16)
+project(incremental_oracle LANGUAGES CXX)
+find_package(PNG REQUIRED)
+find_library(HEIF_LIBRARY NAMES heif PATHS "${HEIF_BUILD_DIR}/libheif" NO_DEFAULT_PATH REQUIRED)
+add_executable(primary-oracle primary-oracle.cc)
+target_compile_features(primary-oracle PRIVATE cxx_std_17)
+target_include_directories(primary-oracle PRIVATE "${HEIF_SOURCE_DIR}/libheif/api" "${HEIF_BUILD_DIR}")
+target_link_libraries(primary-oracle PRIVATE "${HEIF_LIBRARY}" PNG::PNG)
+if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+    target_compile_options(primary-oracle PRIVATE -Wall -Wextra -Werror)
+endif()

scripts/incremental/generate.rb

@@ -0,0 +1,74 @@
+require 'digest'
+require 'fileutils'
+require 'json'
+require 'open3'
+require 'rbconfig'
+
+abort 'Usage: generate.rb ORACLE LIBHEIF_CORPUS_IMAGE [OUTPUT_DIRECTORY]' unless (2..3).cover?(ARGV.size)
+oracle, source = ARGV.take(2).map { |path| File.expand_path(path) }
+root = File.expand_path('../..', __dir__)
+output = File.expand_path(ARGV[2] || File.join(root, '.heic-test-assets/incremental-corpus'))
+FileUtils.mkdir_p(output)
+FileUtils.rm_f(File.join(output, 'manifest.json'))
+
+def execute(*args)
+  stdout, stderr, status = Open3.capture3(*args)
+  raise "Command failed: #{args.inspect}\n#{stderr}\n#{stdout}" unless status.success?
+  stdout
+end
+
+def oracle_png(oracle, input, output)
+  metadata = JSON.parse(execute(oracle, input, output, 'strict'))
+  raise 'Oracle recovered from errors' unless metadata.fetch('strict') && metadata.fetch('warnings').zero?
+end
+
+seed = File.join(output, 'source.png')
+oracle_png(oracle, source, seed)
+encodes = [['direct', 256, 256], ['tall', 256, 2048], ['pipeline', 1024, 1088], ['edge', 256, 256], ['vertical', 256, 256]]
+encodes.each do |name, width, height|
+  filters = "scale=#{width}:#{height},format=yuv420p"
+  filters += ",geq=lum='lum(X,Y)':cb='16+mod(X*13+Y*7,224)':cr='16+mod(X*5+Y*17,224)'" if %w[edge vertical].include?(name)
+  filters += ',transpose=clock' if name == 'vertical'
+  parameters = 'keyint=1:wpp=0:repeat-headers=1:pools=none'
+  parameters += ':lossless=1' if name == 'vertical'
+  execute('ffmpeg', '-v', 'error', '-y', '-i', seed, '-vf', filters, '-frames:v', '1',
+          '-c:v', 'libx265', '-preset', 'fast', '-crf', '28', '-x265-params', parameters,
+          '-f', 'hevc', File.join(output, "#{name}.hevc"))
+end
+
+cases = [
+  ['direct', 'direct', 1, 1, {}],
+  ['tall', 'tall', 1, 1, {}],
+  ['pipeline', 'pipeline', 1, 1, {}],
+  ['grid', 'direct', 2, 2, {}],
+  ['grid-pipeline', 'pipeline', 2, 2, {}],
+  ['crop', 'tall', 1, 1, { 'CROP' => '252x2040' }],
+  ['oriented', 'direct', 1, 1, { 'CROP' => '130x190', 'ROTATION' => '1', 'MIRROR' => '1' }],
+  ['odd-short', 'direct', 1, 1, { 'CROP' => '130x190' }],
+  ['odd-tall', 'tall', 1, 1, { 'CROP' => '130x1982' }],
+  ['odd-pipeline', 'pipeline', 1, 1, { 'CROP' => '898x1022' }],
+  ['odd-width-grid', 'edge', 1, 1, { 'GRID' => '1', 'CANVAS' => '255x256', 'CROP' => '253x254' }],
+  ['odd-height-grid', 'vertical', 1, 1, { 'GRID' => '1', 'CANVAS' => '256x255', 'CROP' => '254x253' }],
+  ['undefined-grid-nclx', 'direct', 2, 2, { 'TILE_NCLX' => '1/13/1/0', 'PRIMARY_NCLX' => '2/2/2/1' }],
+  ['invalid-grid-references', 'direct', 2, 2, { 'REFERENCE_COUNT' => '65535' }]
+]
+cases.each do |name, encoded, columns, rows, options|
+  _, width, height = encodes.find { |entry| entry[0] == encoded }
+  extension = name == 'invalid-grid-references' ? 'bin' : 'heic'
+  path = File.join(output, "#{name}.#{extension}")
+  environment = %w[GRID CANVAS CROP ROTATION MIRROR TILE_NCLX PRIMARY_NCLX REFERENCE_COUNT].to_h { |key| [key, nil] }.merge(options)
+  execute(environment, RbConfig.ruby, File.join(__dir__, 'wrap.rb'), File.join(output, "#{encoded}.hevc"),
+          path, width.to_s, height.to_s, columns.to_s, rows.to_s)
+  oracle_png(oracle, path, path.sub(/\.heic$/, '.png')) if extension == 'heic'
+  puts "Generated #{path}"
+end
+manifest = {
+  source: { path: source, sha256: Digest::SHA256.file(source).hexdigest },
+  oracle_sha256: Digest::SHA256.file(oracle).hexdigest,
+  ffmpeg: execute('ffmpeg', '-version').lines.first.strip,
+  inputs: cases.to_h do |name, *_|
+    path = File.join(output, "#{name}.#{name == 'invalid-grid-references' ? 'bin' : 'heic'}")
+    [File.basename(path), Digest::SHA256.file(path).hexdigest]
+  end
+}
+File.write(File.join(output, 'manifest.json'), JSON.pretty_generate(manifest))