From ac3d5216173dc5c2fedd1ac61e2199f21880a703 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Fri, 23 Oct 2026 14:54:51 +0200 Subject: [PATCH 1/2] Keep the H.264 PPS id readable in E2EE keyframes (#2180) Chrome reads pic_parameter_set_id from an H.264 IDR slice before the decrypt transform runs, and drops the keyframe if the id names an unknown PPS. Encoders such as Intel Quick Sync code slice_type in 7 bits, so the id lies past the 2 unencrypted slice bytes and gets encrypted. The sender now encrypts again with a fresh IV, up to 16 times, until the sent bytes still read as the true PPS id. The receiver takes the IV from the frame trailer, so the wire format and receivers do not change. Co-Authored-By: Claude Opus 5.5 --- .changeset/e2ee-h264-pps-id.md | 5 ++ src/e2ee/worker/FrameCryptor.test.ts | 16 ++++++ src/e2ee/worker/FrameCryptor.ts | 74 +++++++++++++++++++++------- src/e2ee/worker/naluUtils.test.ts | 22 ++++++++- src/e2ee/worker/naluUtils.ts | 30 +++++++++++ 5 files changed, 127 insertions(+), 20 deletions(-) create mode 100644 .changeset/e2ee-h264-pps-id.md diff --git a/.changeset/e2ee-h264-pps-id.md b/.changeset/e2ee-h264-pps-id.md new file mode 100644 index 0000000000..3c7acce3d0 --- /dev/null +++ b/.changeset/e2ee-h264-pps-id.md @@ -0,0 +1,5 @@ +--- +"livekit-client": patch +--- + +Keep the H.264 PPS id readable in E2EE keyframes, so that Chrome no longer drops keyframes from encoders such as Intel Quick Sync diff --git a/src/e2ee/worker/FrameCryptor.test.ts b/src/e2ee/worker/FrameCryptor.test.ts index ccefd124ba..f4e415c495 100644 --- a/src/e2ee/worker/FrameCryptor.test.ts +++ b/src/e2ee/worker/FrameCryptor.test.ts @@ -10,6 +10,7 @@ import type { KeyProviderOptions } from '../types'; import { createKeyMaterialFromString } from '../utils'; import { FrameCryptor, encryptionEnabledMap, isFrameServerInjected } from './FrameCryptor'; import { ParticipantKeyHandler } from './ParticipantKeyHandler'; +import { parseH264IdrPpsId } from './naluUtils'; function mockEncryptedRTCEncodedVideoFrame(keyIndex: number): RTCEncodedVideoFrame { const trailer = mockFrameTrailer(keyIndex); @@ -311,6 +312,21 @@ describe('FrameCryptor', () => { ); expect(Array.from(decryptedData)).toEqual(plainTextData); }); + it('keeps the PPS id readable when it lies past the unencrypted bytes (#2180)', async () => { + // Intel Quick Sync IDR slice: the PPS id is the first encrypted bit, so a single + // encryption leaves it at 0 only half the time. + const qsvIdr = [0x65, 0x88, 0x80, 0x40, 0x87, 0x0a, 0x55, 0x66]; + const plainTextData = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, ...qsvIdr]; + const sliceStart = plainTextData.length - qsvIdr.length; + for (let i = 0; i < 16; i++) { + const { encryptedData, decryptedData } = await encryptAndDecrypt( + new Uint8Array(plainTextData), + ); + expect(parseH264IdrPpsId(encryptedData.subarray(sliceStart))).toBe(0); + expect(Array.from(decryptedData)).toEqual(plainTextData); + } + }); + it('logs the frame bytes once when NALU processing fails', async () => { vitest.useFakeTimers(); const warn = vitest.spyOn(workerLogger, 'warn'); diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index 20d1f9d313..e93790c0b4 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -25,9 +25,19 @@ import type { import { deriveKeys, isVideoFrame, needsRbspUnescaping, parseRbsp, writeRbsp } from '../utils'; import { ErrorRateLimiter } from './ErrorRateLimiter'; import type { ParticipantKeyHandler } from './ParticipantKeyHandler'; -import { normalizeAnnexBStartCodes, processNALUsForEncryption } from './naluUtils'; +import { + normalizeAnnexBStartCodes, + parseH264IdrPpsId, + processNALUsForEncryption, +} from './naluUtils'; import { identifySifPayload } from './sifPayload'; +/** + * With one bit of the PPS id encrypted, each attempt matches with probability 1/2, so 16 attempts + * all miss with probability 2^-16. After the last attempt the frame goes out as it is. + */ +const MAX_PPS_ID_ENCRYPT_ATTEMPTS = 16; + export const encryptionEnabledMap: Map = new Map(); export interface FrameCryptorConstructor { @@ -483,7 +493,7 @@ export class FrameCryptor extends BaseFrameCryptor { const keyIndex = this.keys.getCurrentKeyIndex(); if (encryptionKey) { - const iv = this.makeIV( + let iv = this.makeIV( encodedFrame.getMetadata().synchronizationSource ?? -1, encodedFrame.timestamp, ); @@ -515,25 +525,51 @@ export class FrameCryptor extends BaseFrameCryptor { // payload |IV...(length = IV_LENGTH)|R|IV_LENGTH|KID | // ---------+-------------------------+-+---------+---- try { - const cipherText = await crypto.subtle.encrypt( - { - name: ENCRYPTION_ALGORITHM, - iv, - additionalData: frameHeader, - }, - encryptionKey, - new Uint8Array(encodedFrame.data, frameInfo.unencryptedBytes), - ); + // Chrome reads pic_parameter_set_id from an H.264 IDR slice before the decrypt transform, + // and drops the keyframe if the id names an unknown PPS. Some encoders put the id past the + // unencrypted bytes (#2180), so encrypt again with a fresh IV until the sent bits still + // read as the true id. The receiver takes the IV from the trailer, so it needs no change. + const sliceStart = frameInfo.unencryptedBytes - 2; + const ppsId = frameInfo.requiresNALUProcessing + ? parseH264IdrPpsId(new Uint8Array(encodedFrame.data, sliceStart)) + : null; + let newDataWithoutHeader: NonSharedUint8Array; + for (let attempt = 1; ; attempt++) { + const cipherText = await crypto.subtle.encrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: frameHeader, + }, + encryptionKey, + new Uint8Array(encodedFrame.data, frameInfo.unencryptedBytes), + ); - let newDataWithoutHeader: NonSharedUint8Array = new Uint8Array( - cipherText.byteLength + iv.byteLength + frameTrailer.byteLength, - ); - newDataWithoutHeader.set(new Uint8Array(cipherText)); // add ciphertext. - newDataWithoutHeader.set(new Uint8Array(iv), cipherText.byteLength); // append IV. - newDataWithoutHeader.set(frameTrailer, cipherText.byteLength + iv.byteLength); // append frame trailer. + newDataWithoutHeader = new Uint8Array( + cipherText.byteLength + iv.byteLength + frameTrailer.byteLength, + ); + newDataWithoutHeader.set(new Uint8Array(cipherText)); // add ciphertext. + newDataWithoutHeader.set(new Uint8Array(iv), cipherText.byteLength); // append IV. + newDataWithoutHeader.set(frameTrailer, cipherText.byteLength + iv.byteLength); // append frame trailer. + + if (frameInfo.requiresNALUProcessing) { + newDataWithoutHeader = writeRbsp(newDataWithoutHeader); + } - if (frameInfo.requiresNALUProcessing) { - newDataWithoutHeader = writeRbsp(newDataWithoutHeader); + if (ppsId === null || attempt === MAX_PPS_ID_ENCRYPT_ATTEMPTS) { + break; + } + // The slice as the receiver sees it: its 2 unencrypted bytes, then the escaped ciphertext. + const sentSlice = new Uint8Array(18); + sentSlice.set(new Uint8Array(encodedFrame.data, sliceStart, 2)); + sentSlice.set(newDataWithoutHeader.subarray(0, 16), 2); + if (parseH264IdrPpsId(sentSlice) === ppsId) { + break; + } + iv = this.makeIV( + encodedFrame.getMetadata().synchronizationSource ?? -1, + encodedFrame.timestamp, + ); } var newData = new Uint8Array(frameHeader.byteLength + newDataWithoutHeader.byteLength); diff --git a/src/e2ee/worker/naluUtils.test.ts b/src/e2ee/worker/naluUtils.test.ts index dbed67cb82..f0ee6db6b5 100644 --- a/src/e2ee/worker/naluUtils.test.ts +++ b/src/e2ee/worker/naluUtils.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { normalizeAnnexBStartCodes, processNALUsForEncryption } from './naluUtils'; +import { + normalizeAnnexBStartCodes, + parseH264IdrPpsId, + processNALUsForEncryption, +} from './naluUtils'; /** Annex-B frame: a 4-byte start code before every NALU. */ function annexB(...nalus: number[][]) { @@ -121,3 +125,19 @@ describe('processNALUsForEncryption', () => { }); }); }); + +describe('parseH264IdrPpsId', () => { + it('reads the PPS id past the second byte of the slice (#2180)', () => { + // Intel Quick Sync: slice_type 7 takes 7 bits, so the id is the first bit of the third byte. + expect(parseH264IdrPpsId(Uint8Array.from([0x65, 0x88, 0x80, 0x40, 0x87, 0x0a]))).toBe(0); + // VideoToolbox: slice_type 2 takes 3 bits, so the id sits in the second byte. + expect(parseH264IdrPpsId(Uint8Array.from([0x25, 0xb8, 0x00, 0x40, 0x00, 0x84]))).toBe(0); + // first_mb_in_slice 0, slice_type 7, pic_parameter_set_id 1 (bits 1 0001000 010). + expect(parseH264IdrPpsId(Uint8Array.from([0x65, 0x88, 0x5f, 0xff]))).toBe(1); + }); + + it('returns null for a non-IDR slice or a slice that ends inside the field', () => { + expect(parseH264IdrPpsId(Uint8Array.from([0x41, 0x9a, 0x21, 0x43]))).toBeNull(); + expect(parseH264IdrPpsId(Uint8Array.from([0x65, 0x88]))).toBeNull(); + }); +}); diff --git a/src/e2ee/worker/naluUtils.ts b/src/e2ee/worker/naluUtils.ts index 4466d8a72c..d2433ec1b2 100644 --- a/src/e2ee/worker/naluUtils.ts +++ b/src/e2ee/worker/naluUtils.ts @@ -3,6 +3,7 @@ * Contains functions for parsing and working with NALUs in video frames */ import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays'; +import { parseRbsp } from '../utils'; /** * Mask for extracting NALU type from H.264 header byte @@ -243,6 +244,35 @@ function findSliceNALUUnencryptedBytes( return null; } +/** + * Read pic_parameter_set_id from an H.264 IDR slice NALU, the way a WebRTC receiver does before + * its decrypt transform runs. An h265 NALU header byte is even, so it never reads as an IDR here. + * @param nalu Bytes from the NALU header on + * @returns The PPS id, or null if `nalu` is not an IDR slice or ends inside the field + */ +export function parseH264IdrPpsId(nalu: NonSharedUint8Array): number | null { + if (parseH264NALUType(nalu[0]) !== H264NALUType.SLICE_IDR) { + return null; + } + // first_mb_in_slice, slice_type and pic_parameter_set_id are the first three ue(v) fields. + // 16 bytes hold all three. + const rbsp = parseRbsp(nalu.subarray(1, 17)); + const bitLength = rbsp.length * 8; + let bit = 0; + const readBit = () => (rbsp[bit >> 3] >> (7 - (bit++ & 7))) & 1; + const readUe = () => { + let zeros = 0; + while (readBit() === 0 && bit < bitLength) zeros++; + let value = 1; + for (let i = 0; i < zeros; i++) value = value * 2 + readBit(); + return value - 1; + }; + readUe(); + readUe(); + const ppsId = readUe(); + return bit <= bitLength ? ppsId : null; +} + /** * Find all NALU start indices in a byte stream * Supports both H.264 and H.265 with 3-byte and 4-byte start codes From fb142866f379fe5aa6c8a3edda62e4b0e7d298ee Mon Sep 17 00:00:00 2001 From: lukasIO Date: Sun, 25 Oct 2026 12:18:53 +0100 Subject: [PATCH 2/2] Check the PPS id before escaping the frame writeRbsp takes about 95% of an encryption attempt on a large keyframe. The retry loop escaped the whole frame on each attempt, but the PPS id check needs only the first 16 bytes. Escape those for the check, and escape the whole frame once, after the loop. A retry then costs only the AES-GCM encryption. Co-Authored-By: Claude Opus 5.5 --- src/e2ee/worker/FrameCryptor.ts | 35 ++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index e93790c0b4..c3ae04a7b8 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -533,9 +533,9 @@ export class FrameCryptor extends BaseFrameCryptor { const ppsId = frameInfo.requiresNALUProcessing ? parseH264IdrPpsId(new Uint8Array(encodedFrame.data, sliceStart)) : null; - let newDataWithoutHeader: NonSharedUint8Array; + let cipherText: ArrayBuffer; for (let attempt = 1; ; attempt++) { - const cipherText = await crypto.subtle.encrypt( + cipherText = await crypto.subtle.encrypt( { name: ENCRYPTION_ALGORITHM, iv, @@ -545,24 +545,16 @@ export class FrameCryptor extends BaseFrameCryptor { new Uint8Array(encodedFrame.data, frameInfo.unencryptedBytes), ); - newDataWithoutHeader = new Uint8Array( - cipherText.byteLength + iv.byteLength + frameTrailer.byteLength, - ); - newDataWithoutHeader.set(new Uint8Array(cipherText)); // add ciphertext. - newDataWithoutHeader.set(new Uint8Array(iv), cipherText.byteLength); // append IV. - newDataWithoutHeader.set(frameTrailer, cipherText.byteLength + iv.byteLength); // append frame trailer. - - if (frameInfo.requiresNALUProcessing) { - newDataWithoutHeader = writeRbsp(newDataWithoutHeader); - } - if (ppsId === null || attempt === MAX_PPS_ID_ENCRYPT_ATTEMPTS) { break; } - // The slice as the receiver sees it: its 2 unencrypted bytes, then the escaped ciphertext. - const sentSlice = new Uint8Array(18); + // The slice as the receiver sees it: its 2 unencrypted bytes, then the escaped + // ciphertext. writeRbsp works byte by byte, so escaping the first 16 bytes gives the + // start of the escaped frame, and a retry costs only the encryption. + const escapedStart = writeRbsp(new Uint8Array(cipherText, 0, 16)); + const sentSlice = new Uint8Array(2 + escapedStart.byteLength); sentSlice.set(new Uint8Array(encodedFrame.data, sliceStart, 2)); - sentSlice.set(newDataWithoutHeader.subarray(0, 16), 2); + sentSlice.set(escapedStart, 2); if (parseH264IdrPpsId(sentSlice) === ppsId) { break; } @@ -572,6 +564,17 @@ export class FrameCryptor extends BaseFrameCryptor { ); } + let newDataWithoutHeader: NonSharedUint8Array = new Uint8Array( + cipherText.byteLength + iv.byteLength + frameTrailer.byteLength, + ); + newDataWithoutHeader.set(new Uint8Array(cipherText)); // add ciphertext. + newDataWithoutHeader.set(new Uint8Array(iv), cipherText.byteLength); // append IV. + newDataWithoutHeader.set(frameTrailer, cipherText.byteLength + iv.byteLength); // append frame trailer. + + if (frameInfo.requiresNALUProcessing) { + newDataWithoutHeader = writeRbsp(newDataWithoutHeader); + } + var newData = new Uint8Array(frameHeader.byteLength + newDataWithoutHeader.byteLength); newData.set(frameHeader); newData.set(newDataWithoutHeader, frameHeader.byteLength);