diff --git a/.changeset/e2ee-h264-pps-id.md b/.changeset/e2ee-h264-pps-id.md new file mode 100644 index 0000000000..3c7acce3d0 --- /dev/null +++ b/.changeset/e2ee-h264-pps-id.md @@ -0,0 +1,5 @@ +--- +"livekit-client": patch +--- + +Keep the H.264 PPS id readable in E2EE keyframes, so that Chrome no longer drops keyframes from encoders such as Intel Quick Sync diff --git a/src/e2ee/worker/FrameCryptor.test.ts b/src/e2ee/worker/FrameCryptor.test.ts index ccefd124ba..f4e415c495 100644 --- a/src/e2ee/worker/FrameCryptor.test.ts +++ b/src/e2ee/worker/FrameCryptor.test.ts @@ -10,6 +10,7 @@ import type { KeyProviderOptions } from '../types'; import { createKeyMaterialFromString } from '../utils'; import { FrameCryptor, encryptionEnabledMap, isFrameServerInjected } from './FrameCryptor'; import { ParticipantKeyHandler } from './ParticipantKeyHandler'; +import { parseH264IdrPpsId } from './naluUtils'; function mockEncryptedRTCEncodedVideoFrame(keyIndex: number): RTCEncodedVideoFrame { const trailer = mockFrameTrailer(keyIndex); @@ -311,6 +312,21 @@ describe('FrameCryptor', () => { ); expect(Array.from(decryptedData)).toEqual(plainTextData); }); + it('keeps the PPS id readable when it lies past the unencrypted bytes (#2180)', async () => { + // Intel Quick Sync IDR slice: the PPS id is the first encrypted bit, so a single + // encryption leaves it at 0 only half the time. + const qsvIdr = [0x65, 0x88, 0x80, 0x40, 0x87, 0x0a, 0x55, 0x66]; + const plainTextData = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, ...qsvIdr]; + const sliceStart = plainTextData.length - qsvIdr.length; + for (let i = 0; i < 16; i++) { + const { encryptedData, decryptedData } = await encryptAndDecrypt( + new Uint8Array(plainTextData), + ); + expect(parseH264IdrPpsId(encryptedData.subarray(sliceStart))).toBe(0); + expect(Array.from(decryptedData)).toEqual(plainTextData); + } + }); + it('logs the frame bytes once when NALU processing fails', async () => { vitest.useFakeTimers(); const warn = vitest.spyOn(workerLogger, 'warn'); diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index 20d1f9d313..c3ae04a7b8 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -25,9 +25,19 @@ import type { import { deriveKeys, isVideoFrame, needsRbspUnescaping, parseRbsp, writeRbsp } from '../utils'; import { ErrorRateLimiter } from './ErrorRateLimiter'; import type { ParticipantKeyHandler } from './ParticipantKeyHandler'; -import { normalizeAnnexBStartCodes, processNALUsForEncryption } from './naluUtils'; +import { + normalizeAnnexBStartCodes, + parseH264IdrPpsId, + processNALUsForEncryption, +} from './naluUtils'; import { identifySifPayload } from './sifPayload'; +/** + * With one bit of the PPS id encrypted, each attempt matches with probability 1/2, so 16 attempts + * all miss with probability 2^-16. After the last attempt the frame goes out as it is. + */ +const MAX_PPS_ID_ENCRYPT_ATTEMPTS = 16; + export const encryptionEnabledMap: Map = new Map(); export interface FrameCryptorConstructor { @@ -483,7 +493,7 @@ export class FrameCryptor extends BaseFrameCryptor { const keyIndex = this.keys.getCurrentKeyIndex(); if (encryptionKey) { - const iv = this.makeIV( + let iv = this.makeIV( encodedFrame.getMetadata().synchronizationSource ?? -1, encodedFrame.timestamp, ); @@ -515,15 +525,44 @@ export class FrameCryptor extends BaseFrameCryptor { // payload |IV...(length = IV_LENGTH)|R|IV_LENGTH|KID | // ---------+-------------------------+-+---------+---- try { - const cipherText = await crypto.subtle.encrypt( - { - name: ENCRYPTION_ALGORITHM, - iv, - additionalData: frameHeader, - }, - encryptionKey, - new Uint8Array(encodedFrame.data, frameInfo.unencryptedBytes), - ); + // Chrome reads pic_parameter_set_id from an H.264 IDR slice before the decrypt transform, + // and drops the keyframe if the id names an unknown PPS. Some encoders put the id past the + // unencrypted bytes (#2180), so encrypt again with a fresh IV until the sent bits still + // read as the true id. The receiver takes the IV from the trailer, so it needs no change. + const sliceStart = frameInfo.unencryptedBytes - 2; + const ppsId = frameInfo.requiresNALUProcessing + ? parseH264IdrPpsId(new Uint8Array(encodedFrame.data, sliceStart)) + : null; + let cipherText: ArrayBuffer; + for (let attempt = 1; ; attempt++) { + cipherText = await crypto.subtle.encrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: frameHeader, + }, + encryptionKey, + new Uint8Array(encodedFrame.data, frameInfo.unencryptedBytes), + ); + + if (ppsId === null || attempt === MAX_PPS_ID_ENCRYPT_ATTEMPTS) { + break; + } + // The slice as the receiver sees it: its 2 unencrypted bytes, then the escaped + // ciphertext. writeRbsp works byte by byte, so escaping the first 16 bytes gives the + // start of the escaped frame, and a retry costs only the encryption. + const escapedStart = writeRbsp(new Uint8Array(cipherText, 0, 16)); + const sentSlice = new Uint8Array(2 + escapedStart.byteLength); + sentSlice.set(new Uint8Array(encodedFrame.data, sliceStart, 2)); + sentSlice.set(escapedStart, 2); + if (parseH264IdrPpsId(sentSlice) === ppsId) { + break; + } + iv = this.makeIV( + encodedFrame.getMetadata().synchronizationSource ?? -1, + encodedFrame.timestamp, + ); + } let newDataWithoutHeader: NonSharedUint8Array = new Uint8Array( cipherText.byteLength + iv.byteLength + frameTrailer.byteLength, diff --git a/src/e2ee/worker/naluUtils.test.ts b/src/e2ee/worker/naluUtils.test.ts index dbed67cb82..f0ee6db6b5 100644 --- a/src/e2ee/worker/naluUtils.test.ts +++ b/src/e2ee/worker/naluUtils.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { normalizeAnnexBStartCodes, processNALUsForEncryption } from './naluUtils'; +import { + normalizeAnnexBStartCodes, + parseH264IdrPpsId, + processNALUsForEncryption, +} from './naluUtils'; /** Annex-B frame: a 4-byte start code before every NALU. */ function annexB(...nalus: number[][]) { @@ -121,3 +125,19 @@ describe('processNALUsForEncryption', () => { }); }); }); + +describe('parseH264IdrPpsId', () => { + it('reads the PPS id past the second byte of the slice (#2180)', () => { + // Intel Quick Sync: slice_type 7 takes 7 bits, so the id is the first bit of the third byte. + expect(parseH264IdrPpsId(Uint8Array.from([0x65, 0x88, 0x80, 0x40, 0x87, 0x0a]))).toBe(0); + // VideoToolbox: slice_type 2 takes 3 bits, so the id sits in the second byte. + expect(parseH264IdrPpsId(Uint8Array.from([0x25, 0xb8, 0x00, 0x40, 0x00, 0x84]))).toBe(0); + // first_mb_in_slice 0, slice_type 7, pic_parameter_set_id 1 (bits 1 0001000 010). + expect(parseH264IdrPpsId(Uint8Array.from([0x65, 0x88, 0x5f, 0xff]))).toBe(1); + }); + + it('returns null for a non-IDR slice or a slice that ends inside the field', () => { + expect(parseH264IdrPpsId(Uint8Array.from([0x41, 0x9a, 0x21, 0x43]))).toBeNull(); + expect(parseH264IdrPpsId(Uint8Array.from([0x65, 0x88]))).toBeNull(); + }); +}); diff --git a/src/e2ee/worker/naluUtils.ts b/src/e2ee/worker/naluUtils.ts index 4466d8a72c..d2433ec1b2 100644 --- a/src/e2ee/worker/naluUtils.ts +++ b/src/e2ee/worker/naluUtils.ts @@ -3,6 +3,7 @@ * Contains functions for parsing and working with NALUs in video frames */ import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays'; +import { parseRbsp } from '../utils'; /** * Mask for extracting NALU type from H.264 header byte @@ -243,6 +244,35 @@ function findSliceNALUUnencryptedBytes( return null; } +/** + * Read pic_parameter_set_id from an H.264 IDR slice NALU, the way a WebRTC receiver does before + * its decrypt transform runs. An h265 NALU header byte is even, so it never reads as an IDR here. + * @param nalu Bytes from the NALU header on + * @returns The PPS id, or null if `nalu` is not an IDR slice or ends inside the field + */ +export function parseH264IdrPpsId(nalu: NonSharedUint8Array): number | null { + if (parseH264NALUType(nalu[0]) !== H264NALUType.SLICE_IDR) { + return null; + } + // first_mb_in_slice, slice_type and pic_parameter_set_id are the first three ue(v) fields. + // 16 bytes hold all three. + const rbsp = parseRbsp(nalu.subarray(1, 17)); + const bitLength = rbsp.length * 8; + let bit = 0; + const readBit = () => (rbsp[bit >> 3] >> (7 - (bit++ & 7))) & 1; + const readUe = () => { + let zeros = 0; + while (readBit() === 0 && bit < bitLength) zeros++; + let value = 1; + for (let i = 0; i < zeros; i++) value = value * 2 + readBit(); + return value - 1; + }; + readUe(); + readUe(); + const ppsId = readUe(); + return bit <= bitLength ? ppsId : null; +} + /** * Find all NALU start indices in a byte stream * Supports both H.264 and H.265 with 3-byte and 4-byte start codes