Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/e2ee-h264-pps-id.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"livekit-client": patch
---

Keep the H.264 PPS id readable in E2EE keyframes, so that Chrome no longer drops keyframes from encoders such as Intel Quick Sync
16 changes: 16 additions & 0 deletions src/e2ee/worker/FrameCryptor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import type { KeyProviderOptions } from '../types';
import { createKeyMaterialFromString } from '../utils';
import { FrameCryptor, encryptionEnabledMap, isFrameServerInjected } from './FrameCryptor';
import { ParticipantKeyHandler } from './ParticipantKeyHandler';
import { parseH264IdrPpsId } from './naluUtils';

function mockEncryptedRTCEncodedVideoFrame(keyIndex: number): RTCEncodedVideoFrame {
const trailer = mockFrameTrailer(keyIndex);
Expand Down Expand Up @@ -311,6 +312,21 @@ describe('FrameCryptor', () => {
);
expect(Array.from(decryptedData)).toEqual(plainTextData);
});
it('keeps the PPS id readable when it lies past the unencrypted bytes (#2180)', async () => {
// Intel Quick Sync IDR slice: the PPS id is the first encrypted bit, so a single
// encryption leaves it at 0 only half the time.
const qsvIdr = [0x65, 0x88, 0x80, 0x40, 0x87, 0x0a, 0x55, 0x66];
const plainTextData = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, ...qsvIdr];
const sliceStart = plainTextData.length - qsvIdr.length;
for (let i = 0; i < 16; i++) {
const { encryptedData, decryptedData } = await encryptAndDecrypt(
new Uint8Array(plainTextData),
);
expect(parseH264IdrPpsId(encryptedData.subarray(sliceStart))).toBe(0);
expect(Array.from(decryptedData)).toEqual(plainTextData);
}
});

it('logs the frame bytes once when NALU processing fails', async () => {
vitest.useFakeTimers();
const warn = vitest.spyOn(workerLogger, 'warn');
Expand Down
61 changes: 50 additions & 11 deletions src/e2ee/worker/FrameCryptor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,19 @@ import type {
import { deriveKeys, isVideoFrame, needsRbspUnescaping, parseRbsp, writeRbsp } from '../utils';
import { ErrorRateLimiter } from './ErrorRateLimiter';
import type { ParticipantKeyHandler } from './ParticipantKeyHandler';
import { normalizeAnnexBStartCodes, processNALUsForEncryption } from './naluUtils';
import {
normalizeAnnexBStartCodes,
parseH264IdrPpsId,
processNALUsForEncryption,
} from './naluUtils';
import { identifySifPayload } from './sifPayload';

/**
* With one bit of the PPS id encrypted, each attempt matches with probability 1/2, so 16 attempts
* all miss with probability 2^-16. After the last attempt the frame goes out as it is.
*/
const MAX_PPS_ID_ENCRYPT_ATTEMPTS = 16;

export const encryptionEnabledMap: Map<string, boolean> = new Map();

export interface FrameCryptorConstructor {
Expand Down Expand Up @@ -483,7 +493,7 @@ export class FrameCryptor extends BaseFrameCryptor {
const keyIndex = this.keys.getCurrentKeyIndex();

if (encryptionKey) {
const iv = this.makeIV(
let iv = this.makeIV(
encodedFrame.getMetadata().synchronizationSource ?? -1,
encodedFrame.timestamp,
);
Expand Down Expand Up @@ -515,15 +525,44 @@ export class FrameCryptor extends BaseFrameCryptor {
// payload |IV...(length = IV_LENGTH)|R|IV_LENGTH|KID |
// ---------+-------------------------+-+---------+----
try {
const cipherText = await crypto.subtle.encrypt(
{
name: ENCRYPTION_ALGORITHM,
iv,
additionalData: frameHeader,
},
encryptionKey,
new Uint8Array(encodedFrame.data, frameInfo.unencryptedBytes),
);
// Chrome reads pic_parameter_set_id from an H.264 IDR slice before the decrypt transform,
// and drops the keyframe if the id names an unknown PPS. Some encoders put the id past the
// unencrypted bytes (#2180), so encrypt again with a fresh IV until the sent bits still
// read as the true id. The receiver takes the IV from the trailer, so it needs no change.
const sliceStart = frameInfo.unencryptedBytes - 2;
const ppsId = frameInfo.requiresNALUProcessing
? parseH264IdrPpsId(new Uint8Array(encodedFrame.data, sliceStart))
: null;
let cipherText: ArrayBuffer;
for (let attempt = 1; ; attempt++) {
cipherText = await crypto.subtle.encrypt(
{
name: ENCRYPTION_ALGORITHM,
iv,
additionalData: frameHeader,
},
encryptionKey,
new Uint8Array(encodedFrame.data, frameInfo.unencryptedBytes),
);

if (ppsId === null || attempt === MAX_PPS_ID_ENCRYPT_ATTEMPTS) {
break;
}
// The slice as the receiver sees it: its 2 unencrypted bytes, then the escaped
// ciphertext. writeRbsp works byte by byte, so escaping the first 16 bytes gives the
// start of the escaped frame, and a retry costs only the encryption.
const escapedStart = writeRbsp(new Uint8Array(cipherText, 0, 16));
const sentSlice = new Uint8Array(2 + escapedStart.byteLength);
sentSlice.set(new Uint8Array(encodedFrame.data, sliceStart, 2));
sentSlice.set(escapedStart, 2);
if (parseH264IdrPpsId(sentSlice) === ppsId) {
break;
}
iv = this.makeIV(
encodedFrame.getMetadata().synchronizationSource ?? -1,
encodedFrame.timestamp,
);
}

let newDataWithoutHeader: NonSharedUint8Array = new Uint8Array(
cipherText.byteLength + iv.byteLength + frameTrailer.byteLength,
Expand Down
22 changes: 21 additions & 1 deletion src/e2ee/worker/naluUtils.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import { describe, expect, it } from 'vitest';
import { normalizeAnnexBStartCodes, processNALUsForEncryption } from './naluUtils';
import {
normalizeAnnexBStartCodes,
parseH264IdrPpsId,
processNALUsForEncryption,
} from './naluUtils';

/** Annex-B frame: a 4-byte start code before every NALU. */
function annexB(...nalus: number[][]) {
Expand Down Expand Up @@ -121,3 +125,19 @@ describe('processNALUsForEncryption', () => {
});
});
});

describe('parseH264IdrPpsId', () => {
it('reads the PPS id past the second byte of the slice (#2180)', () => {
// Intel Quick Sync: slice_type 7 takes 7 bits, so the id is the first bit of the third byte.
expect(parseH264IdrPpsId(Uint8Array.from([0x65, 0x88, 0x80, 0x40, 0x87, 0x0a]))).toBe(0);
// VideoToolbox: slice_type 2 takes 3 bits, so the id sits in the second byte.
expect(parseH264IdrPpsId(Uint8Array.from([0x25, 0xb8, 0x00, 0x40, 0x00, 0x84]))).toBe(0);
// first_mb_in_slice 0, slice_type 7, pic_parameter_set_id 1 (bits 1 0001000 010).
expect(parseH264IdrPpsId(Uint8Array.from([0x65, 0x88, 0x5f, 0xff]))).toBe(1);
});

it('returns null for a non-IDR slice or a slice that ends inside the field', () => {
expect(parseH264IdrPpsId(Uint8Array.from([0x41, 0x9a, 0x21, 0x43]))).toBeNull();
expect(parseH264IdrPpsId(Uint8Array.from([0x65, 0x88]))).toBeNull();
});
});
30 changes: 30 additions & 0 deletions src/e2ee/worker/naluUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
* Contains functions for parsing and working with NALUs in video frames
*/
import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
import { parseRbsp } from '../utils';

/**
* Mask for extracting NALU type from H.264 header byte
Expand Down Expand Up @@ -243,6 +244,35 @@ function findSliceNALUUnencryptedBytes(
return null;
}

/**
* Read pic_parameter_set_id from an H.264 IDR slice NALU, the way a WebRTC receiver does before
* its decrypt transform runs. An h265 NALU header byte is even, so it never reads as an IDR here.
* @param nalu Bytes from the NALU header on
* @returns The PPS id, or null if `nalu` is not an IDR slice or ends inside the field
*/
export function parseH264IdrPpsId(nalu: NonSharedUint8Array): number | null {
if (parseH264NALUType(nalu[0]) !== H264NALUType.SLICE_IDR) {
return null;
}
// first_mb_in_slice, slice_type and pic_parameter_set_id are the first three ue(v) fields.
// 16 bytes hold all three.
const rbsp = parseRbsp(nalu.subarray(1, 17));
const bitLength = rbsp.length * 8;
let bit = 0;
const readBit = () => (rbsp[bit >> 3] >> (7 - (bit++ & 7))) & 1;
const readUe = () => {
let zeros = 0;
while (readBit() === 0 && bit < bitLength) zeros++;
let value = 1;
for (let i = 0; i < zeros; i++) value = value * 2 + readBit();
return value - 1;
};
readUe();
readUe();
const ppsId = readUe();
return bit <= bitLength ? ppsId : null;
}

/**
* Find all NALU start indices in a byte stream
* Supports both H.264 and H.265 with 3-byte and 4-byte start codes
Expand Down
Loading