Skip to content

fix(extension): release failed camera preview connections #3443

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vedanth-jadhav wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(extension): release failed camera preview connections #3443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 1 addition & 66 deletions apps/chrome-extension/src/content/overlay.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
useState,
} from "react";
import { createRoot } from "react-dom/client";
import { connectCameraPreview } from "../shared/camera-preview-connection";
import { isOverlayMessage } from "../shared/messages";
import { sendServiceWorkerMessage } from "../shared/runtime";
import {
Expand Down Expand Up @@ -35,10 +36,6 @@ import type {
WebcamSettings,
WebcamShape,
} from "../shared/types";
import {
toSessionDescriptionInit,
waitForIceGatheringComplete,
} from "../shared/webrtc";
import { ConfirmOverlay } from "./confirm-overlay";
import { CountdownOverlay } from "./countdown-overlay";
import overlayCss from "./overlay.css?inline";
Expand Down Expand Up @@ -245,75 +242,13 @@ const isSameWebcamSettings = (
current.shape === next.shape &&
current.mirror === next.mirror;

const waitForRemoteStream = (peer: RTCPeerConnection) =>
new Promise<MediaStream>((resolve, reject) => {
let settled = false;
const timeout = window.setTimeout(() => {
if (settled) return;
settled = true;
reject(new Error("Camera preview timed out."));
}, 10000);

const finish = (stream: MediaStream) => {
if (settled) return;
settled = true;
window.clearTimeout(timeout);
resolve(stream);
};

peer.addEventListener("track", (event) => {
finish(event.streams[0] ?? new MediaStream([event.track]));
});

peer.addEventListener("connectionstatechange", () => {
if (
settled ||
(peer.connectionState !== "failed" && peer.connectionState !== "closed")
) {
return;
}
settled = true;
window.clearTimeout(timeout);
reject(new Error("Camera preview connection failed."));
});
});

const stopStream = (stream: MediaStream | null) => {
if (!stream) return;
for (const track of stream.getTracks()) {
track.stop();
}
};

const connectCameraPreview = async (
settings: WebcamSettings,
sessionId: string,
) => {
const peer = new RTCPeerConnection();
peer.addTransceiver("video", { direction: "recvonly" });
const remoteStreamPromise = waitForRemoteStream(peer);
await peer.setLocalDescription(await peer.createOffer());
await waitForIceGatheringComplete(peer);

const response = await sendServiceWorkerMessage({
target: "service-worker",
type: "connect-camera-preview",
sessionId,
settings,
offer: toSessionDescriptionInit(peer.localDescription),
});

if (!response.ok || !response.answer) {
peer.close();
throw new Error(response.ok ? "Missing camera answer" : response.error);
}
await peer.setRemoteDescription(response.answer);
return {
peer,
stream: await remoteStreamPromise,
};
};

function OverlayApp() {
const [extensionSettings, setExtensionSettings] =
useState<ExtensionSettings | null>(null);
Expand Down
72 changes: 1 addition & 71 deletions apps/chrome-extension/src/preview/camera-preview.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { PictureInPicture } from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import { createRoot } from "react-dom/client";
import { connectCameraPreview } from "../shared/camera-preview-connection";
import { toCameraDevices } from "../shared/devices";
import { sendServiceWorkerMessage } from "../shared/runtime";
import type {
Expand All @@ -9,10 +10,6 @@ import type {
WebcamPreviewFrame,
WebcamSettings,
} from "../shared/types";
import {
toSessionDescriptionInit,
waitForIceGatheringComplete,
} from "../shared/webrtc";
import "./styles.css";

const FRAME_CAPTURE_INTERVAL_MS = 700;
Expand Down Expand Up @@ -87,73 +84,6 @@ const postParent = (event: CameraPreviewEvent) => {
);
};

const waitForRemoteStream = (peer: RTCPeerConnection) =>
new Promise<MediaStream>((resolve, reject) => {
let settled = false;
const timeout = window.setTimeout(() => {
if (settled) return;
settled = true;
reject(new Error("Camera preview timed out."));
}, 10000);

const finish = (stream: MediaStream) => {
if (settled) return;
settled = true;
window.clearTimeout(timeout);
resolve(stream);
};

peer.addEventListener("track", (event) => {
finish(event.streams[0] ?? new MediaStream([event.track]));
});

peer.addEventListener("connectionstatechange", () => {
if (
settled ||
(peer.connectionState !== "failed" && peer.connectionState !== "closed")
) {
return;
}
settled = true;
window.clearTimeout(timeout);
reject(new Error("Camera preview connection failed."));
});
});

const connectCameraPreview = async (
settings: WebcamSettings,
sessionId: string,
) => {
const peer = new RTCPeerConnection();
peer.addTransceiver("video", { direction: "recvonly" });
const remoteStreamPromise = waitForRemoteStream(peer);
await peer.setLocalDescription(await peer.createOffer());
await waitForIceGatheringComplete(peer);

const response = await sendServiceWorkerMessage({
target: "service-worker",
type: "connect-camera-preview",
sessionId,
settings,
offer: toSessionDescriptionInit(peer.localDescription),
});

if (!response.ok) {
peer.close();
throw new Error(response.error);
}
if (!response.answer) {
peer.close();
throw new Error("Camera preview did not return an answer.");
}

await peer.setRemoteDescription(response.answer);
return {
peer,
stream: await remoteStreamPromise,
};
};

let frameCanvas: HTMLCanvasElement | null = null;

const captureVideoFrame = (
Expand Down
195 changes: 195 additions & 0 deletions apps/chrome-extension/src/shared/camera-preview-connection.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { connectCameraPreview } from "./camera-preview-connection";
import { sendServiceWorkerMessage } from "./runtime";
import type { WebcamSettings } from "./types";

vi.mock("./runtime", () => ({ sendServiceWorkerMessage: vi.fn() }));

const settings: WebcamSettings = {
enabled: true,
deviceId: null,
position: "bottom-left",
size: 200,
shape: "round",
mirror: true,
};

let failedStage: string | null = null;
let failure = new Error("Negotiation failed");

class Peer extends EventTarget {
static latest: Peer;
iceGatheringState = "complete";
connectionState = "new";
localDescription = { type: "offer", sdp: "local" };
addTransceiver = vi.fn();
createOffer = vi.fn(async () => {
if (failedStage === "offer") throw failure;
return this.localDescription;
});
setLocalDescription = vi.fn(async () => {
if (failedStage === "local description") throw failure;
});
setRemoteDescription = vi.fn(async () => {});
close = vi.fn(() => {
this.connectionState = "closed";
});
constructor() {
super();
Peer.latest = this;
}
track(stream: MediaStream) {
const event = new Event("track");
Object.defineProperty(event, "streams", { value: [stream] });
this.dispatchEvent(event);
}
}

const flush = async () => {
for (let i = 0; i < 8; i++) await Promise.resolve();
};

beforeEach(() => {
failedStage = null;
vi.useFakeTimers();
vi.stubGlobal("RTCPeerConnection", Peer);
vi.stubGlobal("window", globalThis);
vi.mocked(sendServiceWorkerMessage).mockReset();
vi.mocked(sendServiceWorkerMessage).mockResolvedValue({
ok: true,
answer: { type: "answer", sdp: "remote" },
});
});

afterEach(() => {
vi.clearAllTimers();
vi.useRealTimers();
vi.restoreAllMocks();
vi.unstubAllGlobals();
});

describe("camera preview connection ownership", () => {
it.each(["offer", "local description", "messaging", "remote description"])(
"closes the peer and cancels its waiter after %s fails",
async (stage) => {
failure = new Error(`${stage} failed`);
failedStage = stage;
const starting = connectCameraPreview(settings, "session").catch(
(error: unknown) => error,
);
const peer = Peer.latest;
const remove = vi.spyOn(peer, "removeEventListener");
if (stage === "messaging")
vi.mocked(sendServiceWorkerMessage).mockRejectedValueOnce(failure);
if (stage === "remote description")
peer.setRemoteDescription.mockRejectedValueOnce(failure);
expect(await starting).toBe(failure);
expect(peer.close).toHaveBeenCalledTimes(1);
expect(vi.getTimerCount()).toBe(0);
expect(remove).toHaveBeenCalledWith("track", expect.any(Function));
expect(remove).toHaveBeenCalledWith(
"connectionstatechange",
expect.any(Function),
);
},
);

it.each(["rejected", "missing answer"])(
"cleans up after a %s response",
async (outcome) => {
vi.mocked(sendServiceWorkerMessage).mockResolvedValueOnce(
outcome === "rejected"
? { ok: false, error: "Camera busy" }
: { ok: true },
);
await expect(connectCameraPreview(settings, "session")).rejects.toThrow(
outcome === "rejected"
? "Camera busy"
: "Camera preview did not return an answer.",
);
expect(Peer.latest.close).toHaveBeenCalledTimes(1);
expect(vi.getTimerCount()).toBe(0);
},
);

it("closes a peer when no track arrives before the timeout", async () => {
const starting = connectCameraPreview(settings, "session").catch(
(error: unknown) => error,
);
await flush();
await vi.advanceTimersByTimeAsync(10_000);
expect(await starting).toMatchObject({
message: "Camera preview timed out.",
});
expect(Peer.latest.close).toHaveBeenCalledTimes(1);
expect(vi.getTimerCount()).toBe(0);
});

it("releases an early track when applying the answer fails", async () => {
const stop = vi.fn();
const stream = { getTracks: () => [{ stop }] } as unknown as MediaStream;
const starting = connectCameraPreview(settings, "session").catch(
(error: unknown) => error,
);
const failure = new Error("Invalid answer");
Peer.latest.setRemoteDescription.mockImplementationOnce(async () => {
Peer.latest.track(stream);
throw failure;
});
expect(await starting).toBe(failure);
expect(stop).toHaveBeenCalledTimes(1);
expect(Peer.latest.close).toHaveBeenCalledTimes(1);
});

it("closes the peer and removes listeners when the connection fails", async () => {
const starting = connectCameraPreview(settings, "session").catch(
(error: unknown) => error,
);
await flush();
const peer = Peer.latest;
const remove = vi.spyOn(peer, "removeEventListener");
peer.connectionState = "failed";
peer.dispatchEvent(new Event("connectionstatechange"));
expect(await starting).toMatchObject({
message: "Camera preview connection failed.",
});
expect(peer.close).toHaveBeenCalledTimes(1);
expect(remove).toHaveBeenCalledWith("track", expect.any(Function));
expect(remove).toHaveBeenCalledWith(
"connectionstatechange",
expect.any(Function),
);
expect(vi.getTimerCount()).toBe(0);
});

it("preserves a setup error after the remote waiter has already timed out", async () => {
let rejectMessage!: (error: Error) => void;
vi.mocked(sendServiceWorkerMessage).mockReturnValueOnce(
new Promise((_, reject) => {
rejectMessage = reject;
}),
);
const starting = connectCameraPreview(settings, "session").catch(
(error: unknown) => error,
);
await flush();
await vi.advanceTimersByTimeAsync(10_000);
const failure = new Error("Service worker disconnected");
rejectMessage(failure);
expect(await starting).toBe(failure);
expect(Peer.latest.close).toHaveBeenCalledTimes(1);
expect(vi.getTimerCount()).toBe(0);
});

it("keeps a successful peer and stream owned by the caller", async () => {
const stop = vi.fn();
const stream = { getTracks: () => [{ stop }] } as unknown as MediaStream;
const starting = connectCameraPreview(settings, "session");
await flush();
Peer.latest.track(stream);
await expect(starting).resolves.toEqual({ peer: Peer.latest, stream });
expect(Peer.latest.close).not.toHaveBeenCalled();
expect(stop).not.toHaveBeenCalled();
expect(vi.getTimerCount()).toBe(0);
});
});
Loading
Loading