Support video and native context-menu media paste in Markdown

.github/workflows/pr.yml

@@ -90,10 +90,11 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Check Markdown editor manifests
+      - name: Check Markdown editor manifests and clipboard
         run: |
           npm run markdown-editor-package-json-check
           npm --prefix extensions/markdown-language-features run test-markdown-editor-package-json
+          npm --prefix extensions/markdown-language-features run test-markdown-editor-clipboard
 
       - name: Compile & Hygiene
         run: npm exec -- npm-run-all2 -lp core-ci hygiene eslint valid-layers-check define-class-fields-check vscode-dts-compile-check tsec-compile-check test-build-scripts

extensions/markdown-language-features/README.md

@@ -5,18 +5,3 @@
 ## Features
 
 See [Markdown in Visual Studio Code](https://code.visualstudio.com/docs/languages/markdown) to learn about the features of this extension.
-
-### Markdown editor commit links
-
-Commit links such as `[commit](commit:1234567)` render the commit subject and
-addition/deletion counts. Opening one shows the commit's changes in the multi-diff
-editor. Without a repository hint, the Git extension searches open repositories.
-
-Use `[commit](commit:1234567?path=../../my-repo)` to select a repository relative to
-the Markdown file, including repositories that are not open yet. Absolute file
-URIs are also supported. URL-encode paths containing `?`, `#`, or `&`. A hinted
-repository must contain the commit; an invalid hint does not fall back to another
-repository. Opening a new repository requires a trusted workspace.
-
-Forge links such as `https://github.com/owner/repo/commit/1234567?path=../repo`
-also accept this hint, but the local repository's remote must match the URL.

extensions/markdown-language-features/markdown-editor-src/clipboardMedia.ts

@@ -0,0 +1,16 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { NativeClipboardData } from '@vscode/markdown-editor';
+
+export const clipboardFormatsUnavailableMessage = 'The clipboard formats are not available to this editor. Try copying the media itself instead of its file.';
+
+export function getNativeClipboardMedia(data: NativeClipboardData, plainText: boolean): { readonly text: string; readonly images: readonly File[] } {
+	const images = plainText ? [] : data.files.filter(file => /^(image|video|audio)\//.test(file.type));
+	if (!plainText && data.files.length && !images.length && !data.text) {
+		throw new Error(clipboardFormatsUnavailableMessage);
+	}
+	return { text: data.text, images };
+}

extensions/markdown-language-features/markdown-editor-src/editor.ts

@@ -123,6 +123,7 @@ class Editor extends Disposable {
 	#rename: RenameController | undefined;
 	#completion: CompletionController | undefined;
 	#diagnostics: DiagnosticsController | undefined;
+	#imagePaste: ImagePasteController | undefined;
 	#view: EditorView | undefined;
 	#embeddedCodeEditorFactory: LazyCodeBlockEditorFactory | undefined;
 	/** Identifies the authoritative text baseline against which local edits are computed. */
@@ -182,6 +183,10 @@ class Editor extends Disposable {
 		this.model.readonlyMode.set(initialState.readonly, undefined);
 
 		this._register(this.#connection.register(markdownEditorRenderer, {
+			readClipboardMedia: (request, _context, { signal }) => {
+				if (!this.#imagePaste) { throw new Error('The Markdown editor is not ready to paste media.'); }
+				return this.#imagePaste.readClipboardMedia(request, signal);
+			},
 			diagnosticsChanged: () => this.#diagnostics?.refresh(),
 			update: ({ content, editEpoch }) => {
 				// Applying authoritative text maps selection and clears stale pending-paragraph state.
@@ -447,7 +452,7 @@ class Editor extends Disposable {
 		// the TextDocument owns the history, and a second local stack would drift
 		// from the Edit menu, dirty state and hot exit.
 		this.#controller = this._register(new EditorController(model, view, {
-			clipboardStrategy: this._register(new ImagePasteController(model, view, this.#host, () => this.#editEpoch)),
+			clipboardStrategy: this.#imagePaste = this._register(new ImagePasteController(model, view, this.#host, () => this.#editEpoch)),
 			keyboardProfile: vscodeLocalKeyboardProfile,
 			forwardedKeyboardProfile: vscodeHostKeyboardProfile,
 			historyStrategy: {

extensions/markdown-language-features/markdown-editor-src/imagePasteController.ts

@@ -3,16 +3,19 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { AsyncClipboardStrategy, EditorModel, EditorView, Selection, readNativeClipboard, type IClipboardContext, type IClipboardStrategy } from '@vscode/markdown-editor';
+import { AsyncClipboardStrategy, EditorModel, EditorView, Selection, readNativeClipboard, type IClipboardContext, type IClipboardStrategy, type NativeClipboardData } from '@vscode/markdown-editor';
 import { Disposable } from '@vscode/observables';
-import type { MarkdownEditorHost } from '../src/preview/markdownEditorProtocol';
+import type { MarkdownEditorHost, MarkdownEditorRenderer } from '../src/preview/markdownEditorProtocol';
+import { ClipboardMediaStore } from '../src/preview/markdownEditorMediaTransfer';
+import { clipboardFormatsUnavailableMessage, getNativeClipboardMedia } from './clipboardMedia';
 
 export class ImagePasteController extends Disposable implements IClipboardStrategy {
 	readonly #strategy: AsyncClipboardStrategy;
 	readonly #status = document.createElement('div');
 	#disposed = false;
 	#request = 0;
 	#interaction = 0;
+	#media: ClipboardMediaStore | undefined;
 
 	constructor(
 		readonly editor: EditorModel,
@@ -22,27 +25,28 @@ export class ImagePasteController extends Disposable implements IClipboardStrate
 		readonly clipboard: Clipboard = navigator.clipboard,
 	) {
 		super();
-		this.#strategy = new AsyncClipboardStrategy(clipboard, (context, plainText) => this.#paste(context, async () => {
-			const native = readNativeClipboard(context.element);
+		this.#strategy = new AsyncClipboardStrategy(clipboard, (context, plainText, nativeData?: NativeClipboardData) => this.#paste(context, async () => {
+			const native = nativeData ?? readNativeClipboard(context.element);
 			if (native) {
-				return { text: native.text, images: plainText ? [] : native.files.filter(file => file.type.startsWith('image/')) };
+				return getNativeClipboardMedia(native, plainText);
 			}
 			if (plainText || !clipboard.read) { return { text: await clipboard.readText(), images: [] }; }
 			const items = await clipboard.read();
 			const images: File[] = [];
 			let text = '';
 			for (const item of items) {
-				const type = item.types.find(type => type === 'image/png') ?? item.types.find(type => type.startsWith('image/'));
+				const type = item.types.find(type => type === 'image/png') ?? item.types.find(type => /^(image|video|audio)\//.test(type));
 				if (type) {
 					const blob = await item.getType(type);
-					const extension = type.slice('image/'.length).split('+')[0];
-					images.push(new File([blob], `image.${extension}`, { type }));
+					const [kind, subtype] = type.split('/');
+					const extension = subtype.split('+')[0];
+					images.push(new File([blob], `${kind}.${extension}`, { type }));
 				} else if (item.types.includes('text/plain')) {
 					text += await (await item.getType('text/plain')).text();
 				}
 			}
 			if (items.length && !images.length && !text && items.every(item => !item.types.includes('text/plain'))) {
-				throw new Error('The clipboard formats are not available to this editor. Try copying the image itself instead of its file.');
+				throw new Error(clipboardFormatsUnavailableMessage);
 			}
 			return { text, images };
 		}, plainText));
@@ -54,32 +58,31 @@ export class ImagePasteController extends Disposable implements IClipboardStrate
 
 	writeText(text: string): Promise<void> { return this.#strategy.writeText(text); }
 
+	readClipboardMedia(request: Parameters<MarkdownEditorRenderer['readClipboardMedia']>[0], signal: AbortSignal): ReturnType<MarkdownEditorRenderer['readClipboardMedia']> {
+		if (!this.#media || this.#disposed) {
+			throw new Error('The clipboard media paste has expired.');
+		}
+		return this.#media.read(request, signal);
+	}
+
 	connect(context: IClipboardContext): { dispose(): void } {
 		const interact = (): void => { this.#interaction++; };
 		context.element.addEventListener('pointerdown', interact);
 		context.element.addEventListener('keydown', interact);
 		const connection = this.#strategy.connect(context);
-		const paste = (event: ClipboardEvent): void => {
-			if (event.target !== context.element || !event.clipboardData) { return; }
-			const images = [...event.clipboardData.files].filter(file => file.type.startsWith('image/'));
-			const text = event.clipboardData.getData('text/plain');
-			event.preventDefault();
-			event.stopPropagation();
-			void this.#paste(context, async () => ({ images, text }));
-		};
-		context.element.addEventListener('paste', paste);
 		return {
 			dispose: () => {
 				connection.dispose();
 				context.element.removeEventListener('pointerdown', interact);
 				context.element.removeEventListener('keydown', interact);
-				context.element.removeEventListener('paste', paste);
 			}
 		};
 	}
 
 	async #paste(context: IClipboardContext, read: () => Promise<{ readonly images: readonly File[]; readonly text: string }>, plainText = false): Promise<void> {
 		const request = ++this.#request;
+		this.#media?.dispose();
+		this.#media = undefined;
 		const source = this.editor.sourceText.get();
 		const selection = this.editor.selection.get();
 		const epoch = this.epoch();
@@ -88,33 +91,24 @@ export class ImagePasteController extends Disposable implements IClipboardStrate
 			this.#showError('Switch to editing mode and place the caret before pasting.');
 			return;
 		}
+		let media: ClipboardMediaStore | undefined;
 		try {
 			const { images, text } = await read();
-			const payload = await Promise.all(images.map(async image => ({
-				name: image.name,
-				mime: image.type,
-				base64: await new Promise<string>((resolve, reject) => {
-					const reader = new FileReader();
-					reader.onload = () => resolve(String(reader.result).split(',')[1]);
-					reader.onerror = () => reject(reader.error ?? new Error('Could not read clipboard image.'));
-					reader.onabort = () => reject(new Error('Clipboard image reading was cancelled.'));
-					reader.readAsDataURL(image);
-				}),
-			})));
 			if (this.#disposed || request !== this.#request) { return; }
 			if (source !== this.editor.sourceText.get() || selection !== this.editor.selection.get() || epoch !== this.epoch()
 				|| this.editor.readonlyMode.get() || document.activeElement !== context.element) {
 				throw new Error('The editor changed while reading the clipboard. Paste again.');
 			}
 			this.#status.hidden = true;
-			if (!payload.length) {
+			if (!images.length) {
 				if (text) { context.insertText(text, plainText); }
 				return;
 			}
-			this.#status.textContent = 'Pasting image...';
+			this.#status.textContent = 'Pasting media...';
 			this.#status.hidden = false;
+			media = this.#media = new ClipboardMediaStore(images);
 			const result = await this.host.pasteImages({
-				start: selection.range.start, endExclusive: selection.range.endExclusive, editEpoch: epoch, images: payload,
+				start: selection.range.start, endExclusive: selection.range.endExclusive, editEpoch: epoch, images: [...media.images],
 			});
 			if (this.#disposed || request !== this.#request) { return; }
 			this.#status.hidden = true;
@@ -126,6 +120,9 @@ export class ImagePasteController extends Disposable implements IClipboardStrate
 			if (!this.#disposed && request === this.#request) {
 				this.#showError(`Could not paste: ${error instanceof Error ? error.message : String(error)}`);
 			}
+		} finally {
+			media?.dispose();
+			if (this.#media === media) { this.#media = undefined; }
 		}
 	}
 
@@ -136,6 +133,8 @@ export class ImagePasteController extends Disposable implements IClipboardStrate
 
 	override dispose(): void {
 		this.#disposed = true;
+		this.#media?.dispose();
+		this.#media = undefined;
 		super.dispose();
 	}
 }

extensions/markdown-language-features/package.json

@@ -1982,6 +1982,7 @@
     "update-markdown-editor-package-json": "node ./scripts/updateMarkdownEditorPackageJson.mts --write",
     "check-markdown-editor-package-json": "node ./scripts/updateMarkdownEditorPackageJson.mts --check",
     "test-markdown-editor-package-json": "node --test ./scripts/updateMarkdownEditorPackageJson.test.mts",
+    "test-markdown-editor-clipboard": "node --test ./scripts/clipboardMedia.test.mts ./scripts/markdownEditorMediaTransfer.test.mts",
     "compile-web": "npm-run-all2 -lp bundle-web typecheck-web",
     "bundle-web": "node ./esbuild.browser.mts",
     "typecheck-web": "node ../../node_modules/@typescript/native/lib/tsc.js --project ./tsconfig.browser.json --noEmit",
@@ -1991,7 +1992,7 @@
   },
   "dependencies": {
     "@vscode/extension-telemetry": "^0.9.8",
-    "@vscode/markdown-editor": "0.0.2-121",
+    "@vscode/markdown-editor": "0.0.2-127",
     "@vscode/observables": "^0.1.1-0",
     "@vscode/web-editors": "^0.0.2-41",
     "dompurify": "^3.4.10",

extensions/markdown-language-features/scripts/clipboardMedia.test.mts

@@ -0,0 +1,48 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { getNativeClipboardMedia } from '../markdown-editor-src/clipboardMedia.ts';
+
+test('preserves native image, video, and audio file identity and bytes', async () => {
+	const files = [
+		new File([new Uint8Array([1, 2, 3])], 'image.png', { type: 'image/png' }),
+		new File([new Uint8Array([4, 5, 6])], 'clip.mp4', { type: 'video/mp4' }),
+		new File([new Uint8Array([7, 8, 9])], 'sound.mp3', { type: 'audio/mpeg' }),
+	];
+	const result = getNativeClipboardMedia({ text: 'fallback', files }, false);
+	assert.deepStrictEqual({
+		text: result.text,
+		files: await Promise.all(result.images.map(async file => ({ name: file.name, type: file.type, bytes: [...new Uint8Array(await file.arrayBuffer())] }))),
+	}, {
+		text: 'fallback',
+		files: [
+			{ name: 'image.png', type: 'image/png', bytes: [1, 2, 3] },
+			{ name: 'clip.mp4', type: 'video/mp4', bytes: [4, 5, 6] },
+			{ name: 'sound.mp3', type: 'audio/mpeg', bytes: [7, 8, 9] },
+		],
+	});
+	assert.ok(result.images.every((file, index) => file === files[index]));
+});
+
+test('reports unsupported and untyped native file-only payloads explicitly', () => {
+	for (const type of ['application/octet-stream', '']) {
+		assert.throws(() => getNativeClipboardMedia({ text: '', files: [new File(['data'], 'file.bin', { type })] }, false), /clipboard formats are not available/);
+	}
+});
+
+test('preserves plain-text fallback, empty clipboard, and explicit plain-text paste', () => {
+	const files = [new File(['data'], 'file.bin')];
+	assert.deepStrictEqual([
+		getNativeClipboardMedia({ text: 'fallback', files }, false),
+		getNativeClipboardMedia({ text: '', files }, true),
+		getNativeClipboardMedia({ text: '', files: [] }, false),
+	], [
+		{ text: 'fallback', images: [] },
+		{ text: '', images: [] },
+		{ text: '', images: [] },
+	]);
+});