Skip to content

make integrated browser attach to chat feature kb accessible #300465

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
meganrogge wants to merge 1 commit into
base: main
Choose a base branch
from
+216 −45
Open

make integrated browser attach to chat feature kb accessible #300465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/vs/platform/browserElements/common/browserElements.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ export interface INativeBrowserElementsService {

getElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator, cancellationId?: number): Promise<IElementData | undefined>;

getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator, cancellationId?: number): Promise<IElementData | undefined>;

startDebugSession(token: CancellationToken, locator: IBrowserTargetLocator, cancelAndDetachId?: number): Promise<void>;

startConsoleSession(token: CancellationToken, locator: IBrowserTargetLocator, cancelAndDetachId?: number): Promise<void>;
Expand Down
115 changes: 115 additions & 0 deletions src/vs/platform/browserElements/electron-main/nativeBrowserElementsMainService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,121 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
};
}

async getFocusedElementData(windowId: number | undefined, rect: IRectangle, _token: CancellationToken, locator: IBrowserTargetLocator, _cancellationId?: number): Promise<IElementData | undefined> {
const window = this.windowById(windowId);
if (!window?.win) {
return undefined;
}

const allWebContents = webContents.getAllWebContents();
const simpleBrowserWebview = allWebContents.find(webContent => webContent.id === window.id);
if (!simpleBrowserWebview) {
return undefined;
}

const debuggers = simpleBrowserWebview.debugger;
if (!debuggers.isAttached()) {
debuggers.attach();
}

let sessionId: string | undefined;
try {
const targetId = await this.findWebviewTarget(debuggers, locator);
if (!targetId) {
return undefined;
}

const attach = await debuggers.sendCommand('Target.attachToTarget', { targetId, flatten: true });
sessionId = attach.sessionId;
await debuggers.sendCommand('Runtime.enable', {}, sessionId);

const { result } = await debuggers.sendCommand('Runtime.evaluate', {
expression: `(() => {
const el = document.activeElement;
Copy link
Contributor

@kycutler kycutler Mar 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we get the node ID of the focused element to avoid duplicating the attachment data logic? Copilot claims something like this might work:

const { result } = await debuggers.sendCommand('Runtime.evaluate', {
  expression: 'document.activeElement',
  returnByValue: false
});

const { nodeId } = await debuggers.sendCommand('DOM.requestNode', {
  objectId: result.objectId!
});

if (!el || el.nodeType !== 1) {
return undefined;
}
const r = el.getBoundingClientRect();
const attrs = {};
for (let i = 0; i < el.attributes.length; i++) {
attrs[el.attributes[i].name] = el.attributes[i].value;
}
const ancestors = [];
let n = el;
while (n && n.nodeType === 1) {
const entry = { tagName: n.tagName.toLowerCase() };
if (n.id) {
entry.id = n.id;
}
if (typeof n.className === 'string' && n.className.trim().length > 0) {
entry.classNames = n.className.trim().split(/\\s+/).filter(Boolean);
}
ancestors.unshift(entry);
n = n.parentElement;
}
const css = getComputedStyle(el);
const computedStyles = {};
for (let i = 0; i < css.length; i++) {
const name = css[i];
computedStyles[name] = css.getPropertyValue(name);
}
const text = (el.innerText || '').trim();
return {
outerHTML: el.outerHTML,
computedStyle: '',
bounds: { x: r.x, y: r.y, width: r.width, height: r.height },
ancestors,
attributes: attrs,
computedStyles,
dimensions: { top: r.top, left: r.left, width: r.width, height: r.height },
innerText: text.length > 100 ? text.slice(0, 100) + '\\u2026' : text
};
})();`,
returnByValue: true
}, sessionId);

const focusedData = result?.value as NodeDataResponse | undefined;
if (!focusedData) {
return undefined;
}

const zoomFactor = simpleBrowserWebview.getZoomFactor();
const absoluteBounds = {
x: rect.x + focusedData.bounds.x,
y: rect.y + focusedData.bounds.y,
width: focusedData.bounds.width,
height: focusedData.bounds.height
};

const clippedBounds = {
x: Math.max(absoluteBounds.x, rect.x),
y: Math.max(absoluteBounds.y, rect.y),
width: Math.max(0, Math.min(absoluteBounds.x + absoluteBounds.width, rect.x + rect.width) - Math.max(absoluteBounds.x, rect.x)),
height: Math.max(0, Math.min(absoluteBounds.y + absoluteBounds.height, rect.y + rect.height) - Math.max(absoluteBounds.y, rect.y))
};

return {
outerHTML: focusedData.outerHTML,
computedStyle: focusedData.computedStyle,
bounds: {
x: clippedBounds.x * zoomFactor,
y: clippedBounds.y * zoomFactor,
width: clippedBounds.width * zoomFactor,
height: clippedBounds.height * zoomFactor
},
ancestors: focusedData.ancestors,
attributes: focusedData.attributes,
computedStyles: focusedData.computedStyles,
dimensions: focusedData.dimensions,
innerText: focusedData.innerText,
};
} finally {
if (debuggers.isAttached()) {
debuggers.detach();
}
}
}

async getNodeData(sessionId: string, debuggers: Electron.Debugger, window: BrowserWindow, cancellationId?: number): Promise<NodeDataResponse> {
return new Promise((resolve, reject) => {
const onMessage = async (event: Electron.Event, method: string, params: { backendNodeId: number }) => {
Expand Down
1 change: 1 addition & 0 deletions src/vs/platform/browserView/electron-main/browserView.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -645,6 +645,7 @@ export class BrowserView extends Disposable implements ICDPTarget {

const isArrowKey = keyCode >= KeyCode.LeftArrow && keyCode <= KeyCode.DownArrow;
const isNonEditingKey =
keyCode === KeyCode.Enter ||
keyCode === KeyCode.Escape ||
keyCode >= KeyCode.F1 && keyCode <= KeyCode.F24 ||
keyCode >= KeyCode.AudioVolumeMute;
Expand Down
121 changes: 76 additions & 45 deletions src/vs/workbench/contrib/browserView/electron-browser/browserEditor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -803,51 +803,7 @@ export class BrowserEditor extends EditorPane {
throw new Error('Element data not found');
}

const bounds = elementData.bounds;
const toAttach: IChatRequestVariableEntry[] = [];

// Prepare HTML/CSS context
const displayName = getDisplayNameFromOuterHTML(elementData.outerHTML);
const attachCss = this.configurationService.getValue<boolean>('chat.sendElementsToChat.attachCSS');
const value = this.createElementContextValue(elementData, displayName, attachCss);

toAttach.push({
id: 'element-' + Date.now(),
name: displayName,
fullName: displayName,
value: value,
modelDescription: attachCss
? 'Structured browser element context with HTML path, attributes, and computed styles.'
: 'Structured browser element context with HTML path and attributes.',
kind: 'element',
icon: ThemeIcon.fromId(Codicon.layout.id),
ancestors: elementData.ancestors,
attributes: elementData.attributes,
computedStyles: attachCss ? elementData.computedStyles : undefined,
dimensions: elementData.dimensions,
innerText: elementData.innerText,
});

// Attach screenshot if enabled
const attachImages = this.configurationService.getValue<boolean>('chat.sendElementsToChat.attachImages');
if (attachImages && this._model) {
const screenshotBuffer = await this._model.captureScreenshot({
quality: 90,
rect: bounds
});

toAttach.push({
id: 'element-screenshot-' + Date.now(),
name: 'Element Screenshot',
fullName: 'Element Screenshot',
kind: 'image',
value: screenshotBuffer.buffer
});
}

// Attach to chat widget
const widget = await this.chatWidgetService.revealWidget() ?? this.chatWidgetService.lastFocusedWidget;
widget?.attachmentModel?.addContext(...toAttach);
const { attachCss, attachImages } = await this.attachElementDataToChat(elementData);

type IntegratedBrowserAddElementToChatAddedEvent = {
attachCss: boolean;
Expand Down Expand Up @@ -992,6 +948,53 @@ export class BrowserEditor extends EditorPane {
return sections.join('\n\n');
}

private async attachElementDataToChat(elementData: IElementData): Promise<{ attachCss: boolean; attachImages: boolean }> {
const bounds = elementData.bounds;
const toAttach: IChatRequestVariableEntry[] = [];

const displayName = getDisplayNameFromOuterHTML(elementData.outerHTML);
const attachCss = this.configurationService.getValue<boolean>('chat.sendElementsToChat.attachCSS');
const value = this.createElementContextValue(elementData, displayName, attachCss);

toAttach.push({
id: 'element-' + Date.now(),
name: displayName,
fullName: displayName,
value: value,
modelDescription: attachCss
? 'Structured browser element context with HTML path, attributes, and computed styles.'
: 'Structured browser element context with HTML path and attributes.',
kind: 'element',
icon: ThemeIcon.fromId(Codicon.layout.id),
ancestors: elementData.ancestors,
attributes: elementData.attributes,
computedStyles: attachCss ? elementData.computedStyles : undefined,
dimensions: elementData.dimensions,
innerText: elementData.innerText,
});

const attachImages = this.configurationService.getValue<boolean>('chat.sendElementsToChat.attachImages');
if (attachImages && this._model) {
const screenshotBuffer = await this._model.captureScreenshot({
quality: 90,
rect: bounds
});

toAttach.push({
id: 'element-screenshot-' + Date.now(),
name: 'Element Screenshot',
fullName: 'Element Screenshot',
kind: 'image',
value: screenshotBuffer.buffer
});
}

const widget = await this.chatWidgetService.revealWidget() ?? this.chatWidgetService.lastFocusedWidget;
widget?.attachmentModel?.addContext(...toAttach);

return { attachCss, attachImages };
}

private formatElementPath(ancestors: readonly IElementAncestor[] | undefined): string | undefined {
if (!ancestors || ancestors.length === 0) {
return undefined;
Expand Down Expand Up @@ -1149,6 +1152,34 @@ export class BrowserEditor extends EditorPane {
this._currentKeyDownEvent = keyEvent;

try {
const isEnterKey =
keyEvent.code === 'Enter' ||
keyEvent.code === 'NumpadEnter' ||
keyEvent.key === 'Enter' ||
keyEvent.key === 'Return';
if (this._elementSelectionCts && isEnterKey) {
const cts = this._elementSelectionCts;
const resourceUri = this.input?.resource;
if (!resourceUri) {
return;
}

const locator: IBrowserTargetLocator = { browserViewId: BrowserViewUri.getId(resourceUri) };
const { width, height } = this._browserContainer.getBoundingClientRect();
const elementData = await this.browserElementsService.getFocusedElementData({ x: 0, y: 0, width, height }, cts.token, locator);
if (!elementData) {
return;
}

await this.attachElementDataToChat(elementData);
cts.dispose();
if (this._elementSelectionCts === cts) {
this._elementSelectionCts = undefined;
this._elementSelectionActiveContext.set(false);
}
return;
}

const syntheticEvent = new KeyboardEvent('keydown', keyEvent);
const standardEvent = new StandardKeyboardEvent(syntheticEvent);

Expand Down
2 changes: 2 additions & 0 deletions src/vs/workbench/services/browserElements/browser/browserElementsService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ export interface IBrowserElementsService {
// no browser implementation yet
getElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise<IElementData | undefined>;

getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise<IElementData | undefined>;

startDebugSession(token: CancellationToken, locator: IBrowserTargetLocator): Promise<void>;

startConsoleSession(token: CancellationToken, locator: IBrowserTargetLocator): Promise<void>;
Expand Down
4 changes: 4 additions & 0 deletions src/vs/workbench/services/browserElements/browser/webBrowserElementsService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ class WebBrowserElementsService implements IBrowserElementsService {
throw new Error('Not implemented');
}

async getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise<IElementData | undefined> {
throw new Error('Not implemented');
}

async startDebugSession(token: CancellationToken, locator: IBrowserTargetLocator): Promise<void> {
throw new Error('Not implemented');
}
Expand Down
16 changes: 16 additions & 0 deletions src/vs/workbench/services/browserElements/electron-browser/browserElementsService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,22 @@ class WorkbenchBrowserElementsService implements IBrowserElementsService {
disposable.dispose();
}
}

async getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise<IElementData | undefined> {
if (!locator) {
return undefined;
}
const cancelSelectionId = cancelSelectionIdPool++;
const onCancelChannel = `vscode:cancelElementSelection${cancelSelectionId}`;
const disposable = token.onCancellationRequested(() => {
ipcRenderer.send(onCancelChannel, cancelSelectionId);
});
try {
return await this.simpleBrowser.getFocusedElementData(rect, token, locator, cancelSelectionId);
} finally {
disposable.dispose();
}
}
}

registerSingleton(IBrowserElementsService, WorkbenchBrowserElementsService, InstantiationType.Delayed);
Expand Down
Loading