diff --git a/src/vs/platform/browserElements/common/browserElements.ts b/src/vs/platform/browserElements/common/browserElements.ts index 2973d9db93918..2973512713422 100644 --- a/src/vs/platform/browserElements/common/browserElements.ts +++ b/src/vs/platform/browserElements/common/browserElements.ts @@ -56,6 +56,8 @@ export interface INativeBrowserElementsService { getElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator, cancellationId?: number): Promise; + getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator, cancellationId?: number): Promise; + startDebugSession(token: CancellationToken, locator: IBrowserTargetLocator, cancelAndDetachId?: number): Promise; startConsoleSession(token: CancellationToken, locator: IBrowserTargetLocator, cancelAndDetachId?: number): Promise; diff --git a/src/vs/platform/browserElements/electron-main/nativeBrowserElementsMainService.ts b/src/vs/platform/browserElements/electron-main/nativeBrowserElementsMainService.ts index 5e018a6d718a2..84a64f6d43bfc 100644 --- a/src/vs/platform/browserElements/electron-main/nativeBrowserElementsMainService.ts +++ b/src/vs/platform/browserElements/electron-main/nativeBrowserElementsMainService.ts @@ -431,6 +431,121 @@ export class NativeBrowserElementsMainService extends Disposable implements INat }; } + async getFocusedElementData(windowId: number | undefined, rect: IRectangle, _token: CancellationToken, locator: IBrowserTargetLocator, _cancellationId?: number): Promise { + const window = this.windowById(windowId); + if (!window?.win) { + return undefined; + } + + const allWebContents = webContents.getAllWebContents(); + const simpleBrowserWebview = allWebContents.find(webContent => webContent.id === window.id); + if (!simpleBrowserWebview) { + return undefined; + } + + const debuggers = simpleBrowserWebview.debugger; + if (!debuggers.isAttached()) { + debuggers.attach(); + } + + let sessionId: string | undefined; + try { + const targetId = await this.findWebviewTarget(debuggers, locator); + if (!targetId) { + return undefined; + } + + const attach = await debuggers.sendCommand('Target.attachToTarget', { targetId, flatten: true }); + sessionId = attach.sessionId; + await debuggers.sendCommand('Runtime.enable', {}, sessionId); + + const { result } = await debuggers.sendCommand('Runtime.evaluate', { + expression: `(() => { + const el = document.activeElement; + if (!el || el.nodeType !== 1) { + return undefined; + } + const r = el.getBoundingClientRect(); + const attrs = {}; + for (let i = 0; i < el.attributes.length; i++) { + attrs[el.attributes[i].name] = el.attributes[i].value; + } + const ancestors = []; + let n = el; + while (n && n.nodeType === 1) { + const entry = { tagName: n.tagName.toLowerCase() }; + if (n.id) { + entry.id = n.id; + } + if (typeof n.className === 'string' && n.className.trim().length > 0) { + entry.classNames = n.className.trim().split(/\\s+/).filter(Boolean); + } + ancestors.unshift(entry); + n = n.parentElement; + } + const css = getComputedStyle(el); + const computedStyles = {}; + for (let i = 0; i < css.length; i++) { + const name = css[i]; + computedStyles[name] = css.getPropertyValue(name); + } + const text = (el.innerText || '').trim(); + return { + outerHTML: el.outerHTML, + computedStyle: '', + bounds: { x: r.x, y: r.y, width: r.width, height: r.height }, + ancestors, + attributes: attrs, + computedStyles, + dimensions: { top: r.top, left: r.left, width: r.width, height: r.height }, + innerText: text.length > 100 ? text.slice(0, 100) + '\\u2026' : text + }; + })();`, + returnByValue: true + }, sessionId); + + const focusedData = result?.value as NodeDataResponse | undefined; + if (!focusedData) { + return undefined; + } + + const zoomFactor = simpleBrowserWebview.getZoomFactor(); + const absoluteBounds = { + x: rect.x + focusedData.bounds.x, + y: rect.y + focusedData.bounds.y, + width: focusedData.bounds.width, + height: focusedData.bounds.height + }; + + const clippedBounds = { + x: Math.max(absoluteBounds.x, rect.x), + y: Math.max(absoluteBounds.y, rect.y), + width: Math.max(0, Math.min(absoluteBounds.x + absoluteBounds.width, rect.x + rect.width) - Math.max(absoluteBounds.x, rect.x)), + height: Math.max(0, Math.min(absoluteBounds.y + absoluteBounds.height, rect.y + rect.height) - Math.max(absoluteBounds.y, rect.y)) + }; + + return { + outerHTML: focusedData.outerHTML, + computedStyle: focusedData.computedStyle, + bounds: { + x: clippedBounds.x * zoomFactor, + y: clippedBounds.y * zoomFactor, + width: clippedBounds.width * zoomFactor, + height: clippedBounds.height * zoomFactor + }, + ancestors: focusedData.ancestors, + attributes: focusedData.attributes, + computedStyles: focusedData.computedStyles, + dimensions: focusedData.dimensions, + innerText: focusedData.innerText, + }; + } finally { + if (debuggers.isAttached()) { + debuggers.detach(); + } + } + } + async getNodeData(sessionId: string, debuggers: Electron.Debugger, window: BrowserWindow, cancellationId?: number): Promise { return new Promise((resolve, reject) => { const onMessage = async (event: Electron.Event, method: string, params: { backendNodeId: number }) => { diff --git a/src/vs/platform/browserView/electron-main/browserView.ts b/src/vs/platform/browserView/electron-main/browserView.ts index bcac609f8e6ec..b7d702720e40f 100644 --- a/src/vs/platform/browserView/electron-main/browserView.ts +++ b/src/vs/platform/browserView/electron-main/browserView.ts @@ -645,6 +645,7 @@ export class BrowserView extends Disposable implements ICDPTarget { const isArrowKey = keyCode >= KeyCode.LeftArrow && keyCode <= KeyCode.DownArrow; const isNonEditingKey = + keyCode === KeyCode.Enter || keyCode === KeyCode.Escape || keyCode >= KeyCode.F1 && keyCode <= KeyCode.F24 || keyCode >= KeyCode.AudioVolumeMute; diff --git a/src/vs/workbench/contrib/browserView/electron-browser/browserEditor.ts b/src/vs/workbench/contrib/browserView/electron-browser/browserEditor.ts index a3af60577018d..a4f9acae41241 100644 --- a/src/vs/workbench/contrib/browserView/electron-browser/browserEditor.ts +++ b/src/vs/workbench/contrib/browserView/electron-browser/browserEditor.ts @@ -803,51 +803,7 @@ export class BrowserEditor extends EditorPane { throw new Error('Element data not found'); } - const bounds = elementData.bounds; - const toAttach: IChatRequestVariableEntry[] = []; - - // Prepare HTML/CSS context - const displayName = getDisplayNameFromOuterHTML(elementData.outerHTML); - const attachCss = this.configurationService.getValue('chat.sendElementsToChat.attachCSS'); - const value = this.createElementContextValue(elementData, displayName, attachCss); - - toAttach.push({ - id: 'element-' + Date.now(), - name: displayName, - fullName: displayName, - value: value, - modelDescription: attachCss - ? 'Structured browser element context with HTML path, attributes, and computed styles.' - : 'Structured browser element context with HTML path and attributes.', - kind: 'element', - icon: ThemeIcon.fromId(Codicon.layout.id), - ancestors: elementData.ancestors, - attributes: elementData.attributes, - computedStyles: attachCss ? elementData.computedStyles : undefined, - dimensions: elementData.dimensions, - innerText: elementData.innerText, - }); - - // Attach screenshot if enabled - const attachImages = this.configurationService.getValue('chat.sendElementsToChat.attachImages'); - if (attachImages && this._model) { - const screenshotBuffer = await this._model.captureScreenshot({ - quality: 90, - rect: bounds - }); - - toAttach.push({ - id: 'element-screenshot-' + Date.now(), - name: 'Element Screenshot', - fullName: 'Element Screenshot', - kind: 'image', - value: screenshotBuffer.buffer - }); - } - - // Attach to chat widget - const widget = await this.chatWidgetService.revealWidget() ?? this.chatWidgetService.lastFocusedWidget; - widget?.attachmentModel?.addContext(...toAttach); + const { attachCss, attachImages } = await this.attachElementDataToChat(elementData); type IntegratedBrowserAddElementToChatAddedEvent = { attachCss: boolean; @@ -992,6 +948,53 @@ export class BrowserEditor extends EditorPane { return sections.join('\n\n'); } + private async attachElementDataToChat(elementData: IElementData): Promise<{ attachCss: boolean; attachImages: boolean }> { + const bounds = elementData.bounds; + const toAttach: IChatRequestVariableEntry[] = []; + + const displayName = getDisplayNameFromOuterHTML(elementData.outerHTML); + const attachCss = this.configurationService.getValue('chat.sendElementsToChat.attachCSS'); + const value = this.createElementContextValue(elementData, displayName, attachCss); + + toAttach.push({ + id: 'element-' + Date.now(), + name: displayName, + fullName: displayName, + value: value, + modelDescription: attachCss + ? 'Structured browser element context with HTML path, attributes, and computed styles.' + : 'Structured browser element context with HTML path and attributes.', + kind: 'element', + icon: ThemeIcon.fromId(Codicon.layout.id), + ancestors: elementData.ancestors, + attributes: elementData.attributes, + computedStyles: attachCss ? elementData.computedStyles : undefined, + dimensions: elementData.dimensions, + innerText: elementData.innerText, + }); + + const attachImages = this.configurationService.getValue('chat.sendElementsToChat.attachImages'); + if (attachImages && this._model) { + const screenshotBuffer = await this._model.captureScreenshot({ + quality: 90, + rect: bounds + }); + + toAttach.push({ + id: 'element-screenshot-' + Date.now(), + name: 'Element Screenshot', + fullName: 'Element Screenshot', + kind: 'image', + value: screenshotBuffer.buffer + }); + } + + const widget = await this.chatWidgetService.revealWidget() ?? this.chatWidgetService.lastFocusedWidget; + widget?.attachmentModel?.addContext(...toAttach); + + return { attachCss, attachImages }; + } + private formatElementPath(ancestors: readonly IElementAncestor[] | undefined): string | undefined { if (!ancestors || ancestors.length === 0) { return undefined; @@ -1149,6 +1152,34 @@ export class BrowserEditor extends EditorPane { this._currentKeyDownEvent = keyEvent; try { + const isEnterKey = + keyEvent.code === 'Enter' || + keyEvent.code === 'NumpadEnter' || + keyEvent.key === 'Enter' || + keyEvent.key === 'Return'; + if (this._elementSelectionCts && isEnterKey) { + const cts = this._elementSelectionCts; + const resourceUri = this.input?.resource; + if (!resourceUri) { + return; + } + + const locator: IBrowserTargetLocator = { browserViewId: BrowserViewUri.getId(resourceUri) }; + const { width, height } = this._browserContainer.getBoundingClientRect(); + const elementData = await this.browserElementsService.getFocusedElementData({ x: 0, y: 0, width, height }, cts.token, locator); + if (!elementData) { + return; + } + + await this.attachElementDataToChat(elementData); + cts.dispose(); + if (this._elementSelectionCts === cts) { + this._elementSelectionCts = undefined; + this._elementSelectionActiveContext.set(false); + } + return; + } + const syntheticEvent = new KeyboardEvent('keydown', keyEvent); const standardEvent = new StandardKeyboardEvent(syntheticEvent); diff --git a/src/vs/workbench/services/browserElements/browser/browserElementsService.ts b/src/vs/workbench/services/browserElements/browser/browserElementsService.ts index 0f56a1edf3a4d..1a94f8a529cc8 100644 --- a/src/vs/workbench/services/browserElements/browser/browserElementsService.ts +++ b/src/vs/workbench/services/browserElements/browser/browserElementsService.ts @@ -16,6 +16,8 @@ export interface IBrowserElementsService { // no browser implementation yet getElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise; + getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise; + startDebugSession(token: CancellationToken, locator: IBrowserTargetLocator): Promise; startConsoleSession(token: CancellationToken, locator: IBrowserTargetLocator): Promise; diff --git a/src/vs/workbench/services/browserElements/browser/webBrowserElementsService.ts b/src/vs/workbench/services/browserElements/browser/webBrowserElementsService.ts index a98c4de1ba587..614ffc444f953 100644 --- a/src/vs/workbench/services/browserElements/browser/webBrowserElementsService.ts +++ b/src/vs/workbench/services/browserElements/browser/webBrowserElementsService.ts @@ -18,6 +18,10 @@ class WebBrowserElementsService implements IBrowserElementsService { throw new Error('Not implemented'); } + async getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise { + throw new Error('Not implemented'); + } + async startDebugSession(token: CancellationToken, locator: IBrowserTargetLocator): Promise { throw new Error('Not implemented'); } diff --git a/src/vs/workbench/services/browserElements/electron-browser/browserElementsService.ts b/src/vs/workbench/services/browserElements/electron-browser/browserElementsService.ts index fa2ddc772882d..1d88e15309bbe 100644 --- a/src/vs/workbench/services/browserElements/electron-browser/browserElementsService.ts +++ b/src/vs/workbench/services/browserElements/electron-browser/browserElementsService.ts @@ -90,6 +90,22 @@ class WorkbenchBrowserElementsService implements IBrowserElementsService { disposable.dispose(); } } + + async getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise { + if (!locator) { + return undefined; + } + const cancelSelectionId = cancelSelectionIdPool++; + const onCancelChannel = `vscode:cancelElementSelection${cancelSelectionId}`; + const disposable = token.onCancellationRequested(() => { + ipcRenderer.send(onCancelChannel, cancelSelectionId); + }); + try { + return await this.simpleBrowser.getFocusedElementData(rect, token, locator, cancelSelectionId); + } finally { + disposable.dispose(); + } + } } registerSingleton(IBrowserElementsService, WorkbenchBrowserElementsService, InstantiationType.Delayed);