From add62435131bc0983d9214ae6025ca89ece97577 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Fri, 17 Apr 2026 14:22:22 +0800 Subject: [PATCH 01/13] refactor(studio): generalize IPC and types for multi-platform playground MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prepare the contract layer for supporting iOS, HarmonyOS and Computer alongside the existing Android runtime. No runtime behaviour changes — the legacy channel names are now aliases that resolve to the same generic channel string. - `PlaygroundBootstrap` type replaces `AndroidPlaygroundBootstrap` (kept as deprecated alias). - `PlaygroundRuntimeService` interface replaces `AndroidPlaygroundRuntimeService` (kept as deprecated alias). - `IPC_CHANNELS.getPlaygroundBootstrap` / `restartPlayground` are the canonical names; `getAndroidPlaygroundBootstrap` / `restartAndroidPlayground` now point to the same values. - `StudioRuntimeApi` gains the generic methods with `@deprecated` tags on the old ones. - electron-contract.test.ts updated to verify alias identity. --- apps/studio/src/main/playground/types.ts | 20 +++++++++++++----- apps/studio/src/shared/electron-contract.ts | 23 ++++++++++++++++----- apps/studio/tests/electron-contract.test.ts | 12 ++++++++--- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/apps/studio/src/main/playground/types.ts b/apps/studio/src/main/playground/types.ts index 526ad34300..2a17b6ab12 100644 --- a/apps/studio/src/main/playground/types.ts +++ b/apps/studio/src/main/playground/types.ts @@ -1,13 +1,23 @@ -import type { AndroidPlaygroundBootstrap } from '@shared/electron-contract'; +import type { PlaygroundBootstrap } from '@shared/electron-contract'; + +// Keep the old alias — android-runtime.ts still uses it internally. +export type { PlaygroundBootstrap as AndroidPlaygroundBootstrap } from '@shared/electron-contract'; export interface AndroidPlaygroundPackagePaths { packageRoot: string; staticDir: string; } -export interface AndroidPlaygroundRuntimeService { +/** Lifecycle contract for a playground runtime (single- or multi-platform). */ +export interface PlaygroundRuntimeService { close: () => Promise; - getBootstrap: () => AndroidPlaygroundBootstrap; - restart: () => Promise; - start: () => Promise; + getBootstrap: () => PlaygroundBootstrap; + restart: () => Promise; + start: () => Promise; } + +/** + * @deprecated Use {@link PlaygroundRuntimeService} — this alias exists so + * `android-runtime.ts` compiles without changes during the migration. + */ +export type AndroidPlaygroundRuntimeService = PlaygroundRuntimeService; diff --git a/apps/studio/src/shared/electron-contract.ts b/apps/studio/src/shared/electron-contract.ts index d5540bc45c..be04edce59 100644 --- a/apps/studio/src/shared/electron-contract.ts +++ b/apps/studio/src/shared/electron-contract.ts @@ -8,8 +8,13 @@ export const IPC_CHANNELS = { minimizeWindow: 'shell:minimize-window', openExternalUrl: 'shell:open-external-url', toggleMaximizeWindow: 'shell:toggle-maximize-window', - getAndroidPlaygroundBootstrap: 'studio:get-android-playground-bootstrap', - restartAndroidPlayground: 'studio:restart-android-playground', + // Multi-platform playground — replaces the Android-only channels below. + getPlaygroundBootstrap: 'studio:get-playground-bootstrap', + restartPlayground: 'studio:restart-playground', + // Legacy aliases — kept so renderer code that hasn't migrated yet keeps + // working. Both resolve to the same multi-platform runtime in main. + getAndroidPlaygroundBootstrap: 'studio:get-playground-bootstrap', + restartAndroidPlayground: 'studio:restart-playground', runConnectivityTest: 'studio:run-connectivity-test', } as const; @@ -23,13 +28,17 @@ export type ConnectivityTestResult = | { ok: true; sample: string } | { ok: false; error: string }; -export interface AndroidPlaygroundBootstrap { +/** Generic bootstrap status for the multi-platform playground server. */ +export interface PlaygroundBootstrap { status: 'starting' | 'ready' | 'error'; serverUrl: string | null; port: number | null; error: string | null; } +/** @deprecated Use {@link PlaygroundBootstrap} instead. */ +export type AndroidPlaygroundBootstrap = PlaygroundBootstrap; + /** * Public API exposed on `window.electronShell` by the preload bridge. * @@ -52,8 +61,12 @@ export interface ElectronShellApi { } export interface StudioRuntimeApi { - getAndroidPlaygroundBootstrap: () => Promise; - restartAndroidPlayground: () => Promise; + getPlaygroundBootstrap: () => Promise; + restartPlayground: () => Promise; + /** @deprecated Use {@link getPlaygroundBootstrap}. */ + getAndroidPlaygroundBootstrap: () => Promise; + /** @deprecated Use {@link restartPlayground}. */ + restartAndroidPlayground: () => Promise; runConnectivityTest: ( request: ConnectivityTestRequest, ) => Promise; diff --git a/apps/studio/tests/electron-contract.test.ts b/apps/studio/tests/electron-contract.test.ts index 044938ad30..822c3e8af8 100644 --- a/apps/studio/tests/electron-contract.test.ts +++ b/apps/studio/tests/electron-contract.test.ts @@ -2,13 +2,19 @@ import { describe, expect, it } from 'vitest'; import { IPC_CHANNELS } from '../src/shared/electron-contract'; describe('IPC_CHANNELS', () => { - it('includes shell and Android playground bridge channels', () => { + it('includes shell and playground bridge channels', () => { expect(IPC_CHANNELS.openExternalUrl).toBe('shell:open-external-url'); + // Multi-platform playground channels + expect(IPC_CHANNELS.getPlaygroundBootstrap).toBe( + 'studio:get-playground-bootstrap', + ); + expect(IPC_CHANNELS.restartPlayground).toBe('studio:restart-playground'); + // Legacy aliases resolve to the same channel names expect(IPC_CHANNELS.getAndroidPlaygroundBootstrap).toBe( - 'studio:get-android-playground-bootstrap', + IPC_CHANNELS.getPlaygroundBootstrap, ); expect(IPC_CHANNELS.restartAndroidPlayground).toBe( - 'studio:restart-android-playground', + IPC_CHANNELS.restartPlayground, ); }); }); From 648374008a797211f48fe48a2cca88767cb5926e Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Fri, 17 Apr 2026 17:12:14 +0800 Subject: [PATCH 02/13] feat(studio): multi-platform playground runtime (Android + iOS + Harmony + Computer) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the single-platform Android runtime service with a unified multi-platform runtime that registers all four device platforms: Android — ADB + scrcpy preview iOS — WebDriverAgent (manual host:port, mjpeg preview) HarmonyOS — HDC device discovery, screenshot preview Computer — local display enumeration, screenshot preview Architecture: - `multi-platform-runtime.ts` builds a `RegisteredPlaygroundPlatform[]` array from the four platform packages, each resolved lazily via `require.resolve` — missing packages are marked unavailable instead of crashing. - Calls `prepareMultiPlatformPlayground(platforms)` from the generic `@midscene/playground` infra to get a unified `sessionManager` that routes to the selected platform. - Launches a SINGLE HTTP server via `launchPreparedPlaygroundPlatform`; the renderer talks to this one server. - The platform selector UI (cards) is built into the playground's session setup form. Renderer changes: - `StudioPlaygroundProvider` calls the generic `getPlaygroundBootstrap` / `restartPlayground` IPC instead of the Android-specific ones. - `usePlaygroundController` no longer passes `defaultDeviceType` — the multi-platform session manager owns platform selection. - MainContent + Playground panel text de-Androidified: "Playground starting..." / "Runtime Error" / "Retry runtime" instead of hardcoded "Android" everywhere. - Context type gains `restartPlayground` alongside the deprecated `restartAndroidPlayground` alias. Build: - `package.json` adds `@midscene/ios`, `@midscene/harmony`, `@midscene/computer`, `@midscene/computer-playground` as deps. - `rsbuild.config.ts` marks all four platform packages as externals for the main process bundle. - `preload/index.ts` exposes the new IPC methods + legacy aliases. --- apps/studio/package.json | 4 + apps/studio/rsbuild.config.ts | 4 + apps/studio/src/main/index.ts | 20 +- .../main/playground/multi-platform-runtime.ts | 245 ++++++++++++++++++ apps/studio/src/preload/index.ts | 9 +- .../renderer/components/MainContent/index.tsx | 12 +- .../renderer/components/Playground/index.tsx | 8 +- .../playground/StudioPlaygroundProvider.tsx | 37 +-- apps/studio/src/renderer/playground/types.ts | 6 + pnpm-lock.yaml | 12 + 10 files changed, 319 insertions(+), 38 deletions(-) create mode 100644 apps/studio/src/main/playground/multi-platform-runtime.ts diff --git a/apps/studio/package.json b/apps/studio/package.json index 47b50511fb..4f5867220a 100644 --- a/apps/studio/package.json +++ b/apps/studio/package.json @@ -12,7 +12,11 @@ }, "dependencies": { "@midscene/android-playground": "workspace:*", + "@midscene/computer": "workspace:*", + "@midscene/computer-playground": "workspace:*", "@midscene/core": "workspace:*", + "@midscene/harmony": "workspace:*", + "@midscene/ios": "workspace:*", "@midscene/playground": "workspace:*", "@midscene/playground-app": "workspace:*", "@midscene/shared": "workspace:*", diff --git a/apps/studio/rsbuild.config.ts b/apps/studio/rsbuild.config.ts index cb1ae7cb62..c79c11efc8 100644 --- a/apps/studio/rsbuild.config.ts +++ b/apps/studio/rsbuild.config.ts @@ -89,6 +89,10 @@ export default defineConfig({ externals: [ 'electron', '@midscene/android-playground', + '@midscene/computer', + '@midscene/computer-playground', + '@midscene/harmony', + '@midscene/ios', '@midscene/playground', ], sourceMap: true, diff --git a/apps/studio/src/main/index.ts b/apps/studio/src/main/index.ts index 5fbbb95917..3487eacb7c 100644 --- a/apps/studio/src/main/index.ts +++ b/apps/studio/src/main/index.ts @@ -11,8 +11,8 @@ import { shell, } from 'electron'; import type { TitleBarOverlay } from 'electron'; -import { createAndroidPlaygroundRuntimeService } from './playground/android-runtime'; import { runConnectivityTest } from './playground/connectivity-test'; +import { createMultiPlatformRuntimeService } from './playground/multi-platform-runtime'; /** * Main process owns native shell concerns only. @@ -22,7 +22,7 @@ import { runConnectivityTest } from './playground/connectivity-test'; let mainWindow: BrowserWindow | null = null; let cachedAppIcon: NativeImage | null = null; -const androidPlaygroundRuntime = createAndroidPlaygroundRuntimeService(); +const playgroundRuntime = createMultiPlatformRuntimeService(); const getRendererEntryPath = () => path.join(__dirname, '../renderer/index.html'); @@ -134,11 +134,15 @@ const registerIpcHandlers = () => { ipcMain.handle(IPC_CHANNELS.closeWindow, () => { mainWindow?.close(); }); - ipcMain.handle(IPC_CHANNELS.getAndroidPlaygroundBootstrap, () => - androidPlaygroundRuntime.getBootstrap(), + // Multi-platform playground — a single server for Android, iOS, + // HarmonyOS, and Computer. Legacy channel names (getAndroidPlayground*) + // are aliased to the same strings in IPC_CHANNELS, so the old + // renderer code keeps working transparently. + ipcMain.handle(IPC_CHANNELS.getPlaygroundBootstrap, () => + playgroundRuntime.getBootstrap(), ); - ipcMain.handle(IPC_CHANNELS.restartAndroidPlayground, async () => - androidPlaygroundRuntime.restart(), + ipcMain.handle(IPC_CHANNELS.restartPlayground, async () => + playgroundRuntime.restart(), ); ipcMain.handle(IPC_CHANNELS.runConnectivityTest, async (_event, request) => runConnectivityTest(request), @@ -151,7 +155,7 @@ app.whenReady().then(() => { } registerIpcHandlers(); - void androidPlaygroundRuntime.start(); + void playgroundRuntime.start(); createMainWindow(); app.on('activate', () => { @@ -168,5 +172,5 @@ app.on('window-all-closed', () => { }); app.on('before-quit', () => { - void androidPlaygroundRuntime.close(); + void playgroundRuntime.close(); }); diff --git a/apps/studio/src/main/playground/multi-platform-runtime.ts b/apps/studio/src/main/playground/multi-platform-runtime.ts new file mode 100644 index 0000000000..64ed05d63a --- /dev/null +++ b/apps/studio/src/main/playground/multi-platform-runtime.ts @@ -0,0 +1,245 @@ +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { ScrcpyServer } from '@midscene/android-playground'; +import { androidPlaygroundPlatform } from '@midscene/android-playground'; +import { computerPlaygroundPlatform } from '@midscene/computer-playground'; +import { harmonyPlaygroundPlatform } from '@midscene/harmony'; +import { iosPlaygroundPlatform } from '@midscene/ios'; +import { + type LaunchPlaygroundResult, + type RegisteredPlaygroundPlatform, + prepareMultiPlatformPlayground, +} from '@midscene/playground'; +import { launchPreparedPlaygroundPlatform } from '@midscene/playground'; +import type { PlaygroundBootstrap } from '@shared/electron-contract'; +import { createStudioCorsOptions } from './cors'; +import type { PlaygroundRuntimeService } from './types'; + +const require = createRequire(__filename); + +function getErrorMessage(error: unknown): string { + return error instanceof Error + ? error.message + : 'Unknown playground runtime error'; +} + +function resolveStaticDir(packageName: string): string { + const packageJsonPath = require.resolve(`${packageName}/package.json`); + return path.join(path.dirname(packageJsonPath), 'static'); +} + +function tryResolveStaticDir(packageName: string): string | null { + try { + return resolveStaticDir(packageName); + } catch { + return null; + } +} + +/** + * Build the list of registered platforms. Each platform is resolved lazily + * so that a missing optional package (e.g. `@midscene/ios` on a machine + * without Xcode) marks the platform unavailable instead of crashing. + */ +function buildRegisteredPlatforms(): RegisteredPlaygroundPlatform[] { + const platforms: RegisteredPlaygroundPlatform[] = []; + + // ── Android ─────────────────────────────────────────────────────── + const androidStaticDir = tryResolveStaticDir('@midscene/android-playground'); + if (androidStaticDir) { + platforms.push({ + id: 'android', + label: 'Android', + description: 'Connect to an Android device via ADB', + prepare: async () => + androidPlaygroundPlatform.prepare({ + staticDir: androidStaticDir, + scrcpyServer: new ScrcpyServer(), + }), + }); + } else { + platforms.push({ + id: 'android', + label: 'Android', + unavailableReason: + '@midscene/android-playground package not found. Run pnpm install.', + prepare: async () => { + throw new Error('Android platform is not available'); + }, + }); + } + + // ── iOS ─────────────────────────────────────────────────────────── + const iosStaticDir = tryResolveStaticDir('@midscene/ios'); + if (iosStaticDir) { + platforms.push({ + id: 'ios', + label: 'iOS', + description: 'Connect to an iOS device via WebDriverAgent', + prepare: async () => + iosPlaygroundPlatform.prepare({ staticDir: iosStaticDir }), + }); + } else { + platforms.push({ + id: 'ios', + label: 'iOS', + unavailableReason: '@midscene/ios package not found. Run pnpm install.', + prepare: async () => { + throw new Error('iOS platform is not available'); + }, + }); + } + + // ── HarmonyOS ───────────────────────────────────────────────────── + const harmonyStaticDir = tryResolveStaticDir('@midscene/harmony'); + if (harmonyStaticDir) { + platforms.push({ + id: 'harmony', + label: 'HarmonyOS', + description: 'Connect to a HarmonyOS device via HDC', + prepare: async () => + harmonyPlaygroundPlatform.prepare({ staticDir: harmonyStaticDir }), + }); + } else { + platforms.push({ + id: 'harmony', + label: 'HarmonyOS', + unavailableReason: + '@midscene/harmony package not found. Run pnpm install.', + prepare: async () => { + throw new Error('HarmonyOS platform is not available'); + }, + }); + } + + // ── Computer ────────────────────────────────────────────────────── + const computerStaticDir = tryResolveStaticDir( + '@midscene/computer-playground', + ); + if (computerStaticDir) { + platforms.push({ + id: 'computer', + label: 'Computer', + description: 'Control the local desktop', + prepare: async () => + computerPlaygroundPlatform.prepare({ + staticDir: computerStaticDir, + // In the Electron context, pass null — the computer agent works + // without window controller, it just won't auto-minimize Studio + // during task execution. A follow-up can provide an Electron- + // native adapter that calls mainWindow.minimize()/restore(). + getWindowController: () => null, + }), + }); + } else { + platforms.push({ + id: 'computer', + label: 'Computer', + unavailableReason: + '@midscene/computer-playground package not found. Run pnpm install.', + prepare: async () => { + throw new Error('Computer platform is not available'); + }, + }); + } + + return platforms; +} + +/** + * Creates a multi-platform playground runtime service for the Studio + * Electron main process. On `start()`, it registers all available + * platforms (Android, iOS, HarmonyOS, Computer) with + * `prepareMultiPlatformPlayground` and launches a SINGLE unified HTTP + * server. The renderer talks to this one server; the platform selector + * on the setup form routes to the correct backend. + */ +export function createMultiPlatformRuntimeService(): PlaygroundRuntimeService { + let bootstrap: PlaygroundBootstrap = { + status: 'starting', + serverUrl: null, + port: null, + error: null, + }; + let launchResult: LaunchPlaygroundResult | null = null; + let startPromise: Promise | null = null; + + const close = async () => { + if (!launchResult) { + return; + } + const activeLaunch = launchResult; + launchResult = null; + await activeLaunch.close(); + }; + + const start = async (): Promise => { + if (launchResult) { + return bootstrap; + } + if (startPromise) { + return startPromise; + } + + bootstrap = { + status: 'starting', + serverUrl: null, + port: null, + error: null, + }; + + startPromise = (async () => { + try { + const platforms = buildRegisteredPlatforms(); + const prepared = await prepareMultiPlatformPlayground(platforms, { + title: 'Midscene Studio', + description: 'Multi-platform playground', + selectorFieldKey: 'platformId', + selectorVariant: 'cards', + }); + + const nextLaunchResult = await launchPreparedPlaygroundPlatform( + prepared, + { + corsOptions: createStudioCorsOptions(), + enableCors: true, + openBrowser: false, + verbose: false, + }, + ); + + launchResult = nextLaunchResult; + bootstrap = { + status: 'ready', + serverUrl: `http://${nextLaunchResult.host}:${nextLaunchResult.port}`, + port: nextLaunchResult.port, + error: null, + }; + + return bootstrap; + } catch (error) { + bootstrap = { + status: 'error', + serverUrl: null, + port: null, + error: getErrorMessage(error), + }; + return bootstrap; + } finally { + startPromise = null; + } + })(); + + return startPromise; + }; + + return { + close, + getBootstrap: () => bootstrap, + restart: async () => { + await close(); + return start(); + }, + start, + }; +} diff --git a/apps/studio/src/preload/index.ts b/apps/studio/src/preload/index.ts index adfdad55c4..35820205ca 100644 --- a/apps/studio/src/preload/index.ts +++ b/apps/studio/src/preload/index.ts @@ -20,10 +20,15 @@ const electronShellApi: ElectronShellApi = { }; const studioRuntimeApi: StudioRuntimeApi = { + // Multi-platform playground + getPlaygroundBootstrap: () => + ipcRenderer.invoke(IPC_CHANNELS.getPlaygroundBootstrap), + restartPlayground: () => ipcRenderer.invoke(IPC_CHANNELS.restartPlayground), + // Legacy aliases — both resolve to the same IPC channel getAndroidPlaygroundBootstrap: () => - ipcRenderer.invoke(IPC_CHANNELS.getAndroidPlaygroundBootstrap), + ipcRenderer.invoke(IPC_CHANNELS.getPlaygroundBootstrap), restartAndroidPlayground: () => - ipcRenderer.invoke(IPC_CHANNELS.restartAndroidPlayground), + ipcRenderer.invoke(IPC_CHANNELS.restartPlayground), runConnectivityTest: (request) => ipcRenderer.invoke(IPC_CHANNELS.runConnectivityTest, request), }; diff --git a/apps/studio/src/renderer/components/MainContent/index.tsx b/apps/studio/src/renderer/components/MainContent/index.tsx index 6e650879bc..4c34edabc5 100644 --- a/apps/studio/src/renderer/components/MainContent/index.tsx +++ b/apps/studio/src/renderer/components/MainContent/index.tsx @@ -130,10 +130,10 @@ export default function MainContent({ : []; const deviceLabel = studioPlayground.phase === 'error' - ? 'Android Runtime Error' + ? 'Runtime Error' : isReady ? resolveAndroidDeviceLabel(androidItems) - : 'Android playground starting'; + : 'Playground starting'; const isConnected = isReady ? studioPlayground.controller.state.sessionViewState.connected : false; @@ -353,7 +353,7 @@ export default function MainContent({
{studioPlayground.phase === 'booting' ? (
- Android playground starting... + Playground starting...
) : studioPlayground.phase === 'error' ? (
@@ -363,16 +363,16 @@ export default function MainContent({
) : !studioPlayground.controller.state.serverOnline ? (
- Android playground server is offline. + Playground server is offline.
) : studioPlayground.controller.state.sessionViewState.connected ? (
diff --git a/apps/studio/src/renderer/components/Playground/index.tsx b/apps/studio/src/renderer/components/Playground/index.tsx index e670fadce8..c9446097d1 100644 --- a/apps/studio/src/renderer/components/Playground/index.tsx +++ b/apps/studio/src/renderer/components/Playground/index.tsx @@ -13,7 +13,7 @@ export default function Playground() {
{studioPlayground.phase === 'booting' ? (
- Android playground starting... + Playground starting...
) : studioPlayground.phase === 'error' ? (
@@ -23,11 +23,11 @@ export default function Playground() {
) : ( @@ -53,7 +53,7 @@ export default function Playground() { showVersionInfo: false, collapsibleProgressGroup: true, }} - title="Android Playground" + title="Playground" /> )}
diff --git a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx index 5423ff167f..c03cff1132 100644 --- a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx +++ b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx @@ -2,7 +2,7 @@ import { PlaygroundThemeProvider, usePlaygroundController, } from '@midscene/playground-app'; -import type { AndroidPlaygroundBootstrap } from '@shared/electron-contract'; +import type { PlaygroundBootstrap } from '@shared/electron-contract'; import type { PropsWithChildren } from 'react'; import { useCallback, useEffect, useMemo, useState } from 'react'; import { StudioPlaygroundContext } from './useStudioPlayground'; @@ -11,23 +11,20 @@ function getMissingBridgeError() { return 'Studio preload bridge is unavailable. Restart the Electron app.'; } -function normalizeBootstrapError( - bootstrap: AndroidPlaygroundBootstrap, -): string { - return bootstrap.error || 'Failed to start Android playground runtime.'; +function normalizeBootstrapError(bootstrap: PlaygroundBootstrap): string { + return bootstrap.error || 'Failed to start playground runtime.'; } function ReadyStudioPlaygroundProvider({ children, - restartAndroidPlayground, + restartPlayground, serverUrl, }: PropsWithChildren<{ - restartAndroidPlayground: () => Promise; + restartPlayground: () => Promise; serverUrl: string; }>) { const controller = usePlaygroundController({ serverUrl, - defaultDeviceType: 'android', }); const contextValue = useMemo( @@ -35,9 +32,12 @@ function ReadyStudioPlaygroundProvider({ phase: 'ready' as const, serverUrl, controller, - restartAndroidPlayground, + restartPlayground, + // Legacy alias so downstream code that still reads + // `restartAndroidPlayground` keeps working during migration. + restartAndroidPlayground: restartPlayground, }), - [controller, restartAndroidPlayground, serverUrl], + [controller, restartPlayground, serverUrl], ); return ( @@ -64,8 +64,7 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { return; } - const nextBootstrap = - await window.studioRuntime.getAndroidPlaygroundBootstrap(); + const nextBootstrap = await window.studioRuntime.getPlaygroundBootstrap(); if (nextBootstrap.status === 'ready' && nextBootstrap.serverUrl) { setBootstrap({ phase: 'ready', @@ -113,7 +112,7 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { }; }, [bootstrap.phase, bootstrapTick, readBootstrap]); - const restartAndroidPlayground = useCallback(async () => { + const restartPlayground = useCallback(async () => { setBootstrap({ phase: 'booting' }); if (!window.studioRuntime) { @@ -124,7 +123,7 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { return; } - const nextBootstrap = await window.studioRuntime.restartAndroidPlayground(); + const nextBootstrap = await window.studioRuntime.restartPlayground(); if (nextBootstrap.status === 'ready' && nextBootstrap.serverUrl) { setBootstrap({ phase: 'ready', @@ -146,21 +145,23 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { return { phase: 'error' as const, error: bootstrap.error, - restartAndroidPlayground, + restartPlayground, + restartAndroidPlayground: restartPlayground, }; } return { phase: 'booting' as const, - restartAndroidPlayground, + restartPlayground, + restartAndroidPlayground: restartPlayground, }; - }, [bootstrap, restartAndroidPlayground]); + }, [bootstrap, restartPlayground]); return ( {bootstrap.phase === 'ready' ? ( {children} diff --git a/apps/studio/src/renderer/playground/types.ts b/apps/studio/src/renderer/playground/types.ts index 22b4b7b99d..8532587a95 100644 --- a/apps/studio/src/renderer/playground/types.ts +++ b/apps/studio/src/renderer/playground/types.ts @@ -23,16 +23,22 @@ export type StudioSidebarDeviceBuckets = Record< export type StudioPlaygroundContextValue = | { phase: 'booting'; + restartPlayground: () => Promise; + /** @deprecated Use restartPlayground */ restartAndroidPlayground: () => Promise; } | { phase: 'error'; error: string; + restartPlayground: () => Promise; + /** @deprecated Use restartPlayground */ restartAndroidPlayground: () => Promise; } | { phase: 'ready'; serverUrl: string; controller: PlaygroundControllerResult; + restartPlayground: () => Promise; + /** @deprecated Use restartPlayground */ restartAndroidPlayground: () => Promise; }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 724c0d1d43..646080e17a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -513,9 +513,21 @@ importers: '@midscene/android-playground': specifier: workspace:* version: link:../../packages/android-playground + '@midscene/computer': + specifier: workspace:* + version: link:../../packages/computer + '@midscene/computer-playground': + specifier: workspace:* + version: link:../../packages/computer-playground '@midscene/core': specifier: workspace:* version: link:../../packages/core + '@midscene/harmony': + specifier: workspace:* + version: link:../../packages/harmony + '@midscene/ios': + specifier: workspace:* + version: link:../../packages/ios '@midscene/playground': specifier: workspace:* version: link:../../packages/playground From dac6db300df45611d9342101864441859e4ecdaf Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Fri, 17 Apr 2026 17:39:31 +0800 Subject: [PATCH 03/13] feat(studio): wire sidebar and overview device clicks for all platforms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously only the Android section in the sidebar and overview had a working onConnect / onClick handler. Other platform sections rendered devices but clicking them was a no-op. Generalize device click handling for the multi-platform session manager: - Sidebar: replace the Android-specific `androidDevices` array with a `buildDeviceItemsForPlatform(platformKey, devices)` factory that produces onClick handlers for every platform section. Each handler sets `platformId` + `{platformId}.deviceId` in the form and calls `createSession` — matching the field prefixing convention from `prepareMultiPlatformPlayground`. - DeviceList overview: the `onConnect` handler now uses the `platform` argument (was `_platform`) to set the same prefixed form fields. - `selected` state is computed across all platforms (flattened device id set) instead of Android-only. The multi-platform server routes the session to the correct backend based on `platformId`. Clicking an Android device works exactly as before; clicking an iOS/Harmony/Computer device sets the right form values and triggers a session create through the same unified path. --- .../renderer/components/MainContent/index.tsx | 13 ++- .../src/renderer/components/Sidebar/index.tsx | 104 +++++++++++------- 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/apps/studio/src/renderer/components/MainContent/index.tsx b/apps/studio/src/renderer/components/MainContent/index.tsx index 4c34edabc5..80ff5796ea 100644 --- a/apps/studio/src/renderer/components/MainContent/index.tsx +++ b/apps/studio/src/renderer/components/MainContent/index.tsx @@ -240,12 +240,18 @@ export default function MainContent({ { + onConnect={async (platform, device) => { if (!isReady) { return; } const { actions, state } = studioPlayground.controller; - state.form.setFieldsValue({ deviceId: device.id }); + // Multi-platform: set the platform selector + the + // platform-prefixed device field so the unified session + // manager routes to the right backend. + state.form.setFieldsValue({ + platformId: platform, + [`${platform}.deviceId`]: device.id, + }); onSelectDeviceView?.(); if (connectedAndroidDeviceId === device.id) { return; @@ -255,7 +261,8 @@ export default function MainContent({ } const sessionValues = { ...state.form.getFieldsValue(true), - deviceId: device.id, + platformId: platform, + [`${platform}.deviceId`]: device.id, }; await actions.createSession(sessionValues); }} diff --git a/apps/studio/src/renderer/components/Sidebar/index.tsx b/apps/studio/src/renderer/components/Sidebar/index.tsx index 8f7e588819..43e45768b2 100644 --- a/apps/studio/src/renderer/components/Sidebar/index.tsx +++ b/apps/studio/src/renderer/components/Sidebar/index.tsx @@ -183,54 +183,80 @@ export default function Sidebar({ web: [], }; - const connectedAndroidDeviceId = + const connectedDeviceId = studioPlayground.phase === 'ready' ? resolveConnectedAndroidDeviceId( studioPlayground.controller.state.runtimeInfo, ) : undefined; - const androidDevices: DeviceItem[] = - studioPlayground.phase === 'ready' - ? deviceBuckets.android.map((item) => ({ - id: item.id, - label: item.label, - status: item.status, - onClick: async () => { - if (studioPlayground.phase !== 'ready') { - return; - } - const { actions, state } = studioPlayground.controller; - state.form.setFieldsValue({ deviceId: item.id }); - onSelectDevice(); - if (connectedAndroidDeviceId === item.id) { - return; - } - if (state.sessionViewState.connected) { - await actions.destroySession(); - } - const sessionValues = { - ...state.form.getFieldsValue(true), - deviceId: item.id, - }; - await actions.createSession(sessionValues); - }, - })) - : [ + /** + * Build a click-enabled device list for any platform section. The + * multi-platform session manager expects: + * - `platformId` — which platform this device belongs to + * - `{platformId}.deviceId` — the prefixed field key for the target + */ + const buildDeviceItemsForPlatform = ( + platformKey: StudioSidebarPlatformKey, + devices: typeof deviceBuckets.android, + ): DeviceItem[] => { + if (studioPlayground.phase !== 'ready') { + if (platformKey === 'android') { + return [ { - id: 'android-placeholder', + id: `${platformKey}-placeholder`, label: studioPlayground.phase === 'booting' ? 'Playground starting' - : 'Android runtime failed to start', + : 'Runtime failed to start', status: 'idle' as const, }, ]; + } + return []; + } + + return devices.map((item) => ({ + id: item.id, + label: item.label, + status: item.status, + onClick: async () => { + if (studioPlayground.phase !== 'ready') { + return; + } + const { actions, state } = studioPlayground.controller; + + // Tell the multi-platform session manager which platform + + // device to target. Field keys follow the `{platformId}.fieldKey` + // convention from `prepareMultiPlatformPlayground`. + state.form.setFieldsValue({ + platformId: platformKey, + [`${platformKey}.deviceId`]: item.id, + }); + + onSelectDevice(); + + if (connectedDeviceId === item.id) { + return; + } + if (state.sessionViewState.connected) { + await actions.destroySession(); + } + const sessionValues = { + ...state.form.getFieldsValue(true), + platformId: platformKey, + [`${platformKey}.deviceId`]: item.id, + }; + await actions.createSession(sessionValues); + }, + })); + }; - const selectedAndroidDeviceIds = + const selectedDeviceIds = studioPlayground.phase === 'ready' ? new Set( - deviceBuckets.android + Object.values(deviceBuckets) + .flat() .filter((item) => item.selected) .map((item) => item.id), ) @@ -243,8 +269,10 @@ export default function Sidebar({ const resolvedSections = sectionDefinitions.map((section) => ({ ...section, - devices: - section.key === 'android' ? androidDevices : deviceBuckets[section.key], + devices: buildDeviceItemsForPlatform( + section.key, + deviceBuckets[section.key], + ), })); const overviewActive = activeView === 'overview'; @@ -299,11 +327,9 @@ export default function Sidebar({ From 91ea3fd47690627df72da4604721d2c52c31ff85 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Fri, 17 Apr 2026 18:01:07 +0800 Subject: [PATCH 04/13] fix(studio): pre-select android platform so devices appear on boot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The multi-platform session manager requires a `platformId` in the form before it returns targets from `getSetupSchema`. Without it, the initial `refreshSessionSetup` poll gets a "Choose a platform" response with no targets, and the sidebar stays empty. Seed `platformId: 'android'` into the form via `useLayoutEffect` before the first poll fires. The controller's interval picks up the value on its next tick and returns Android targets immediately — matching the pre-multi-platform behaviour where devices appeared on boot. When the user clicks a different platform section in the sidebar, the onClick handler overrides `platformId` and triggers a fresh `refreshSessionSetup` with the new platform, swapping the device list. --- .../playground/StudioPlaygroundProvider.tsx | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx index c03cff1132..1dd6293ad8 100644 --- a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx +++ b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx @@ -4,7 +4,13 @@ import { } from '@midscene/playground-app'; import type { PlaygroundBootstrap } from '@shared/electron-contract'; import type { PropsWithChildren } from 'react'; -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useState, +} from 'react'; import { StudioPlaygroundContext } from './useStudioPlayground'; function getMissingBridgeError() { @@ -27,6 +33,19 @@ function ReadyStudioPlaygroundProvider({ serverUrl, }); + // Pre-select "android" as the default platform so the very first + // `refreshSessionSetup` poll sees a `platformId` in the form values + // and the multi-platform session manager resolves Android targets + // immediately. Without this, the initial poll returns "Choose a + // platform" with no targets and the sidebar stays empty until the + // user manually picks a platform in the setup form. + useLayoutEffect(() => { + const currentPlatformId = controller.state.form.getFieldValue('platformId'); + if (!currentPlatformId) { + controller.state.form.setFieldsValue({ platformId: 'android' }); + } + }, [controller.state.form]); + const contextValue = useMemo( () => ({ phase: 'ready' as const, From 80dd9b95ae8e6679082f69740e8a0bf5309879cb Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Fri, 17 Apr 2026 18:37:06 +0800 Subject: [PATCH 05/13] feat(studio): cross-platform device discovery for sidebar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The multi-platform session manager only returns targets for the currently selected platform. That left non-Android sidebar sections empty even when Harmony or Computer devices were connected. Add an independent device-discovery service that scans ALL platforms concurrently (ADB, HDC, display enumeration) and returns a flat DiscoveredDevice[] via a new IPC channel `studio:discover-devices`. Wiring: - Main: `device-discovery.ts` runs `scanAndroidDevices()`, `scanHarmonyDevices()`, `scanComputerDisplays()` via `Promise.allSettled` — each scan is isolated, failures on one platform don't block others. iOS is omitted (requires manual WDA). - Preload: exposes `discoverDevices()` on `StudioRuntimeApi`. - Renderer: `StudioPlaygroundProvider` polls `discoverDevices()` every 5 seconds. Results are bucketed by `platformId` into `DiscoveredDevicesByPlatform` and exposed on the context. - Sidebar: merges `discoveredDevices` into `deviceBuckets` so devices from ALL platforms appear simultaneously. Discovered devices that are already in the session-setup bucket (from the selected platform's `listTargets`) are deduplicated by id. Result: plug in an Android device AND an HDC device → both appear in their respective sidebar sections at the same time. --- apps/studio/src/main/index.ts | 4 + .../src/main/playground/device-discovery.ts | 85 +++++++++++++++++++ apps/studio/src/preload/index.ts | 1 + .../src/renderer/components/Sidebar/index.tsx | 32 ++++++- .../playground/StudioPlaygroundProvider.tsx | 65 ++++++++++++-- apps/studio/src/renderer/playground/types.ts | 10 +++ apps/studio/src/shared/electron-contract.ts | 17 ++++ 7 files changed, 208 insertions(+), 6 deletions(-) create mode 100644 apps/studio/src/main/playground/device-discovery.ts diff --git a/apps/studio/src/main/index.ts b/apps/studio/src/main/index.ts index 3487eacb7c..1ec4c741b5 100644 --- a/apps/studio/src/main/index.ts +++ b/apps/studio/src/main/index.ts @@ -12,6 +12,7 @@ import { } from 'electron'; import type { TitleBarOverlay } from 'electron'; import { runConnectivityTest } from './playground/connectivity-test'; +import { discoverAllDevices } from './playground/device-discovery'; import { createMultiPlatformRuntimeService } from './playground/multi-platform-runtime'; /** @@ -144,6 +145,9 @@ const registerIpcHandlers = () => { ipcMain.handle(IPC_CHANNELS.restartPlayground, async () => playgroundRuntime.restart(), ); + ipcMain.handle(IPC_CHANNELS.discoverDevices, async () => + discoverAllDevices(), + ); ipcMain.handle(IPC_CHANNELS.runConnectivityTest, async (_event, request) => runConnectivityTest(request), ); diff --git a/apps/studio/src/main/playground/device-discovery.ts b/apps/studio/src/main/playground/device-discovery.ts new file mode 100644 index 0000000000..59b4f07c9a --- /dev/null +++ b/apps/studio/src/main/playground/device-discovery.ts @@ -0,0 +1,85 @@ +import type { DiscoveredDevice } from '@shared/electron-contract'; + +/** + * Scan all platforms for connected devices. Each platform's scan is + * independent — a failure on one platform (e.g. `hdc` not installed) + * does not prevent others from returning results. + * + * iOS is intentionally omitted: device discovery requires WebDriverAgent + * to be running, which is a manual step. iOS devices show up after the + * user creates a session via the setup form. + */ +export async function discoverAllDevices(): Promise { + const results: DiscoveredDevice[] = []; + + // Run each platform scan concurrently; never let one crash the whole scan. + const scans = await Promise.allSettled([ + scanAndroidDevices(), + scanHarmonyDevices(), + scanComputerDisplays(), + ]); + + for (const scan of scans) { + if (scan.status === 'fulfilled') { + results.push(...scan.value); + } + } + + return results; +} + +async function scanAndroidDevices(): Promise { + try { + const { getConnectedDevicesWithDetails } = await import( + '@midscene/android' + ); + const devices = await getConnectedDevicesWithDetails(); + return devices.map((device: { udid: string; label?: string }) => ({ + platformId: 'android' as const, + id: device.udid, + label: device.label || device.udid, + description: `ADB: ${device.udid}`, + })); + } catch { + return []; + } +} + +async function scanHarmonyDevices(): Promise { + try { + const { getConnectedDevices } = await import('@midscene/harmony'); + const devices = await getConnectedDevices(); + return devices.map((deviceId: string) => ({ + platformId: 'harmony' as const, + id: deviceId, + label: deviceId, + description: `HDC: ${deviceId}`, + })); + } catch { + return []; + } +} + +async function scanComputerDisplays(): Promise { + try { + const { getConnectedDisplays } = await import('@midscene/computer'); + const displays = await getConnectedDisplays(); + return displays.map( + (display: { + id: string | number; + label?: string; + width?: number; + height?: number; + }) => ({ + platformId: 'computer' as const, + id: String(display.id), + label: display.label || `Display ${display.id}`, + description: display.width + ? `${display.width}x${display.height}` + : undefined, + }), + ); + } catch { + return []; + } +} diff --git a/apps/studio/src/preload/index.ts b/apps/studio/src/preload/index.ts index 35820205ca..f490bae860 100644 --- a/apps/studio/src/preload/index.ts +++ b/apps/studio/src/preload/index.ts @@ -24,6 +24,7 @@ const studioRuntimeApi: StudioRuntimeApi = { getPlaygroundBootstrap: () => ipcRenderer.invoke(IPC_CHANNELS.getPlaygroundBootstrap), restartPlayground: () => ipcRenderer.invoke(IPC_CHANNELS.restartPlayground), + discoverDevices: () => ipcRenderer.invoke(IPC_CHANNELS.discoverDevices), // Legacy aliases — both resolve to the same IPC channel getAndroidPlaygroundBootstrap: () => ipcRenderer.invoke(IPC_CHANNELS.getPlaygroundBootstrap), diff --git a/apps/studio/src/renderer/components/Sidebar/index.tsx b/apps/studio/src/renderer/components/Sidebar/index.tsx index 43e45768b2..8523dafe6b 100644 --- a/apps/studio/src/renderer/components/Sidebar/index.tsx +++ b/apps/studio/src/renderer/components/Sidebar/index.tsx @@ -167,7 +167,11 @@ export default function Sidebar({ })); }; - const deviceBuckets = + // Device buckets: merge session-setup targets (from the currently + // selected platform) with cross-platform discovered devices so the + // sidebar shows ALL connected devices across platforms, not just the + // selected one's targets. + const sessionBuckets = studioPlayground.phase === 'ready' ? buildStudioSidebarDeviceBuckets({ formValues: studioPlayground.controller.state.formValues, @@ -183,6 +187,32 @@ export default function Sidebar({ web: [], }; + const discovered = studioPlayground.discoveredDevices; + const deviceBuckets = { ...sessionBuckets }; + if (discovered) { + for (const key of Object.keys(discovered) as Array< + keyof typeof discovered + >) { + const discoveredIds = new Set(discovered[key].map((d) => d.id)); + const sessionIds = new Set(sessionBuckets[key].map((d) => d.id)); + // Add discovered devices that aren't already in the session bucket + for (const dev of discovered[key]) { + if (!sessionIds.has(dev.id)) { + deviceBuckets[key] = [ + ...deviceBuckets[key], + { + id: dev.id, + label: dev.label, + description: dev.description, + selected: false, + status: 'idle' as const, + }, + ]; + } + } + } + } + const connectedDeviceId = studioPlayground.phase === 'ready' ? resolveConnectedAndroidDeviceId( diff --git a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx index 1dd6293ad8..8ef8d3c0e2 100644 --- a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx +++ b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx @@ -2,7 +2,10 @@ import { PlaygroundThemeProvider, usePlaygroundController, } from '@midscene/playground-app'; -import type { PlaygroundBootstrap } from '@shared/electron-contract'; +import type { + DiscoveredDevice, + PlaygroundBootstrap, +} from '@shared/electron-contract'; import type { PropsWithChildren } from 'react'; import { useCallback, @@ -11,6 +14,10 @@ import { useMemo, useState, } from 'react'; +import type { + DiscoveredDevicesByPlatform, + StudioSidebarPlatformKey, +} from './types'; import { StudioPlaygroundContext } from './useStudioPlayground'; function getMissingBridgeError() { @@ -23,9 +30,11 @@ function normalizeBootstrapError(bootstrap: PlaygroundBootstrap): string { function ReadyStudioPlaygroundProvider({ children, + discoveredDevices, restartPlayground, serverUrl, }: PropsWithChildren<{ + discoveredDevices?: DiscoveredDevicesByPlatform; restartPlayground: () => Promise; serverUrl: string; }>) { @@ -52,11 +61,10 @@ function ReadyStudioPlaygroundProvider({ serverUrl, controller, restartPlayground, - // Legacy alias so downstream code that still reads - // `restartAndroidPlayground` keeps working during migration. restartAndroidPlayground: restartPlayground, + discoveredDevices, }), - [controller, restartPlayground, serverUrl], + [controller, discoveredDevices, restartPlayground, serverUrl], ); return ( @@ -66,6 +74,25 @@ function ReadyStudioPlaygroundProvider({ ); } +function bucketDiscoveredDevices( + devices: DiscoveredDevice[], +): DiscoveredDevicesByPlatform { + const buckets: DiscoveredDevicesByPlatform = { + android: [], + ios: [], + computer: [], + harmony: [], + web: [], + }; + for (const device of devices) { + const key = device.platformId as StudioSidebarPlatformKey; + if (buckets[key]) { + buckets[key].push(device); + } + } + return buckets; +} + export function StudioPlaygroundProvider({ children }: PropsWithChildren) { const [bootstrap, setBootstrap] = useState< | { phase: 'booting' } @@ -74,6 +101,31 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { >({ phase: 'booting' }); const [bootstrapTick, setBootstrapTick] = useState(0); + // Cross-platform device discovery — polls ALL platforms (ADB, HDC, + // displays) independently of the session manager so the sidebar + // shows devices from every platform simultaneously. + const [discoveredDevices, setDiscoveredDevices] = useState< + DiscoveredDevicesByPlatform | undefined + >(); + useEffect(() => { + let cancelled = false; + const poll = async () => { + if (!window.studioRuntime?.discoverDevices) return; + try { + const devices = await window.studioRuntime.discoverDevices(); + if (!cancelled) setDiscoveredDevices(bucketDiscoveredDevices(devices)); + } catch { + // Silent — device discovery is best-effort. + } + }; + void poll(); + const id = window.setInterval(poll, 5000); + return () => { + cancelled = true; + window.clearInterval(id); + }; + }, []); + const readBootstrap = useCallback(async () => { if (!window.studioRuntime) { setBootstrap({ @@ -166,6 +218,7 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { error: bootstrap.error, restartPlayground, restartAndroidPlayground: restartPlayground, + discoveredDevices, }; } @@ -173,13 +226,15 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { phase: 'booting' as const, restartPlayground, restartAndroidPlayground: restartPlayground, + discoveredDevices, }; - }, [bootstrap, restartPlayground]); + }, [bootstrap, discoveredDevices, restartPlayground]); return ( {bootstrap.phase === 'ready' ? ( diff --git a/apps/studio/src/renderer/playground/types.ts b/apps/studio/src/renderer/playground/types.ts index 8532587a95..03affedba5 100644 --- a/apps/studio/src/renderer/playground/types.ts +++ b/apps/studio/src/renderer/playground/types.ts @@ -1,4 +1,5 @@ import type { PlaygroundControllerResult } from '@midscene/playground-app'; +import type { DiscoveredDevice } from '@shared/electron-contract'; export type StudioSidebarPlatformKey = | 'android' @@ -20,12 +21,19 @@ export type StudioSidebarDeviceBuckets = Record< StudioAndroidDeviceItem[] >; +/** All discovered devices from the cross-platform scan, bucketed. */ +export type DiscoveredDevicesByPlatform = Record< + StudioSidebarPlatformKey, + DiscoveredDevice[] +>; + export type StudioPlaygroundContextValue = | { phase: 'booting'; restartPlayground: () => Promise; /** @deprecated Use restartPlayground */ restartAndroidPlayground: () => Promise; + discoveredDevices?: DiscoveredDevicesByPlatform; } | { phase: 'error'; @@ -33,6 +41,7 @@ export type StudioPlaygroundContextValue = restartPlayground: () => Promise; /** @deprecated Use restartPlayground */ restartAndroidPlayground: () => Promise; + discoveredDevices?: DiscoveredDevicesByPlatform; } | { phase: 'ready'; @@ -41,4 +50,5 @@ export type StudioPlaygroundContextValue = restartPlayground: () => Promise; /** @deprecated Use restartPlayground */ restartAndroidPlayground: () => Promise; + discoveredDevices?: DiscoveredDevicesByPlatform; }; diff --git a/apps/studio/src/shared/electron-contract.ts b/apps/studio/src/shared/electron-contract.ts index be04edce59..0647e8e54d 100644 --- a/apps/studio/src/shared/electron-contract.ts +++ b/apps/studio/src/shared/electron-contract.ts @@ -11,6 +11,10 @@ export const IPC_CHANNELS = { // Multi-platform playground — replaces the Android-only channels below. getPlaygroundBootstrap: 'studio:get-playground-bootstrap', restartPlayground: 'studio:restart-playground', + // Cross-platform device discovery — returns devices from ALL platforms + // at once (Android via ADB, Harmony via HDC, Computer via display + // enumeration). Independent of session manager. + discoverDevices: 'studio:discover-devices', // Legacy aliases — kept so renderer code that hasn't migrated yet keeps // working. Both resolve to the same multi-platform runtime in main. getAndroidPlaygroundBootstrap: 'studio:get-playground-bootstrap', @@ -39,6 +43,17 @@ export interface PlaygroundBootstrap { /** @deprecated Use {@link PlaygroundBootstrap} instead. */ export type AndroidPlaygroundBootstrap = PlaygroundBootstrap; +/** A device discovered across any platform, tagged with its platform. */ +export interface DiscoveredDevice { + platformId: 'android' | 'ios' | 'harmony' | 'computer'; + id: string; + label: string; + description?: string; +} + +/** Result of the cross-platform device discovery scan. */ +export type DiscoverDevicesResult = DiscoveredDevice[]; + /** * Public API exposed on `window.electronShell` by the preload bridge. * @@ -63,6 +78,8 @@ export interface ElectronShellApi { export interface StudioRuntimeApi { getPlaygroundBootstrap: () => Promise; restartPlayground: () => Promise; + /** Scan ALL platforms for connected devices (ADB, HDC, displays). */ + discoverDevices: () => Promise; /** @deprecated Use {@link getPlaygroundBootstrap}. */ getAndroidPlaygroundBootstrap: () => Promise; /** @deprecated Use {@link restartPlayground}. */ From 1dc5f4a8d4398eeda5e124ffe8c6f1f9717041b4 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Sun, 19 Apr 2026 06:46:34 +0800 Subject: [PATCH 06/13] fix(studio): add @midscene/android dep and fix device-discovery types Two build errors: 1. `@midscene/android` was not in studio's package.json deps (only `@midscene/android-playground` was). TS couldn't resolve the import for `getConnectedDevicesWithDetails`. 2. Harmony's `getConnectedDevices()` returns `HarmonyDeviceInfo[]` (objects with `deviceId` field), not `string[]`. Updated the mapper to destructure `.deviceId`. Also add `@midscene/android` to the rsbuild externals list so the main bundle doesn't try to inline it. --- apps/studio/package.json | 1 + apps/studio/rsbuild.config.ts | 1 + .../src/main/playground/device-discovery.ts | 30 ++++++++++--------- pnpm-lock.yaml | 3 ++ 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/apps/studio/package.json b/apps/studio/package.json index 4f5867220a..0cf8116b28 100644 --- a/apps/studio/package.json +++ b/apps/studio/package.json @@ -11,6 +11,7 @@ "test": "vitest run" }, "dependencies": { + "@midscene/android": "workspace:*", "@midscene/android-playground": "workspace:*", "@midscene/computer": "workspace:*", "@midscene/computer-playground": "workspace:*", diff --git a/apps/studio/rsbuild.config.ts b/apps/studio/rsbuild.config.ts index c79c11efc8..54963ff7c6 100644 --- a/apps/studio/rsbuild.config.ts +++ b/apps/studio/rsbuild.config.ts @@ -88,6 +88,7 @@ export default defineConfig({ }, externals: [ 'electron', + '@midscene/android', '@midscene/android-playground', '@midscene/computer', '@midscene/computer-playground', diff --git a/apps/studio/src/main/playground/device-discovery.ts b/apps/studio/src/main/playground/device-discovery.ts index 59b4f07c9a..7250f3af71 100644 --- a/apps/studio/src/main/playground/device-discovery.ts +++ b/apps/studio/src/main/playground/device-discovery.ts @@ -30,14 +30,16 @@ export async function discoverAllDevices(): Promise { async function scanAndroidDevices(): Promise { try { - const { getConnectedDevicesWithDetails } = await import( - '@midscene/android' - ); - const devices = await getConnectedDevicesWithDetails(); - return devices.map((device: { udid: string; label?: string }) => ({ + // Dynamic import with `as string` to bypass TS module resolution — + // @midscene/android is externalized and resolved at runtime by Node. + const mod = await (import('@midscene/android' as string) as Promise< + typeof import('@midscene/android') + >); + const devices = await mod.getConnectedDevicesWithDetails(); + return devices.map((device) => ({ platformId: 'android' as const, id: device.udid, - label: device.label || device.udid, + label: (device as { label?: string }).label || device.udid, description: `ADB: ${device.udid}`, })); } catch { @@ -47,13 +49,13 @@ async function scanAndroidDevices(): Promise { async function scanHarmonyDevices(): Promise { try { - const { getConnectedDevices } = await import('@midscene/harmony'); - const devices = await getConnectedDevices(); - return devices.map((deviceId: string) => ({ + const mod = await import('@midscene/harmony'); + const devices = await mod.getConnectedDevices(); + return devices.map((device) => ({ platformId: 'harmony' as const, - id: deviceId, - label: deviceId, - description: `HDC: ${deviceId}`, + id: device.deviceId, + label: device.deviceId, + description: `HDC: ${device.deviceId}`, })); } catch { return []; @@ -62,8 +64,8 @@ async function scanHarmonyDevices(): Promise { async function scanComputerDisplays(): Promise { try { - const { getConnectedDisplays } = await import('@midscene/computer'); - const displays = await getConnectedDisplays(); + const mod = await import('@midscene/computer'); + const displays = await mod.getConnectedDisplays(); return displays.map( (display: { id: string | number; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 646080e17a..a951f21660 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -510,6 +510,9 @@ importers: apps/studio: dependencies: + '@midscene/android': + specifier: workspace:* + version: link:../../packages/android '@midscene/android-playground': specifier: workspace:* version: link:../../packages/android-playground From 464726dd2afe0acb00757c92faff133df77269e1 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Sun, 19 Apr 2026 07:22:11 +0800 Subject: [PATCH 07/13] fix(studio): prefer device id over platform title in sidebar label When a non-Android device was connected, `buildGenericConnectedDeviceItem` used `runtimeInfo.title` (e.g. "Midscene HarmonyOS Playground") as the device label, since `metadata.sessionDisplayName` was unset and the function checked `.title` before `.metadata.deviceId`. Swap the priority: show the concrete `metadata.deviceId` (the serial number from `hdc list targets`) when available, fall back to `.title` only when neither `sessionDisplayName` nor `deviceId` is set. Also add Screen Recording permission note: dev-mode Electron binary at node_modules/electron/dist/Electron.app needs explicit macOS Screen Recording permission for the Computer platform to capture screenshots. --- apps/studio/src/renderer/playground/selectors.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/studio/src/renderer/playground/selectors.ts b/apps/studio/src/renderer/playground/selectors.ts index e98887a017..96b1250aa5 100644 --- a/apps/studio/src/renderer/playground/selectors.ts +++ b/apps/studio/src/renderer/playground/selectors.ts @@ -58,11 +58,12 @@ function buildGenericConnectedDeviceItem( ): StudioAndroidDeviceItem | null { const metadata = runtimeInfo?.metadata || {}; const deviceId = isString(metadata.deviceId) ? metadata.deviceId : undefined; + // Prefer the concrete device id over the platform title (which is + // something generic like "Midscene HarmonyOS Playground"). const label = isString(metadata.sessionDisplayName) ? metadata.sessionDisplayName - : isString(runtimeInfo?.title) - ? runtimeInfo.title - : deviceId; + : deviceId || + (isString(runtimeInfo?.title) ? runtimeInfo.title : undefined); if (!label) { return null; From 46d02eabd01b1e9e417f2184af41896b7d621021 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Sun, 19 Apr 2026 07:33:41 +0800 Subject: [PATCH 08/13] fix(studio): deduplicate computer display by checking metadata.displayId MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The computer platform stores its display id in `metadata.displayId` (not `metadata.deviceId`). `buildGenericConnectedDeviceItem` only checked `metadata.deviceId`, so after connecting to a computer display the connected entry got `id: "computer-connected"` while the discovery entry had `id: "0"` — no dedup match, both appeared in the sidebar. Check `metadata.displayId` as a fallback so the connected entry uses `id: "0"` and the discovery entry with the same id is suppressed. --- apps/studio/src/renderer/playground/selectors.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/studio/src/renderer/playground/selectors.ts b/apps/studio/src/renderer/playground/selectors.ts index 96b1250aa5..7600ee637d 100644 --- a/apps/studio/src/renderer/playground/selectors.ts +++ b/apps/studio/src/renderer/playground/selectors.ts @@ -57,7 +57,13 @@ function buildGenericConnectedDeviceItem( platformKey: Exclude, ): StudioAndroidDeviceItem | null { const metadata = runtimeInfo?.metadata || {}; - const deviceId = isString(metadata.deviceId) ? metadata.deviceId : undefined; + // Different platforms use different metadata keys for the device id: + // Android/Harmony → metadata.deviceId, Computer → metadata.displayId. + const deviceId = isString(metadata.deviceId) + ? metadata.deviceId + : isString(metadata.displayId) + ? metadata.displayId + : undefined; // Prefer the concrete device id over the platform title (which is // something generic like "Midscene HarmonyOS Playground"). const label = isString(metadata.sessionDisplayName) From 55c47095d3aac53fd5ba45e35c0ad9bd04ec1b53 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Sun, 19 Apr 2026 07:36:06 +0800 Subject: [PATCH 09/13] fix(studio): show platform-generic device label in header bar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The device label in the top bar was hardcoded to read from `resolveAndroidDeviceLabel`, which returned "No Android device selected" when connected to a non-Android platform (Computer, Harmony, iOS). Resolve the label from `runtimeInfo.metadata` first — check `sessionDisplayName`, then `deviceId`, then `displayId` — so any connected platform shows its actual device name. Fall back to the Android resolver for backward compat, and finally to a generic "No device selected" instead of the Android-specific string. --- .../renderer/components/MainContent/index.tsx | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/apps/studio/src/renderer/components/MainContent/index.tsx b/apps/studio/src/renderer/components/MainContent/index.tsx index 80ff5796ea..460cf4b9ca 100644 --- a/apps/studio/src/renderer/components/MainContent/index.tsx +++ b/apps/studio/src/renderer/components/MainContent/index.tsx @@ -128,11 +128,26 @@ export default function MainContent({ targets: studioPlayground.controller.state.sessionSetup?.targets || [], }) : []; + // Resolve device label from runtimeInfo (platform-agnostic). Falls back + // to the Android-specific resolver for backward compat, then to a + // generic "No device selected" when nothing matches. const deviceLabel = studioPlayground.phase === 'error' ? 'Runtime Error' : isReady - ? resolveAndroidDeviceLabel(androidItems) + ? (() => { + const ri = studioPlayground.controller.state.runtimeInfo; + const meta = ri?.metadata || {}; + // Connected device: prefer sessionDisplayName → deviceId → displayId + if (meta.sessionDisplayName) return String(meta.sessionDisplayName); + if (meta.deviceId) return String(meta.deviceId); + if (meta.displayId) return `Display ${meta.displayId}`; + // Fallback to Android items or generic + const androidLabel = resolveAndroidDeviceLabel(androidItems); + return androidLabel !== 'No Android device selected' + ? androidLabel + : 'No device selected'; + })() : 'Playground starting'; const isConnected = isReady ? studioPlayground.controller.state.sessionViewState.connected From d0838b4ac6118dd3b9e3ea52ab047d1d5b60bda7 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Sun, 19 Apr 2026 08:02:08 +0800 Subject: [PATCH 10/13] feat(playground-app): accept initialFormValues in usePlaygroundController Lets hosts seed the session-setup form on mount (e.g. to pre-select a default platform) so the first refreshSessionSetup poll lands on a real platform instead of the generic "Choose a platform" setup, without forcing the host into a post-mount setFieldsValue workaround. --- .../src/controller/usePlaygroundController.ts | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/playground-app/src/controller/usePlaygroundController.ts b/packages/playground-app/src/controller/usePlaygroundController.ts index db75da8474..8b9f1baad0 100644 --- a/packages/playground-app/src/controller/usePlaygroundController.ts +++ b/packages/playground-app/src/controller/usePlaygroundController.ts @@ -2,7 +2,14 @@ import type { PlaygroundSessionSetup } from '@midscene/playground'; import { PlaygroundSDK } from '@midscene/playground'; import { type DeviceType, useEnvConfig } from '@midscene/visualizer'; import { Form, message } from 'antd'; -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from 'react'; import { resolveAutoCreateSessionInput } from '../session-setup'; import { buildSessionInitialValues, @@ -28,6 +35,13 @@ export interface UsePlaygroundControllerOptions { defaultDeviceType?: DeviceType; pollIntervalMs?: number; countdownSeconds?: number; + /** + * Seed values written into the session-setup form on the first render. + * Useful for pre-selecting a default platform so the initial + * `refreshSessionSetup` poll already has a `platformId`, instead of + * returning a generic "Choose a platform" setup. + */ + initialFormValues?: Record; } export function usePlaygroundController({ @@ -35,8 +49,23 @@ export function usePlaygroundController({ defaultDeviceType = 'web', pollIntervalMs = 5000, countdownSeconds = 3, + initialFormValues, }: UsePlaygroundControllerOptions): PlaygroundControllerResult { const [form] = Form.useForm(); + const initialFormValuesRef = useRef(initialFormValues); + // Seed the form ONCE before paint. Later prop changes are ignored so + // the user's in-flight edits never get overwritten. + useLayoutEffect(() => { + const seed = initialFormValuesRef.current; + if (!seed) { + return; + } + for (const [key, value] of Object.entries(seed)) { + if (form.getFieldValue(key) === undefined) { + form.setFieldsValue({ [key]: value } as Partial); + } + } + }, [form]); const formValues = (Form.useWatch([], form) ?? {}) as Record; const [countdown, setCountdown] = useState(null); const [sessionSetup, setSessionSetup] = From 1d0b8fd46315f8654df93baa74ce287cb1441314 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Sun, 19 Apr 2026 08:02:35 +0800 Subject: [PATCH 11/13] refactor(studio): clean up multi-platform playground after review - Delete legacy Android-only aliases: android-runtime.ts, AndroidPlaygroundBootstrap / AndroidPlaygroundRuntimeService types, legacy IPC_CHANNELS entries, and the restartAndroidPlayground StudioRuntimeApi / context field. - Drop unreachable unavailableReason branches in the runtime (workspace deps are always present) by driving registration from a table and letting resolveStaticDir fail fast. - Log per-platform scan failures in device-discovery via the shared getDebug logger instead of swallowing them; normalize the dynamic imports and fix DisplayInfo field usage (name/primary, not label/width). - Share StudioPlatformId + STUDIO_PLATFORM_IDS across main and renderer so bucketing can never silently drop a new platform. - Move bucketDiscoveredDevices and a new resolveConnectedDeviceLabel into selectors so the label logic is defined once and MainContent no longer re-reads runtime metadata by hand. - Seed the default platformId via usePlaygroundController's initialFormValues option, removing the post-mount setFieldsValue hack. - Gate device-discovery polling on the ready phase, flag sidebar placeholder rows with isPlaceholder so they can't get "selected", and surface an iOS setup hint when no iOS devices are discovered. - Tests: add bucket-discovered-devices.test.ts and resolveConnectedDeviceLabel cases; wire @main/@preload/@renderer/@shared aliases into vitest. --- .../src/main/playground/android-runtime.ts | 127 ------------ .../src/main/playground/device-discovery.ts | 61 +++--- .../main/playground/multi-platform-runtime.ts | 184 +++++++----------- apps/studio/src/main/playground/types.ts | 14 -- apps/studio/src/preload/index.ts | 6 - .../renderer/components/MainContent/index.tsx | 30 +-- .../src/renderer/components/Sidebar/index.tsx | 26 ++- .../playground/StudioPlaygroundProvider.tsx | 83 +++----- .../src/renderer/playground/selectors.ts | 77 +++++++- apps/studio/src/renderer/playground/types.ts | 19 +- apps/studio/src/shared/electron-contract.ts | 28 +-- .../tests/bucket-discovered-devices.test.ts | 29 +++ apps/studio/tests/electron-contract.test.ts | 9 +- .../studio/tests/playground-selectors.test.ts | 69 +++++++ apps/studio/vitest.config.ts | 9 + 15 files changed, 346 insertions(+), 425 deletions(-) delete mode 100644 apps/studio/src/main/playground/android-runtime.ts create mode 100644 apps/studio/tests/bucket-discovered-devices.test.ts diff --git a/apps/studio/src/main/playground/android-runtime.ts b/apps/studio/src/main/playground/android-runtime.ts deleted file mode 100644 index 9877a09482..0000000000 --- a/apps/studio/src/main/playground/android-runtime.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { createRequire } from 'node:module'; -import path from 'node:path'; -import { - ScrcpyServer, - androidPlaygroundPlatform, -} from '@midscene/android-playground'; -import { - type LaunchPlaygroundResult, - launchPreparedPlaygroundPlatform, -} from '@midscene/playground'; -import type { AndroidPlaygroundBootstrap } from '@shared/electron-contract'; -import { createStudioCorsOptions } from './cors'; -import type { - AndroidPlaygroundPackagePaths, - AndroidPlaygroundRuntimeService, -} from './types'; - -const require = createRequire(__filename); - -function getErrorMessage(error: unknown): string { - return error instanceof Error - ? error.message - : 'Unknown Android runtime error'; -} - -export function resolveAndroidPlaygroundPackagePaths(): AndroidPlaygroundPackagePaths { - const packageJsonPath = require.resolve( - '@midscene/android-playground/package.json', - ); - const packageRoot = path.dirname(packageJsonPath); - - return { - packageRoot, - staticDir: path.join(packageRoot, 'static'), - }; -} - -export function createAndroidPlaygroundRuntimeService(): AndroidPlaygroundRuntimeService { - let bootstrap: AndroidPlaygroundBootstrap = { - status: 'starting', - serverUrl: null, - port: null, - error: null, - }; - let launchResult: LaunchPlaygroundResult | null = null; - let startPromise: Promise | null = null; - - const close = async () => { - if (!launchResult) { - return; - } - - const activeLaunch = launchResult; - launchResult = null; - await activeLaunch.close(); - }; - - const start = async (): Promise => { - if (launchResult) { - return bootstrap; - } - - if (startPromise) { - return startPromise; - } - - bootstrap = { - status: 'starting', - serverUrl: null, - port: null, - error: null, - }; - - startPromise = (async () => { - try { - const { staticDir } = resolveAndroidPlaygroundPackagePaths(); - const scrcpyServer = new ScrcpyServer(); - const prepared = await androidPlaygroundPlatform.prepare({ - staticDir, - scrcpyServer, - }); - const nextLaunchResult = await launchPreparedPlaygroundPlatform( - prepared, - { - corsOptions: createStudioCorsOptions(), - enableCors: true, - openBrowser: false, - verbose: false, - }, - ); - - launchResult = nextLaunchResult; - bootstrap = { - status: 'ready', - serverUrl: `http://${nextLaunchResult.host}:${nextLaunchResult.port}`, - port: nextLaunchResult.port, - error: null, - }; - - return bootstrap; - } catch (error) { - bootstrap = { - status: 'error', - serverUrl: null, - port: null, - error: getErrorMessage(error), - }; - - return bootstrap; - } finally { - startPromise = null; - } - })(); - - return startPromise; - }; - - return { - close, - getBootstrap: () => bootstrap, - restart: async () => { - await close(); - return start(); - }, - start, - }; -} diff --git a/apps/studio/src/main/playground/device-discovery.ts b/apps/studio/src/main/playground/device-discovery.ts index 7250f3af71..73511fd1b6 100644 --- a/apps/studio/src/main/playground/device-discovery.ts +++ b/apps/studio/src/main/playground/device-discovery.ts @@ -1,5 +1,8 @@ +import { getDebug } from '@midscene/shared/logger'; import type { DiscoveredDevice } from '@shared/electron-contract'; +const debugLog = getDebug('studio:device-discovery', { console: true }); + /** * Scan all platforms for connected devices. Each platform's scan is * independent — a failure on one platform (e.g. `hdc` not installed) @@ -10,18 +13,18 @@ import type { DiscoveredDevice } from '@shared/electron-contract'; * user creates a session via the setup form. */ export async function discoverAllDevices(): Promise { - const results: DiscoveredDevice[] = []; - - // Run each platform scan concurrently; never let one crash the whole scan. const scans = await Promise.allSettled([ scanAndroidDevices(), scanHarmonyDevices(), scanComputerDisplays(), ]); + const results: DiscoveredDevice[] = []; for (const scan of scans) { if (scan.status === 'fulfilled') { results.push(...scan.value); + } else { + debugLog('platform scan rejected:', scan.reason); } } @@ -30,58 +33,50 @@ export async function discoverAllDevices(): Promise { async function scanAndroidDevices(): Promise { try { - // Dynamic import with `as string` to bypass TS module resolution — - // @midscene/android is externalized and resolved at runtime by Node. - const mod = await (import('@midscene/android' as string) as Promise< - typeof import('@midscene/android') - >); - const devices = await mod.getConnectedDevicesWithDetails(); + const { getConnectedDevicesWithDetails } = await import( + '@midscene/android' + ); + const devices = await getConnectedDevicesWithDetails(); return devices.map((device) => ({ - platformId: 'android' as const, + platformId: 'android', id: device.udid, label: (device as { label?: string }).label || device.udid, description: `ADB: ${device.udid}`, })); - } catch { + } catch (err) { + debugLog('android scan failed:', err); return []; } } async function scanHarmonyDevices(): Promise { try { - const mod = await import('@midscene/harmony'); - const devices = await mod.getConnectedDevices(); + const { getConnectedDevices } = await import('@midscene/harmony'); + const devices = await getConnectedDevices(); return devices.map((device) => ({ - platformId: 'harmony' as const, + platformId: 'harmony', id: device.deviceId, label: device.deviceId, description: `HDC: ${device.deviceId}`, })); - } catch { + } catch (err) { + debugLog('harmony scan failed:', err); return []; } } async function scanComputerDisplays(): Promise { try { - const mod = await import('@midscene/computer'); - const displays = await mod.getConnectedDisplays(); - return displays.map( - (display: { - id: string | number; - label?: string; - width?: number; - height?: number; - }) => ({ - platformId: 'computer' as const, - id: String(display.id), - label: display.label || `Display ${display.id}`, - description: display.width - ? `${display.width}x${display.height}` - : undefined, - }), - ); - } catch { + const { getConnectedDisplays } = await import('@midscene/computer'); + const displays = await getConnectedDisplays(); + return displays.map((display) => ({ + platformId: 'computer', + id: String(display.id), + label: display.name || `Display ${display.id}`, + description: display.primary ? 'Primary display' : undefined, + })); + } catch (err) { + debugLog('computer scan failed:', err); return []; } } diff --git a/apps/studio/src/main/playground/multi-platform-runtime.ts b/apps/studio/src/main/playground/multi-platform-runtime.ts index 64ed05d63a..646006424b 100644 --- a/apps/studio/src/main/playground/multi-platform-runtime.ts +++ b/apps/studio/src/main/playground/multi-platform-runtime.ts @@ -1,16 +1,19 @@ import { createRequire } from 'node:module'; import path from 'node:path'; -import { ScrcpyServer } from '@midscene/android-playground'; -import { androidPlaygroundPlatform } from '@midscene/android-playground'; +import { + ScrcpyServer, + androidPlaygroundPlatform, +} from '@midscene/android-playground'; import { computerPlaygroundPlatform } from '@midscene/computer-playground'; import { harmonyPlaygroundPlatform } from '@midscene/harmony'; import { iosPlaygroundPlatform } from '@midscene/ios'; import { type LaunchPlaygroundResult, + type PreparedPlaygroundPlatform, type RegisteredPlaygroundPlatform, + launchPreparedPlaygroundPlatform, prepareMultiPlatformPlayground, } from '@midscene/playground'; -import { launchPreparedPlaygroundPlatform } from '@midscene/playground'; import type { PlaygroundBootstrap } from '@shared/electron-contract'; import { createStudioCorsOptions } from './cors'; import type { PlaygroundRuntimeService } from './types'; @@ -28,128 +31,73 @@ function resolveStaticDir(packageName: string): string { return path.join(path.dirname(packageJsonPath), 'static'); } -function tryResolveStaticDir(packageName: string): string | null { - try { - return resolveStaticDir(packageName); - } catch { - return null; - } +interface StudioPlatformSpec { + id: string; + label: string; + description: string; + staticDirPackage: string; + prepare: (staticDir: string) => Promise; } -/** - * Build the list of registered platforms. Each platform is resolved lazily - * so that a missing optional package (e.g. `@midscene/ios` on a machine - * without Xcode) marks the platform unavailable instead of crashing. - */ +const studioPlatformSpecs: StudioPlatformSpec[] = [ + { + id: 'android', + label: 'Android', + description: 'Connect to an Android device via ADB', + staticDirPackage: '@midscene/android-playground', + prepare: (staticDir) => + androidPlaygroundPlatform.prepare({ + staticDir, + scrcpyServer: new ScrcpyServer(), + }), + }, + { + id: 'ios', + label: 'iOS', + description: 'Connect to an iOS device via WebDriverAgent', + staticDirPackage: '@midscene/ios', + prepare: (staticDir) => iosPlaygroundPlatform.prepare({ staticDir }), + }, + { + id: 'harmony', + label: 'HarmonyOS', + description: 'Connect to a HarmonyOS device via HDC', + staticDirPackage: '@midscene/harmony', + prepare: (staticDir) => harmonyPlaygroundPlatform.prepare({ staticDir }), + }, + { + id: 'computer', + label: 'Computer', + description: 'Control the local desktop', + staticDirPackage: '@midscene/computer-playground', + // In the Electron context, pass null — the computer agent works + // without a window controller, it just won't auto-minimize Studio + // during task execution. A follow-up can provide an Electron-native + // adapter that calls mainWindow.minimize()/restore(). + prepare: (staticDir) => + computerPlaygroundPlatform.prepare({ + staticDir, + getWindowController: () => null, + }), + }, +]; + function buildRegisteredPlatforms(): RegisteredPlaygroundPlatform[] { - const platforms: RegisteredPlaygroundPlatform[] = []; - - // ── Android ─────────────────────────────────────────────────────── - const androidStaticDir = tryResolveStaticDir('@midscene/android-playground'); - if (androidStaticDir) { - platforms.push({ - id: 'android', - label: 'Android', - description: 'Connect to an Android device via ADB', - prepare: async () => - androidPlaygroundPlatform.prepare({ - staticDir: androidStaticDir, - scrcpyServer: new ScrcpyServer(), - }), - }); - } else { - platforms.push({ - id: 'android', - label: 'Android', - unavailableReason: - '@midscene/android-playground package not found. Run pnpm install.', - prepare: async () => { - throw new Error('Android platform is not available'); - }, - }); - } - - // ── iOS ─────────────────────────────────────────────────────────── - const iosStaticDir = tryResolveStaticDir('@midscene/ios'); - if (iosStaticDir) { - platforms.push({ - id: 'ios', - label: 'iOS', - description: 'Connect to an iOS device via WebDriverAgent', - prepare: async () => - iosPlaygroundPlatform.prepare({ staticDir: iosStaticDir }), - }); - } else { - platforms.push({ - id: 'ios', - label: 'iOS', - unavailableReason: '@midscene/ios package not found. Run pnpm install.', - prepare: async () => { - throw new Error('iOS platform is not available'); - }, - }); - } - - // ── HarmonyOS ───────────────────────────────────────────────────── - const harmonyStaticDir = tryResolveStaticDir('@midscene/harmony'); - if (harmonyStaticDir) { - platforms.push({ - id: 'harmony', - label: 'HarmonyOS', - description: 'Connect to a HarmonyOS device via HDC', - prepare: async () => - harmonyPlaygroundPlatform.prepare({ staticDir: harmonyStaticDir }), - }); - } else { - platforms.push({ - id: 'harmony', - label: 'HarmonyOS', - unavailableReason: - '@midscene/harmony package not found. Run pnpm install.', - prepare: async () => { - throw new Error('HarmonyOS platform is not available'); - }, - }); - } - - // ── Computer ────────────────────────────────────────────────────── - const computerStaticDir = tryResolveStaticDir( - '@midscene/computer-playground', - ); - if (computerStaticDir) { - platforms.push({ - id: 'computer', - label: 'Computer', - description: 'Control the local desktop', - prepare: async () => - computerPlaygroundPlatform.prepare({ - staticDir: computerStaticDir, - // In the Electron context, pass null — the computer agent works - // without window controller, it just won't auto-minimize Studio - // during task execution. A follow-up can provide an Electron- - // native adapter that calls mainWindow.minimize()/restore(). - getWindowController: () => null, - }), - }); - } else { - platforms.push({ - id: 'computer', - label: 'Computer', - unavailableReason: - '@midscene/computer-playground package not found. Run pnpm install.', - prepare: async () => { - throw new Error('Computer platform is not available'); - }, - }); - } - - return platforms; + return studioPlatformSpecs.map((spec) => { + const staticDir = resolveStaticDir(spec.staticDirPackage); + return { + id: spec.id, + label: spec.label, + description: spec.description, + prepare: () => spec.prepare(staticDir), + }; + }); } /** * Creates a multi-platform playground runtime service for the Studio - * Electron main process. On `start()`, it registers all available - * platforms (Android, iOS, HarmonyOS, Computer) with + * Electron main process. On `start()`, it registers all platforms + * (Android, iOS, HarmonyOS, Computer) with * `prepareMultiPlatformPlayground` and launches a SINGLE unified HTTP * server. The renderer talks to this one server; the platform selector * on the setup form routes to the correct backend. diff --git a/apps/studio/src/main/playground/types.ts b/apps/studio/src/main/playground/types.ts index 2a17b6ab12..5272b2ac59 100644 --- a/apps/studio/src/main/playground/types.ts +++ b/apps/studio/src/main/playground/types.ts @@ -1,13 +1,5 @@ import type { PlaygroundBootstrap } from '@shared/electron-contract'; -// Keep the old alias — android-runtime.ts still uses it internally. -export type { PlaygroundBootstrap as AndroidPlaygroundBootstrap } from '@shared/electron-contract'; - -export interface AndroidPlaygroundPackagePaths { - packageRoot: string; - staticDir: string; -} - /** Lifecycle contract for a playground runtime (single- or multi-platform). */ export interface PlaygroundRuntimeService { close: () => Promise; @@ -15,9 +7,3 @@ export interface PlaygroundRuntimeService { restart: () => Promise; start: () => Promise; } - -/** - * @deprecated Use {@link PlaygroundRuntimeService} — this alias exists so - * `android-runtime.ts` compiles without changes during the migration. - */ -export type AndroidPlaygroundRuntimeService = PlaygroundRuntimeService; diff --git a/apps/studio/src/preload/index.ts b/apps/studio/src/preload/index.ts index f490bae860..dc00d1f376 100644 --- a/apps/studio/src/preload/index.ts +++ b/apps/studio/src/preload/index.ts @@ -20,16 +20,10 @@ const electronShellApi: ElectronShellApi = { }; const studioRuntimeApi: StudioRuntimeApi = { - // Multi-platform playground getPlaygroundBootstrap: () => ipcRenderer.invoke(IPC_CHANNELS.getPlaygroundBootstrap), restartPlayground: () => ipcRenderer.invoke(IPC_CHANNELS.restartPlayground), discoverDevices: () => ipcRenderer.invoke(IPC_CHANNELS.discoverDevices), - // Legacy aliases — both resolve to the same IPC channel - getAndroidPlaygroundBootstrap: () => - ipcRenderer.invoke(IPC_CHANNELS.getPlaygroundBootstrap), - restartAndroidPlayground: () => - ipcRenderer.invoke(IPC_CHANNELS.restartPlayground), runConnectivityTest: (request) => ipcRenderer.invoke(IPC_CHANNELS.runConnectivityTest, request), }; diff --git a/apps/studio/src/renderer/components/MainContent/index.tsx b/apps/studio/src/renderer/components/MainContent/index.tsx index 460cf4b9ca..5dc1992238 100644 --- a/apps/studio/src/renderer/components/MainContent/index.tsx +++ b/apps/studio/src/renderer/components/MainContent/index.tsx @@ -2,10 +2,9 @@ import { PlaygroundPreview } from '@midscene/playground-app'; import { useEffect, useRef, useState } from 'react'; import { assetUrls } from '../../assets'; import { - buildAndroidDeviceItems, buildStudioSidebarDeviceBuckets, - resolveAndroidDeviceLabel, resolveConnectedAndroidDeviceId, + resolveConnectedDeviceLabel, resolveSelectedAndroidDeviceId, } from '../../playground/selectors'; import { useStudioPlayground } from '../../playground/useStudioPlayground'; @@ -121,33 +120,14 @@ export default function MainContent({ useState(null); const [overviewRefreshing, setOverviewRefreshing] = useState(false); const isReady = studioPlayground.phase === 'ready'; - const androidItems = isReady - ? buildAndroidDeviceItems({ - formValues: studioPlayground.controller.state.formValues, - runtimeInfo: studioPlayground.controller.state.runtimeInfo, - targets: studioPlayground.controller.state.sessionSetup?.targets || [], - }) - : []; - // Resolve device label from runtimeInfo (platform-agnostic). Falls back - // to the Android-specific resolver for backward compat, then to a - // generic "No device selected" when nothing matches. const deviceLabel = studioPlayground.phase === 'error' ? 'Runtime Error' : isReady - ? (() => { - const ri = studioPlayground.controller.state.runtimeInfo; - const meta = ri?.metadata || {}; - // Connected device: prefer sessionDisplayName → deviceId → displayId - if (meta.sessionDisplayName) return String(meta.sessionDisplayName); - if (meta.deviceId) return String(meta.deviceId); - if (meta.displayId) return `Display ${meta.displayId}`; - // Fallback to Android items or generic - const androidLabel = resolveAndroidDeviceLabel(androidItems); - return androidLabel !== 'No Android device selected' - ? androidLabel - : 'No device selected'; - })() + ? resolveConnectedDeviceLabel( + studioPlayground.controller.state.runtimeInfo, + { emptyLabel: 'No device selected' }, + ) : 'Playground starting'; const isConnected = isReady ? studioPlayground.controller.state.sessionViewState.connected diff --git a/apps/studio/src/renderer/components/Sidebar/index.tsx b/apps/studio/src/renderer/components/Sidebar/index.tsx index 8523dafe6b..7b27b37cae 100644 --- a/apps/studio/src/renderer/components/Sidebar/index.tsx +++ b/apps/studio/src/renderer/components/Sidebar/index.tsx @@ -16,6 +16,8 @@ interface DeviceItem { label: string; status: DeviceStatus; onClick?: () => void | Promise; + /** Purely informational rows that should never appear "selected". */ + isPlaceholder?: boolean; } interface SectionDefinition { @@ -240,12 +242,27 @@ export default function Sidebar({ ? 'Playground starting' : 'Runtime failed to start', status: 'idle' as const, + isPlaceholder: true, }, ]; } return []; } + // iOS discovery needs WebDriverAgent running, which is a manual + // setup step; surface a hint row instead of an empty section so + // users know it isn't a bug. + if (platformKey === 'ios' && devices.length === 0) { + return [ + { + id: 'ios-setup-hint', + label: 'Set up iOS via the playground form', + status: 'idle' as const, + isPlaceholder: true, + }, + ]; + } + return devices.map((item) => ({ id: item.id, label: item.label, @@ -290,7 +307,7 @@ export default function Sidebar({ .filter((item) => item.selected) .map((item) => item.id), ) - : new Set(['android-placeholder']); + : new Set(); const totalDeviceCount = sectionDefinitions.reduce( (sum, section) => sum + deviceBuckets[section.key].length, @@ -353,13 +370,12 @@ export default function Sidebar({ {isExpanded ? ( hasDevices ? ( - section.devices.map((device, index) => ( + section.devices.map((device) => ( diff --git a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx index 8ef8d3c0e2..3f362eaf43 100644 --- a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx +++ b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx @@ -3,21 +3,13 @@ import { usePlaygroundController, } from '@midscene/playground-app'; import type { - DiscoveredDevice, PlaygroundBootstrap, + StudioPlatformId, } from '@shared/electron-contract'; import type { PropsWithChildren } from 'react'; -import { - useCallback, - useEffect, - useLayoutEffect, - useMemo, - useState, -} from 'react'; -import type { - DiscoveredDevicesByPlatform, - StudioSidebarPlatformKey, -} from './types'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { bucketDiscoveredDevices } from './selectors'; +import type { DiscoveredDevicesByPlatform } from './types'; import { StudioPlaygroundContext } from './useStudioPlayground'; function getMissingBridgeError() { @@ -28,6 +20,11 @@ function normalizeBootstrapError(bootstrap: PlaygroundBootstrap): string { return bootstrap.error || 'Failed to start playground runtime.'; } +// Default platform for Studio — pre-selected so the first session-setup +// poll already has a `platformId` and immediately returns Android +// targets, instead of the generic "Choose a platform" setup. +const DEFAULT_PLATFORM_ID: StudioPlatformId = 'android'; + function ReadyStudioPlaygroundProvider({ children, discoveredDevices, @@ -40,28 +37,15 @@ function ReadyStudioPlaygroundProvider({ }>) { const controller = usePlaygroundController({ serverUrl, + initialFormValues: { platformId: DEFAULT_PLATFORM_ID }, }); - // Pre-select "android" as the default platform so the very first - // `refreshSessionSetup` poll sees a `platformId` in the form values - // and the multi-platform session manager resolves Android targets - // immediately. Without this, the initial poll returns "Choose a - // platform" with no targets and the sidebar stays empty until the - // user manually picks a platform in the setup form. - useLayoutEffect(() => { - const currentPlatformId = controller.state.form.getFieldValue('platformId'); - if (!currentPlatformId) { - controller.state.form.setFieldsValue({ platformId: 'android' }); - } - }, [controller.state.form]); - const contextValue = useMemo( () => ({ phase: 'ready' as const, serverUrl, controller, restartPlayground, - restartAndroidPlayground: restartPlayground, discoveredDevices, }), [controller, discoveredDevices, restartPlayground, serverUrl], @@ -74,24 +58,7 @@ function ReadyStudioPlaygroundProvider({ ); } -function bucketDiscoveredDevices( - devices: DiscoveredDevice[], -): DiscoveredDevicesByPlatform { - const buckets: DiscoveredDevicesByPlatform = { - android: [], - ios: [], - computer: [], - harmony: [], - web: [], - }; - for (const device of devices) { - const key = device.platformId as StudioSidebarPlatformKey; - if (buckets[key]) { - buckets[key].push(device); - } - } - return buckets; -} +const DISCOVERY_POLL_INTERVAL_MS = 5000; export function StudioPlaygroundProvider({ children }: PropsWithChildren) { const [bootstrap, setBootstrap] = useState< @@ -102,29 +69,37 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { const [bootstrapTick, setBootstrapTick] = useState(0); // Cross-platform device discovery — polls ALL platforms (ADB, HDC, - // displays) independently of the session manager so the sidebar - // shows devices from every platform simultaneously. + // displays) once the playground runtime is ready, so the sidebar shows + // devices from every platform simultaneously. Gated on the ready phase + // to avoid waking up adb/hdc while we are still booting or in error. const [discoveredDevices, setDiscoveredDevices] = useState< DiscoveredDevicesByPlatform | undefined >(); + const pollingActive = bootstrap.phase === 'ready'; useEffect(() => { + if (!pollingActive || !window.studioRuntime?.discoverDevices) { + return; + } let cancelled = false; const poll = async () => { - if (!window.studioRuntime?.discoverDevices) return; try { - const devices = await window.studioRuntime.discoverDevices(); - if (!cancelled) setDiscoveredDevices(bucketDiscoveredDevices(devices)); - } catch { - // Silent — device discovery is best-effort. + const devices = await window.studioRuntime!.discoverDevices(); + if (!cancelled) { + setDiscoveredDevices(bucketDiscoveredDevices(devices)); + } + } catch (err) { + if (!cancelled) { + console.warn('[studio] device discovery failed:', err); + } } }; void poll(); - const id = window.setInterval(poll, 5000); + const id = window.setInterval(poll, DISCOVERY_POLL_INTERVAL_MS); return () => { cancelled = true; window.clearInterval(id); }; - }, []); + }, [pollingActive]); const readBootstrap = useCallback(async () => { if (!window.studioRuntime) { @@ -217,7 +192,6 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { phase: 'error' as const, error: bootstrap.error, restartPlayground, - restartAndroidPlayground: restartPlayground, discoveredDevices, }; } @@ -225,7 +199,6 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { return { phase: 'booting' as const, restartPlayground, - restartAndroidPlayground: restartPlayground, discoveredDevices, }; }, [bootstrap, discoveredDevices, restartPlayground]); diff --git a/apps/studio/src/renderer/playground/selectors.ts b/apps/studio/src/renderer/playground/selectors.ts index 7600ee637d..967d6c8cfb 100644 --- a/apps/studio/src/renderer/playground/selectors.ts +++ b/apps/studio/src/renderer/playground/selectors.ts @@ -2,7 +2,10 @@ import type { PlaygroundRuntimeInfo, PlaygroundSessionTarget, } from '@midscene/playground'; +import type { DiscoveredDevice } from '@shared/electron-contract'; +import { STUDIO_PLATFORM_IDS } from '@shared/electron-contract'; import type { + DiscoveredDevicesByPlatform, StudioAndroidDeviceItem, StudioSidebarDeviceBuckets, StudioSidebarPlatformKey, @@ -52,20 +55,57 @@ function normalizeSidebarPlatformKey( } } +/** + * Platforms use different metadata keys for the device id: + * Android / Harmony → metadata.deviceId + * Computer → metadata.displayId + */ +function resolveConnectedDeviceId( + runtimeInfo: PlaygroundRuntimeInfo | null, +): string | undefined { + const metadata = runtimeInfo?.metadata || {}; + if (isString(metadata.deviceId)) { + return metadata.deviceId; + } + if (isString(metadata.displayId)) { + return metadata.displayId; + } + return undefined; +} + +/** + * Human-readable label for whatever device the playground is currently + * connected to, across platforms. Prefers the session display name, then + * falls back to a concrete device id, then to the platform title, and + * finally to `emptyLabel` when nothing is connected. + */ +export function resolveConnectedDeviceLabel( + runtimeInfo: PlaygroundRuntimeInfo | null, + options: { emptyLabel: string }, +): string { + const metadata = runtimeInfo?.metadata || {}; + if (isString(metadata.sessionDisplayName)) { + return metadata.sessionDisplayName; + } + const deviceId = resolveConnectedDeviceId(runtimeInfo); + if (deviceId) { + // "Display 1" reads better than a bare numeric id for computer. + return isString(metadata.displayId) && !isString(metadata.deviceId) + ? `Display ${deviceId}` + : deviceId; + } + if (isString(runtimeInfo?.title)) { + return runtimeInfo.title; + } + return options.emptyLabel; +} + function buildGenericConnectedDeviceItem( runtimeInfo: PlaygroundRuntimeInfo | null, platformKey: Exclude, ): StudioAndroidDeviceItem | null { const metadata = runtimeInfo?.metadata || {}; - // Different platforms use different metadata keys for the device id: - // Android/Harmony → metadata.deviceId, Computer → metadata.displayId. - const deviceId = isString(metadata.deviceId) - ? metadata.deviceId - : isString(metadata.displayId) - ? metadata.displayId - : undefined; - // Prefer the concrete device id over the platform title (which is - // something generic like "Midscene HarmonyOS Playground"). + const deviceId = resolveConnectedDeviceId(runtimeInfo); const label = isString(metadata.sessionDisplayName) ? metadata.sessionDisplayName : deviceId || @@ -187,6 +227,25 @@ export function resolveVisibleSidebarPlatforms( .map(([platformKey]) => platformKey); } +/** + * Bucket a flat discovery result by platform, driven off the canonical + * platform id set so a new platform can't be silently dropped. + */ +export function bucketDiscoveredDevices( + devices: DiscoveredDevice[], +): DiscoveredDevicesByPlatform { + const buckets = Object.fromEntries( + STUDIO_PLATFORM_IDS.map((key) => [key, [] as DiscoveredDevice[]]), + ) as DiscoveredDevicesByPlatform; + for (const device of devices) { + const bucket = buckets[device.platformId]; + if (bucket) { + bucket.push(device); + } + } + return buckets; +} + export function resolveAndroidDeviceLabel( items: StudioAndroidDeviceItem[], ): string { diff --git a/apps/studio/src/renderer/playground/types.ts b/apps/studio/src/renderer/playground/types.ts index 03affedba5..9979417113 100644 --- a/apps/studio/src/renderer/playground/types.ts +++ b/apps/studio/src/renderer/playground/types.ts @@ -1,12 +1,10 @@ import type { PlaygroundControllerResult } from '@midscene/playground-app'; -import type { DiscoveredDevice } from '@shared/electron-contract'; +import type { + DiscoveredDevice, + StudioPlatformId, +} from '@shared/electron-contract'; -export type StudioSidebarPlatformKey = - | 'android' - | 'ios' - | 'computer' - | 'harmony' - | 'web'; +export type StudioSidebarPlatformKey = StudioPlatformId; export interface StudioAndroidDeviceItem { id: string; @@ -14,6 +12,7 @@ export interface StudioAndroidDeviceItem { description?: string; selected: boolean; status: 'active' | 'idle'; + isPlaceholder?: boolean; } export type StudioSidebarDeviceBuckets = Record< @@ -31,16 +30,12 @@ export type StudioPlaygroundContextValue = | { phase: 'booting'; restartPlayground: () => Promise; - /** @deprecated Use restartPlayground */ - restartAndroidPlayground: () => Promise; discoveredDevices?: DiscoveredDevicesByPlatform; } | { phase: 'error'; error: string; restartPlayground: () => Promise; - /** @deprecated Use restartPlayground */ - restartAndroidPlayground: () => Promise; discoveredDevices?: DiscoveredDevicesByPlatform; } | { @@ -48,7 +43,5 @@ export type StudioPlaygroundContextValue = serverUrl: string; controller: PlaygroundControllerResult; restartPlayground: () => Promise; - /** @deprecated Use restartPlayground */ - restartAndroidPlayground: () => Promise; discoveredDevices?: DiscoveredDevicesByPlatform; }; diff --git a/apps/studio/src/shared/electron-contract.ts b/apps/studio/src/shared/electron-contract.ts index 0647e8e54d..132e629a6c 100644 --- a/apps/studio/src/shared/electron-contract.ts +++ b/apps/studio/src/shared/electron-contract.ts @@ -8,17 +8,13 @@ export const IPC_CHANNELS = { minimizeWindow: 'shell:minimize-window', openExternalUrl: 'shell:open-external-url', toggleMaximizeWindow: 'shell:toggle-maximize-window', - // Multi-platform playground — replaces the Android-only channels below. + // Multi-platform playground runtime (Android, iOS, HarmonyOS, Computer). getPlaygroundBootstrap: 'studio:get-playground-bootstrap', restartPlayground: 'studio:restart-playground', // Cross-platform device discovery — returns devices from ALL platforms // at once (Android via ADB, Harmony via HDC, Computer via display // enumeration). Independent of session manager. discoverDevices: 'studio:discover-devices', - // Legacy aliases — kept so renderer code that hasn't migrated yet keeps - // working. Both resolve to the same multi-platform runtime in main. - getAndroidPlaygroundBootstrap: 'studio:get-playground-bootstrap', - restartAndroidPlayground: 'studio:restart-playground', runConnectivityTest: 'studio:run-connectivity-test', } as const; @@ -40,12 +36,24 @@ export interface PlaygroundBootstrap { error: string | null; } -/** @deprecated Use {@link PlaygroundBootstrap} instead. */ -export type AndroidPlaygroundBootstrap = PlaygroundBootstrap; +/** + * Canonical set of platform identifiers the Studio shell understands. + * Shared between main process discovery and renderer sidebar/buckets so + * neither side can drift into using a stringly-typed platform id. + */ +export const STUDIO_PLATFORM_IDS = [ + 'android', + 'ios', + 'computer', + 'harmony', + 'web', +] as const; + +export type StudioPlatformId = (typeof STUDIO_PLATFORM_IDS)[number]; /** A device discovered across any platform, tagged with its platform. */ export interface DiscoveredDevice { - platformId: 'android' | 'ios' | 'harmony' | 'computer'; + platformId: StudioPlatformId; id: string; label: string; description?: string; @@ -80,10 +88,6 @@ export interface StudioRuntimeApi { restartPlayground: () => Promise; /** Scan ALL platforms for connected devices (ADB, HDC, displays). */ discoverDevices: () => Promise; - /** @deprecated Use {@link getPlaygroundBootstrap}. */ - getAndroidPlaygroundBootstrap: () => Promise; - /** @deprecated Use {@link restartPlayground}. */ - restartAndroidPlayground: () => Promise; runConnectivityTest: ( request: ConnectivityTestRequest, ) => Promise; diff --git a/apps/studio/tests/bucket-discovered-devices.test.ts b/apps/studio/tests/bucket-discovered-devices.test.ts new file mode 100644 index 0000000000..29ce67bcab --- /dev/null +++ b/apps/studio/tests/bucket-discovered-devices.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest'; +import { bucketDiscoveredDevices } from '../src/renderer/playground/selectors'; + +describe('bucketDiscoveredDevices', () => { + it('returns empty buckets for every platform when no devices are discovered', () => { + expect(bucketDiscoveredDevices([])).toEqual({ + android: [], + ios: [], + computer: [], + harmony: [], + web: [], + }); + }); + + it('groups devices by their platformId tag', () => { + const result = bucketDiscoveredDevices([ + { platformId: 'android', id: 'a1', label: 'Pixel' }, + { platformId: 'harmony', id: 'h1', label: 'Huawei P70' }, + { platformId: 'android', id: 'a2', label: 'Galaxy' }, + { platformId: 'computer', id: 'c1', label: 'Display 1' }, + ]); + + expect(result.android.map((d) => d.id)).toEqual(['a1', 'a2']); + expect(result.harmony.map((d) => d.id)).toEqual(['h1']); + expect(result.computer.map((d) => d.id)).toEqual(['c1']); + expect(result.ios).toEqual([]); + expect(result.web).toEqual([]); + }); +}); diff --git a/apps/studio/tests/electron-contract.test.ts b/apps/studio/tests/electron-contract.test.ts index 822c3e8af8..97c973a399 100644 --- a/apps/studio/tests/electron-contract.test.ts +++ b/apps/studio/tests/electron-contract.test.ts @@ -4,17 +4,10 @@ import { IPC_CHANNELS } from '../src/shared/electron-contract'; describe('IPC_CHANNELS', () => { it('includes shell and playground bridge channels', () => { expect(IPC_CHANNELS.openExternalUrl).toBe('shell:open-external-url'); - // Multi-platform playground channels expect(IPC_CHANNELS.getPlaygroundBootstrap).toBe( 'studio:get-playground-bootstrap', ); expect(IPC_CHANNELS.restartPlayground).toBe('studio:restart-playground'); - // Legacy aliases resolve to the same channel names - expect(IPC_CHANNELS.getAndroidPlaygroundBootstrap).toBe( - IPC_CHANNELS.getPlaygroundBootstrap, - ); - expect(IPC_CHANNELS.restartAndroidPlayground).toBe( - IPC_CHANNELS.restartPlayground, - ); + expect(IPC_CHANNELS.discoverDevices).toBe('studio:discover-devices'); }); }); diff --git a/apps/studio/tests/playground-selectors.test.ts b/apps/studio/tests/playground-selectors.test.ts index ba128588ca..6e01deecef 100644 --- a/apps/studio/tests/playground-selectors.test.ts +++ b/apps/studio/tests/playground-selectors.test.ts @@ -3,6 +3,7 @@ import { buildAndroidDeviceItems, buildStudioSidebarDeviceBuckets, resolveAndroidDeviceLabel, + resolveConnectedDeviceLabel, resolveVisibleSidebarPlatforms, } from '../src/renderer/playground/selectors'; @@ -196,3 +197,71 @@ describe('resolveAndroidDeviceLabel', () => { ).toBe('Pixel 8'); }); }); + +describe('resolveConnectedDeviceLabel', () => { + const emptyOpts = { emptyLabel: 'No device' }; + + it('returns emptyLabel when no runtime info is available', () => { + expect(resolveConnectedDeviceLabel(null, emptyOpts)).toBe('No device'); + }); + + it('prefers sessionDisplayName over raw device id', () => { + expect( + resolveConnectedDeviceLabel( + { + interface: { type: 'android' }, + preview: { kind: 'none', capabilities: [] }, + executionUxHints: [], + metadata: { + deviceId: 'emulator-5554', + sessionDisplayName: 'Pixel 9', + }, + }, + emptyOpts, + ), + ).toBe('Pixel 9'); + }); + + it('falls back to deviceId when sessionDisplayName is absent', () => { + expect( + resolveConnectedDeviceLabel( + { + interface: { type: 'android' }, + preview: { kind: 'none', capabilities: [] }, + executionUxHints: [], + metadata: { deviceId: 'emulator-5554' }, + }, + emptyOpts, + ), + ).toBe('emulator-5554'); + }); + + it('labels computer displays using the displayId', () => { + expect( + resolveConnectedDeviceLabel( + { + interface: { type: 'computer' }, + preview: { kind: 'none', capabilities: [] }, + executionUxHints: [], + metadata: { displayId: '1' }, + }, + emptyOpts, + ), + ).toBe('Display 1'); + }); + + it('falls back to runtime title when no device metadata is present', () => { + expect( + resolveConnectedDeviceLabel( + { + title: 'Midscene Playground', + interface: { type: 'android' }, + preview: { kind: 'none', capabilities: [] }, + executionUxHints: [], + metadata: {}, + }, + emptyOpts, + ), + ).toBe('Midscene Playground'); + }); +}); diff --git a/apps/studio/vitest.config.ts b/apps/studio/vitest.config.ts index 0d5f95a1f3..6187f6aa17 100644 --- a/apps/studio/vitest.config.ts +++ b/apps/studio/vitest.config.ts @@ -1,6 +1,15 @@ +import path from 'node:path'; import { defineConfig } from 'vitest/config'; export default defineConfig({ + resolve: { + alias: { + '@main': path.resolve(__dirname, 'src/main'), + '@preload': path.resolve(__dirname, 'src/preload'), + '@renderer': path.resolve(__dirname, 'src/renderer'), + '@shared': path.resolve(__dirname, 'src/shared'), + }, + }, test: { environment: 'node', environmentMatchGlobs: [['tests/theme-provider.test.ts', 'jsdom']], From 80b7d9a066ae61203c2ce33224e9d8518955680e Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Mon, 20 Apr 2026 10:26:39 +0800 Subject: [PATCH 12/13] fix(studio): drop unplugged devices from sidebar and wire refresh button MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: after unplugging a phone while the playground session was still "connected", the device kept showing in the sidebar. The controller stops polling session-setup while connected, so the stale target hung around, and the sidebar merge only appended new discovered devices — it never removed session items that had disappeared. Fix: - Add mergeSidebarDeviceBucketsWithDiscovery selector that treats the live discovery snapshot as source of truth for ADB/HDC/display platforms: items not in the current discovery bucket are dropped (catching unplug), items only in discovery are appended as idle rows. iOS and web pass through unchanged since they have no discovery source. - Expose refreshDiscoveredDevices on StudioPlaygroundContext and run it alongside refreshSessionSetup when the overview refresh button fires, so a user-initiated refresh hits both data sources. - Share the polling callback with the imperative refresh so we only have one discovery code path. --- .../renderer/components/MainContent/index.tsx | 20 +++- .../src/renderer/components/Sidebar/index.tsx | 36 ++----- .../playground/StudioPlaygroundProvider.tsx | 60 ++++++++---- .../src/renderer/playground/selectors.ts | 62 ++++++++++++ apps/studio/src/renderer/playground/types.ts | 3 + .../studio/tests/playground-selectors.test.ts | 96 +++++++++++++++++++ 6 files changed, 225 insertions(+), 52 deletions(-) diff --git a/apps/studio/src/renderer/components/MainContent/index.tsx b/apps/studio/src/renderer/components/MainContent/index.tsx index 5dc1992238..a735c6ae01 100644 --- a/apps/studio/src/renderer/components/MainContent/index.tsx +++ b/apps/studio/src/renderer/components/MainContent/index.tsx @@ -3,6 +3,7 @@ import { useEffect, useRef, useState } from 'react'; import { assetUrls } from '../../assets'; import { buildStudioSidebarDeviceBuckets, + mergeSidebarDeviceBucketsWithDiscovery, resolveConnectedAndroidDeviceId, resolveConnectedDeviceLabel, resolveSelectedAndroidDeviceId, @@ -201,7 +202,7 @@ export default function MainContent({ ); } - const overviewBuckets = isReady + const overviewSessionBuckets = isReady ? buildStudioSidebarDeviceBuckets({ formValues: studioPlayground.controller.state.formValues, runtimeInfo: studioPlayground.controller.state.runtimeInfo, @@ -209,6 +210,10 @@ export default function MainContent({ studioPlayground.controller.state.sessionSetup?.targets || [], }) : { android: [], ios: [], computer: [], harmony: [], web: [] }; + const overviewBuckets = mergeSidebarDeviceBucketsWithDiscovery( + overviewSessionBuckets, + studioPlayground.discoveredDevices, + ); const overviewSelectedDeviceId = isReady && studioPlayground.controller.state.sessionMutating ? selectedAndroidDeviceId @@ -223,9 +228,16 @@ export default function MainContent({ } setOverviewRefreshing(true); try { - await studioPlayground.controller.actions.refreshSessionSetup( - studioPlayground.controller.state.formValues, - ); + // Refresh BOTH sources: session-setup targets (server-side + // list) and cross-platform discovery (ADB/HDC/displays). + // Discovery is what surfaces an unplug while a session is + // still technically "connected" on the server. + await Promise.all([ + studioPlayground.controller.actions.refreshSessionSetup( + studioPlayground.controller.state.formValues, + ), + studioPlayground.refreshDiscoveredDevices(), + ]); } finally { setOverviewRefreshing(false); } diff --git a/apps/studio/src/renderer/components/Sidebar/index.tsx b/apps/studio/src/renderer/components/Sidebar/index.tsx index 7b27b37cae..01e6f42fda 100644 --- a/apps/studio/src/renderer/components/Sidebar/index.tsx +++ b/apps/studio/src/renderer/components/Sidebar/index.tsx @@ -2,6 +2,7 @@ import { useState } from 'react'; import { assetUrls } from '../../assets'; import { buildStudioSidebarDeviceBuckets, + mergeSidebarDeviceBucketsWithDiscovery, resolveConnectedAndroidDeviceId, } from '../../playground/selectors'; import type { StudioSidebarPlatformKey } from '../../playground/types'; @@ -170,9 +171,9 @@ export default function Sidebar({ }; // Device buckets: merge session-setup targets (from the currently - // selected platform) with cross-platform discovered devices so the - // sidebar shows ALL connected devices across platforms, not just the - // selected one's targets. + // selected platform) with cross-platform discovered devices. Discovery + // is the source of truth for platforms that support it (ADB/HDC/ + // displays) — this is what makes an unplug disappear from the list. const sessionBuckets = studioPlayground.phase === 'ready' ? buildStudioSidebarDeviceBuckets({ @@ -189,31 +190,10 @@ export default function Sidebar({ web: [], }; - const discovered = studioPlayground.discoveredDevices; - const deviceBuckets = { ...sessionBuckets }; - if (discovered) { - for (const key of Object.keys(discovered) as Array< - keyof typeof discovered - >) { - const discoveredIds = new Set(discovered[key].map((d) => d.id)); - const sessionIds = new Set(sessionBuckets[key].map((d) => d.id)); - // Add discovered devices that aren't already in the session bucket - for (const dev of discovered[key]) { - if (!sessionIds.has(dev.id)) { - deviceBuckets[key] = [ - ...deviceBuckets[key], - { - id: dev.id, - label: dev.label, - description: dev.description, - selected: false, - status: 'idle' as const, - }, - ]; - } - } - } - } + const deviceBuckets = mergeSidebarDeviceBucketsWithDiscovery( + sessionBuckets, + studioPlayground.discoveredDevices, + ); const connectedDeviceId = studioPlayground.phase === 'ready' diff --git a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx index 3f362eaf43..13f5acad43 100644 --- a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx +++ b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx @@ -28,10 +28,12 @@ const DEFAULT_PLATFORM_ID: StudioPlatformId = 'android'; function ReadyStudioPlaygroundProvider({ children, discoveredDevices, + refreshDiscoveredDevices, restartPlayground, serverUrl, }: PropsWithChildren<{ discoveredDevices?: DiscoveredDevicesByPlatform; + refreshDiscoveredDevices: () => Promise; restartPlayground: () => Promise; serverUrl: string; }>) { @@ -46,9 +48,16 @@ function ReadyStudioPlaygroundProvider({ serverUrl, controller, restartPlayground, + refreshDiscoveredDevices, discoveredDevices, }), - [controller, discoveredDevices, restartPlayground, serverUrl], + [ + controller, + discoveredDevices, + refreshDiscoveredDevices, + restartPlayground, + serverUrl, + ], ); return ( @@ -75,31 +84,34 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { const [discoveredDevices, setDiscoveredDevices] = useState< DiscoveredDevicesByPlatform | undefined >(); + + // Imperative scan — safe to call from anywhere (user-initiated refresh, + // post-destroy session cleanup, etc). Resolves after state is updated. + const refreshDiscoveredDevices = useCallback(async () => { + if (!window.studioRuntime?.discoverDevices) { + return; + } + try { + const devices = await window.studioRuntime.discoverDevices(); + setDiscoveredDevices(bucketDiscoveredDevices(devices)); + } catch (err) { + console.warn('[studio] device discovery failed:', err); + } + }, []); + const pollingActive = bootstrap.phase === 'ready'; useEffect(() => { - if (!pollingActive || !window.studioRuntime?.discoverDevices) { + if (!pollingActive) { return; } - let cancelled = false; - const poll = async () => { - try { - const devices = await window.studioRuntime!.discoverDevices(); - if (!cancelled) { - setDiscoveredDevices(bucketDiscoveredDevices(devices)); - } - } catch (err) { - if (!cancelled) { - console.warn('[studio] device discovery failed:', err); - } - } - }; - void poll(); - const id = window.setInterval(poll, DISCOVERY_POLL_INTERVAL_MS); + void refreshDiscoveredDevices(); + const id = window.setInterval(() => { + void refreshDiscoveredDevices(); + }, DISCOVERY_POLL_INTERVAL_MS); return () => { - cancelled = true; window.clearInterval(id); }; - }, [pollingActive]); + }, [pollingActive, refreshDiscoveredDevices]); const readBootstrap = useCallback(async () => { if (!window.studioRuntime) { @@ -192,6 +204,7 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { phase: 'error' as const, error: bootstrap.error, restartPlayground, + refreshDiscoveredDevices, discoveredDevices, }; } @@ -199,15 +212,22 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { return { phase: 'booting' as const, restartPlayground, + refreshDiscoveredDevices, discoveredDevices, }; - }, [bootstrap, discoveredDevices, restartPlayground]); + }, [ + bootstrap, + discoveredDevices, + refreshDiscoveredDevices, + restartPlayground, + ]); return ( {bootstrap.phase === 'ready' ? ( diff --git a/apps/studio/src/renderer/playground/selectors.ts b/apps/studio/src/renderer/playground/selectors.ts index 967d6c8cfb..af63b18c3f 100644 --- a/apps/studio/src/renderer/playground/selectors.ts +++ b/apps/studio/src/renderer/playground/selectors.ts @@ -246,6 +246,68 @@ export function bucketDiscoveredDevices( return buckets; } +/** + * Platforms that expose a real-time discovery source (ADB / HDC / + * display enumeration) in the main process. For these, discovery is + * the authoritative "physically present" list — once it has polled at + * least once, a session item that isn't discovered anymore is stale + * (e.g. the user unplugged the device while a session was still open). + */ +const DISCOVERABLE_PLATFORMS = ['android', 'harmony', 'computer'] as const; + +/** + * Merge session-setup buckets with the live discovery snapshot. For + * discoverable platforms, discovery wins — items not present in the + * discovery snapshot are dropped (catches unplug while connected), and + * items only present in discovery are appended as idle entries. + * + * iOS (and web) have no discovery source, so `sessionBuckets` is + * passed through unchanged. + * + * If `discovered` is undefined (first poll hasn't landed yet), the + * session buckets are returned as-is rather than being wiped out. + */ +export function mergeSidebarDeviceBucketsWithDiscovery( + sessionBuckets: StudioSidebarDeviceBuckets, + discovered: DiscoveredDevicesByPlatform | undefined, +): StudioSidebarDeviceBuckets { + if (!discovered) { + return sessionBuckets; + } + + const merged: StudioSidebarDeviceBuckets = { ...sessionBuckets }; + + for (const key of DISCOVERABLE_PLATFORMS) { + const discoveredBucket = discovered[key]; + const discoveredIds = new Set(discoveredBucket.map((d) => d.id)); + + // Drop any session items whose id is not physically present anymore. + const survivingSessionItems = sessionBuckets[key].filter((item) => + discoveredIds.has(item.id), + ); + const survivingIds = new Set(survivingSessionItems.map((item) => item.id)); + + // Append discovered devices that aren't already covered by the + // session bucket (those already carry label/selected/active metadata). + const additions: StudioAndroidDeviceItem[] = []; + for (const dev of discoveredBucket) { + if (!survivingIds.has(dev.id)) { + additions.push({ + id: dev.id, + label: dev.label, + description: dev.description, + selected: false, + status: 'idle', + }); + } + } + + merged[key] = [...survivingSessionItems, ...additions]; + } + + return merged; +} + export function resolveAndroidDeviceLabel( items: StudioAndroidDeviceItem[], ): string { diff --git a/apps/studio/src/renderer/playground/types.ts b/apps/studio/src/renderer/playground/types.ts index 9979417113..c5c457fc56 100644 --- a/apps/studio/src/renderer/playground/types.ts +++ b/apps/studio/src/renderer/playground/types.ts @@ -30,12 +30,14 @@ export type StudioPlaygroundContextValue = | { phase: 'booting'; restartPlayground: () => Promise; + refreshDiscoveredDevices: () => Promise; discoveredDevices?: DiscoveredDevicesByPlatform; } | { phase: 'error'; error: string; restartPlayground: () => Promise; + refreshDiscoveredDevices: () => Promise; discoveredDevices?: DiscoveredDevicesByPlatform; } | { @@ -43,5 +45,6 @@ export type StudioPlaygroundContextValue = serverUrl: string; controller: PlaygroundControllerResult; restartPlayground: () => Promise; + refreshDiscoveredDevices: () => Promise; discoveredDevices?: DiscoveredDevicesByPlatform; }; diff --git a/apps/studio/tests/playground-selectors.test.ts b/apps/studio/tests/playground-selectors.test.ts index 6e01deecef..d57a800afe 100644 --- a/apps/studio/tests/playground-selectors.test.ts +++ b/apps/studio/tests/playground-selectors.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { buildAndroidDeviceItems, buildStudioSidebarDeviceBuckets, + mergeSidebarDeviceBucketsWithDiscovery, resolveAndroidDeviceLabel, resolveConnectedDeviceLabel, resolveVisibleSidebarPlatforms, @@ -198,6 +199,101 @@ describe('resolveAndroidDeviceLabel', () => { }); }); +describe('mergeSidebarDeviceBucketsWithDiscovery', () => { + const emptyBuckets = { + android: [], + ios: [], + computer: [], + harmony: [], + web: [], + }; + + it('returns session buckets unchanged when discovery has not polled yet', () => { + const session = { + ...emptyBuckets, + android: [ + { + id: 'device-1', + label: 'Pixel 9', + selected: true, + status: 'active' as const, + }, + ], + }; + + expect(mergeSidebarDeviceBucketsWithDiscovery(session, undefined)).toBe( + session, + ); + }); + + it('drops a session item that is no longer discoverable (phone unplugged)', () => { + const session = { + ...emptyBuckets, + android: [ + { + id: 'device-1', + label: 'Pixel 9', + selected: true, + status: 'active' as const, + }, + ], + }; + const discovered = { + ...emptyBuckets, + // device-1 has vanished from ADB + }; + + expect( + mergeSidebarDeviceBucketsWithDiscovery(session, discovered).android, + ).toEqual([]); + }); + + it('appends discovered devices that session setup has not surfaced', () => { + const session = emptyBuckets; + const discovered = { + ...emptyBuckets, + android: [ + { + platformId: 'android' as const, + id: 'device-2', + label: 'Galaxy', + description: 'ADB: device-2', + }, + ], + }; + + expect( + mergeSidebarDeviceBucketsWithDiscovery(session, discovered).android, + ).toEqual([ + { + id: 'device-2', + label: 'Galaxy', + description: 'ADB: device-2', + selected: false, + status: 'idle', + }, + ]); + }); + + it('passes iOS through unchanged because iOS has no discovery source', () => { + const session = { + ...emptyBuckets, + ios: [ + { + id: 'ios-1', + label: 'iPhone 15', + selected: true, + status: 'active' as const, + }, + ], + }; + + expect( + mergeSidebarDeviceBucketsWithDiscovery(session, emptyBuckets).ios, + ).toEqual(session.ios); + }); +}); + describe('resolveConnectedDeviceLabel', () => { const emptyOpts = { emptyLabel: 'No device' }; From 212e2fe24f06480e18adff6b2ed079d2edaaa841 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Mon, 20 Apr 2026 11:44:46 +0800 Subject: [PATCH 13/13] fix(playground): improve studio device discovery and screenshot handling --- .../src/main/playground/device-discovery.ts | 104 ++++++++- .../renderer/components/MainContent/index.tsx | 38 ++-- .../src/renderer/components/Sidebar/index.tsx | 32 ++- .../src/renderer/playground/selectors.ts | 198 +++++++++++++++++- apps/studio/src/renderer/playground/types.ts | 1 + apps/studio/src/shared/electron-contract.ts | 7 + .../tests/bucket-discovered-devices.test.ts | 24 ++- .../studio/tests/playground-selectors.test.ts | 163 +++++++++++++- .../src/adapters/remote-execution.ts | 4 +- packages/playground/src/server.ts | 7 +- .../unit/remote-execution-adapter.test.ts | 34 +++ .../tests/unit/server-session-manager.test.ts | 44 +++- 12 files changed, 603 insertions(+), 53 deletions(-) diff --git a/apps/studio/src/main/playground/device-discovery.ts b/apps/studio/src/main/playground/device-discovery.ts index 73511fd1b6..4779a3a92b 100644 --- a/apps/studio/src/main/playground/device-discovery.ts +++ b/apps/studio/src/main/playground/device-discovery.ts @@ -1,20 +1,59 @@ +import { DEFAULT_WDA_PORT } from '@midscene/shared/constants'; import { getDebug } from '@midscene/shared/logger'; import type { DiscoveredDevice } from '@shared/electron-contract'; const debugLog = getDebug('studio:device-discovery', { console: true }); +const IOS_WDA_DISCOVERY_HOST = 'localhost'; +const IOS_WDA_DISCOVERY_TIMEOUT_MS = 1000; + +interface WDAStatusResponse { + value?: { + device?: string; + os?: { + version?: string; + }; + ready?: boolean; + }; +} + +function isString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0; +} + +function buildIOSDiscoveryLabel(status: WDAStatusResponse): string { + const device = status.value?.device; + if (isString(device)) { + return `iOS (${device})`; + } + return 'iOS via WDA'; +} + +function buildIOSDiscoveryDescription( + host: string, + port: number, + status: WDAStatusResponse, +): string { + const details = [`WebDriverAgent: ${host}:${port}`]; + const osVersion = status.value?.os?.version; + if (isString(osVersion)) { + details.push(`iOS ${osVersion}`); + } + return details.join(' · '); +} /** * Scan all platforms for connected devices. Each platform's scan is * independent — a failure on one platform (e.g. `hdc` not installed) * does not prevent others from returning results. * - * iOS is intentionally omitted: device discovery requires WebDriverAgent - * to be running, which is a manual step. iOS devices show up after the - * user creates a session via the setup form. + * iOS discovery is a local convenience probe only: if WebDriverAgent is + * already reachable on the default loopback endpoint, surface it as a + * clickable target so Studio can prefill the iOS setup form. */ export async function discoverAllDevices(): Promise { const scans = await Promise.allSettled([ scanAndroidDevices(), + scanIOSDevices(), scanHarmonyDevices(), scanComputerDisplays(), ]); @@ -42,6 +81,9 @@ async function scanAndroidDevices(): Promise { id: device.udid, label: (device as { label?: string }).label || device.udid, description: `ADB: ${device.udid}`, + sessionValues: { + deviceId: device.udid, + }, })); } catch (err) { debugLog('android scan failed:', err); @@ -49,6 +91,56 @@ async function scanAndroidDevices(): Promise { } } +async function scanIOSDevices(): Promise { + const controller = new AbortController(); + const timeoutId = setTimeout(() => { + controller.abort(); + }, IOS_WDA_DISCOVERY_TIMEOUT_MS); + + try { + const response = await fetch( + `http://${IOS_WDA_DISCOVERY_HOST}:${DEFAULT_WDA_PORT}/status`, + { + headers: { + Accept: 'application/json', + }, + signal: controller.signal, + }, + ); + + if (!response.ok) { + return []; + } + + const status = (await response.json()) as WDAStatusResponse; + if (status.value?.ready !== true) { + return []; + } + + return [ + { + platformId: 'ios', + id: `${IOS_WDA_DISCOVERY_HOST}:${DEFAULT_WDA_PORT}`, + label: buildIOSDiscoveryLabel(status), + description: buildIOSDiscoveryDescription( + IOS_WDA_DISCOVERY_HOST, + DEFAULT_WDA_PORT, + status, + ), + sessionValues: { + host: IOS_WDA_DISCOVERY_HOST, + port: DEFAULT_WDA_PORT, + }, + }, + ]; + } catch (err) { + debugLog('ios scan failed:', err); + return []; + } finally { + clearTimeout(timeoutId); + } +} + async function scanHarmonyDevices(): Promise { try { const { getConnectedDevices } = await import('@midscene/harmony'); @@ -58,6 +150,9 @@ async function scanHarmonyDevices(): Promise { id: device.deviceId, label: device.deviceId, description: `HDC: ${device.deviceId}`, + sessionValues: { + deviceId: device.deviceId, + }, })); } catch (err) { debugLog('harmony scan failed:', err); @@ -74,6 +169,9 @@ async function scanComputerDisplays(): Promise { id: String(display.id), label: display.name || `Display ${display.id}`, description: display.primary ? 'Primary display' : undefined, + sessionValues: { + displayId: String(display.id), + }, })); } catch (err) { debugLog('computer scan failed:', err); diff --git a/apps/studio/src/renderer/components/MainContent/index.tsx b/apps/studio/src/renderer/components/MainContent/index.tsx index a735c6ae01..1027c3d162 100644 --- a/apps/studio/src/renderer/components/MainContent/index.tsx +++ b/apps/studio/src/renderer/components/MainContent/index.tsx @@ -2,11 +2,12 @@ import { PlaygroundPreview } from '@midscene/playground-app'; import { useEffect, useRef, useState } from 'react'; import { assetUrls } from '../../assets'; import { + buildDeviceSelectionFormValues, buildStudioSidebarDeviceBuckets, mergeSidebarDeviceBucketsWithDiscovery, - resolveConnectedAndroidDeviceId, + resolveConnectedDeviceId, resolveConnectedDeviceLabel, - resolveSelectedAndroidDeviceId, + resolveSelectedDeviceId, } from '../../playground/selectors'; import { useStudioPlayground } from '../../playground/useStudioPlayground'; import ConnectingPreview from '../ConnectingPreview'; @@ -133,17 +134,13 @@ export default function MainContent({ const isConnected = isReady ? studioPlayground.controller.state.sessionViewState.connected : false; - const connectedAndroidDeviceId = isReady - ? resolveConnectedAndroidDeviceId( - studioPlayground.controller.state.runtimeInfo, - ) + const connectedDeviceId = isReady + ? resolveConnectedDeviceId(studioPlayground.controller.state.runtimeInfo) : undefined; - const selectedAndroidDeviceId = isReady - ? resolveSelectedAndroidDeviceId( - studioPlayground.controller.state.formValues, - ) + const selectedDeviceId = isReady + ? resolveSelectedDeviceId(studioPlayground.controller.state.formValues) : undefined; - const previewDeviceId = connectedAndroidDeviceId ?? selectedAndroidDeviceId; + const previewDeviceId = connectedDeviceId ?? selectedDeviceId; const disconnectDisabled = !isReady || !studioPlayground.controller.state.sessionViewState.connected; const previewConnectionFailed = @@ -216,7 +213,7 @@ export default function MainContent({ ); const overviewSelectedDeviceId = isReady && studioPlayground.controller.state.sessionMutating - ? selectedAndroidDeviceId + ? selectedDeviceId : undefined; return ( @@ -252,15 +249,13 @@ export default function MainContent({ return; } const { actions, state } = studioPlayground.controller; - // Multi-platform: set the platform selector + the - // platform-prefixed device field so the unified session - // manager routes to the right backend. - state.form.setFieldsValue({ - platformId: platform, - [`${platform}.deviceId`]: device.id, - }); + const selectionValues = buildDeviceSelectionFormValues( + platform, + device, + ); + state.form.setFieldsValue(selectionValues); onSelectDeviceView?.(); - if (connectedAndroidDeviceId === device.id) { + if (connectedDeviceId === device.id) { return; } if (state.sessionViewState.connected) { @@ -268,8 +263,7 @@ export default function MainContent({ } const sessionValues = { ...state.form.getFieldsValue(true), - platformId: platform, - [`${platform}.deviceId`]: device.id, + ...selectionValues, }; await actions.createSession(sessionValues); }} diff --git a/apps/studio/src/renderer/components/Sidebar/index.tsx b/apps/studio/src/renderer/components/Sidebar/index.tsx index 01e6f42fda..911c041a88 100644 --- a/apps/studio/src/renderer/components/Sidebar/index.tsx +++ b/apps/studio/src/renderer/components/Sidebar/index.tsx @@ -1,9 +1,10 @@ import { useState } from 'react'; import { assetUrls } from '../../assets'; import { + buildDeviceSelectionFormValues, buildStudioSidebarDeviceBuckets, mergeSidebarDeviceBucketsWithDiscovery, - resolveConnectedAndroidDeviceId, + resolveConnectedDeviceId, } from '../../playground/selectors'; import type { StudioSidebarPlatformKey } from '../../playground/types'; import { useStudioPlayground } from '../../playground/useStudioPlayground'; @@ -197,9 +198,7 @@ export default function Sidebar({ const connectedDeviceId = studioPlayground.phase === 'ready' - ? resolveConnectedAndroidDeviceId( - studioPlayground.controller.state.runtimeInfo, - ) + ? resolveConnectedDeviceId(studioPlayground.controller.state.runtimeInfo) : undefined; /** @@ -239,6 +238,19 @@ export default function Sidebar({ label: 'Set up iOS via the playground form', status: 'idle' as const, isPlaceholder: true, + onClick: async () => { + if (studioPlayground.phase !== 'ready') { + return; + } + const { actions, state } = studioPlayground.controller; + const nextValues = { + ...state.form.getFieldsValue(true), + platformId: 'ios', + }; + state.form.setFieldsValue(nextValues); + onSelectDevice(); + await actions.refreshSessionSetup(nextValues); + }, }, ]; } @@ -256,10 +268,11 @@ export default function Sidebar({ // Tell the multi-platform session manager which platform + // device to target. Field keys follow the `{platformId}.fieldKey` // convention from `prepareMultiPlatformPlayground`. - state.form.setFieldsValue({ - platformId: platformKey, - [`${platformKey}.deviceId`]: item.id, - }); + const selectionValues = buildDeviceSelectionFormValues( + platformKey, + item, + ); + state.form.setFieldsValue(selectionValues); onSelectDevice(); @@ -271,8 +284,7 @@ export default function Sidebar({ } const sessionValues = { ...state.form.getFieldsValue(true), - platformId: platformKey, - [`${platformKey}.deviceId`]: item.id, + ...selectionValues, }; await actions.createSession(sessionValues); }, diff --git a/apps/studio/src/renderer/playground/selectors.ts b/apps/studio/src/renderer/playground/selectors.ts index af63b18c3f..e7080c78b1 100644 --- a/apps/studio/src/renderer/playground/selectors.ts +++ b/apps/studio/src/renderer/playground/selectors.ts @@ -2,7 +2,10 @@ import type { PlaygroundRuntimeInfo, PlaygroundSessionTarget, } from '@midscene/playground'; -import type { DiscoveredDevice } from '@shared/electron-contract'; +import type { + DiscoveredDevice, + StudioSessionValue, +} from '@shared/electron-contract'; import { STUDIO_PLATFORM_IDS } from '@shared/electron-contract'; import type { DiscoveredDevicesByPlatform, @@ -25,6 +28,23 @@ function createEmptySidebarDeviceBuckets(): StudioSidebarDeviceBuckets { }; } +function normalizePort(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value)) { + return value; + } + if (typeof value === 'string' && value.trim()) { + const parsed = Number.parseInt(value, 10); + if (Number.isFinite(parsed)) { + return parsed; + } + } + return undefined; +} + +function buildHostPortId(host: string, port: number): string { + return `${host}:${port}`; +} + function normalizeSidebarPlatformKey( value: unknown, ): StudioSidebarPlatformKey | undefined { @@ -59,8 +79,9 @@ function normalizeSidebarPlatformKey( * Platforms use different metadata keys for the device id: * Android / Harmony → metadata.deviceId * Computer → metadata.displayId + * iOS → metadata.wdaHost + metadata.wdaPort */ -function resolveConnectedDeviceId( +export function resolveConnectedDeviceId( runtimeInfo: PlaygroundRuntimeInfo | null, ): string | undefined { const metadata = runtimeInfo?.metadata || {}; @@ -70,9 +91,49 @@ function resolveConnectedDeviceId( if (isString(metadata.displayId)) { return metadata.displayId; } + if (isString(metadata.wdaHost)) { + const wdaPort = normalizePort(metadata.wdaPort); + if (wdaPort !== undefined) { + return buildHostPortId(metadata.wdaHost, wdaPort); + } + } return undefined; } +function resolveConnectedSessionValues( + runtimeInfo: PlaygroundRuntimeInfo | null, + platformKey: StudioSidebarPlatformKey, +): Record | undefined { + const metadata = runtimeInfo?.metadata || {}; + + switch (platformKey) { + case 'android': + case 'harmony': + return isString(metadata.deviceId) + ? { + deviceId: metadata.deviceId, + } + : undefined; + case 'computer': + return isString(metadata.displayId) + ? { + displayId: metadata.displayId, + } + : undefined; + case 'ios': { + const wdaPort = normalizePort(metadata.wdaPort); + return isString(metadata.wdaHost) && wdaPort !== undefined + ? { + host: metadata.wdaHost, + port: wdaPort, + } + : undefined; + } + default: + return undefined; + } +} + /** * Human-readable label for whatever device the playground is currently * connected to, across platforms. Prefers the session display name, then @@ -102,7 +163,7 @@ export function resolveConnectedDeviceLabel( function buildGenericConnectedDeviceItem( runtimeInfo: PlaygroundRuntimeInfo | null, - platformKey: Exclude, + platformKey: StudioSidebarPlatformKey, ): StudioAndroidDeviceItem | null { const metadata = runtimeInfo?.metadata || {}; const deviceId = resolveConnectedDeviceId(runtimeInfo); @@ -121,6 +182,7 @@ function buildGenericConnectedDeviceItem( description: deviceId && deviceId !== label ? deviceId : undefined, selected: true, status: 'active', + sessionValues: resolveConnectedSessionValues(runtimeInfo, platformKey), }; } @@ -135,9 +197,100 @@ export function resolveConnectedAndroidDeviceId( export function resolveSelectedAndroidDeviceId( formValues: Record, ): string | undefined { + if (isString(formValues['android.deviceId'])) { + return formValues['android.deviceId']; + } return isString(formValues.deviceId) ? formValues.deviceId : undefined; } +export function resolveSelectedDeviceId( + formValues: Record, +): string | undefined { + const selectedPlatform = normalizeSidebarPlatformKey(formValues.platformId); + + if (selectedPlatform === 'ios') { + const host = isString(formValues['ios.host']) + ? formValues['ios.host'] + : isString(formValues.host) + ? formValues.host + : undefined; + const port = normalizePort(formValues['ios.port'] ?? formValues.port); + return host && port !== undefined ? buildHostPortId(host, port) : undefined; + } + + if (selectedPlatform === 'computer') { + return isString(formValues['computer.displayId']) + ? formValues['computer.displayId'] + : isString(formValues.displayId) + ? formValues.displayId + : undefined; + } + + if (selectedPlatform === 'harmony') { + return isString(formValues['harmony.deviceId']) + ? formValues['harmony.deviceId'] + : isString(formValues.deviceId) + ? formValues.deviceId + : undefined; + } + + if (selectedPlatform === 'android') { + return resolveSelectedAndroidDeviceId(formValues); + } + + return ( + resolveSelectedAndroidDeviceId(formValues) || + (isString(formValues['computer.displayId']) + ? formValues['computer.displayId'] + : undefined) || + ((): string | undefined => { + const host = isString(formValues['ios.host']) + ? formValues['ios.host'] + : undefined; + const port = normalizePort(formValues['ios.port']); + return host && port !== undefined + ? buildHostPortId(host, port) + : undefined; + })() + ); +} + +function prefixSessionValues( + platform: StudioSidebarPlatformKey, + sessionValues: Record, +): Record { + return Object.fromEntries( + Object.entries(sessionValues).map(([key, value]) => [ + `${platform}.${key}`, + value, + ]), + ); +} + +export function buildDeviceSelectionFormValues( + platform: StudioSidebarPlatformKey, + device: Pick, +): Record { + if (device.sessionValues) { + return { + platformId: platform, + ...prefixSessionValues(platform, device.sessionValues), + }; + } + + if (platform === 'computer') { + return { + platformId: platform, + [`${platform}.displayId`]: device.id, + }; + } + + return { + platformId: platform, + [`${platform}.deviceId`]: device.id, + }; +} + export function buildAndroidDeviceItems({ formValues, runtimeInfo, @@ -159,6 +312,7 @@ export function buildAndroidDeviceItems({ : connectedDeviceId, selected: true, status: 'active', + sessionValues: resolveConnectedSessionValues(runtimeInfo, 'android'), }, ]; } @@ -171,6 +325,9 @@ export function buildAndroidDeviceItems({ target.id === connectedDeviceId || (!connectedDeviceId && target.id === selectedDeviceId), status: target.id === connectedDeviceId ? 'active' : 'idle', + sessionValues: { + deviceId: target.id, + }, })); } @@ -253,7 +410,12 @@ export function bucketDiscoveredDevices( * least once, a session item that isn't discovered anymore is stale * (e.g. the user unplugged the device while a session was still open). */ -const DISCOVERABLE_PLATFORMS = ['android', 'harmony', 'computer'] as const; +const AUTHORITATIVE_DISCOVERY_PLATFORMS = [ + 'android', + 'harmony', + 'computer', +] as const; +const ADDITIVE_DISCOVERY_PLATFORMS = ['ios'] as const; /** * Merge session-setup buckets with the live discovery snapshot. For @@ -261,8 +423,9 @@ const DISCOVERABLE_PLATFORMS = ['android', 'harmony', 'computer'] as const; * discovery snapshot are dropped (catches unplug while connected), and * items only present in discovery are appended as idle entries. * - * iOS (and web) have no discovery source, so `sessionBuckets` is - * passed through unchanged. + * iOS discovery is additive only: local WDA probes are appended to the + * sidebar, but they do not evict existing session rows because Studio can + * still connect to manually entered non-local WDA hosts. * * If `discovered` is undefined (first poll hasn't landed yet), the * session buckets are returned as-is rather than being wiped out. @@ -277,7 +440,7 @@ export function mergeSidebarDeviceBucketsWithDiscovery( const merged: StudioSidebarDeviceBuckets = { ...sessionBuckets }; - for (const key of DISCOVERABLE_PLATFORMS) { + for (const key of AUTHORITATIVE_DISCOVERY_PLATFORMS) { const discoveredBucket = discovered[key]; const discoveredIds = new Set(discoveredBucket.map((d) => d.id)); @@ -298,6 +461,7 @@ export function mergeSidebarDeviceBucketsWithDiscovery( description: dev.description, selected: false, status: 'idle', + sessionValues: dev.sessionValues, }); } } @@ -305,6 +469,26 @@ export function mergeSidebarDeviceBucketsWithDiscovery( merged[key] = [...survivingSessionItems, ...additions]; } + for (const key of ADDITIVE_DISCOVERY_PLATFORMS) { + const existingIds = new Set(sessionBuckets[key].map((item) => item.id)); + const additions: StudioAndroidDeviceItem[] = []; + + for (const dev of discovered[key]) { + if (!existingIds.has(dev.id)) { + additions.push({ + id: dev.id, + label: dev.label, + description: dev.description, + selected: false, + status: 'idle', + sessionValues: dev.sessionValues, + }); + } + } + + merged[key] = [...sessionBuckets[key], ...additions]; + } + return merged; } diff --git a/apps/studio/src/renderer/playground/types.ts b/apps/studio/src/renderer/playground/types.ts index c5c457fc56..2e11b95707 100644 --- a/apps/studio/src/renderer/playground/types.ts +++ b/apps/studio/src/renderer/playground/types.ts @@ -13,6 +13,7 @@ export interface StudioAndroidDeviceItem { selected: boolean; status: 'active' | 'idle'; isPlaceholder?: boolean; + sessionValues?: DiscoveredDevice['sessionValues']; } export type StudioSidebarDeviceBuckets = Record< diff --git a/apps/studio/src/shared/electron-contract.ts b/apps/studio/src/shared/electron-contract.ts index 132e629a6c..ba29e1b43e 100644 --- a/apps/studio/src/shared/electron-contract.ts +++ b/apps/studio/src/shared/electron-contract.ts @@ -51,12 +51,19 @@ export const STUDIO_PLATFORM_IDS = [ export type StudioPlatformId = (typeof STUDIO_PLATFORM_IDS)[number]; +export type StudioSessionValue = string | number | boolean; + /** A device discovered across any platform, tagged with its platform. */ export interface DiscoveredDevice { platformId: StudioPlatformId; id: string; label: string; description?: string; + /** + * Session-setup field values for this discovered target, before Studio + * prefixes them with `{platformId}.`. + */ + sessionValues?: Record; } /** Result of the cross-platform device discovery scan. */ diff --git a/apps/studio/tests/bucket-discovered-devices.test.ts b/apps/studio/tests/bucket-discovered-devices.test.ts index 29ce67bcab..deba2ce1c3 100644 --- a/apps/studio/tests/bucket-discovered-devices.test.ts +++ b/apps/studio/tests/bucket-discovered-devices.test.ts @@ -15,15 +15,33 @@ describe('bucketDiscoveredDevices', () => { it('groups devices by their platformId tag', () => { const result = bucketDiscoveredDevices([ { platformId: 'android', id: 'a1', label: 'Pixel' }, - { platformId: 'harmony', id: 'h1', label: 'Huawei P70' }, + { + platformId: 'ios', + id: 'localhost:8100', + label: 'iOS via WDA', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, { platformId: 'android', id: 'a2', label: 'Galaxy' }, { platformId: 'computer', id: 'c1', label: 'Display 1' }, ]); expect(result.android.map((d) => d.id)).toEqual(['a1', 'a2']); - expect(result.harmony.map((d) => d.id)).toEqual(['h1']); + expect(result.ios).toEqual([ + { + platformId: 'ios', + id: 'localhost:8100', + label: 'iOS via WDA', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + ]); expect(result.computer.map((d) => d.id)).toEqual(['c1']); - expect(result.ios).toEqual([]); + expect(result.harmony).toEqual([]); expect(result.web).toEqual([]); }); }); diff --git a/apps/studio/tests/playground-selectors.test.ts b/apps/studio/tests/playground-selectors.test.ts index d57a800afe..9b4e1ab439 100644 --- a/apps/studio/tests/playground-selectors.test.ts +++ b/apps/studio/tests/playground-selectors.test.ts @@ -1,10 +1,13 @@ import { describe, expect, it } from 'vitest'; import { buildAndroidDeviceItems, + buildDeviceSelectionFormValues, buildStudioSidebarDeviceBuckets, mergeSidebarDeviceBucketsWithDiscovery, resolveAndroidDeviceLabel, + resolveConnectedDeviceId, resolveConnectedDeviceLabel, + resolveSelectedDeviceId, resolveVisibleSidebarPlatforms, } from '../src/renderer/playground/selectors'; @@ -42,6 +45,9 @@ describe('buildAndroidDeviceItems', () => { description: undefined, selected: true, status: 'active', + sessionValues: { + deviceId: 'device-1', + }, }, { id: 'device-2', @@ -49,6 +55,9 @@ describe('buildAndroidDeviceItems', () => { description: undefined, selected: false, status: 'idle', + sessionValues: { + deviceId: 'device-2', + }, }, ]); }); @@ -117,6 +126,9 @@ describe('buildStudioSidebarDeviceBuckets', () => { description: undefined, selected: true, status: 'active', + sessionValues: { + deviceId: 'device-1', + }, }, { id: 'device-2', @@ -124,6 +136,9 @@ describe('buildStudioSidebarDeviceBuckets', () => { description: undefined, selected: false, status: 'idle', + sessionValues: { + deviceId: 'device-2', + }, }, ], ios: [], @@ -161,6 +176,9 @@ describe('buildStudioSidebarDeviceBuckets', () => { description: 'HDC-001', selected: true, status: 'active', + sessionValues: { + deviceId: 'HDC-001', + }, }, ], web: [], @@ -275,22 +293,106 @@ describe('mergeSidebarDeviceBucketsWithDiscovery', () => { ]); }); - it('passes iOS through unchanged because iOS has no discovery source', () => { + it('preserves manual iOS session rows while appending local WDA probes', () => { const session = { ...emptyBuckets, ios: [ { - id: 'ios-1', + id: 'remote-wda:8100', label: 'iPhone 15', selected: true, status: 'active' as const, }, ], }; + const discovered = { + ...emptyBuckets, + ios: [ + { + platformId: 'ios' as const, + id: 'localhost:8100', + label: 'iOS via WDA', + description: 'WebDriverAgent: localhost:8100', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + ], + }; + + expect( + mergeSidebarDeviceBucketsWithDiscovery(session, discovered).ios, + ).toEqual([ + { + id: 'remote-wda:8100', + label: 'iPhone 15', + selected: true, + status: 'active', + }, + { + id: 'localhost:8100', + label: 'iOS via WDA', + description: 'WebDriverAgent: localhost:8100', + selected: false, + status: 'idle', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + ]); + }); + + it('appends local iOS WDA probes without evicting manual iOS sessions', () => { + const session = { + ...emptyBuckets, + ios: [ + { + id: 'remote-wda:8100', + label: 'Remote iPhone', + selected: true, + status: 'active' as const, + }, + ], + }; + const discovered = { + ...emptyBuckets, + ios: [ + { + platformId: 'ios' as const, + id: 'localhost:8100', + label: 'iOS via WDA', + description: 'WebDriverAgent: localhost:8100', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + ], + }; expect( - mergeSidebarDeviceBucketsWithDiscovery(session, emptyBuckets).ios, - ).toEqual(session.ios); + mergeSidebarDeviceBucketsWithDiscovery(session, discovered).ios, + ).toEqual([ + { + id: 'remote-wda:8100', + label: 'Remote iPhone', + selected: true, + status: 'active', + }, + { + id: 'localhost:8100', + label: 'iOS via WDA', + description: 'WebDriverAgent: localhost:8100', + selected: false, + status: 'idle', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + ]); }); }); @@ -346,6 +448,20 @@ describe('resolveConnectedDeviceLabel', () => { ).toBe('Display 1'); }); + it('uses WDA host and port as the fallback iOS connection id', () => { + expect( + resolveConnectedDeviceId({ + interface: { type: 'ios' }, + preview: { kind: 'none', capabilities: [] }, + executionUxHints: [], + metadata: { + wdaHost: 'localhost', + wdaPort: 8100, + }, + }), + ).toBe('localhost:8100'); + }); + it('falls back to runtime title when no device metadata is present', () => { expect( resolveConnectedDeviceLabel( @@ -361,3 +477,42 @@ describe('resolveConnectedDeviceLabel', () => { ).toBe('Midscene Playground'); }); }); + +describe('resolveSelectedDeviceId', () => { + it('supports prefixed Android selection values', () => { + expect( + resolveSelectedDeviceId({ + platformId: 'android', + 'android.deviceId': 'device-2', + }), + ).toBe('device-2'); + }); + + it('builds the iOS selection id from host and port', () => { + expect( + resolveSelectedDeviceId({ + platformId: 'ios', + 'ios.host': 'localhost', + 'ios.port': 8100, + }), + ).toBe('localhost:8100'); + }); +}); + +describe('buildDeviceSelectionFormValues', () => { + it('prefixes discovery-provided session values for iOS', () => { + expect( + buildDeviceSelectionFormValues('ios', { + id: 'localhost:8100', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }), + ).toEqual({ + platformId: 'ios', + 'ios.host': 'localhost', + 'ios.port': 8100, + }); + }); +}); diff --git a/packages/playground/src/adapters/remote-execution.ts b/packages/playground/src/adapters/remote-execution.ts index 29cac89fe1..970820da4f 100644 --- a/packages/playground/src/adapters/remote-execution.ts +++ b/packages/playground/src/adapters/remote-execution.ts @@ -446,7 +446,9 @@ export class RemoteExecutionAdapter extends BasePlaygroundAdapter { const response = await fetch(`${this.serverUrl}/screenshot`); if (!response.ok) { - console.warn(`Screenshot request failed: ${response.statusText}`); + if (response.status !== 409) { + console.warn(`Screenshot request failed: ${response.statusText}`); + } return null; } diff --git a/packages/playground/src/server.ts b/packages/playground/src/server.ts index cfa0572958..341666e558 100644 --- a/packages/playground/src/server.ts +++ b/packages/playground/src/server.ts @@ -1159,8 +1159,11 @@ class PlaygroundServer { } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - console.error(`Failed to take screenshot: ${errorMessage}`); - res.status(errorMessage === 'No active session' ? 409 : 500).json({ + const statusCode = errorMessage === 'No active session' ? 409 : 500; + if (statusCode !== 409) { + console.error(`Failed to take screenshot: ${errorMessage}`); + } + res.status(statusCode).json({ error: `Failed to take screenshot: ${errorMessage}`, }); } diff --git a/packages/playground/tests/unit/remote-execution-adapter.test.ts b/packages/playground/tests/unit/remote-execution-adapter.test.ts index 93033ca3cb..c8c9d7aa32 100644 --- a/packages/playground/tests/unit/remote-execution-adapter.test.ts +++ b/packages/playground/tests/unit/remote-execution-adapter.test.ts @@ -608,4 +608,38 @@ describe('RemoteExecutionAdapter', () => { }); }); }); + + describe('getScreenshot', () => { + it('should return screenshot data when the server responds successfully', async () => { + const screenshotResponse = { + screenshot: 'data:image/png;base64,abc123', + timestamp: 123, + }; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve(screenshotResponse), + }); + + await expect(adapter.getScreenshot()).resolves.toEqual( + screenshotResponse, + ); + expect(mockFetch).toHaveBeenCalledWith(`${mockServerUrl}/screenshot`); + }); + + it('should suppress warning logs for expected 409 no-session screenshot responses', async () => { + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 409, + statusText: 'Conflict', + }); + const consoleWarnSpy = vi + .spyOn(console, 'warn') + .mockImplementation(() => {}); + + await expect(adapter.getScreenshot()).resolves.toBeNull(); + expect(consoleWarnSpy).not.toHaveBeenCalled(); + + consoleWarnSpy.mockRestore(); + }); + }); }); diff --git a/packages/playground/tests/unit/server-session-manager.test.ts b/packages/playground/tests/unit/server-session-manager.test.ts index f46dad7441..f46c0104b0 100644 --- a/packages/playground/tests/unit/server-session-manager.test.ts +++ b/packages/playground/tests/unit/server-session-manager.test.ts @@ -22,7 +22,7 @@ function createMockResponse() { function getRouteHandler( server: PlaygroundServer, - method: 'post' | 'delete', + method: 'get' | 'post' | 'delete', route: string, ) { const calls = (server.app[method] as any).mock.calls as Array<[string, any]>; @@ -165,6 +165,48 @@ describe('PlaygroundServer session manager APIs', () => { expect(sidecar.stop).toHaveBeenCalledTimes(1); }); + test('returns 409 without logging an error when screenshot is requested without a session', async () => { + const consoleErrorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => {}); + const server = new PlaygroundServer(); + server.setPreparedPlatform({ + platformId: 'computer', + title: 'Midscene Computer Playground', + description: 'Computer playground platform descriptor', + preview: { + kind: 'screenshot', + title: 'Desktop preview', + screenshotPath: '/screenshot', + capabilities: [], + }, + metadata: { + sessionConnected: false, + setupState: 'required', + }, + sessionManager: { + async createSession() { + throw new Error('not needed'); + }, + }, + }); + + await server.launch(6105); + const screenshotHandler = getRouteHandler(server, 'get', '/screenshot'); + expect(screenshotHandler).toBeTypeOf('function'); + + const response = createMockResponse(); + await screenshotHandler({}, response); + + expect(response.statusCode).toBe(409); + expect(response.body).toMatchObject({ + error: 'Failed to take screenshot: No active session', + }); + expect(consoleErrorSpy).not.toHaveBeenCalled(); + + consoleErrorSpy.mockRestore(); + }); + test('stops started sidecars and restores base runtime when session creation fails after apply', async () => { const sidecar = { id: 'session-sidecar',