diff --git a/apps/studio/package.json b/apps/studio/package.json index 47b50511fb..0cf8116b28 100644 --- a/apps/studio/package.json +++ b/apps/studio/package.json @@ -11,8 +11,13 @@ "test": "vitest run" }, "dependencies": { + "@midscene/android": "workspace:*", "@midscene/android-playground": "workspace:*", + "@midscene/computer": "workspace:*", + "@midscene/computer-playground": "workspace:*", "@midscene/core": "workspace:*", + "@midscene/harmony": "workspace:*", + "@midscene/ios": "workspace:*", "@midscene/playground": "workspace:*", "@midscene/playground-app": "workspace:*", "@midscene/shared": "workspace:*", diff --git a/apps/studio/rsbuild.config.ts b/apps/studio/rsbuild.config.ts index cb1ae7cb62..54963ff7c6 100644 --- a/apps/studio/rsbuild.config.ts +++ b/apps/studio/rsbuild.config.ts @@ -88,7 +88,12 @@ export default defineConfig({ }, externals: [ 'electron', + '@midscene/android', '@midscene/android-playground', + '@midscene/computer', + '@midscene/computer-playground', + '@midscene/harmony', + '@midscene/ios', '@midscene/playground', ], sourceMap: true, diff --git a/apps/studio/src/main/index.ts b/apps/studio/src/main/index.ts index 5fbbb95917..1ec4c741b5 100644 --- a/apps/studio/src/main/index.ts +++ b/apps/studio/src/main/index.ts @@ -11,8 +11,9 @@ import { shell, } from 'electron'; import type { TitleBarOverlay } from 'electron'; -import { createAndroidPlaygroundRuntimeService } from './playground/android-runtime'; import { runConnectivityTest } from './playground/connectivity-test'; +import { discoverAllDevices } from './playground/device-discovery'; +import { createMultiPlatformRuntimeService } from './playground/multi-platform-runtime'; /** * Main process owns native shell concerns only. @@ -22,7 +23,7 @@ import { runConnectivityTest } from './playground/connectivity-test'; let mainWindow: BrowserWindow | null = null; let cachedAppIcon: NativeImage | null = null; -const androidPlaygroundRuntime = createAndroidPlaygroundRuntimeService(); +const playgroundRuntime = createMultiPlatformRuntimeService(); const getRendererEntryPath = () => path.join(__dirname, '../renderer/index.html'); @@ -134,11 +135,18 @@ const registerIpcHandlers = () => { ipcMain.handle(IPC_CHANNELS.closeWindow, () => { mainWindow?.close(); }); - ipcMain.handle(IPC_CHANNELS.getAndroidPlaygroundBootstrap, () => - androidPlaygroundRuntime.getBootstrap(), + // Multi-platform playground — a single server for Android, iOS, + // HarmonyOS, and Computer. Legacy channel names (getAndroidPlayground*) + // are aliased to the same strings in IPC_CHANNELS, so the old + // renderer code keeps working transparently. + ipcMain.handle(IPC_CHANNELS.getPlaygroundBootstrap, () => + playgroundRuntime.getBootstrap(), ); - ipcMain.handle(IPC_CHANNELS.restartAndroidPlayground, async () => - androidPlaygroundRuntime.restart(), + ipcMain.handle(IPC_CHANNELS.restartPlayground, async () => + playgroundRuntime.restart(), + ); + ipcMain.handle(IPC_CHANNELS.discoverDevices, async () => + discoverAllDevices(), ); ipcMain.handle(IPC_CHANNELS.runConnectivityTest, async (_event, request) => runConnectivityTest(request), @@ -151,7 +159,7 @@ app.whenReady().then(() => { } registerIpcHandlers(); - void androidPlaygroundRuntime.start(); + void playgroundRuntime.start(); createMainWindow(); app.on('activate', () => { @@ -168,5 +176,5 @@ app.on('window-all-closed', () => { }); app.on('before-quit', () => { - void androidPlaygroundRuntime.close(); + void playgroundRuntime.close(); }); diff --git a/apps/studio/src/main/playground/android-runtime.ts b/apps/studio/src/main/playground/android-runtime.ts deleted file mode 100644 index 9877a09482..0000000000 --- a/apps/studio/src/main/playground/android-runtime.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { createRequire } from 'node:module'; -import path from 'node:path'; -import { - ScrcpyServer, - androidPlaygroundPlatform, -} from '@midscene/android-playground'; -import { - type LaunchPlaygroundResult, - launchPreparedPlaygroundPlatform, -} from '@midscene/playground'; -import type { AndroidPlaygroundBootstrap } from '@shared/electron-contract'; -import { createStudioCorsOptions } from './cors'; -import type { - AndroidPlaygroundPackagePaths, - AndroidPlaygroundRuntimeService, -} from './types'; - -const require = createRequire(__filename); - -function getErrorMessage(error: unknown): string { - return error instanceof Error - ? error.message - : 'Unknown Android runtime error'; -} - -export function resolveAndroidPlaygroundPackagePaths(): AndroidPlaygroundPackagePaths { - const packageJsonPath = require.resolve( - '@midscene/android-playground/package.json', - ); - const packageRoot = path.dirname(packageJsonPath); - - return { - packageRoot, - staticDir: path.join(packageRoot, 'static'), - }; -} - -export function createAndroidPlaygroundRuntimeService(): AndroidPlaygroundRuntimeService { - let bootstrap: AndroidPlaygroundBootstrap = { - status: 'starting', - serverUrl: null, - port: null, - error: null, - }; - let launchResult: LaunchPlaygroundResult | null = null; - let startPromise: Promise | null = null; - - const close = async () => { - if (!launchResult) { - return; - } - - const activeLaunch = launchResult; - launchResult = null; - await activeLaunch.close(); - }; - - const start = async (): Promise => { - if (launchResult) { - return bootstrap; - } - - if (startPromise) { - return startPromise; - } - - bootstrap = { - status: 'starting', - serverUrl: null, - port: null, - error: null, - }; - - startPromise = (async () => { - try { - const { staticDir } = resolveAndroidPlaygroundPackagePaths(); - const scrcpyServer = new ScrcpyServer(); - const prepared = await androidPlaygroundPlatform.prepare({ - staticDir, - scrcpyServer, - }); - const nextLaunchResult = await launchPreparedPlaygroundPlatform( - prepared, - { - corsOptions: createStudioCorsOptions(), - enableCors: true, - openBrowser: false, - verbose: false, - }, - ); - - launchResult = nextLaunchResult; - bootstrap = { - status: 'ready', - serverUrl: `http://${nextLaunchResult.host}:${nextLaunchResult.port}`, - port: nextLaunchResult.port, - error: null, - }; - - return bootstrap; - } catch (error) { - bootstrap = { - status: 'error', - serverUrl: null, - port: null, - error: getErrorMessage(error), - }; - - return bootstrap; - } finally { - startPromise = null; - } - })(); - - return startPromise; - }; - - return { - close, - getBootstrap: () => bootstrap, - restart: async () => { - await close(); - return start(); - }, - start, - }; -} diff --git a/apps/studio/src/main/playground/device-discovery.ts b/apps/studio/src/main/playground/device-discovery.ts new file mode 100644 index 0000000000..4779a3a92b --- /dev/null +++ b/apps/studio/src/main/playground/device-discovery.ts @@ -0,0 +1,180 @@ +import { DEFAULT_WDA_PORT } from '@midscene/shared/constants'; +import { getDebug } from '@midscene/shared/logger'; +import type { DiscoveredDevice } from '@shared/electron-contract'; + +const debugLog = getDebug('studio:device-discovery', { console: true }); +const IOS_WDA_DISCOVERY_HOST = 'localhost'; +const IOS_WDA_DISCOVERY_TIMEOUT_MS = 1000; + +interface WDAStatusResponse { + value?: { + device?: string; + os?: { + version?: string; + }; + ready?: boolean; + }; +} + +function isString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0; +} + +function buildIOSDiscoveryLabel(status: WDAStatusResponse): string { + const device = status.value?.device; + if (isString(device)) { + return `iOS (${device})`; + } + return 'iOS via WDA'; +} + +function buildIOSDiscoveryDescription( + host: string, + port: number, + status: WDAStatusResponse, +): string { + const details = [`WebDriverAgent: ${host}:${port}`]; + const osVersion = status.value?.os?.version; + if (isString(osVersion)) { + details.push(`iOS ${osVersion}`); + } + return details.join(' · '); +} + +/** + * Scan all platforms for connected devices. Each platform's scan is + * independent — a failure on one platform (e.g. `hdc` not installed) + * does not prevent others from returning results. + * + * iOS discovery is a local convenience probe only: if WebDriverAgent is + * already reachable on the default loopback endpoint, surface it as a + * clickable target so Studio can prefill the iOS setup form. + */ +export async function discoverAllDevices(): Promise { + const scans = await Promise.allSettled([ + scanAndroidDevices(), + scanIOSDevices(), + scanHarmonyDevices(), + scanComputerDisplays(), + ]); + + const results: DiscoveredDevice[] = []; + for (const scan of scans) { + if (scan.status === 'fulfilled') { + results.push(...scan.value); + } else { + debugLog('platform scan rejected:', scan.reason); + } + } + + return results; +} + +async function scanAndroidDevices(): Promise { + try { + const { getConnectedDevicesWithDetails } = await import( + '@midscene/android' + ); + const devices = await getConnectedDevicesWithDetails(); + return devices.map((device) => ({ + platformId: 'android', + id: device.udid, + label: (device as { label?: string }).label || device.udid, + description: `ADB: ${device.udid}`, + sessionValues: { + deviceId: device.udid, + }, + })); + } catch (err) { + debugLog('android scan failed:', err); + return []; + } +} + +async function scanIOSDevices(): Promise { + const controller = new AbortController(); + const timeoutId = setTimeout(() => { + controller.abort(); + }, IOS_WDA_DISCOVERY_TIMEOUT_MS); + + try { + const response = await fetch( + `http://${IOS_WDA_DISCOVERY_HOST}:${DEFAULT_WDA_PORT}/status`, + { + headers: { + Accept: 'application/json', + }, + signal: controller.signal, + }, + ); + + if (!response.ok) { + return []; + } + + const status = (await response.json()) as WDAStatusResponse; + if (status.value?.ready !== true) { + return []; + } + + return [ + { + platformId: 'ios', + id: `${IOS_WDA_DISCOVERY_HOST}:${DEFAULT_WDA_PORT}`, + label: buildIOSDiscoveryLabel(status), + description: buildIOSDiscoveryDescription( + IOS_WDA_DISCOVERY_HOST, + DEFAULT_WDA_PORT, + status, + ), + sessionValues: { + host: IOS_WDA_DISCOVERY_HOST, + port: DEFAULT_WDA_PORT, + }, + }, + ]; + } catch (err) { + debugLog('ios scan failed:', err); + return []; + } finally { + clearTimeout(timeoutId); + } +} + +async function scanHarmonyDevices(): Promise { + try { + const { getConnectedDevices } = await import('@midscene/harmony'); + const devices = await getConnectedDevices(); + return devices.map((device) => ({ + platformId: 'harmony', + id: device.deviceId, + label: device.deviceId, + description: `HDC: ${device.deviceId}`, + sessionValues: { + deviceId: device.deviceId, + }, + })); + } catch (err) { + debugLog('harmony scan failed:', err); + return []; + } +} + +async function scanComputerDisplays(): Promise { + try { + const { getConnectedDisplays } = await import('@midscene/computer'); + const displays = await getConnectedDisplays(); + return displays.map((display) => ({ + platformId: 'computer', + id: String(display.id), + label: display.name || `Display ${display.id}`, + description: display.primary ? 'Primary display' : undefined, + sessionValues: { + displayId: String(display.id), + }, + })); + } catch (err) { + debugLog('computer scan failed:', err); + return []; + } +} diff --git a/apps/studio/src/main/playground/multi-platform-runtime.ts b/apps/studio/src/main/playground/multi-platform-runtime.ts new file mode 100644 index 0000000000..646006424b --- /dev/null +++ b/apps/studio/src/main/playground/multi-platform-runtime.ts @@ -0,0 +1,193 @@ +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { + ScrcpyServer, + androidPlaygroundPlatform, +} from '@midscene/android-playground'; +import { computerPlaygroundPlatform } from '@midscene/computer-playground'; +import { harmonyPlaygroundPlatform } from '@midscene/harmony'; +import { iosPlaygroundPlatform } from '@midscene/ios'; +import { + type LaunchPlaygroundResult, + type PreparedPlaygroundPlatform, + type RegisteredPlaygroundPlatform, + launchPreparedPlaygroundPlatform, + prepareMultiPlatformPlayground, +} from '@midscene/playground'; +import type { PlaygroundBootstrap } from '@shared/electron-contract'; +import { createStudioCorsOptions } from './cors'; +import type { PlaygroundRuntimeService } from './types'; + +const require = createRequire(__filename); + +function getErrorMessage(error: unknown): string { + return error instanceof Error + ? error.message + : 'Unknown playground runtime error'; +} + +function resolveStaticDir(packageName: string): string { + const packageJsonPath = require.resolve(`${packageName}/package.json`); + return path.join(path.dirname(packageJsonPath), 'static'); +} + +interface StudioPlatformSpec { + id: string; + label: string; + description: string; + staticDirPackage: string; + prepare: (staticDir: string) => Promise; +} + +const studioPlatformSpecs: StudioPlatformSpec[] = [ + { + id: 'android', + label: 'Android', + description: 'Connect to an Android device via ADB', + staticDirPackage: '@midscene/android-playground', + prepare: (staticDir) => + androidPlaygroundPlatform.prepare({ + staticDir, + scrcpyServer: new ScrcpyServer(), + }), + }, + { + id: 'ios', + label: 'iOS', + description: 'Connect to an iOS device via WebDriverAgent', + staticDirPackage: '@midscene/ios', + prepare: (staticDir) => iosPlaygroundPlatform.prepare({ staticDir }), + }, + { + id: 'harmony', + label: 'HarmonyOS', + description: 'Connect to a HarmonyOS device via HDC', + staticDirPackage: '@midscene/harmony', + prepare: (staticDir) => harmonyPlaygroundPlatform.prepare({ staticDir }), + }, + { + id: 'computer', + label: 'Computer', + description: 'Control the local desktop', + staticDirPackage: '@midscene/computer-playground', + // In the Electron context, pass null — the computer agent works + // without a window controller, it just won't auto-minimize Studio + // during task execution. A follow-up can provide an Electron-native + // adapter that calls mainWindow.minimize()/restore(). + prepare: (staticDir) => + computerPlaygroundPlatform.prepare({ + staticDir, + getWindowController: () => null, + }), + }, +]; + +function buildRegisteredPlatforms(): RegisteredPlaygroundPlatform[] { + return studioPlatformSpecs.map((spec) => { + const staticDir = resolveStaticDir(spec.staticDirPackage); + return { + id: spec.id, + label: spec.label, + description: spec.description, + prepare: () => spec.prepare(staticDir), + }; + }); +} + +/** + * Creates a multi-platform playground runtime service for the Studio + * Electron main process. On `start()`, it registers all platforms + * (Android, iOS, HarmonyOS, Computer) with + * `prepareMultiPlatformPlayground` and launches a SINGLE unified HTTP + * server. The renderer talks to this one server; the platform selector + * on the setup form routes to the correct backend. + */ +export function createMultiPlatformRuntimeService(): PlaygroundRuntimeService { + let bootstrap: PlaygroundBootstrap = { + status: 'starting', + serverUrl: null, + port: null, + error: null, + }; + let launchResult: LaunchPlaygroundResult | null = null; + let startPromise: Promise | null = null; + + const close = async () => { + if (!launchResult) { + return; + } + const activeLaunch = launchResult; + launchResult = null; + await activeLaunch.close(); + }; + + const start = async (): Promise => { + if (launchResult) { + return bootstrap; + } + if (startPromise) { + return startPromise; + } + + bootstrap = { + status: 'starting', + serverUrl: null, + port: null, + error: null, + }; + + startPromise = (async () => { + try { + const platforms = buildRegisteredPlatforms(); + const prepared = await prepareMultiPlatformPlayground(platforms, { + title: 'Midscene Studio', + description: 'Multi-platform playground', + selectorFieldKey: 'platformId', + selectorVariant: 'cards', + }); + + const nextLaunchResult = await launchPreparedPlaygroundPlatform( + prepared, + { + corsOptions: createStudioCorsOptions(), + enableCors: true, + openBrowser: false, + verbose: false, + }, + ); + + launchResult = nextLaunchResult; + bootstrap = { + status: 'ready', + serverUrl: `http://${nextLaunchResult.host}:${nextLaunchResult.port}`, + port: nextLaunchResult.port, + error: null, + }; + + return bootstrap; + } catch (error) { + bootstrap = { + status: 'error', + serverUrl: null, + port: null, + error: getErrorMessage(error), + }; + return bootstrap; + } finally { + startPromise = null; + } + })(); + + return startPromise; + }; + + return { + close, + getBootstrap: () => bootstrap, + restart: async () => { + await close(); + return start(); + }, + start, + }; +} diff --git a/apps/studio/src/main/playground/types.ts b/apps/studio/src/main/playground/types.ts index 526ad34300..5272b2ac59 100644 --- a/apps/studio/src/main/playground/types.ts +++ b/apps/studio/src/main/playground/types.ts @@ -1,13 +1,9 @@ -import type { AndroidPlaygroundBootstrap } from '@shared/electron-contract'; +import type { PlaygroundBootstrap } from '@shared/electron-contract'; -export interface AndroidPlaygroundPackagePaths { - packageRoot: string; - staticDir: string; -} - -export interface AndroidPlaygroundRuntimeService { +/** Lifecycle contract for a playground runtime (single- or multi-platform). */ +export interface PlaygroundRuntimeService { close: () => Promise; - getBootstrap: () => AndroidPlaygroundBootstrap; - restart: () => Promise; - start: () => Promise; + getBootstrap: () => PlaygroundBootstrap; + restart: () => Promise; + start: () => Promise; } diff --git a/apps/studio/src/preload/index.ts b/apps/studio/src/preload/index.ts index adfdad55c4..dc00d1f376 100644 --- a/apps/studio/src/preload/index.ts +++ b/apps/studio/src/preload/index.ts @@ -20,10 +20,10 @@ const electronShellApi: ElectronShellApi = { }; const studioRuntimeApi: StudioRuntimeApi = { - getAndroidPlaygroundBootstrap: () => - ipcRenderer.invoke(IPC_CHANNELS.getAndroidPlaygroundBootstrap), - restartAndroidPlayground: () => - ipcRenderer.invoke(IPC_CHANNELS.restartAndroidPlayground), + getPlaygroundBootstrap: () => + ipcRenderer.invoke(IPC_CHANNELS.getPlaygroundBootstrap), + restartPlayground: () => ipcRenderer.invoke(IPC_CHANNELS.restartPlayground), + discoverDevices: () => ipcRenderer.invoke(IPC_CHANNELS.discoverDevices), runConnectivityTest: (request) => ipcRenderer.invoke(IPC_CHANNELS.runConnectivityTest, request), }; diff --git a/apps/studio/src/renderer/components/MainContent/index.tsx b/apps/studio/src/renderer/components/MainContent/index.tsx index 6e650879bc..1027c3d162 100644 --- a/apps/studio/src/renderer/components/MainContent/index.tsx +++ b/apps/studio/src/renderer/components/MainContent/index.tsx @@ -2,11 +2,12 @@ import { PlaygroundPreview } from '@midscene/playground-app'; import { useEffect, useRef, useState } from 'react'; import { assetUrls } from '../../assets'; import { - buildAndroidDeviceItems, + buildDeviceSelectionFormValues, buildStudioSidebarDeviceBuckets, - resolveAndroidDeviceLabel, - resolveConnectedAndroidDeviceId, - resolveSelectedAndroidDeviceId, + mergeSidebarDeviceBucketsWithDiscovery, + resolveConnectedDeviceId, + resolveConnectedDeviceLabel, + resolveSelectedDeviceId, } from '../../playground/selectors'; import { useStudioPlayground } from '../../playground/useStudioPlayground'; import ConnectingPreview from '../ConnectingPreview'; @@ -121,33 +122,25 @@ export default function MainContent({ useState(null); const [overviewRefreshing, setOverviewRefreshing] = useState(false); const isReady = studioPlayground.phase === 'ready'; - const androidItems = isReady - ? buildAndroidDeviceItems({ - formValues: studioPlayground.controller.state.formValues, - runtimeInfo: studioPlayground.controller.state.runtimeInfo, - targets: studioPlayground.controller.state.sessionSetup?.targets || [], - }) - : []; const deviceLabel = studioPlayground.phase === 'error' - ? 'Android Runtime Error' + ? 'Runtime Error' : isReady - ? resolveAndroidDeviceLabel(androidItems) - : 'Android playground starting'; + ? resolveConnectedDeviceLabel( + studioPlayground.controller.state.runtimeInfo, + { emptyLabel: 'No device selected' }, + ) + : 'Playground starting'; const isConnected = isReady ? studioPlayground.controller.state.sessionViewState.connected : false; - const connectedAndroidDeviceId = isReady - ? resolveConnectedAndroidDeviceId( - studioPlayground.controller.state.runtimeInfo, - ) + const connectedDeviceId = isReady + ? resolveConnectedDeviceId(studioPlayground.controller.state.runtimeInfo) : undefined; - const selectedAndroidDeviceId = isReady - ? resolveSelectedAndroidDeviceId( - studioPlayground.controller.state.formValues, - ) + const selectedDeviceId = isReady + ? resolveSelectedDeviceId(studioPlayground.controller.state.formValues) : undefined; - const previewDeviceId = connectedAndroidDeviceId ?? selectedAndroidDeviceId; + const previewDeviceId = connectedDeviceId ?? selectedDeviceId; const disconnectDisabled = !isReady || !studioPlayground.controller.state.sessionViewState.connected; const previewConnectionFailed = @@ -206,7 +199,7 @@ export default function MainContent({ ); } - const overviewBuckets = isReady + const overviewSessionBuckets = isReady ? buildStudioSidebarDeviceBuckets({ formValues: studioPlayground.controller.state.formValues, runtimeInfo: studioPlayground.controller.state.runtimeInfo, @@ -214,9 +207,13 @@ export default function MainContent({ studioPlayground.controller.state.sessionSetup?.targets || [], }) : { android: [], ios: [], computer: [], harmony: [], web: [] }; + const overviewBuckets = mergeSidebarDeviceBucketsWithDiscovery( + overviewSessionBuckets, + studioPlayground.discoveredDevices, + ); const overviewSelectedDeviceId = isReady && studioPlayground.controller.state.sessionMutating - ? selectedAndroidDeviceId + ? selectedDeviceId : undefined; return ( @@ -228,9 +225,16 @@ export default function MainContent({ } setOverviewRefreshing(true); try { - await studioPlayground.controller.actions.refreshSessionSetup( - studioPlayground.controller.state.formValues, - ); + // Refresh BOTH sources: session-setup targets (server-side + // list) and cross-platform discovery (ADB/HDC/displays). + // Discovery is what surfaces an unplug while a session is + // still technically "connected" on the server. + await Promise.all([ + studioPlayground.controller.actions.refreshSessionSetup( + studioPlayground.controller.state.formValues, + ), + studioPlayground.refreshDiscoveredDevices(), + ]); } finally { setOverviewRefreshing(false); } @@ -240,14 +244,18 @@ export default function MainContent({ { + onConnect={async (platform, device) => { if (!isReady) { return; } const { actions, state } = studioPlayground.controller; - state.form.setFieldsValue({ deviceId: device.id }); + const selectionValues = buildDeviceSelectionFormValues( + platform, + device, + ); + state.form.setFieldsValue(selectionValues); onSelectDeviceView?.(); - if (connectedAndroidDeviceId === device.id) { + if (connectedDeviceId === device.id) { return; } if (state.sessionViewState.connected) { @@ -255,7 +263,7 @@ export default function MainContent({ } const sessionValues = { ...state.form.getFieldsValue(true), - deviceId: device.id, + ...selectionValues, }; await actions.createSession(sessionValues); }} @@ -353,7 +361,7 @@ export default function MainContent({
{studioPlayground.phase === 'booting' ? (
- Android playground starting... + Playground starting...
) : studioPlayground.phase === 'error' ? (
@@ -363,16 +371,16 @@ export default function MainContent({
) : !studioPlayground.controller.state.serverOnline ? (
- Android playground server is offline. + Playground server is offline.
) : studioPlayground.controller.state.sessionViewState.connected ? (
diff --git a/apps/studio/src/renderer/components/Playground/index.tsx b/apps/studio/src/renderer/components/Playground/index.tsx index e670fadce8..c9446097d1 100644 --- a/apps/studio/src/renderer/components/Playground/index.tsx +++ b/apps/studio/src/renderer/components/Playground/index.tsx @@ -13,7 +13,7 @@ export default function Playground() {
{studioPlayground.phase === 'booting' ? (
- Android playground starting... + Playground starting...
) : studioPlayground.phase === 'error' ? (
@@ -23,11 +23,11 @@ export default function Playground() {
) : ( @@ -53,7 +53,7 @@ export default function Playground() { showVersionInfo: false, collapsibleProgressGroup: true, }} - title="Android Playground" + title="Playground" /> )}
diff --git a/apps/studio/src/renderer/components/Sidebar/index.tsx b/apps/studio/src/renderer/components/Sidebar/index.tsx index 8f7e588819..911c041a88 100644 --- a/apps/studio/src/renderer/components/Sidebar/index.tsx +++ b/apps/studio/src/renderer/components/Sidebar/index.tsx @@ -1,8 +1,10 @@ import { useState } from 'react'; import { assetUrls } from '../../assets'; import { + buildDeviceSelectionFormValues, buildStudioSidebarDeviceBuckets, - resolveConnectedAndroidDeviceId, + mergeSidebarDeviceBucketsWithDiscovery, + resolveConnectedDeviceId, } from '../../playground/selectors'; import type { StudioSidebarPlatformKey } from '../../playground/types'; import { useStudioPlayground } from '../../playground/useStudioPlayground'; @@ -16,6 +18,8 @@ interface DeviceItem { label: string; status: DeviceStatus; onClick?: () => void | Promise; + /** Purely informational rows that should never appear "selected". */ + isPlaceholder?: boolean; } interface SectionDefinition { @@ -167,7 +171,11 @@ export default function Sidebar({ })); }; - const deviceBuckets = + // Device buckets: merge session-setup targets (from the currently + // selected platform) with cross-platform discovered devices. Discovery + // is the source of truth for platforms that support it (ADB/HDC/ + // displays) — this is what makes an unplug disappear from the list. + const sessionBuckets = studioPlayground.phase === 'ready' ? buildStudioSidebarDeviceBuckets({ formValues: studioPlayground.controller.state.formValues, @@ -183,58 +191,115 @@ export default function Sidebar({ web: [], }; - const connectedAndroidDeviceId = + const deviceBuckets = mergeSidebarDeviceBucketsWithDiscovery( + sessionBuckets, + studioPlayground.discoveredDevices, + ); + + const connectedDeviceId = studioPlayground.phase === 'ready' - ? resolveConnectedAndroidDeviceId( - studioPlayground.controller.state.runtimeInfo, - ) + ? resolveConnectedDeviceId(studioPlayground.controller.state.runtimeInfo) : undefined; - const androidDevices: DeviceItem[] = - studioPlayground.phase === 'ready' - ? deviceBuckets.android.map((item) => ({ - id: item.id, - label: item.label, - status: item.status, + /** + * Build a click-enabled device list for any platform section. The + * multi-platform session manager expects: + * - `platformId` — which platform this device belongs to + * - `{platformId}.deviceId` — the prefixed field key for the target + */ + const buildDeviceItemsForPlatform = ( + platformKey: StudioSidebarPlatformKey, + devices: typeof deviceBuckets.android, + ): DeviceItem[] => { + if (studioPlayground.phase !== 'ready') { + if (platformKey === 'android') { + return [ + { + id: `${platformKey}-placeholder`, + label: + studioPlayground.phase === 'booting' + ? 'Playground starting' + : 'Runtime failed to start', + status: 'idle' as const, + isPlaceholder: true, + }, + ]; + } + return []; + } + + // iOS discovery needs WebDriverAgent running, which is a manual + // setup step; surface a hint row instead of an empty section so + // users know it isn't a bug. + if (platformKey === 'ios' && devices.length === 0) { + return [ + { + id: 'ios-setup-hint', + label: 'Set up iOS via the playground form', + status: 'idle' as const, + isPlaceholder: true, onClick: async () => { if (studioPlayground.phase !== 'ready') { return; } const { actions, state } = studioPlayground.controller; - state.form.setFieldsValue({ deviceId: item.id }); - onSelectDevice(); - if (connectedAndroidDeviceId === item.id) { - return; - } - if (state.sessionViewState.connected) { - await actions.destroySession(); - } - const sessionValues = { + const nextValues = { ...state.form.getFieldsValue(true), - deviceId: item.id, + platformId: 'ios', }; - await actions.createSession(sessionValues); - }, - })) - : [ - { - id: 'android-placeholder', - label: - studioPlayground.phase === 'booting' - ? 'Playground starting' - : 'Android runtime failed to start', - status: 'idle' as const, + state.form.setFieldsValue(nextValues); + onSelectDevice(); + await actions.refreshSessionSetup(nextValues); }, - ]; + }, + ]; + } + + return devices.map((item) => ({ + id: item.id, + label: item.label, + status: item.status, + onClick: async () => { + if (studioPlayground.phase !== 'ready') { + return; + } + const { actions, state } = studioPlayground.controller; + + // Tell the multi-platform session manager which platform + + // device to target. Field keys follow the `{platformId}.fieldKey` + // convention from `prepareMultiPlatformPlayground`. + const selectionValues = buildDeviceSelectionFormValues( + platformKey, + item, + ); + state.form.setFieldsValue(selectionValues); + + onSelectDevice(); + + if (connectedDeviceId === item.id) { + return; + } + if (state.sessionViewState.connected) { + await actions.destroySession(); + } + const sessionValues = { + ...state.form.getFieldsValue(true), + ...selectionValues, + }; + await actions.createSession(sessionValues); + }, + })); + }; - const selectedAndroidDeviceIds = + const selectedDeviceIds = studioPlayground.phase === 'ready' ? new Set( - deviceBuckets.android + Object.values(deviceBuckets) + .flat() .filter((item) => item.selected) .map((item) => item.id), ) - : new Set(['android-placeholder']); + : new Set(); const totalDeviceCount = sectionDefinitions.reduce( (sum, section) => sum + deviceBuckets[section.key].length, @@ -243,8 +308,10 @@ export default function Sidebar({ const resolvedSections = sectionDefinitions.map((section) => ({ ...section, - devices: - section.key === 'android' ? androidDevices : deviceBuckets[section.key], + devices: buildDeviceItemsForPlatform( + section.key, + deviceBuckets[section.key], + ), })); const overviewActive = activeView === 'overview'; @@ -295,15 +362,12 @@ export default function Sidebar({ {isExpanded ? ( hasDevices ? ( - section.devices.map((device, index) => ( + section.devices.map((device) => ( diff --git a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx index 5423ff167f..13f5acad43 100644 --- a/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx +++ b/apps/studio/src/renderer/playground/StudioPlaygroundProvider.tsx @@ -2,32 +2,44 @@ import { PlaygroundThemeProvider, usePlaygroundController, } from '@midscene/playground-app'; -import type { AndroidPlaygroundBootstrap } from '@shared/electron-contract'; +import type { + PlaygroundBootstrap, + StudioPlatformId, +} from '@shared/electron-contract'; import type { PropsWithChildren } from 'react'; import { useCallback, useEffect, useMemo, useState } from 'react'; +import { bucketDiscoveredDevices } from './selectors'; +import type { DiscoveredDevicesByPlatform } from './types'; import { StudioPlaygroundContext } from './useStudioPlayground'; function getMissingBridgeError() { return 'Studio preload bridge is unavailable. Restart the Electron app.'; } -function normalizeBootstrapError( - bootstrap: AndroidPlaygroundBootstrap, -): string { - return bootstrap.error || 'Failed to start Android playground runtime.'; +function normalizeBootstrapError(bootstrap: PlaygroundBootstrap): string { + return bootstrap.error || 'Failed to start playground runtime.'; } +// Default platform for Studio — pre-selected so the first session-setup +// poll already has a `platformId` and immediately returns Android +// targets, instead of the generic "Choose a platform" setup. +const DEFAULT_PLATFORM_ID: StudioPlatformId = 'android'; + function ReadyStudioPlaygroundProvider({ children, - restartAndroidPlayground, + discoveredDevices, + refreshDiscoveredDevices, + restartPlayground, serverUrl, }: PropsWithChildren<{ - restartAndroidPlayground: () => Promise; + discoveredDevices?: DiscoveredDevicesByPlatform; + refreshDiscoveredDevices: () => Promise; + restartPlayground: () => Promise; serverUrl: string; }>) { const controller = usePlaygroundController({ serverUrl, - defaultDeviceType: 'android', + initialFormValues: { platformId: DEFAULT_PLATFORM_ID }, }); const contextValue = useMemo( @@ -35,9 +47,17 @@ function ReadyStudioPlaygroundProvider({ phase: 'ready' as const, serverUrl, controller, - restartAndroidPlayground, + restartPlayground, + refreshDiscoveredDevices, + discoveredDevices, }), - [controller, restartAndroidPlayground, serverUrl], + [ + controller, + discoveredDevices, + refreshDiscoveredDevices, + restartPlayground, + serverUrl, + ], ); return ( @@ -47,6 +67,8 @@ function ReadyStudioPlaygroundProvider({ ); } +const DISCOVERY_POLL_INTERVAL_MS = 5000; + export function StudioPlaygroundProvider({ children }: PropsWithChildren) { const [bootstrap, setBootstrap] = useState< | { phase: 'booting' } @@ -55,6 +77,42 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { >({ phase: 'booting' }); const [bootstrapTick, setBootstrapTick] = useState(0); + // Cross-platform device discovery — polls ALL platforms (ADB, HDC, + // displays) once the playground runtime is ready, so the sidebar shows + // devices from every platform simultaneously. Gated on the ready phase + // to avoid waking up adb/hdc while we are still booting or in error. + const [discoveredDevices, setDiscoveredDevices] = useState< + DiscoveredDevicesByPlatform | undefined + >(); + + // Imperative scan — safe to call from anywhere (user-initiated refresh, + // post-destroy session cleanup, etc). Resolves after state is updated. + const refreshDiscoveredDevices = useCallback(async () => { + if (!window.studioRuntime?.discoverDevices) { + return; + } + try { + const devices = await window.studioRuntime.discoverDevices(); + setDiscoveredDevices(bucketDiscoveredDevices(devices)); + } catch (err) { + console.warn('[studio] device discovery failed:', err); + } + }, []); + + const pollingActive = bootstrap.phase === 'ready'; + useEffect(() => { + if (!pollingActive) { + return; + } + void refreshDiscoveredDevices(); + const id = window.setInterval(() => { + void refreshDiscoveredDevices(); + }, DISCOVERY_POLL_INTERVAL_MS); + return () => { + window.clearInterval(id); + }; + }, [pollingActive, refreshDiscoveredDevices]); + const readBootstrap = useCallback(async () => { if (!window.studioRuntime) { setBootstrap({ @@ -64,8 +122,7 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { return; } - const nextBootstrap = - await window.studioRuntime.getAndroidPlaygroundBootstrap(); + const nextBootstrap = await window.studioRuntime.getPlaygroundBootstrap(); if (nextBootstrap.status === 'ready' && nextBootstrap.serverUrl) { setBootstrap({ phase: 'ready', @@ -113,7 +170,7 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { }; }, [bootstrap.phase, bootstrapTick, readBootstrap]); - const restartAndroidPlayground = useCallback(async () => { + const restartPlayground = useCallback(async () => { setBootstrap({ phase: 'booting' }); if (!window.studioRuntime) { @@ -124,7 +181,7 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { return; } - const nextBootstrap = await window.studioRuntime.restartAndroidPlayground(); + const nextBootstrap = await window.studioRuntime.restartPlayground(); if (nextBootstrap.status === 'ready' && nextBootstrap.serverUrl) { setBootstrap({ phase: 'ready', @@ -146,21 +203,32 @@ export function StudioPlaygroundProvider({ children }: PropsWithChildren) { return { phase: 'error' as const, error: bootstrap.error, - restartAndroidPlayground, + restartPlayground, + refreshDiscoveredDevices, + discoveredDevices, }; } return { phase: 'booting' as const, - restartAndroidPlayground, + restartPlayground, + refreshDiscoveredDevices, + discoveredDevices, }; - }, [bootstrap, restartAndroidPlayground]); + }, [ + bootstrap, + discoveredDevices, + refreshDiscoveredDevices, + restartPlayground, + ]); return ( {bootstrap.phase === 'ready' ? ( {children} diff --git a/apps/studio/src/renderer/playground/selectors.ts b/apps/studio/src/renderer/playground/selectors.ts index e98887a017..e7080c78b1 100644 --- a/apps/studio/src/renderer/playground/selectors.ts +++ b/apps/studio/src/renderer/playground/selectors.ts @@ -3,6 +3,12 @@ import type { PlaygroundSessionTarget, } from '@midscene/playground'; import type { + DiscoveredDevice, + StudioSessionValue, +} from '@shared/electron-contract'; +import { STUDIO_PLATFORM_IDS } from '@shared/electron-contract'; +import type { + DiscoveredDevicesByPlatform, StudioAndroidDeviceItem, StudioSidebarDeviceBuckets, StudioSidebarPlatformKey, @@ -22,6 +28,23 @@ function createEmptySidebarDeviceBuckets(): StudioSidebarDeviceBuckets { }; } +function normalizePort(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value)) { + return value; + } + if (typeof value === 'string' && value.trim()) { + const parsed = Number.parseInt(value, 10); + if (Number.isFinite(parsed)) { + return parsed; + } + } + return undefined; +} + +function buildHostPortId(host: string, port: number): string { + return `${host}:${port}`; +} + function normalizeSidebarPlatformKey( value: unknown, ): StudioSidebarPlatformKey | undefined { @@ -52,17 +75,102 @@ function normalizeSidebarPlatformKey( } } +/** + * Platforms use different metadata keys for the device id: + * Android / Harmony → metadata.deviceId + * Computer → metadata.displayId + * iOS → metadata.wdaHost + metadata.wdaPort + */ +export function resolveConnectedDeviceId( + runtimeInfo: PlaygroundRuntimeInfo | null, +): string | undefined { + const metadata = runtimeInfo?.metadata || {}; + if (isString(metadata.deviceId)) { + return metadata.deviceId; + } + if (isString(metadata.displayId)) { + return metadata.displayId; + } + if (isString(metadata.wdaHost)) { + const wdaPort = normalizePort(metadata.wdaPort); + if (wdaPort !== undefined) { + return buildHostPortId(metadata.wdaHost, wdaPort); + } + } + return undefined; +} + +function resolveConnectedSessionValues( + runtimeInfo: PlaygroundRuntimeInfo | null, + platformKey: StudioSidebarPlatformKey, +): Record | undefined { + const metadata = runtimeInfo?.metadata || {}; + + switch (platformKey) { + case 'android': + case 'harmony': + return isString(metadata.deviceId) + ? { + deviceId: metadata.deviceId, + } + : undefined; + case 'computer': + return isString(metadata.displayId) + ? { + displayId: metadata.displayId, + } + : undefined; + case 'ios': { + const wdaPort = normalizePort(metadata.wdaPort); + return isString(metadata.wdaHost) && wdaPort !== undefined + ? { + host: metadata.wdaHost, + port: wdaPort, + } + : undefined; + } + default: + return undefined; + } +} + +/** + * Human-readable label for whatever device the playground is currently + * connected to, across platforms. Prefers the session display name, then + * falls back to a concrete device id, then to the platform title, and + * finally to `emptyLabel` when nothing is connected. + */ +export function resolveConnectedDeviceLabel( + runtimeInfo: PlaygroundRuntimeInfo | null, + options: { emptyLabel: string }, +): string { + const metadata = runtimeInfo?.metadata || {}; + if (isString(metadata.sessionDisplayName)) { + return metadata.sessionDisplayName; + } + const deviceId = resolveConnectedDeviceId(runtimeInfo); + if (deviceId) { + // "Display 1" reads better than a bare numeric id for computer. + return isString(metadata.displayId) && !isString(metadata.deviceId) + ? `Display ${deviceId}` + : deviceId; + } + if (isString(runtimeInfo?.title)) { + return runtimeInfo.title; + } + return options.emptyLabel; +} + function buildGenericConnectedDeviceItem( runtimeInfo: PlaygroundRuntimeInfo | null, - platformKey: Exclude, + platformKey: StudioSidebarPlatformKey, ): StudioAndroidDeviceItem | null { const metadata = runtimeInfo?.metadata || {}; - const deviceId = isString(metadata.deviceId) ? metadata.deviceId : undefined; + const deviceId = resolveConnectedDeviceId(runtimeInfo); const label = isString(metadata.sessionDisplayName) ? metadata.sessionDisplayName - : isString(runtimeInfo?.title) - ? runtimeInfo.title - : deviceId; + : deviceId || + (isString(runtimeInfo?.title) ? runtimeInfo.title : undefined); if (!label) { return null; @@ -74,6 +182,7 @@ function buildGenericConnectedDeviceItem( description: deviceId && deviceId !== label ? deviceId : undefined, selected: true, status: 'active', + sessionValues: resolveConnectedSessionValues(runtimeInfo, platformKey), }; } @@ -88,9 +197,100 @@ export function resolveConnectedAndroidDeviceId( export function resolveSelectedAndroidDeviceId( formValues: Record, ): string | undefined { + if (isString(formValues['android.deviceId'])) { + return formValues['android.deviceId']; + } return isString(formValues.deviceId) ? formValues.deviceId : undefined; } +export function resolveSelectedDeviceId( + formValues: Record, +): string | undefined { + const selectedPlatform = normalizeSidebarPlatformKey(formValues.platformId); + + if (selectedPlatform === 'ios') { + const host = isString(formValues['ios.host']) + ? formValues['ios.host'] + : isString(formValues.host) + ? formValues.host + : undefined; + const port = normalizePort(formValues['ios.port'] ?? formValues.port); + return host && port !== undefined ? buildHostPortId(host, port) : undefined; + } + + if (selectedPlatform === 'computer') { + return isString(formValues['computer.displayId']) + ? formValues['computer.displayId'] + : isString(formValues.displayId) + ? formValues.displayId + : undefined; + } + + if (selectedPlatform === 'harmony') { + return isString(formValues['harmony.deviceId']) + ? formValues['harmony.deviceId'] + : isString(formValues.deviceId) + ? formValues.deviceId + : undefined; + } + + if (selectedPlatform === 'android') { + return resolveSelectedAndroidDeviceId(formValues); + } + + return ( + resolveSelectedAndroidDeviceId(formValues) || + (isString(formValues['computer.displayId']) + ? formValues['computer.displayId'] + : undefined) || + ((): string | undefined => { + const host = isString(formValues['ios.host']) + ? formValues['ios.host'] + : undefined; + const port = normalizePort(formValues['ios.port']); + return host && port !== undefined + ? buildHostPortId(host, port) + : undefined; + })() + ); +} + +function prefixSessionValues( + platform: StudioSidebarPlatformKey, + sessionValues: Record, +): Record { + return Object.fromEntries( + Object.entries(sessionValues).map(([key, value]) => [ + `${platform}.${key}`, + value, + ]), + ); +} + +export function buildDeviceSelectionFormValues( + platform: StudioSidebarPlatformKey, + device: Pick, +): Record { + if (device.sessionValues) { + return { + platformId: platform, + ...prefixSessionValues(platform, device.sessionValues), + }; + } + + if (platform === 'computer') { + return { + platformId: platform, + [`${platform}.displayId`]: device.id, + }; + } + + return { + platformId: platform, + [`${platform}.deviceId`]: device.id, + }; +} + export function buildAndroidDeviceItems({ formValues, runtimeInfo, @@ -112,6 +312,7 @@ export function buildAndroidDeviceItems({ : connectedDeviceId, selected: true, status: 'active', + sessionValues: resolveConnectedSessionValues(runtimeInfo, 'android'), }, ]; } @@ -124,6 +325,9 @@ export function buildAndroidDeviceItems({ target.id === connectedDeviceId || (!connectedDeviceId && target.id === selectedDeviceId), status: target.id === connectedDeviceId ? 'active' : 'idle', + sessionValues: { + deviceId: target.id, + }, })); } @@ -180,6 +384,114 @@ export function resolveVisibleSidebarPlatforms( .map(([platformKey]) => platformKey); } +/** + * Bucket a flat discovery result by platform, driven off the canonical + * platform id set so a new platform can't be silently dropped. + */ +export function bucketDiscoveredDevices( + devices: DiscoveredDevice[], +): DiscoveredDevicesByPlatform { + const buckets = Object.fromEntries( + STUDIO_PLATFORM_IDS.map((key) => [key, [] as DiscoveredDevice[]]), + ) as DiscoveredDevicesByPlatform; + for (const device of devices) { + const bucket = buckets[device.platformId]; + if (bucket) { + bucket.push(device); + } + } + return buckets; +} + +/** + * Platforms that expose a real-time discovery source (ADB / HDC / + * display enumeration) in the main process. For these, discovery is + * the authoritative "physically present" list — once it has polled at + * least once, a session item that isn't discovered anymore is stale + * (e.g. the user unplugged the device while a session was still open). + */ +const AUTHORITATIVE_DISCOVERY_PLATFORMS = [ + 'android', + 'harmony', + 'computer', +] as const; +const ADDITIVE_DISCOVERY_PLATFORMS = ['ios'] as const; + +/** + * Merge session-setup buckets with the live discovery snapshot. For + * discoverable platforms, discovery wins — items not present in the + * discovery snapshot are dropped (catches unplug while connected), and + * items only present in discovery are appended as idle entries. + * + * iOS discovery is additive only: local WDA probes are appended to the + * sidebar, but they do not evict existing session rows because Studio can + * still connect to manually entered non-local WDA hosts. + * + * If `discovered` is undefined (first poll hasn't landed yet), the + * session buckets are returned as-is rather than being wiped out. + */ +export function mergeSidebarDeviceBucketsWithDiscovery( + sessionBuckets: StudioSidebarDeviceBuckets, + discovered: DiscoveredDevicesByPlatform | undefined, +): StudioSidebarDeviceBuckets { + if (!discovered) { + return sessionBuckets; + } + + const merged: StudioSidebarDeviceBuckets = { ...sessionBuckets }; + + for (const key of AUTHORITATIVE_DISCOVERY_PLATFORMS) { + const discoveredBucket = discovered[key]; + const discoveredIds = new Set(discoveredBucket.map((d) => d.id)); + + // Drop any session items whose id is not physically present anymore. + const survivingSessionItems = sessionBuckets[key].filter((item) => + discoveredIds.has(item.id), + ); + const survivingIds = new Set(survivingSessionItems.map((item) => item.id)); + + // Append discovered devices that aren't already covered by the + // session bucket (those already carry label/selected/active metadata). + const additions: StudioAndroidDeviceItem[] = []; + for (const dev of discoveredBucket) { + if (!survivingIds.has(dev.id)) { + additions.push({ + id: dev.id, + label: dev.label, + description: dev.description, + selected: false, + status: 'idle', + sessionValues: dev.sessionValues, + }); + } + } + + merged[key] = [...survivingSessionItems, ...additions]; + } + + for (const key of ADDITIVE_DISCOVERY_PLATFORMS) { + const existingIds = new Set(sessionBuckets[key].map((item) => item.id)); + const additions: StudioAndroidDeviceItem[] = []; + + for (const dev of discovered[key]) { + if (!existingIds.has(dev.id)) { + additions.push({ + id: dev.id, + label: dev.label, + description: dev.description, + selected: false, + status: 'idle', + sessionValues: dev.sessionValues, + }); + } + } + + merged[key] = [...sessionBuckets[key], ...additions]; + } + + return merged; +} + export function resolveAndroidDeviceLabel( items: StudioAndroidDeviceItem[], ): string { diff --git a/apps/studio/src/renderer/playground/types.ts b/apps/studio/src/renderer/playground/types.ts index 22b4b7b99d..2e11b95707 100644 --- a/apps/studio/src/renderer/playground/types.ts +++ b/apps/studio/src/renderer/playground/types.ts @@ -1,11 +1,10 @@ import type { PlaygroundControllerResult } from '@midscene/playground-app'; +import type { + DiscoveredDevice, + StudioPlatformId, +} from '@shared/electron-contract'; -export type StudioSidebarPlatformKey = - | 'android' - | 'ios' - | 'computer' - | 'harmony' - | 'web'; +export type StudioSidebarPlatformKey = StudioPlatformId; export interface StudioAndroidDeviceItem { id: string; @@ -13,6 +12,8 @@ export interface StudioAndroidDeviceItem { description?: string; selected: boolean; status: 'active' | 'idle'; + isPlaceholder?: boolean; + sessionValues?: DiscoveredDevice['sessionValues']; } export type StudioSidebarDeviceBuckets = Record< @@ -20,19 +21,31 @@ export type StudioSidebarDeviceBuckets = Record< StudioAndroidDeviceItem[] >; +/** All discovered devices from the cross-platform scan, bucketed. */ +export type DiscoveredDevicesByPlatform = Record< + StudioSidebarPlatformKey, + DiscoveredDevice[] +>; + export type StudioPlaygroundContextValue = | { phase: 'booting'; - restartAndroidPlayground: () => Promise; + restartPlayground: () => Promise; + refreshDiscoveredDevices: () => Promise; + discoveredDevices?: DiscoveredDevicesByPlatform; } | { phase: 'error'; error: string; - restartAndroidPlayground: () => Promise; + restartPlayground: () => Promise; + refreshDiscoveredDevices: () => Promise; + discoveredDevices?: DiscoveredDevicesByPlatform; } | { phase: 'ready'; serverUrl: string; controller: PlaygroundControllerResult; - restartAndroidPlayground: () => Promise; + restartPlayground: () => Promise; + refreshDiscoveredDevices: () => Promise; + discoveredDevices?: DiscoveredDevicesByPlatform; }; diff --git a/apps/studio/src/shared/electron-contract.ts b/apps/studio/src/shared/electron-contract.ts index d5540bc45c..ba29e1b43e 100644 --- a/apps/studio/src/shared/electron-contract.ts +++ b/apps/studio/src/shared/electron-contract.ts @@ -8,8 +8,13 @@ export const IPC_CHANNELS = { minimizeWindow: 'shell:minimize-window', openExternalUrl: 'shell:open-external-url', toggleMaximizeWindow: 'shell:toggle-maximize-window', - getAndroidPlaygroundBootstrap: 'studio:get-android-playground-bootstrap', - restartAndroidPlayground: 'studio:restart-android-playground', + // Multi-platform playground runtime (Android, iOS, HarmonyOS, Computer). + getPlaygroundBootstrap: 'studio:get-playground-bootstrap', + restartPlayground: 'studio:restart-playground', + // Cross-platform device discovery — returns devices from ALL platforms + // at once (Android via ADB, Harmony via HDC, Computer via display + // enumeration). Independent of session manager. + discoverDevices: 'studio:discover-devices', runConnectivityTest: 'studio:run-connectivity-test', } as const; @@ -23,13 +28,47 @@ export type ConnectivityTestResult = | { ok: true; sample: string } | { ok: false; error: string }; -export interface AndroidPlaygroundBootstrap { +/** Generic bootstrap status for the multi-platform playground server. */ +export interface PlaygroundBootstrap { status: 'starting' | 'ready' | 'error'; serverUrl: string | null; port: number | null; error: string | null; } +/** + * Canonical set of platform identifiers the Studio shell understands. + * Shared between main process discovery and renderer sidebar/buckets so + * neither side can drift into using a stringly-typed platform id. + */ +export const STUDIO_PLATFORM_IDS = [ + 'android', + 'ios', + 'computer', + 'harmony', + 'web', +] as const; + +export type StudioPlatformId = (typeof STUDIO_PLATFORM_IDS)[number]; + +export type StudioSessionValue = string | number | boolean; + +/** A device discovered across any platform, tagged with its platform. */ +export interface DiscoveredDevice { + platformId: StudioPlatformId; + id: string; + label: string; + description?: string; + /** + * Session-setup field values for this discovered target, before Studio + * prefixes them with `{platformId}.`. + */ + sessionValues?: Record; +} + +/** Result of the cross-platform device discovery scan. */ +export type DiscoverDevicesResult = DiscoveredDevice[]; + /** * Public API exposed on `window.electronShell` by the preload bridge. * @@ -52,8 +91,10 @@ export interface ElectronShellApi { } export interface StudioRuntimeApi { - getAndroidPlaygroundBootstrap: () => Promise; - restartAndroidPlayground: () => Promise; + getPlaygroundBootstrap: () => Promise; + restartPlayground: () => Promise; + /** Scan ALL platforms for connected devices (ADB, HDC, displays). */ + discoverDevices: () => Promise; runConnectivityTest: ( request: ConnectivityTestRequest, ) => Promise; diff --git a/apps/studio/tests/bucket-discovered-devices.test.ts b/apps/studio/tests/bucket-discovered-devices.test.ts new file mode 100644 index 0000000000..deba2ce1c3 --- /dev/null +++ b/apps/studio/tests/bucket-discovered-devices.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest'; +import { bucketDiscoveredDevices } from '../src/renderer/playground/selectors'; + +describe('bucketDiscoveredDevices', () => { + it('returns empty buckets for every platform when no devices are discovered', () => { + expect(bucketDiscoveredDevices([])).toEqual({ + android: [], + ios: [], + computer: [], + harmony: [], + web: [], + }); + }); + + it('groups devices by their platformId tag', () => { + const result = bucketDiscoveredDevices([ + { platformId: 'android', id: 'a1', label: 'Pixel' }, + { + platformId: 'ios', + id: 'localhost:8100', + label: 'iOS via WDA', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + { platformId: 'android', id: 'a2', label: 'Galaxy' }, + { platformId: 'computer', id: 'c1', label: 'Display 1' }, + ]); + + expect(result.android.map((d) => d.id)).toEqual(['a1', 'a2']); + expect(result.ios).toEqual([ + { + platformId: 'ios', + id: 'localhost:8100', + label: 'iOS via WDA', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + ]); + expect(result.computer.map((d) => d.id)).toEqual(['c1']); + expect(result.harmony).toEqual([]); + expect(result.web).toEqual([]); + }); +}); diff --git a/apps/studio/tests/electron-contract.test.ts b/apps/studio/tests/electron-contract.test.ts index 044938ad30..97c973a399 100644 --- a/apps/studio/tests/electron-contract.test.ts +++ b/apps/studio/tests/electron-contract.test.ts @@ -2,13 +2,12 @@ import { describe, expect, it } from 'vitest'; import { IPC_CHANNELS } from '../src/shared/electron-contract'; describe('IPC_CHANNELS', () => { - it('includes shell and Android playground bridge channels', () => { + it('includes shell and playground bridge channels', () => { expect(IPC_CHANNELS.openExternalUrl).toBe('shell:open-external-url'); - expect(IPC_CHANNELS.getAndroidPlaygroundBootstrap).toBe( - 'studio:get-android-playground-bootstrap', - ); - expect(IPC_CHANNELS.restartAndroidPlayground).toBe( - 'studio:restart-android-playground', + expect(IPC_CHANNELS.getPlaygroundBootstrap).toBe( + 'studio:get-playground-bootstrap', ); + expect(IPC_CHANNELS.restartPlayground).toBe('studio:restart-playground'); + expect(IPC_CHANNELS.discoverDevices).toBe('studio:discover-devices'); }); }); diff --git a/apps/studio/tests/playground-selectors.test.ts b/apps/studio/tests/playground-selectors.test.ts index ba128588ca..9b4e1ab439 100644 --- a/apps/studio/tests/playground-selectors.test.ts +++ b/apps/studio/tests/playground-selectors.test.ts @@ -1,8 +1,13 @@ import { describe, expect, it } from 'vitest'; import { buildAndroidDeviceItems, + buildDeviceSelectionFormValues, buildStudioSidebarDeviceBuckets, + mergeSidebarDeviceBucketsWithDiscovery, resolveAndroidDeviceLabel, + resolveConnectedDeviceId, + resolveConnectedDeviceLabel, + resolveSelectedDeviceId, resolveVisibleSidebarPlatforms, } from '../src/renderer/playground/selectors'; @@ -40,6 +45,9 @@ describe('buildAndroidDeviceItems', () => { description: undefined, selected: true, status: 'active', + sessionValues: { + deviceId: 'device-1', + }, }, { id: 'device-2', @@ -47,6 +55,9 @@ describe('buildAndroidDeviceItems', () => { description: undefined, selected: false, status: 'idle', + sessionValues: { + deviceId: 'device-2', + }, }, ]); }); @@ -115,6 +126,9 @@ describe('buildStudioSidebarDeviceBuckets', () => { description: undefined, selected: true, status: 'active', + sessionValues: { + deviceId: 'device-1', + }, }, { id: 'device-2', @@ -122,6 +136,9 @@ describe('buildStudioSidebarDeviceBuckets', () => { description: undefined, selected: false, status: 'idle', + sessionValues: { + deviceId: 'device-2', + }, }, ], ios: [], @@ -159,6 +176,9 @@ describe('buildStudioSidebarDeviceBuckets', () => { description: 'HDC-001', selected: true, status: 'active', + sessionValues: { + deviceId: 'HDC-001', + }, }, ], web: [], @@ -196,3 +216,303 @@ describe('resolveAndroidDeviceLabel', () => { ).toBe('Pixel 8'); }); }); + +describe('mergeSidebarDeviceBucketsWithDiscovery', () => { + const emptyBuckets = { + android: [], + ios: [], + computer: [], + harmony: [], + web: [], + }; + + it('returns session buckets unchanged when discovery has not polled yet', () => { + const session = { + ...emptyBuckets, + android: [ + { + id: 'device-1', + label: 'Pixel 9', + selected: true, + status: 'active' as const, + }, + ], + }; + + expect(mergeSidebarDeviceBucketsWithDiscovery(session, undefined)).toBe( + session, + ); + }); + + it('drops a session item that is no longer discoverable (phone unplugged)', () => { + const session = { + ...emptyBuckets, + android: [ + { + id: 'device-1', + label: 'Pixel 9', + selected: true, + status: 'active' as const, + }, + ], + }; + const discovered = { + ...emptyBuckets, + // device-1 has vanished from ADB + }; + + expect( + mergeSidebarDeviceBucketsWithDiscovery(session, discovered).android, + ).toEqual([]); + }); + + it('appends discovered devices that session setup has not surfaced', () => { + const session = emptyBuckets; + const discovered = { + ...emptyBuckets, + android: [ + { + platformId: 'android' as const, + id: 'device-2', + label: 'Galaxy', + description: 'ADB: device-2', + }, + ], + }; + + expect( + mergeSidebarDeviceBucketsWithDiscovery(session, discovered).android, + ).toEqual([ + { + id: 'device-2', + label: 'Galaxy', + description: 'ADB: device-2', + selected: false, + status: 'idle', + }, + ]); + }); + + it('preserves manual iOS session rows while appending local WDA probes', () => { + const session = { + ...emptyBuckets, + ios: [ + { + id: 'remote-wda:8100', + label: 'iPhone 15', + selected: true, + status: 'active' as const, + }, + ], + }; + const discovered = { + ...emptyBuckets, + ios: [ + { + platformId: 'ios' as const, + id: 'localhost:8100', + label: 'iOS via WDA', + description: 'WebDriverAgent: localhost:8100', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + ], + }; + + expect( + mergeSidebarDeviceBucketsWithDiscovery(session, discovered).ios, + ).toEqual([ + { + id: 'remote-wda:8100', + label: 'iPhone 15', + selected: true, + status: 'active', + }, + { + id: 'localhost:8100', + label: 'iOS via WDA', + description: 'WebDriverAgent: localhost:8100', + selected: false, + status: 'idle', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + ]); + }); + + it('appends local iOS WDA probes without evicting manual iOS sessions', () => { + const session = { + ...emptyBuckets, + ios: [ + { + id: 'remote-wda:8100', + label: 'Remote iPhone', + selected: true, + status: 'active' as const, + }, + ], + }; + const discovered = { + ...emptyBuckets, + ios: [ + { + platformId: 'ios' as const, + id: 'localhost:8100', + label: 'iOS via WDA', + description: 'WebDriverAgent: localhost:8100', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + ], + }; + + expect( + mergeSidebarDeviceBucketsWithDiscovery(session, discovered).ios, + ).toEqual([ + { + id: 'remote-wda:8100', + label: 'Remote iPhone', + selected: true, + status: 'active', + }, + { + id: 'localhost:8100', + label: 'iOS via WDA', + description: 'WebDriverAgent: localhost:8100', + selected: false, + status: 'idle', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }, + ]); + }); +}); + +describe('resolveConnectedDeviceLabel', () => { + const emptyOpts = { emptyLabel: 'No device' }; + + it('returns emptyLabel when no runtime info is available', () => { + expect(resolveConnectedDeviceLabel(null, emptyOpts)).toBe('No device'); + }); + + it('prefers sessionDisplayName over raw device id', () => { + expect( + resolveConnectedDeviceLabel( + { + interface: { type: 'android' }, + preview: { kind: 'none', capabilities: [] }, + executionUxHints: [], + metadata: { + deviceId: 'emulator-5554', + sessionDisplayName: 'Pixel 9', + }, + }, + emptyOpts, + ), + ).toBe('Pixel 9'); + }); + + it('falls back to deviceId when sessionDisplayName is absent', () => { + expect( + resolveConnectedDeviceLabel( + { + interface: { type: 'android' }, + preview: { kind: 'none', capabilities: [] }, + executionUxHints: [], + metadata: { deviceId: 'emulator-5554' }, + }, + emptyOpts, + ), + ).toBe('emulator-5554'); + }); + + it('labels computer displays using the displayId', () => { + expect( + resolveConnectedDeviceLabel( + { + interface: { type: 'computer' }, + preview: { kind: 'none', capabilities: [] }, + executionUxHints: [], + metadata: { displayId: '1' }, + }, + emptyOpts, + ), + ).toBe('Display 1'); + }); + + it('uses WDA host and port as the fallback iOS connection id', () => { + expect( + resolveConnectedDeviceId({ + interface: { type: 'ios' }, + preview: { kind: 'none', capabilities: [] }, + executionUxHints: [], + metadata: { + wdaHost: 'localhost', + wdaPort: 8100, + }, + }), + ).toBe('localhost:8100'); + }); + + it('falls back to runtime title when no device metadata is present', () => { + expect( + resolveConnectedDeviceLabel( + { + title: 'Midscene Playground', + interface: { type: 'android' }, + preview: { kind: 'none', capabilities: [] }, + executionUxHints: [], + metadata: {}, + }, + emptyOpts, + ), + ).toBe('Midscene Playground'); + }); +}); + +describe('resolveSelectedDeviceId', () => { + it('supports prefixed Android selection values', () => { + expect( + resolveSelectedDeviceId({ + platformId: 'android', + 'android.deviceId': 'device-2', + }), + ).toBe('device-2'); + }); + + it('builds the iOS selection id from host and port', () => { + expect( + resolveSelectedDeviceId({ + platformId: 'ios', + 'ios.host': 'localhost', + 'ios.port': 8100, + }), + ).toBe('localhost:8100'); + }); +}); + +describe('buildDeviceSelectionFormValues', () => { + it('prefixes discovery-provided session values for iOS', () => { + expect( + buildDeviceSelectionFormValues('ios', { + id: 'localhost:8100', + sessionValues: { + host: 'localhost', + port: 8100, + }, + }), + ).toEqual({ + platformId: 'ios', + 'ios.host': 'localhost', + 'ios.port': 8100, + }); + }); +}); diff --git a/apps/studio/vitest.config.ts b/apps/studio/vitest.config.ts index 0d5f95a1f3..6187f6aa17 100644 --- a/apps/studio/vitest.config.ts +++ b/apps/studio/vitest.config.ts @@ -1,6 +1,15 @@ +import path from 'node:path'; import { defineConfig } from 'vitest/config'; export default defineConfig({ + resolve: { + alias: { + '@main': path.resolve(__dirname, 'src/main'), + '@preload': path.resolve(__dirname, 'src/preload'), + '@renderer': path.resolve(__dirname, 'src/renderer'), + '@shared': path.resolve(__dirname, 'src/shared'), + }, + }, test: { environment: 'node', environmentMatchGlobs: [['tests/theme-provider.test.ts', 'jsdom']], diff --git a/packages/playground-app/src/controller/usePlaygroundController.ts b/packages/playground-app/src/controller/usePlaygroundController.ts index db75da8474..8b9f1baad0 100644 --- a/packages/playground-app/src/controller/usePlaygroundController.ts +++ b/packages/playground-app/src/controller/usePlaygroundController.ts @@ -2,7 +2,14 @@ import type { PlaygroundSessionSetup } from '@midscene/playground'; import { PlaygroundSDK } from '@midscene/playground'; import { type DeviceType, useEnvConfig } from '@midscene/visualizer'; import { Form, message } from 'antd'; -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from 'react'; import { resolveAutoCreateSessionInput } from '../session-setup'; import { buildSessionInitialValues, @@ -28,6 +35,13 @@ export interface UsePlaygroundControllerOptions { defaultDeviceType?: DeviceType; pollIntervalMs?: number; countdownSeconds?: number; + /** + * Seed values written into the session-setup form on the first render. + * Useful for pre-selecting a default platform so the initial + * `refreshSessionSetup` poll already has a `platformId`, instead of + * returning a generic "Choose a platform" setup. + */ + initialFormValues?: Record; } export function usePlaygroundController({ @@ -35,8 +49,23 @@ export function usePlaygroundController({ defaultDeviceType = 'web', pollIntervalMs = 5000, countdownSeconds = 3, + initialFormValues, }: UsePlaygroundControllerOptions): PlaygroundControllerResult { const [form] = Form.useForm(); + const initialFormValuesRef = useRef(initialFormValues); + // Seed the form ONCE before paint. Later prop changes are ignored so + // the user's in-flight edits never get overwritten. + useLayoutEffect(() => { + const seed = initialFormValuesRef.current; + if (!seed) { + return; + } + for (const [key, value] of Object.entries(seed)) { + if (form.getFieldValue(key) === undefined) { + form.setFieldsValue({ [key]: value } as Partial); + } + } + }, [form]); const formValues = (Form.useWatch([], form) ?? {}) as Record; const [countdown, setCountdown] = useState(null); const [sessionSetup, setSessionSetup] = diff --git a/packages/playground/src/adapters/remote-execution.ts b/packages/playground/src/adapters/remote-execution.ts index 29cac89fe1..970820da4f 100644 --- a/packages/playground/src/adapters/remote-execution.ts +++ b/packages/playground/src/adapters/remote-execution.ts @@ -446,7 +446,9 @@ export class RemoteExecutionAdapter extends BasePlaygroundAdapter { const response = await fetch(`${this.serverUrl}/screenshot`); if (!response.ok) { - console.warn(`Screenshot request failed: ${response.statusText}`); + if (response.status !== 409) { + console.warn(`Screenshot request failed: ${response.statusText}`); + } return null; } diff --git a/packages/playground/src/server.ts b/packages/playground/src/server.ts index cfa0572958..341666e558 100644 --- a/packages/playground/src/server.ts +++ b/packages/playground/src/server.ts @@ -1159,8 +1159,11 @@ class PlaygroundServer { } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - console.error(`Failed to take screenshot: ${errorMessage}`); - res.status(errorMessage === 'No active session' ? 409 : 500).json({ + const statusCode = errorMessage === 'No active session' ? 409 : 500; + if (statusCode !== 409) { + console.error(`Failed to take screenshot: ${errorMessage}`); + } + res.status(statusCode).json({ error: `Failed to take screenshot: ${errorMessage}`, }); } diff --git a/packages/playground/tests/unit/remote-execution-adapter.test.ts b/packages/playground/tests/unit/remote-execution-adapter.test.ts index 93033ca3cb..c8c9d7aa32 100644 --- a/packages/playground/tests/unit/remote-execution-adapter.test.ts +++ b/packages/playground/tests/unit/remote-execution-adapter.test.ts @@ -608,4 +608,38 @@ describe('RemoteExecutionAdapter', () => { }); }); }); + + describe('getScreenshot', () => { + it('should return screenshot data when the server responds successfully', async () => { + const screenshotResponse = { + screenshot: 'data:image/png;base64,abc123', + timestamp: 123, + }; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve(screenshotResponse), + }); + + await expect(adapter.getScreenshot()).resolves.toEqual( + screenshotResponse, + ); + expect(mockFetch).toHaveBeenCalledWith(`${mockServerUrl}/screenshot`); + }); + + it('should suppress warning logs for expected 409 no-session screenshot responses', async () => { + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 409, + statusText: 'Conflict', + }); + const consoleWarnSpy = vi + .spyOn(console, 'warn') + .mockImplementation(() => {}); + + await expect(adapter.getScreenshot()).resolves.toBeNull(); + expect(consoleWarnSpy).not.toHaveBeenCalled(); + + consoleWarnSpy.mockRestore(); + }); + }); }); diff --git a/packages/playground/tests/unit/server-session-manager.test.ts b/packages/playground/tests/unit/server-session-manager.test.ts index f46dad7441..f46c0104b0 100644 --- a/packages/playground/tests/unit/server-session-manager.test.ts +++ b/packages/playground/tests/unit/server-session-manager.test.ts @@ -22,7 +22,7 @@ function createMockResponse() { function getRouteHandler( server: PlaygroundServer, - method: 'post' | 'delete', + method: 'get' | 'post' | 'delete', route: string, ) { const calls = (server.app[method] as any).mock.calls as Array<[string, any]>; @@ -165,6 +165,48 @@ describe('PlaygroundServer session manager APIs', () => { expect(sidecar.stop).toHaveBeenCalledTimes(1); }); + test('returns 409 without logging an error when screenshot is requested without a session', async () => { + const consoleErrorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => {}); + const server = new PlaygroundServer(); + server.setPreparedPlatform({ + platformId: 'computer', + title: 'Midscene Computer Playground', + description: 'Computer playground platform descriptor', + preview: { + kind: 'screenshot', + title: 'Desktop preview', + screenshotPath: '/screenshot', + capabilities: [], + }, + metadata: { + sessionConnected: false, + setupState: 'required', + }, + sessionManager: { + async createSession() { + throw new Error('not needed'); + }, + }, + }); + + await server.launch(6105); + const screenshotHandler = getRouteHandler(server, 'get', '/screenshot'); + expect(screenshotHandler).toBeTypeOf('function'); + + const response = createMockResponse(); + await screenshotHandler({}, response); + + expect(response.statusCode).toBe(409); + expect(response.body).toMatchObject({ + error: 'Failed to take screenshot: No active session', + }); + expect(consoleErrorSpy).not.toHaveBeenCalled(); + + consoleErrorSpy.mockRestore(); + }); + test('stops started sidecars and restores base runtime when session creation fails after apply', async () => { const sidecar = { id: 'session-sidecar', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 724c0d1d43..a951f21660 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -510,12 +510,27 @@ importers: apps/studio: dependencies: + '@midscene/android': + specifier: workspace:* + version: link:../../packages/android '@midscene/android-playground': specifier: workspace:* version: link:../../packages/android-playground + '@midscene/computer': + specifier: workspace:* + version: link:../../packages/computer + '@midscene/computer-playground': + specifier: workspace:* + version: link:../../packages/computer-playground '@midscene/core': specifier: workspace:* version: link:../../packages/core + '@midscene/harmony': + specifier: workspace:* + version: link:../../packages/harmony + '@midscene/ios': + specifier: workspace:* + version: link:../../packages/ios '@midscene/playground': specifier: workspace:* version: link:../../packages/playground