From 095fdbb147d79bf9f67e6063a08c94e063dd9888 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Fri, 17 Apr 2026 08:56:16 +0200 Subject: [PATCH 1/4] fix: wire soft assertions into Harness runtime --- packages/runtime/src/expect/context.ts | 29 +++++++++++++++ packages/runtime/src/expect/errors.ts | 49 +++++++++++++++++++++++++ packages/runtime/src/expect/expect.ts | 21 ++++++++--- packages/runtime/src/runner/runSuite.ts | 36 ++++++++++++------ 4 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 packages/runtime/src/expect/context.ts create mode 100644 packages/runtime/src/expect/errors.ts diff --git a/packages/runtime/src/expect/context.ts b/packages/runtime/src/expect/context.ts new file mode 100644 index 00000000..47d89eaf --- /dev/null +++ b/packages/runtime/src/expect/context.ts @@ -0,0 +1,29 @@ +type HarnessExpectError = { + name?: string; + message?: string; +}; + +export type HarnessExpectTestState = { + result?: { + state: 'pass' | 'fail'; + errors?: HarnessExpectError[]; + }; + promises?: Promise[]; + onFinished?: Array<() => void | Promise>; +}; + +declare global { + var HARNESS_EXPECT_TEST_STATE: HarnessExpectTestState | undefined; +} + +export const getCurrentExpectTestState = (): + | HarnessExpectTestState + | undefined => { + return globalThis.HARNESS_EXPECT_TEST_STATE; +}; + +export const setCurrentExpectTestState = ( + state: HarnessExpectTestState | undefined, +): void => { + globalThis.HARNESS_EXPECT_TEST_STATE = state; +}; diff --git a/packages/runtime/src/expect/errors.ts b/packages/runtime/src/expect/errors.ts new file mode 100644 index 00000000..6cd510dc --- /dev/null +++ b/packages/runtime/src/expect/errors.ts @@ -0,0 +1,49 @@ +import type { HarnessExpectTestState } from './context.js'; + +const formatErrorMessage = (error: unknown): string => { + if (error instanceof Error) { + return `${error.name}: ${error.message}`; + } + + if (error && typeof error === 'object') { + const maybeError = error as { name?: string; message?: string }; + if (maybeError.name || maybeError.message) { + return [maybeError.name, maybeError.message].filter(Boolean).join(': '); + } + } + + return String(error); +}; + +export const flushExpectTestState = async ( + state: HarnessExpectTestState, +): Promise => { + if (state.promises?.length) { + const results = await Promise.allSettled(state.promises); + const rejected = results + .filter( + (result): result is PromiseRejectedResult => + result.status === 'rejected', + ) + .map((result) => result.reason); + + if (rejected.length > 0) { + throw new Error(rejected.map(formatErrorMessage).join('\n\n')); + } + } + + for (const hook of state.onFinished ?? []) { + await hook(); + } + + const softErrors = state.result?.errors ?? []; + if (softErrors.length === 0) { + return; + } + + throw new Error( + ['Soft assertion failures:', ...softErrors.map(formatErrorMessage)].join( + '\n\n', + ), + ); +}; diff --git a/packages/runtime/src/expect/expect.ts b/packages/runtime/src/expect/expect.ts index beca251a..88a6c25e 100644 --- a/packages/runtime/src/expect/expect.ts +++ b/packages/runtime/src/expect/expect.ts @@ -13,6 +13,7 @@ import { import * as chai from 'chai'; // Setup additional matchers +import { getCurrentExpectTestState } from './context.js'; import './setup.js'; import { toMatchImageSnapshot } from './matchers/toMatchImageSnapshot.js'; @@ -20,12 +21,22 @@ export function createExpect(): ExpectStatic { const expect = ((value: unknown, message?: string): Assertion => { const { assertionCalls } = getState(expect); setState({ assertionCalls: assertionCalls + 1 }, expect); - return chai.expect(value, message) as unknown as Assertion; + + const assertion = chai.expect(value, message) as unknown as Assertion & { + withTest?: (test: unknown) => Assertion; + }; + const currentTest = getCurrentExpectTestState(); + + return currentTest && assertion.withTest + ? assertion.withTest(currentTest) + : assertion; }) as ExpectStatic; Object.assign(expect, chai.expect); Object.assign( expect, - globalThis[ASYMMETRIC_MATCHERS_OBJECT as unknown as keyof typeof globalThis] + globalThis[ + ASYMMETRIC_MATCHERS_OBJECT as unknown as keyof typeof globalThis + ], ); expect.getState = () => getState(expect); @@ -44,7 +55,7 @@ export function createExpect(): ExpectStatic { expectedAssertionsNumber: null, expectedAssertionsNumberErrorGen: null, }, - expect + expect, ); // @ts-expect-error untyped @@ -62,7 +73,7 @@ export function createExpect(): ExpectStatic { // @ts-expect-error untyped expect.unreachable = (message?: string) => { chai.assert.fail( - `expected${message ? ` "${message}" ` : ' '}not to be reached` + `expected${message ? ` "${message}" ` : ' '}not to be reached`, ); }; @@ -71,7 +82,7 @@ export function createExpect(): ExpectStatic { new Error( `expected number of assertions to be ${expected}, but got ${ expect.getState().assertionCalls - }` + }`, ); if (Error.captureStackTrace) { Error.captureStackTrace(errorGen(), assertions); diff --git a/packages/runtime/src/runner/runSuite.ts b/packages/runtime/src/runner/runSuite.ts index c189d373..7dc5c008 100644 --- a/packages/runtime/src/runner/runSuite.ts +++ b/packages/runtime/src/runner/runSuite.ts @@ -4,6 +4,11 @@ import type { TestSuite, TestSuiteResult, } from '@react-native-harness/bridge'; +import { + setCurrentExpectTestState, + type HarnessExpectTestState, +} from '../expect/context.js'; +import { flushExpectTestState } from '../expect/errors.js'; import { runHooks } from './hooks.js'; import { getTestExecutionError } from './errors.js'; import { TestRunnerContext } from './types.js'; @@ -15,7 +20,7 @@ declare global { const runTest = async ( test: TestCase, suite: TestSuite, - context: TestRunnerContext + context: TestRunnerContext, ): Promise => { const startTime = Date.now(); @@ -69,14 +74,23 @@ const runTest = async ( return result; } - // Run all beforeEach hooks from the current suite and its parents - await runHooks(suite, 'beforeEach'); + const expectTestState: HarnessExpectTestState = {}; + setCurrentExpectTestState(expectTestState); - // Run the actual test - await test.fn(); + try { + // Run all beforeEach hooks from the current suite and its parents + await runHooks(suite, 'beforeEach'); - // Run all afterEach hooks from the current suite and its parents - await runHooks(suite, 'afterEach'); + // Run the actual test + await test.fn(); + + // Run all afterEach hooks from the current suite and its parents + await runHooks(suite, 'afterEach'); + + await flushExpectTestState(expectTestState); + } finally { + setCurrentExpectTestState(undefined); + } const duration = Date.now() - startTime; @@ -102,7 +116,7 @@ const runTest = async ( error, context.testFilePath, suite.name, - test.name + test.name, ); const duration = Date.now() - startTime; @@ -130,7 +144,7 @@ const runTest = async ( export const runSuite = async ( suite: TestSuite, - context: TestRunnerContext + context: TestRunnerContext, ): Promise => { const startTime = Date.now(); @@ -212,10 +226,10 @@ export const runSuite = async ( // Check if any tests or child suites failed const hasFailedTests = testResults.some( - (result) => result.status === 'failed' + (result) => result.status === 'failed', ); const hasFailedSuites = suiteResults.some( - (result) => result.status === 'failed' + (result) => result.status === 'failed', ); if (hasFailedTests || hasFailedSuites) { From 67da8a1fb6526d3521f2585f0cc4946cf51f74ae Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Fri, 17 Apr 2026 09:40:23 +0200 Subject: [PATCH 2/4] fix: preserve soft assertion callsites --- .../src/__tests__/withRnHarness.test.ts | 101 ++++++++++++++++++ packages/bundler-metro/src/withRnHarness.ts | 8 +- packages/runtime/src/expect/context.ts | 1 + packages/runtime/src/expect/errors.ts | 39 +++++-- website/src/docs/api/expect.md | 56 +++++----- 5 files changed, 167 insertions(+), 38 deletions(-) create mode 100644 packages/bundler-metro/src/__tests__/withRnHarness.test.ts diff --git a/packages/bundler-metro/src/__tests__/withRnHarness.test.ts b/packages/bundler-metro/src/__tests__/withRnHarness.test.ts new file mode 100644 index 00000000..9d835960 --- /dev/null +++ b/packages/bundler-metro/src/__tests__/withRnHarness.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it, vi } from 'vitest'; + +type MinimalMetroConfig = { + projectRoot: string; + serializer?: { + isThirdPartyModule?: (module: { path: string }) => boolean; + }; + symbolicator?: { + customizeFrame?: (frame: { file?: string | null }) => Promise<{ + collapse: boolean; + }>; + }; +}; + +vi.mock('@react-native-harness/config', () => ({ + getConfig: vi.fn(async () => ({ + config: {}, + })), +})); + +vi.mock('../babel-transformer.js', () => ({ + getHarnessBabelTransformerPath: vi.fn( + () => '/tmp/harness-babel-transformer.js', + ), +})); + +vi.mock('../manifest.js', () => ({ + getHarnessManifest: vi.fn(() => '/tmp/harness-manifest.js'), +})); + +vi.mock('../metro-cache.js', () => ({ + getHarnessCacheStores: vi.fn(() => []), +})); + +vi.mock('../resolvers/resolver.js', () => ({ + getHarnessResolver: vi.fn(() => vi.fn()), +})); + +describe('withRnHarness', () => { + it('treats workspace Harness packages as internal callsites', async () => { + const { withRnHarness } = await import('../withRnHarness.js'); + + const config = (await withRnHarness( + { + projectRoot: '/tmp/app', + serializer: {}, + symbolicator: { + async customizeFrame() { + return {}; + }, + }, + }, + true, + )()) as unknown as MinimalMetroConfig; + + expect( + config.serializer?.isThirdPartyModule?.({ + path: '/repo/packages/runtime/src/expect/errors.ts', + }), + ).toBe(true); + + await expect( + config.symbolicator?.customizeFrame?.({ + file: '/repo/packages/runtime/src/expect/errors.ts', + }), + ).resolves.toEqual({ + collapse: true, + }); + }); + + it('does not collapse app source files', async () => { + const { withRnHarness } = await import('../withRnHarness.js'); + + const config = (await withRnHarness( + { + projectRoot: '/tmp/app', + serializer: {}, + symbolicator: { + async customizeFrame() { + return {}; + }, + }, + }, + true, + )()) as unknown as MinimalMetroConfig; + + expect( + config.serializer?.isThirdPartyModule?.({ + path: '/repo/apps/playground/src/__tests__/smoke.harness.ts', + }), + ).toBe(false); + + await expect( + config.symbolicator?.customizeFrame?.({ + file: '/repo/apps/playground/src/__tests__/smoke.harness.ts', + }), + ).resolves.toEqual({ + collapse: false, + }); + }); +}); diff --git a/packages/bundler-metro/src/withRnHarness.ts b/packages/bundler-metro/src/withRnHarness.ts index 0d51f9a1..55012163 100644 --- a/packages/bundler-metro/src/withRnHarness.ts +++ b/packages/bundler-metro/src/withRnHarness.ts @@ -11,11 +11,11 @@ import type { NotReadOnly } from './utils.js'; const require = createRequire(import.meta.url); const INTERNAL_CALLSITES_REGEX = - /(^|[\\/])(node_modules[/\\]@react-native-harness)([\\/]|$)/; + /(^|[\\/])((node_modules[/\\]@react-native-harness)|packages[/\\](babel-preset|bridge|bundler-metro|cli|config|github-action|jest|metro|platform-android|platform-apple|platform-vega|platform-web|platforms|plugins|react-native-harness|runtime|tools|ui))([\\/]|$)/; export const withRnHarness = ( config: T | Promise, - isInvokedByHarness = false + isInvokedByHarness = false, ): (() => Promise) => { return async () => { if (!isInvokedByHarness) { @@ -42,9 +42,7 @@ export const withRnHarness = ( getPolyfills: (...args) => [ ...(metroConfig.serializer?.getPolyfills?.(...args) ?? []), harnessManifest, - require.resolve( - '@react-native-harness/runtime/polyfills/harness-module-system' - ), + require.resolve('@react-native-harness/runtime/polyfills/harness-module-system'), ], isThirdPartyModule({ path: modulePath }) { const isThirdPartyByDefault = diff --git a/packages/runtime/src/expect/context.ts b/packages/runtime/src/expect/context.ts index 47d89eaf..1edf629d 100644 --- a/packages/runtime/src/expect/context.ts +++ b/packages/runtime/src/expect/context.ts @@ -1,6 +1,7 @@ type HarnessExpectError = { name?: string; message?: string; + stack?: string; }; export type HarnessExpectTestState = { diff --git a/packages/runtime/src/expect/errors.ts b/packages/runtime/src/expect/errors.ts index 6cd510dc..7520ad72 100644 --- a/packages/runtime/src/expect/errors.ts +++ b/packages/runtime/src/expect/errors.ts @@ -1,5 +1,11 @@ import type { HarnessExpectTestState } from './context.js'; +type SerializedExpectError = { + name?: string; + message?: string; + stack?: string; +}; + const formatErrorMessage = (error: unknown): string => { if (error instanceof Error) { return `${error.name}: ${error.message}`; @@ -15,6 +21,31 @@ const formatErrorMessage = (error: unknown): string => { return String(error); }; +const createExpectError = (errors: unknown[], title?: string): Error => { + const message = [title, ...errors.map(formatErrorMessage)] + .filter(Boolean) + .join('\n\n'); + + const error = new Error(message); + const firstError = errors.find( + (value): value is SerializedExpectError => + !!value && typeof value === 'object', + ); + + if (firstError?.name) { + error.name = firstError.name; + } + + if (firstError?.stack) { + error.stack = firstError.stack.replace( + /^([^\n]+)(\n|$)/, + `${error.name}: ${message}$2`, + ); + } + + return error; +}; + export const flushExpectTestState = async ( state: HarnessExpectTestState, ): Promise => { @@ -28,7 +59,7 @@ export const flushExpectTestState = async ( .map((result) => result.reason); if (rejected.length > 0) { - throw new Error(rejected.map(formatErrorMessage).join('\n\n')); + throw createExpectError(rejected); } } @@ -41,9 +72,5 @@ export const flushExpectTestState = async ( return; } - throw new Error( - ['Soft assertion failures:', ...softErrors.map(formatErrorMessage)].join( - '\n\n', - ), - ); + throw createExpectError(softErrors, 'Soft assertion failures:'); }; diff --git a/website/src/docs/api/expect.md b/website/src/docs/api/expect.md index c525e1aa..635fe527 100644 --- a/website/src/docs/api/expect.md +++ b/website/src/docs/api/expect.md @@ -5,38 +5,38 @@ Harness uses Vitest's `expect` function for making assertions in your tests. The ## Basic Usage ```typescript -import { describe, test, expect } from 'react-native-harness' +import { describe, test, expect } from 'react-native-harness'; describe('basic assertions', () => { test('primitive values', () => { - expect(2 + 2).toBe(4) - expect('hello').toBe('hello') - expect(true).toBeTruthy() - expect(false).toBeFalsy() - }) + expect(2 + 2).toBe(4); + expect('hello').toBe('hello'); + expect(true).toBeTruthy(); + expect(false).toBeFalsy(); + }); test('objects and arrays', () => { - expect({ name: 'John', age: 30 }).toEqual({ name: 'John', age: 30 }) - expect([1, 2, 3]).toContain(2) - expect(['apple', 'banana']).toHaveLength(2) - }) + expect({ name: 'John', age: 30 }).toEqual({ name: 'John', age: 30 }); + expect([1, 2, 3]).toContain(2); + expect(['apple', 'banana']).toHaveLength(2); + }); test('strings and numbers', () => { - expect('hello world').toContain('world') - expect('hello world').toMatch(/world/) - expect(3.14).toBeCloseTo(3.1, 1) - expect(10).toBeGreaterThan(5) - }) + expect('hello world').toContain('world'); + expect('hello world').toMatch(/world/); + expect(3.14).toBeCloseTo(3.1, 1); + expect(10).toBeGreaterThan(5); + }); test('exceptions', () => { const throwError = () => { - throw new Error('Something went wrong') - } - - expect(throwError).toThrow() - expect(throwError).toThrow('Something went wrong') - }) -}) + throw new Error('Something went wrong'); + }; + + expect(throwError).toThrow(); + expect(throwError).toThrow('Something went wrong'); + }); +}); ``` ## Soft Assertions @@ -45,13 +45,15 @@ Use `expect.soft` to continue test execution even when assertions fail: ```typescript test('soft assertions', () => { - expect.soft(1 + 1).toBe(3) // This will fail but test continues - expect.soft(2 + 2).toBe(5) // This will also fail but test continues - expect(3 + 3).toBe(6) // This passes + expect.soft(1 + 1).toBe(3); // This will fail but test continues + expect.soft(2 + 2).toBe(5); // This will also fail but test continues + expect(3 + 3).toBe(6); // This passes // Test will be marked as failed due to soft assertion failures -}) +}); ``` +When multiple soft assertions fail, Harness reports all failure messages, but currently only the first failure contributes the highlighted stack trace and code frame. + ## Complete API Reference Harness provides the complete Vitest expect API including: @@ -65,4 +67,4 @@ Harness provides the complete Vitest expect API including: - **Types**: `toBeInstanceOf`, `toBeTypeOf` - **Asymmetric matchers**: `expect.anything()`, `expect.any()`, `expect.arrayContaining()`, etc. -For the complete documentation of all available matchers and advanced features, please refer to the [Vitest expect documentation](https://vitest.dev/api/expect.html). \ No newline at end of file +For the complete documentation of all available matchers and advanced features, please refer to the [Vitest expect documentation](https://vitest.dev/api/expect.html). From 6f97fcea8002d43253781770c55c75581e682571 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Fri, 17 Apr 2026 09:58:27 +0200 Subject: [PATCH 3/4] fix: scope internal Metro frames to installed Harness packages --- packages/bundler-metro/src/__tests__/withRnHarness.test.ts | 6 +++--- packages/bundler-metro/src/withRnHarness.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/bundler-metro/src/__tests__/withRnHarness.test.ts b/packages/bundler-metro/src/__tests__/withRnHarness.test.ts index 9d835960..64ae442f 100644 --- a/packages/bundler-metro/src/__tests__/withRnHarness.test.ts +++ b/packages/bundler-metro/src/__tests__/withRnHarness.test.ts @@ -37,7 +37,7 @@ vi.mock('../resolvers/resolver.js', () => ({ })); describe('withRnHarness', () => { - it('treats workspace Harness packages as internal callsites', async () => { + it('treats installed Harness packages as internal callsites', async () => { const { withRnHarness } = await import('../withRnHarness.js'); const config = (await withRnHarness( @@ -55,13 +55,13 @@ describe('withRnHarness', () => { expect( config.serializer?.isThirdPartyModule?.({ - path: '/repo/packages/runtime/src/expect/errors.ts', + path: '/repo/node_modules/@react-native-harness/runtime/dist/expect/errors.js', }), ).toBe(true); await expect( config.symbolicator?.customizeFrame?.({ - file: '/repo/packages/runtime/src/expect/errors.ts', + file: '/repo/node_modules/@react-native-harness/runtime/dist/expect/errors.js', }), ).resolves.toEqual({ collapse: true, diff --git a/packages/bundler-metro/src/withRnHarness.ts b/packages/bundler-metro/src/withRnHarness.ts index 55012163..d4dfa32e 100644 --- a/packages/bundler-metro/src/withRnHarness.ts +++ b/packages/bundler-metro/src/withRnHarness.ts @@ -11,7 +11,7 @@ import type { NotReadOnly } from './utils.js'; const require = createRequire(import.meta.url); const INTERNAL_CALLSITES_REGEX = - /(^|[\\/])((node_modules[/\\]@react-native-harness)|packages[/\\](babel-preset|bridge|bundler-metro|cli|config|github-action|jest|metro|platform-android|platform-apple|platform-vega|platform-web|platforms|plugins|react-native-harness|runtime|tools|ui))([\\/]|$)/; + /(^|[\\/])(node_modules[/\\]@react-native-harness)([\\/]|$)/; export const withRnHarness = ( config: T | Promise, From c9fbafd8bb3488984e9e505db9923e4d149b4c05 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Fri, 17 Apr 2026 10:19:38 +0200 Subject: [PATCH 4/4] chore: add version plan --- .nx/version-plans/version-plan-1776413769245.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .nx/version-plans/version-plan-1776413769245.md diff --git a/.nx/version-plans/version-plan-1776413769245.md b/.nx/version-plans/version-plan-1776413769245.md new file mode 100644 index 00000000..67f370f3 --- /dev/null +++ b/.nx/version-plans/version-plan-1776413769245.md @@ -0,0 +1,5 @@ +--- +__default__: patch +--- + +Using expect.soft will no longer throw an internal exception.