Support source overrides for composable integration test packages

.buildkite/pipeline.trigger.integration.tests.sh

@@ -199,6 +199,17 @@ echo "          image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
 echo "        artifact_paths:"
 echo "          - build/elastic-stack-dump/install-zip-shellinit/logs/*.log"
 
+echo "      - label: \":go: Integration test: build-install-composable\""
+echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-build-install-packages-composable"
+echo "        agents:"
+echo "          provider: \"gcp\""
+echo "          image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
+echo "        artifact_paths:"
+echo "          - build/test-results/*.xml"
+echo "          - build/elastic-stack-dump/composable/logs/*.log"
+echo "          - build/elastic-stack-dump/composable/logs/fleet-server-internal/**/*"
+echo "          - build/test-coverage/coverage-*.xml"
+
 echo "      - label: \":go: Integration test: system-flags\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-system-test-flags"
 echo "        agents:"

Makefile

@@ -101,7 +101,7 @@ test-stack-command-with-basic-subscription:
 
 test-stack-command: test-stack-command-default test-stack-command-agent-version-flag test-stack-command-7x test-stack-command-800 test-stack-command-8x test-stack-command-9x test-stack-command-with-apm-server
 
-test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash
+test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash test-build-install-packages-composable
 
 test-check-packages-with-kind:
 	PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh
@@ -127,6 +127,9 @@ test-check-packages-parallel:
 test-check-packages-with-custom-agent:
 	PACKAGE_TEST_TYPE=with-custom-agent ./scripts/test-check-packages.sh
 
+test-build-install-packages-composable:
+	./scripts/test-composable-packages.sh
+
 test-build-zip:
 	./scripts/test-build-zip.sh
 

cmd/testrunner.go

@@ -197,16 +197,31 @@ func testRunnerAssetCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	baseURL := stack.PackageRegistryBaseURL(profile, appConfig)
+	eprClient, err := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	if err != nil {
+		return fmt.Errorf("failed to create package registry client: %w", err)
+	}
+	assetOverrides, err := globalTestConfig.Asset.RequiresSourceOverrides(packageRoot)
+	if err != nil {
+		return fmt.Errorf("failed to resolve requires source overrides: %w", err)
+	}
+	requiredInputsResolver := requiredinputs.NewRequiredInputsResolver(
+		eprClient,
+		requiredinputs.WithSourceOverrides(assetOverrides),
+	)
+
 	logger.Info(version.Version())
 	logger.Infof("elastic-stack: %s\n", stackVersion.Version())
 	runner := asset.NewAssetTestRunner(asset.AssetTestRunnerOptions{
-		PackageRoot:      packageRoot,
-		KibanaClient:     kibanaClient,
-		GlobalTestConfig: globalTestConfig.Asset,
-		WithCoverage:     testCoverage,
-		CoverageType:     testCoverageFormat,
-		RepositoryRoot:   repositoryRoot,
-		SchemaURLs:       appConfig.SchemaURLs(),
+		PackageRoot:            packageRoot,
+		KibanaClient:           kibanaClient,
+		GlobalTestConfig:       globalTestConfig.Asset,
+		WithCoverage:           testCoverage,
+		CoverageType:           testCoverageFormat,
+		RepositoryRoot:         repositoryRoot,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 
 	results, err := testrunner.RunSuite(ctx, runner)
@@ -630,29 +645,44 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	baseURL := stack.PackageRegistryBaseURL(profile, appConfig)
+	eprClient, err := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	if err != nil {
+		return fmt.Errorf("failed to create package registry client: %w", err)
+	}
+	systemOverrides, err := globalTestConfig.System.RequiresSourceOverrides(packageRoot)
+	if err != nil {
+		return fmt.Errorf("failed to resolve requires source overrides: %w", err)
+	}
+	requiredInputsResolver := requiredinputs.NewRequiredInputsResolver(
+		eprClient,
+		requiredinputs.WithSourceOverrides(systemOverrides),
+	)
+
 	logger.Info(version.Version())
 	logger.Infof("elastic-stack: %s", info.Version.Number)
 	runner := system.NewSystemTestRunner(system.SystemTestRunnerOptions{
-		Profile:              profile,
-		PackageRoot:          packageRoot,
-		KibanaClient:         kibanaClient,
-		SchemaURLs:           appConfig.SchemaURLs(),
-		API:                  esClient.API,
-		ESClient:             esClient,
-		ConfigFilePath:       configFileFlag,
-		RunSetup:             runSetup,
-		RunTearDown:          runTearDown,
-		RunTestsOnly:         runTestsOnly,
-		DataStreams:          dataStreams,
-		ServiceVariant:       variantFlag,
-		FailOnMissingTests:   failOnMissing,
-		GenerateTestResult:   generateTestResult,
-		DeferCleanup:         deferCleanup,
-		GlobalTestConfig:     globalTestConfig.System,
-		WithCoverage:         testCoverage,
-		CoverageType:         testCoverageFormat,
-		RepositoryRoot:       repositoryRoot,
-		OverrideAgentVersion: agentVersion,
+		Profile:                profile,
+		PackageRoot:            packageRoot,
+		KibanaClient:           kibanaClient,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		API:                    esClient.API,
+		ESClient:               esClient,
+		ConfigFilePath:         configFileFlag,
+		RunSetup:               runSetup,
+		RunTearDown:            runTearDown,
+		RunTestsOnly:           runTestsOnly,
+		DataStreams:            dataStreams,
+		ServiceVariant:         variantFlag,
+		FailOnMissingTests:     failOnMissing,
+		GenerateTestResult:     generateTestResult,
+		DeferCleanup:           deferCleanup,
+		GlobalTestConfig:       globalTestConfig.System,
+		WithCoverage:           testCoverage,
+		CoverageType:           testCoverageFormat,
+		RepositoryRoot:         repositoryRoot,
+		OverrideAgentVersion:   agentVersion,
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 
 	logger.Debugf("Running suite...")
@@ -883,7 +913,14 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("failed to create package registry client: %w", err)
 	}
-	requiredInputsResolver := requiredinputs.NewRequiredInputsResolver(eprClient)
+	policyOverrides, err := globalTestConfig.Policy.RequiresSourceOverrides(packageRoot)
+	if err != nil {
+		return fmt.Errorf("failed to resolve requires source overrides: %w", err)
+	}
+	requiredInputsResolver := requiredinputs.NewRequiredInputsResolver(
+		eprClient,
+		requiredinputs.WithSourceOverrides(policyOverrides),
+	)
 
 	logger.Info(version.Version())
 	logger.Infof("elastic-stack: %s", stackVersion.Version())

docs/howto/dependency_management.md

@@ -127,6 +127,49 @@ package dependencies are fetched from the configured package registry URL
 For details on using a local or custom registry when the required input packages are still
 under development, see [HOWTO: Use a local or custom package registry](./local_package_registry.md).
 
+### Testing composable packages with source overrides
+
+When running `elastic-package test` on a composable integration whose required input packages
+are not yet published to the registry, you can point each test runner at a local copy of the
+input package using the `requires` key in `_dev/test/config.yml`.
+
+Each entry in the `requires` list uses one of two forms:
+
+- **`source`** — a path to a local package directory or `.zip` file. Relative paths are
+  resolved relative to the composable package root. The package name is read from the
+  `manifest.yml` at that path.
+- **`package` + `version`** — forces a specific version to be fetched from the registry
+  (useful in CI where the package is already published and you want to pin a version).
+
+`source` and `package`/`version` are mutually exclusive in the same entry.
+
+The `requires` key is supported under any test runner block: `policy`, `system`, `asset`,
+`pipeline`, and `static`. You may define it in multiple blocks; if the same package appears
+in more than one block, the resolved absolute paths must be identical.
+
+```yaml
+# _dev/test/config.yml — composable integration package
+policy:
+  requires:
+    - source: "../my_input_pkg"       # local directory, relative to this package root
+system:
+  requires:
+    - source: "../my_input_pkg"       # same override reused for system tests
+asset:
+  requires:
+    - package: my_input_pkg           # registry-based override for asset tests
+      version: "0.2.0"
+```
+
+> **Note:** Source overrides only affect the test runners (`elastic-package test`).
+> `elastic-package build` always fetches required input packages from the configured
+> package registry. To use a local registry during builds, see
+> [HOWTO: Use a local or custom package registry](./local_package_registry.md).
+
+A working example lives at
+`test/packages/composable/02_ci_composable_integration/_dev/test/config.yml` (uses
+`source: "../01_ci_input_pkg"`).
+
 ### Linked files (`*.link`) and `template_path`
 
 Some repositories share agent templates using **link files** (files ending in `.link` that
@@ -139,7 +182,7 @@ build directory. In `manifest.yml`, always set `template_path` / `template_paths
 Fleet and the builder resolve templates by the names declared in the manifest; the `.link`
 file exists only in the source tree.
 
-A small manual fixture that combines `requires.input` with a linked policy input template
-lives under `test/manual_packages/required_inputs/with_linked_template_path/`. Automated
+A test fixture that combines `requires.input` with a linked policy input template
+lives under `internal/requiredinputs/testdata/with_linked_template_path/`. Automated
 coverage is in `TestBundleInputPackageTemplates_PreservesLinkedTemplateTargetPath` in
-`internal/requiredinputs/requiredinputs_test.go`.
+`internal/requiredinputs/requiredinputs_test.go`.

docs/howto/policy_testing.md

@@ -43,15 +43,35 @@ or data stream.
 
 ## Global test configuration
 
-Each package could define a configuration file in `_dev/test/config.yml` to skip all the policy tests.
+Each package can define a configuration file in `_dev/test/config.yml` to control how policy
+tests are run. The full shape of the `policy` block is:
 
 ```yaml
 policy:
   skip:
     reason: <reason>
     link: <link_to_issue>
+  requires:
+    - source: "../path/to/input_pkg"   # local directory or .zip (relative to package root)
+    - package: sql_input               # registry-based; requires version
+      version: "0.2.0"
 ```
 
+The `skip` key skips all policy tests with a mandatory reason and optional issue link.
+
+The `requires` key is used for **composable integration packages** that declare
+`requires.input` in their `manifest.yml`. It tells the policy test runner which local source
+path (or registry version) to use for each required input package when bundling the
+integration before running the tests. Without this, the runner fetches dependencies from the
+configured package registry.
+
+- Use `source` when the input package is not yet published to the registry (local development).
+- Use `package` + `version` to pin a specific registry version.
+- `source` and `package`/`version` are mutually exclusive in the same entry.
+
+For full details on composable packages and source overrides, see
+[HOWTO: Enable dependency management](./dependency_management.md#testing-composable-packages-with-source-overrides).
+
 ### Defining the configuration of the policy
 
 Test configuration for the policy is defined in a YAML file prefixed with

internal/builder/packages.go

@@ -234,13 +234,11 @@ func BuildPackage(options BuildOptions) (string, error) {
 		return "", fmt.Errorf("resolving transform manifests failed: %w", err)
 	}
 
-	resolver := options.RequiredInputsResolver
-	if resolver == nil {
-		resolver = &requiredinputs.NoopRequiredInputsResolver{}
-	}
-	err = resolver.Bundle(buildPackageRoot)
-	if err != nil {
-		return "", fmt.Errorf("bundling input package templates failed: %w", err)
+	if options.RequiredInputsResolver != nil {
+		err = options.RequiredInputsResolver.Bundle(buildPackageRoot)
+		if err != nil {
+			return "", fmt.Errorf("bundling input package templates failed: %w", err)
+		}
 	}
 
 	if options.UpdateReadmes {

internal/requiredinputs/requiredinputs.go

@@ -28,26 +28,39 @@ type Resolver interface {
 	Bundle(buildPackageRoot string) error
 }
 
-// NoopRequiredInputsResolver is a no-op implementation of Resolver.
-// TODO: Replace with a resolver that supports test overrides (e.g. local package paths)
-// when implementing local input package resolution for development and testing workflows.
-type NoopRequiredInputsResolver struct{}
+// ResolverOption is a functional option for RequiredInputsResolver.
+type ResolverOption func(*RequiredInputsResolver)
 
-func (r *NoopRequiredInputsResolver) Bundle(_ string) error {
-	return nil
+// WithSourceOverrides returns a ResolverOption that configures local path overrides for
+// specific input packages. When set, the resolver uses the local path instead of
+// downloading from the EPR. The map key is the package name; the value is the local
+// path to the package (zip file or directory with a manifest.yml at the root).
+func WithSourceOverrides(overrides map[string]string) ResolverOption {
+	return func(r *RequiredInputsResolver) {
+		r.sourceOverrides = overrides
+	}
 }
 
 // RequiredInputsResolver implements Resolver by downloading required input packages via an EPR client
 // and applying Bundle to the built package tree.
 type RequiredInputsResolver struct {
 	eprClient eprClient
+	// sourceOverrides maps package name to a local path (zip or directory).
+	// A nil map is safe: lookups return ("", false), so all packages fall back to EPR download.
+	sourceOverrides map[string]string
 }
 
 // NewRequiredInputsResolver returns a Resolver that downloads required input packages from the registry.
-func NewRequiredInputsResolver(eprClient eprClient) *RequiredInputsResolver {
-	return &RequiredInputsResolver{
+// Optional ResolverOption values can be provided to configure additional behaviour such as
+// local source overrides (see WithSourceOverrides).
+func NewRequiredInputsResolver(eprClient eprClient, opts ...ResolverOption) *RequiredInputsResolver {
+	r := &RequiredInputsResolver{
 		eprClient: eprClient,
 	}
+	for _, opt := range opts {
+		opt(r)
+	}
+	return r
 }
 
 // Bundle updates buildPackageRoot (a built package directory) for integrations that declare
@@ -115,16 +128,25 @@ func (r *RequiredInputsResolver) Bundle(buildPackageRoot string) error {
 	return nil
 }
 
-// downloadInputsToTmp downloads required input packages to the temporary directory.
-// It returns a map of package name to zip path.
+// mapRequiredInputPackagesPaths resolves required input packages to local paths.
+// For each dependency it first checks for a source override; if found the local path is used
+// directly (zip or directory). Otherwise it downloads the package from the EPR.
+// It returns a map of package name to path.
 func (r *RequiredInputsResolver) mapRequiredInputPackagesPaths(manifestInputRequires []packages.PackageDependency, tmpDir string) (map[string]string, error) {
 	inputPkgPaths := make(map[string]string, len(manifestInputRequires))
 	errs := make([]error, 0, len(manifestInputRequires))
 	for _, inputDependency := range manifestInputRequires {
 		if _, ok := inputPkgPaths[inputDependency.Package]; ok {
-			// skip if already downloaded
+			// skip if already resolved
 			continue
 		}
+
+		if sourcePath, ok := r.sourceOverrides[inputDependency.Package]; ok {
+			inputPkgPaths[inputDependency.Package] = sourcePath
+			logger.Debugf("Using local source override for input package %q at %s", inputDependency.Package, sourcePath)
+			continue
+		}
+
 		path, err := r.eprClient.DownloadPackage(inputDependency.Package, inputDependency.Version, tmpDir)
 		if err != nil {
 			// all required input packages must be downloaded successfully