From b6a99c60e5129dfac1a33a09b48b6c97dc09d270 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 20 Apr 2026 11:40:30 -0700 Subject: [PATCH 1/2] fix(native-build): harden registry login retries --- .../nativeBuild/__tests__/buildkit.test.ts | 103 +++++++++++++++++- src/server/lib/nativeBuild/engines.ts | 18 ++- 2 files changed, 111 insertions(+), 10 deletions(-) diff --git a/src/server/lib/nativeBuild/__tests__/buildkit.test.ts b/src/server/lib/nativeBuild/__tests__/buildkit.test.ts index 045623c4..665bbc01 100644 --- a/src/server/lib/nativeBuild/__tests__/buildkit.test.ts +++ b/src/server/lib/nativeBuild/__tests__/buildkit.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { buildkitBuild, BuildkitBuildOptions, generateSecretArgsScript } from '../engines'; +import { buildkitBuild, NativeBuildOptions, generateSecretArgsScript } from '../engines'; import { shellPromise } from '../../shell'; import { waitForJobAndGetLogs, getGitHubToken } from '../utils'; import GlobalConfigService from '../../../services/globalConfig'; @@ -68,7 +68,7 @@ describe('buildkitBuild', () => { build: { isStatic: false }, } as any; - const mockOptions: BuildkitBuildOptions = { + const mockOptions: NativeBuildOptions = { ecrRepo: 'test-repo', ecrDomain: '123456789.dkr.ecr.us-east-1.amazonaws.com', envVars: { NODE_ENV: 'production' }, @@ -247,6 +247,35 @@ describe('buildkitBuild', () => { expect(fullCommand).toContain('build-arg:NODE_ENV=production'); }); + it('adds retry settings to registry bootstrap commands', async () => { + await buildkitBuild(mockDeploy, mockOptions); + + const kubectlCalls = (shellPromise as jest.Mock).mock.calls; + const applyCall = kubectlCalls.find((call) => call[0].includes('kubectl apply')); + const fullCommand = applyCall[0]; + + expect(fullCommand).toContain('apk add --no-cache --retries 5 aws-cli docker-cli'); + expect(fullCommand).toContain('apk add --no-cache --retries 5 docker-cli'); + expect(fullCommand).toContain('export AWS_MAX_ATTEMPTS=5'); + expect(fullCommand).toContain('export AWS_RETRY_MODE=adaptive'); + }); + + it('renders registry domain safely for non-ECR buildkit targets', async () => { + const optionsWithCustomRegistry = { + ...mockOptions, + ecrDomain: 'registry.internal.svc.cluster.local', + }; + + await buildkitBuild(mockDeploy, optionsWithCustomRegistry); + + const kubectlCalls = (shellPromise as jest.Mock).mock.calls; + const applyCall = kubectlCalls.find((call) => call[0].includes('kubectl apply')); + const fullCommand = applyCall[0]; + + expect(fullCommand).toContain('REGISTRY_DOMAIN=\\"registry.internal.svc.cluster.local\\"'); + expect(fullCommand).toContain('apk add --no-cache --retries 5 docker-cli'); + }); + it('coerces numeric env var values to strings for Kubernetes compatibility', async () => { const optionsWithNumericEnv = { ...mockOptions, @@ -320,7 +349,7 @@ describe('build resource precedence', () => { build: { isStatic: false }, } as any; - const baseOptions: BuildkitBuildOptions = { + const baseOptions: NativeBuildOptions = { ecrRepo: 'test-repo', ecrDomain: '123456789.dkr.ecr.us-east-1.amazonaws.com', envVars: { NODE_ENV: 'production' }, @@ -495,7 +524,7 @@ describe('build pod annotations', () => { build: { isStatic: false }, } as any; - const baseOptions: BuildkitBuildOptions = { + const baseOptions: NativeBuildOptions = { ecrRepo: 'test-repo', ecrDomain: '123456789.dkr.ecr.us-east-1.amazonaws.com', envVars: { NODE_ENV: 'production' }, @@ -655,3 +684,69 @@ describe('generateSecretArgsScript', () => { expect(result).toContain('--opt build-arg:MY_SECRET='); }); }); + +describe('kaniko registry login bootstrap', () => { + const mockDeploy = { + deployable: { name: 'test-service' }, + $fetchGraph: jest.fn(), + build: { isStatic: false }, + } as any; + + const baseOptions: NativeBuildOptions = { + ecrRepo: 'test-repo', + ecrDomain: '123456789.dkr.ecr.us-east-1.amazonaws.com', + envVars: { NODE_ENV: 'production' }, + dockerfilePath: 'Dockerfile', + tag: 'v1.0.0', + revision: 'abc123def456789', + repo: 'owner/repo', + branch: 'main', + namespace: 'env-test-123', + buildId: '456', + deployUuid: 'test-service-abc123', + jobTimeout: 1800, + }; + + beforeEach(() => { + jest.clearAllMocks(); + (GlobalConfigService.getInstance as jest.Mock).mockReturnValue({ + getAllConfigs: jest.fn().mockResolvedValue({}), + }); + (getGitHubToken as jest.Mock).mockResolvedValue('github-token-123'); + (shellPromise as jest.Mock).mockResolvedValue(''); + (waitForJobAndGetLogs as jest.Mock).mockResolvedValue({ + logs: 'Build completed successfully', + success: true, + }); + }); + + it('adds AWS retry env vars for ECR login', async () => { + const { kanikoBuild } = require('../engines'); + await kanikoBuild(mockDeploy, baseOptions); + + const kubectlCalls = (shellPromise as jest.Mock).mock.calls; + const applyCall = kubectlCalls.find((call) => call[0].includes('kubectl apply')); + const fullCommand = applyCall[0]; + + expect(fullCommand).toContain('image: "amazon/aws-cli:2.13.0"'); + expect(fullCommand).toContain('export AWS_MAX_ATTEMPTS=5'); + expect(fullCommand).toContain('export AWS_RETRY_MODE=adaptive'); + expect(fullCommand).toContain('aws ecr get-login-password --region us-east-1'); + }); + + it('keeps non-ECR login bootstrap generic', async () => { + const { kanikoBuild } = require('../engines'); + await kanikoBuild(mockDeploy, { + ...baseOptions, + ecrDomain: 'registry.internal.svc.cluster.local', + }); + + const kubectlCalls = (shellPromise as jest.Mock).mock.calls; + const applyCall = kubectlCalls.find((call) => call[0].includes('kubectl apply')); + const fullCommand = applyCall[0]; + + expect(fullCommand).toContain('image: "alpine:3.18"'); + expect(fullCommand).toContain('Using in-cluster registry: registry.internal.svc.cluster.local'); + expect(fullCommand).not.toContain('AWS_MAX_ATTEMPTS=5'); + }); +}); diff --git a/src/server/lib/nativeBuild/engines.ts b/src/server/lib/nativeBuild/engines.ts index b1e49c3e..1413769e 100644 --- a/src/server/lib/nativeBuild/engines.ts +++ b/src/server/lib/nativeBuild/engines.ts @@ -132,7 +132,10 @@ if echo "\${REGISTRY_DOMAIN}" | grep -qE "^[0-9]+\\.dkr\\.ecr\\.([a-z0-9-]+)\\.a echo "ECR Region: \${AWS_REGION}" echo "Installing AWS CLI and Docker CLI..." - apk add --no-cache aws-cli docker-cli + apk add --no-cache --retries 5 aws-cli docker-cli + + export AWS_MAX_ATTEMPTS=5 + export AWS_RETRY_MODE=adaptive echo "Testing AWS credentials..." if aws sts get-caller-identity; then @@ -151,7 +154,7 @@ if echo "\${REGISTRY_DOMAIN}" | grep -qE "^[0-9]+\\.dkr\\.ecr\\.([a-z0-9-]+)\\.a else echo "Using in-cluster registry: \${REGISTRY_DOMAIN}" echo "Installing Docker CLI..." - apk add --no-cache docker-cli + apk add --no-cache --retries 5 docker-cli fi echo "Setting DOCKER_CONFIG..." @@ -316,10 +319,13 @@ export async function buildWithEngine( const ecrMatch = registryDomain.match(ecrRegex); if (ecrMatch) { const region = ecrMatch[1] || 'us-west-2'; - registryLoginScript = - `aws ecr get-login-password --region ${region} | ` + - `{ read PASSWORD; mkdir -p /workspace/.docker && ` + - `echo '{"auths":{"${registryDomain}":{"auth":"'$(echo -n "AWS:$PASSWORD" | base64)'"}}}' > /workspace/.docker/config.json; }`; + registryLoginScript = [ + 'set -e', + 'export AWS_MAX_ATTEMPTS=5', + 'export AWS_RETRY_MODE=adaptive', + `aws ecr get-login-password --region ${region} | { read PASSWORD; mkdir -p /workspace/.docker && ` + + `echo '{"auths":{"${registryDomain}":{"auth":"'$(echo -n "AWS:$PASSWORD" | base64)'"}}}' > /workspace/.docker/config.json; }`, + ].join('\n'); } else { registryLoginScript = `echo "Using in-cluster registry: ${registryDomain}"; ` + From 3a0bd7c5cbdbfd2a8fd5a29aaee13c307dd041a2 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 20 Apr 2026 13:19:32 -0700 Subject: [PATCH 2/2] fix(native-build): remove unsupported apk retry flags --- src/server/lib/nativeBuild/__tests__/buildkit.test.ts | 8 ++++---- src/server/lib/nativeBuild/engines.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/server/lib/nativeBuild/__tests__/buildkit.test.ts b/src/server/lib/nativeBuild/__tests__/buildkit.test.ts index 665bbc01..ff8dc943 100644 --- a/src/server/lib/nativeBuild/__tests__/buildkit.test.ts +++ b/src/server/lib/nativeBuild/__tests__/buildkit.test.ts @@ -247,15 +247,15 @@ describe('buildkitBuild', () => { expect(fullCommand).toContain('build-arg:NODE_ENV=production'); }); - it('adds retry settings to registry bootstrap commands', async () => { + it('keeps registry bootstrap installs default and adds AWS retry env vars', async () => { await buildkitBuild(mockDeploy, mockOptions); const kubectlCalls = (shellPromise as jest.Mock).mock.calls; const applyCall = kubectlCalls.find((call) => call[0].includes('kubectl apply')); const fullCommand = applyCall[0]; - expect(fullCommand).toContain('apk add --no-cache --retries 5 aws-cli docker-cli'); - expect(fullCommand).toContain('apk add --no-cache --retries 5 docker-cli'); + expect(fullCommand).toContain('apk add --no-cache aws-cli docker-cli'); + expect(fullCommand).toContain('apk add --no-cache docker-cli'); expect(fullCommand).toContain('export AWS_MAX_ATTEMPTS=5'); expect(fullCommand).toContain('export AWS_RETRY_MODE=adaptive'); }); @@ -273,7 +273,7 @@ describe('buildkitBuild', () => { const fullCommand = applyCall[0]; expect(fullCommand).toContain('REGISTRY_DOMAIN=\\"registry.internal.svc.cluster.local\\"'); - expect(fullCommand).toContain('apk add --no-cache --retries 5 docker-cli'); + expect(fullCommand).toContain('apk add --no-cache docker-cli'); }); it('coerces numeric env var values to strings for Kubernetes compatibility', async () => { diff --git a/src/server/lib/nativeBuild/engines.ts b/src/server/lib/nativeBuild/engines.ts index 1413769e..2f9e59bd 100644 --- a/src/server/lib/nativeBuild/engines.ts +++ b/src/server/lib/nativeBuild/engines.ts @@ -132,7 +132,7 @@ if echo "\${REGISTRY_DOMAIN}" | grep -qE "^[0-9]+\\.dkr\\.ecr\\.([a-z0-9-]+)\\.a echo "ECR Region: \${AWS_REGION}" echo "Installing AWS CLI and Docker CLI..." - apk add --no-cache --retries 5 aws-cli docker-cli + apk add --no-cache aws-cli docker-cli export AWS_MAX_ATTEMPTS=5 export AWS_RETRY_MODE=adaptive @@ -154,7 +154,7 @@ if echo "\${REGISTRY_DOMAIN}" | grep -qE "^[0-9]+\\.dkr\\.ecr\\.([a-z0-9-]+)\\.a else echo "Using in-cluster registry: \${REGISTRY_DOMAIN}" echo "Installing Docker CLI..." - apk add --no-cache --retries 5 docker-cli + apk add --no-cache docker-cli fi echo "Setting DOCKER_CONFIG..."