Skip to content

fix(native-build): harden registry login retries #182

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vigneshrajsb merged 2 commits into from
Apr 20, 2026
Merged

fix(native-build): harden registry login retries #182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b6a99c6
fix(native-build): harden registry login retries
vigneshrajsb Apr 20, 2026
3a0bd7c
fix(native-build): remove unsupported apk retry flags
vigneshrajsb Apr 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 99 additions & 4 deletions src/server/lib/nativeBuild/__tests__/buildkit.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/

import { buildkitBuild, BuildkitBuildOptions, generateSecretArgsScript } from '../engines';
import { buildkitBuild, NativeBuildOptions, generateSecretArgsScript } from '../engines';
import { shellPromise } from '../../shell';
import { waitForJobAndGetLogs, getGitHubToken } from '../utils';
import GlobalConfigService from '../../../services/globalConfig';
Expand Down Expand Up @@ -68,7 +68,7 @@ describe('buildkitBuild', () => {
build: { isStatic: false },
} as any;

const mockOptions: BuildkitBuildOptions = {
const mockOptions: NativeBuildOptions = {
ecrRepo: 'test-repo',
ecrDomain: '123456789.dkr.ecr.us-east-1.amazonaws.com',
envVars: { NODE_ENV: 'production' },
Expand Down Expand Up @@ -247,6 +247,35 @@ describe('buildkitBuild', () => {
expect(fullCommand).toContain('build-arg:NODE_ENV=production');
});

it('keeps registry bootstrap installs default and adds AWS retry env vars', async () => {
await buildkitBuild(mockDeploy, mockOptions);

const kubectlCalls = (shellPromise as jest.Mock).mock.calls;
const applyCall = kubectlCalls.find((call) => call[0].includes('kubectl apply'));
const fullCommand = applyCall[0];

expect(fullCommand).toContain('apk add --no-cache aws-cli docker-cli');
expect(fullCommand).toContain('apk add --no-cache docker-cli');
expect(fullCommand).toContain('export AWS_MAX_ATTEMPTS=5');
expect(fullCommand).toContain('export AWS_RETRY_MODE=adaptive');
});

it('renders registry domain safely for non-ECR buildkit targets', async () => {
const optionsWithCustomRegistry = {
...mockOptions,
ecrDomain: 'registry.internal.svc.cluster.local',
};

await buildkitBuild(mockDeploy, optionsWithCustomRegistry);

const kubectlCalls = (shellPromise as jest.Mock).mock.calls;
const applyCall = kubectlCalls.find((call) => call[0].includes('kubectl apply'));
const fullCommand = applyCall[0];

expect(fullCommand).toContain('REGISTRY_DOMAIN=\\"registry.internal.svc.cluster.local\\"');
expect(fullCommand).toContain('apk add --no-cache docker-cli');
});

it('coerces numeric env var values to strings for Kubernetes compatibility', async () => {
const optionsWithNumericEnv = {
...mockOptions,
Expand Down Expand Up @@ -320,7 +349,7 @@ describe('build resource precedence', () => {
build: { isStatic: false },
} as any;

const baseOptions: BuildkitBuildOptions = {
const baseOptions: NativeBuildOptions = {
ecrRepo: 'test-repo',
ecrDomain: '123456789.dkr.ecr.us-east-1.amazonaws.com',
envVars: { NODE_ENV: 'production' },
Expand Down Expand Up @@ -495,7 +524,7 @@ describe('build pod annotations', () => {
build: { isStatic: false },
} as any;

const baseOptions: BuildkitBuildOptions = {
const baseOptions: NativeBuildOptions = {
ecrRepo: 'test-repo',
ecrDomain: '123456789.dkr.ecr.us-east-1.amazonaws.com',
envVars: { NODE_ENV: 'production' },
Expand Down Expand Up @@ -655,3 +684,69 @@ describe('generateSecretArgsScript', () => {
expect(result).toContain('--opt build-arg:MY_SECRET=');
});
});

describe('kaniko registry login bootstrap', () => {
const mockDeploy = {
deployable: { name: 'test-service' },
$fetchGraph: jest.fn(),
build: { isStatic: false },
} as any;

const baseOptions: NativeBuildOptions = {
ecrRepo: 'test-repo',
ecrDomain: '123456789.dkr.ecr.us-east-1.amazonaws.com',
envVars: { NODE_ENV: 'production' },
dockerfilePath: 'Dockerfile',
tag: 'v1.0.0',
revision: 'abc123def456789',
repo: 'owner/repo',
branch: 'main',
namespace: 'env-test-123',
buildId: '456',
deployUuid: 'test-service-abc123',
jobTimeout: 1800,
};

beforeEach(() => {
jest.clearAllMocks();
(GlobalConfigService.getInstance as jest.Mock).mockReturnValue({
getAllConfigs: jest.fn().mockResolvedValue({}),
});
(getGitHubToken as jest.Mock).mockResolvedValue('github-token-123');
(shellPromise as jest.Mock).mockResolvedValue('');
(waitForJobAndGetLogs as jest.Mock).mockResolvedValue({
logs: 'Build completed successfully',
success: true,
});
});

it('adds AWS retry env vars for ECR login', async () => {
const { kanikoBuild } = require('../engines');
await kanikoBuild(mockDeploy, baseOptions);

const kubectlCalls = (shellPromise as jest.Mock).mock.calls;
const applyCall = kubectlCalls.find((call) => call[0].includes('kubectl apply'));
const fullCommand = applyCall[0];

expect(fullCommand).toContain('image: "amazon/aws-cli:2.13.0"');
expect(fullCommand).toContain('export AWS_MAX_ATTEMPTS=5');
expect(fullCommand).toContain('export AWS_RETRY_MODE=adaptive');
expect(fullCommand).toContain('aws ecr get-login-password --region us-east-1');
});

it('keeps non-ECR login bootstrap generic', async () => {
const { kanikoBuild } = require('../engines');
await kanikoBuild(mockDeploy, {
...baseOptions,
ecrDomain: 'registry.internal.svc.cluster.local',
});

const kubectlCalls = (shellPromise as jest.Mock).mock.calls;
const applyCall = kubectlCalls.find((call) => call[0].includes('kubectl apply'));
const fullCommand = applyCall[0];

expect(fullCommand).toContain('image: "alpine:3.18"');
expect(fullCommand).toContain('Using in-cluster registry: registry.internal.svc.cluster.local');
expect(fullCommand).not.toContain('AWS_MAX_ATTEMPTS=5');
});
});
14 changes: 10 additions & 4 deletions src/server/lib/nativeBuild/engines.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,9 @@ if echo "\${REGISTRY_DOMAIN}" | grep -qE "^[0-9]+\\.dkr\\.ecr\\.([a-z0-9-]+)\\.a

echo "Installing AWS CLI and Docker CLI..."
apk add --no-cache aws-cli docker-cli

export AWS_MAX_ATTEMPTS=5
export AWS_RETRY_MODE=adaptive

echo "Testing AWS credentials..."
if aws sts get-caller-identity; then
Expand Down Expand Up @@ -316,10 +319,13 @@ export async function buildWithEngine(
const ecrMatch = registryDomain.match(ecrRegex);
if (ecrMatch) {
const region = ecrMatch[1] || 'us-west-2';
registryLoginScript =
`aws ecr get-login-password --region ${region} | ` +
`{ read PASSWORD; mkdir -p /workspace/.docker && ` +
`echo '{"auths":{"${registryDomain}":{"auth":"'$(echo -n "AWS:$PASSWORD" | base64)'"}}}' > /workspace/.docker/config.json; }`;
registryLoginScript = [
'set -e',
'export AWS_MAX_ATTEMPTS=5',
'export AWS_RETRY_MODE=adaptive',
`aws ecr get-login-password --region ${region} | { read PASSWORD; mkdir -p /workspace/.docker && ` +
`echo '{"auths":{"${registryDomain}":{"auth":"'$(echo -n "AWS:$PASSWORD" | base64)'"}}}' > /workspace/.docker/config.json; }`,
].join('\n');
} else {
registryLoginScript =
`echo "Using in-cluster registry: ${registryDomain}"; ` +
Expand Down
Loading