chore(ci): optimize size/usage of pre-built CI images

.ci/images/build/Dockerfile

@@ -1,54 +1,69 @@
-FROM registry.ddbuild.io/images/docker:27.3.1
+FROM registry.ddbuild.io/images/base/gbi-ubuntu_2204:release
 
 ARG RUST_VERSION=1.93.0
 ARG TARGETARCH
 
-RUN --mount=type=bind,source=.ci/configure-apt-mirror.sh,target=/tmp/configure-apt-mirror.sh \
-    sh /tmp/configure-apt-mirror.sh
+# Base image defaults to the unprivileged `dog` user; switch to root for all install steps.
+USER root
 
-# Install the basics for building and compiling software projects.
+# Install build basics plus docker CLI only. The DinD runner provides the daemon at /var/run/docker.sock,
+# so we don't ship dockerd, containerd, buildx, or the full credential-helper bundle.
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends build-essential software-properties-common curl ca-certificates git gnupg2 \
-    lsb-release make cmake unzip gcc g++ binutils jq bc bzip2 ninja-build python3 python3-pip && \
-    apt-get clean
+    apt-get install -y --no-install-recommends \
+      build-essential software-properties-common curl ca-certificates git gnupg2 \
+      lsb-release make cmake unzip gcc g++ binutils jq bc bzip2 ninja-build python3 python3-pip && \
+    mkdir -p /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    echo "deb [arch=${TARGETARCH} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu jammy stable" > /etc/apt/sources.list.d/docker.list && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends docker-ce-cli=5:27.3.1-1~ubuntu.22.04~jammy && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
-# Install Go, which we need to build AWS-LC in FIPS-compliant mode.
-ENV PATH="/usr/local/go/bin:${PATH}"
-RUN curl -s -L -o /tmp/go-1.23.0.tar.gz https://go.dev/dl/go1.23.0.linux-${TARGETARCH}.tar.gz
-RUN tar -C /usr/local -xzf /tmp/go-1.23.0.tar.gz
+# Copy the registry.ddbuild.io credential helper from the upstream docker image so `docker pull`
+# against registry.ddbuild.io continues to work. ECR/GCR helpers are intentionally omitted —
+# jobs that pull from those registries perform explicit `docker login` beforehand.
+COPY --from=registry.ddbuild.io/images/docker:27.3.1 /usr/bin/docker-credential-ci /usr/bin/docker-credential-ci
 
-# Install a number of packages/tools by hand, because the versions in Ubuntu are too old, or just aren't available
-# from the normal package manager repositories.
-RUN apt-get update && \
-    dd-package --bucket binaries-ddbuild-io-prod --package devtools/dd-package-dev --distribution "20.04"
+# Minimal docker client config: only the ci helper for registry.ddbuild.io.
+RUN mkdir -p /root/.docker && \
+    printf '{\n    "credHelpers": {\n        "registry-staging.ddbuild.io": "ci",\n        "registry.ddbuild.io": "ci"\n    }\n}\n' > /root/.docker/config.json
 
-COPY .ci/install-pr-commenter.sh /
-RUN chmod +x /install-pr-commenter.sh && /install-pr-commenter.sh
-
-COPY .ci/install-bloaty.sh /
-RUN chmod +x /install-bloaty.sh && /install-bloaty.sh
+# Install Go, which we need to build AWS-LC in FIPS-compliant mode.
+ENV PATH="/usr/local/go/bin:${PATH}"
+RUN curl -s -L -o /tmp/go-1.23.0.tar.gz https://go.dev/dl/go1.23.0.linux-${TARGETARCH}.tar.gz && \
+    tar -C /usr/local -xzf /tmp/go-1.23.0.tar.gz && \
+    rm -f /tmp/go-1.23.0.tar.gz
 
 COPY .ci/install-protoc.sh /
 RUN chmod +x /install-protoc.sh && /install-protoc.sh
 
-COPY .ci/install-awscli.sh /
-RUN chmod +x /install-awscli.sh && /install-awscli.sh
-
-COPY .ci/install-vault.sh /
-RUN chmod +x /install-vault.sh && /install-vault.sh
-
-# Install Rust and common Cargo tooling that we depend on.
+# Install Rust and common Cargo tooling that we depend on. Nightly is installed with a minimal profile
+# since we only use it to run rustfmt with Edition 2024.
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain ${RUST_VERSION}
 ENV PATH="/root/.cargo/bin:${PATH}"
-RUN rustup toolchain add nightly && \
-    rustup component add clippy && \
-    rustup component add --toolchain nightly rustfmt
+RUN rustup component add clippy && \
+    rustup toolchain install nightly --profile minimal --component rustfmt
 
-# Pre-install the relevant Cargo tools we use in the build process.
+# Pre-install the relevant Cargo tools we use in the build process. Drop the Cargo registry cache
+# immediately after, since the tools themselves live in /root/.cargo/bin and jobs repopulate the
+# registry from Cargo.lock on first build.
 WORKDIR /tmp
 COPY ./Makefile /tmp
 COPY ./bin/agent-data-plane/Cargo.toml /tmp/bin/agent-data-plane/Cargo.toml
-RUN CI=true make cargo-preinstall
+RUN CI=true make cargo-preinstall && \
+    rm -rf /root/.cargo/registry /tmp/Makefile /tmp/bin
+
+# Final sweep: remove anything leftover in /tmp, apt metadata, docs/man pages, and ephemeral caches.
+# The nightly toolchain's rust-std (in lib/rustlib/*/lib) is only used when compiling Rust code;
+# since we only use nightly to run rustfmt, we can drop it. rustfmt itself still links dynamically
+# against libLLVM / librustc_driver in lib/, so only the rustlib subtree is safe to remove.
+RUN find /tmp -mindepth 1 -delete && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb && \
+    rm -rf /root/.cache && \
+    rm -rf /root/.cargo/registry && \
+    rm -rf /root/.rustup/toolchains/nightly-*/lib/rustlib/*/lib && \
+    rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/locale/*
 
 COPY .ci/images/build/entrypoint.sh /
 RUN chmod +x /entrypoint.sh

.ci/images/smp/Dockerfile

@@ -1,26 +1,54 @@
+# Builder stage: compile bloaty from source. Its build-only deps (cmake, ninja-build, g++) stay here.
+FROM registry.ddbuild.io/docker:24.0.4-jammy AS bloaty-builder
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    TZ=Etc/UTC
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ca-certificates curl cmake ninja-build g++ bzip2 && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY .ci/install-bloaty.sh /install-bloaty.sh
+RUN chmod +x /install-bloaty.sh && /install-bloaty.sh
+
+# Final stage: runtime-only dependencies. Build toolchain is not present here.
 FROM registry.ddbuild.io/docker:24.0.4-jammy
 
+ARG RUST_VERSION=1.93.0
+
 ENV DEBIAN_FRONTEND=noninteractive \
     TZ=Etc/UTC
 
 RUN --mount=type=bind,source=.ci/configure-apt-mirror.sh,target=/tmp/configure-apt-mirror.sh \
     sh /tmp/configure-apt-mirror.sh
 
-# Install basic utilities and an updated compiler/binutils toolchain, which is necessary for compiling.
+# Runtime utilities for the benchmark/binary-size-analysis jobs. python3 + binutils are needed by
+# analyze-binary-size.py (objcopy / strip). No C/C++ toolchain here — bloaty is built in the
+# dedicated builder stage above and copied in below.
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends curl ca-certificates awscli lsb-release git jq bc bzip2 && \
-    apt-get clean
+    apt-get install -y --no-install-recommends curl ca-certificates lsb-release git jq bc bzip2 python3 binutils && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 RUN apt-get update && \
-    dd-package --bucket binaries-ddbuild-io-prod --package devtools/dd-package-dev --distribution "20.04"
+    dd-package --bucket binaries-ddbuild-io-prod --package devtools/dd-package-dev --distribution "20.04" && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
-# Install pr-commenter which we will need to post comments to Github.
-COPY .ci/images/smp/install-pr-commenter.sh /
+# Install Rust which is required by the Binary Size Analysis job.
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain ${RUST_VERSION}
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install pr-commenter for posting analysis reports back to GitHub.
+COPY .ci/install-pr-commenter.sh /
 RUN chmod +x /install-pr-commenter.sh && /install-pr-commenter.sh
 
-# Build and install bloaty for binary size analysis.
-COPY .ci/images/smp/install-bloaty.sh /
-RUN chmod +x /install-bloaty.sh && /install-bloaty.sh
+# Install AWS CLI for retrieiving credentials and settings for the SMP image registry location.
+COPY .ci/install-awscli.sh /
+RUN chmod +x /install-awscli.sh && /install-awscli.sh
+
+# Copy just the compiled bloaty binary from the builder stage.
+COPY --from=bloaty-builder /usr/local/bin/bloaty /usr/local/bin/bloaty
 
 COPY .ci/images/smp/entrypoint.sh /
 RUN chmod +x /entrypoint.sh

.gitlab/benchmark.yml

@@ -120,7 +120,7 @@ run-benchmarks-adp:
   needs:
     - build-adp-baseline-image
     - build-adp-comparison-image
-  image: "${SALUKI_BUILD_CI_IMAGE}"
+  image: "${SALUKI_SMP_CI_IMAGE}"
   before_script:
     - *setup-smp-env
   artifacts:
@@ -172,7 +172,7 @@ binary-size-analysis:
     - if: !reference [.on_mq_branch, rules, if]
       when: never
     - if: !reference [.on_development_branch, rules, if]
-  image: "${SALUKI_BUILD_CI_IMAGE}"
+  image: "${SALUKI_SMP_CI_IMAGE}"
   needs:
     - build-adp-baseline-image
     - build-adp-comparison-image

.gitlab/fuzz.yml

@@ -15,6 +15,8 @@ fuzz:infra:
     # Install Python dependencies for fuzz_infra.py
     - apt-get update && apt-get install -y python3 python3-pip
     - pip3 install requests toml # parsing of cargo toml files
+    # Install vault (used by fuzz_infra.py to fetch the fuzzing platform auth token).
+    - bash .ci/install-vault.sh
   script:
     # There's a bug with `cargo-fuzz` not actually respecting .cargo/config.toml where we normally
     # define the unstable config flag for `tokio`, which we need to properly compile things.

.gitlab/internal.yml

@@ -1,4 +1,4 @@
-generate-build-ci-image:
+.generate-ci-image-definition:
   stage: internal
   image: ${DOCKER_BUILD_IMAGE}
   id_tokens:
@@ -11,88 +11,48 @@ generate-build-ci-image:
       allow_failure: true
     - if: $CI_PIPELINE_SOURCE == "schedule" && $BUILD_HELPER_IMAGES == "true"
       allow_failure: true
-  before_script:
-    - export RUST_VERSION=$(grep channel rust-toolchain.toml | cut -d '"' -f 2)
-    - echo RUST_VERSION=${RUST_VERSION}
   script:
+    - export IMAGE_REF=${SALUKI_IMAGE_REPO_BASE}/${IMAGE_NAME}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
     - docker buildx build
       --platform linux/amd64,linux/arm64
-      --tag ${SALUKI_IMAGE_REPO_BASE}/build-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
-      --tag ${SALUKI_IMAGE_REPO_BASE}/build-ci:latest
+      --tag ${IMAGE_REF}
       --label git.repository=${CI_PROJECT_NAME}
       --label git.branch=${CI_COMMIT_REF_NAME}
       --label git.commit=${CI_COMMIT_SHA}
       --label ci.pipeline_id=${CI_PIPELINE_ID}
       --label ci.job_id=${CI_JOB_ID}
-      --build-arg CI=true
-      --build-arg RUST_VERSION=${RUST_VERSION}
-      --build-arg DD_AGENT_IMAGE=registry.datadoghq.com/agent:latest-jmx
-      --squash
+      ${BUILD_ARGS}
       --push
-      --metadata-file ./build-ci-metadata
-      --file .ci/images/build/Dockerfile
+      --metadata-file ./build.metadata
+      --file ${DOCKERFILE}
       .
-    - ddsign sign ${SALUKI_IMAGE_REPO_BASE}/build-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
-      --docker-metadata-file ./build-ci-metadata
+    # Now flatten the image, realizing any space savings from duplicate layers, cleaned up cruft, etc.
+    - crane flatten --tag ${IMAGE_REF} ${IMAGE_REF}
+    # Tag this as `latest` so we can immediately take advatange of it.
+    - crane tag ${IMAGE_REF} latest
+    # Recalculate the image digest post-flattening and update the build metadata so we can sign the flattened image.
+    - NEW_DIGEST=$(crane digest ${IMAGE_REF})
+    - jq --arg d "$NEW_DIGEST" '.["containerimage.digest"] = $d' build.metadata > build.metadata.new && mv build.metadata.new build.metadata
+    - ddsign sign ${IMAGE_REF} --docker-metadata-file ./build.metadata
+
+generate-build-ci-image:
+  extends: [.generate-ci-image-definition]
+  before_script:
+    - export RUST_VERSION=$(grep channel rust-toolchain.toml | cut -d '"' -f 2)
+    - echo RUST_VERSION=${RUST_VERSION}
+    - export BUILD_ARGS="--build-arg RUST_VERSION=${RUST_VERSION}"
+  variables:
+    IMAGE_NAME: build-ci
+    DOCKERFILE: .ci/images/build/Dockerfile
 
 generate-general-ci-image:
-  stage: internal
-  image: ${DOCKER_BUILD_IMAGE}
-  id_tokens:
-    DDSIGN_ID_TOKEN:
-      aud: image-integrity
-  needs: []
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "web"
-      when: manual
-      allow_failure: true
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $BUILD_HELPER_IMAGES == "true"
-      allow_failure: true
-  script:
-    - docker buildx build
-      --platform linux/amd64,linux/arm64
-      --tag ${SALUKI_IMAGE_REPO_BASE}/general-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
-      --tag ${SALUKI_IMAGE_REPO_BASE}/general-ci:latest
-      --label git.repository=${CI_PROJECT_NAME}
-      --label git.branch=${CI_COMMIT_REF_NAME}
-      --label git.commit=${CI_COMMIT_SHA}
-      --label ci.pipeline_id=${CI_PIPELINE_ID}
-      --label ci.job_id=${CI_JOB_ID}
-      --squash
-      --push
-      --metadata-file ./general-ci-metadata
-      --file .ci/images/general/Dockerfile
-      .
-    - ddsign sign ${SALUKI_IMAGE_REPO_BASE}/general-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
-      --docker-metadata-file ./general-ci-metadata
+  extends: [.generate-ci-image-definition]
+  variables:
+    IMAGE_NAME: general-ci
+    DOCKERFILE: .ci/images/general/Dockerfile
 
 generate-smp-ci-image:
-  stage: internal
-  image: ${DOCKER_BUILD_IMAGE}
-  id_tokens:
-    DDSIGN_ID_TOKEN:
-      aud: image-integrity
-  needs: []
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "web"
-      when: manual
-      allow_failure: true
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $BUILD_HELPER_IMAGES == "true"
-      allow_failure: true
-  script:
-    - docker buildx build
-      --platform linux/amd64,linux/arm64
-      --tag ${SALUKI_IMAGE_REPO_BASE}/smp-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
-      --tag ${SALUKI_IMAGE_REPO_BASE}/smp-ci:latest
-      --label git.repository=${CI_PROJECT_NAME}
-      --label git.branch=${CI_COMMIT_REF_NAME}
-      --label git.commit=${CI_COMMIT_SHA}
-      --label ci.pipeline_id=${CI_PIPELINE_ID}
-      --label ci.job_id=${CI_JOB_ID}
-      --squash
-      --push
-      --metadata-file ./smp-ci-metadata
-      --file .ci/images/smp/Dockerfile
-      .
-    - ddsign sign ${SALUKI_IMAGE_REPO_BASE}/smp-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
-      --docker-metadata-file ./smp-ci-metadata
+  extends: [.generate-ci-image-definition]
+  variables:
+    IMAGE_NAME: smp-ci
+    DOCKERFILE: .ci/images/smp/Dockerfile