diff --git a/.ci/images/build/Dockerfile b/.ci/images/build/Dockerfile index d1c2e277fc..90f54a219e 100644 --- a/.ci/images/build/Dockerfile +++ b/.ci/images/build/Dockerfile @@ -1,54 +1,69 @@ -FROM registry.ddbuild.io/images/docker:27.3.1 +FROM registry.ddbuild.io/images/base/gbi-ubuntu_2204:release ARG RUST_VERSION=1.93.0 ARG TARGETARCH -RUN --mount=type=bind,source=.ci/configure-apt-mirror.sh,target=/tmp/configure-apt-mirror.sh \ - sh /tmp/configure-apt-mirror.sh +# Base image defaults to the unprivileged `dog` user; switch to root for all install steps. +USER root -# Install the basics for building and compiling software projects. +# Install build basics plus docker CLI only. The DinD runner provides the daemon at /var/run/docker.sock, +# so we don't ship dockerd, containerd, buildx, or the full credential-helper bundle. RUN apt-get update && \ - apt-get install -y --no-install-recommends build-essential software-properties-common curl ca-certificates git gnupg2 \ - lsb-release make cmake unzip gcc g++ binutils jq bc bzip2 ninja-build python3 python3-pip && \ - apt-get clean + apt-get install -y --no-install-recommends \ + build-essential software-properties-common curl ca-certificates git gnupg2 \ + lsb-release make cmake unzip gcc g++ binutils jq bc bzip2 ninja-build python3 python3-pip && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ + echo "deb [arch=${TARGETARCH} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu jammy stable" > /etc/apt/sources.list.d/docker.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends docker-ce-cli=5:27.3.1-1~ubuntu.22.04~jammy && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -# Install Go, which we need to build AWS-LC in FIPS-compliant mode. -ENV PATH="/usr/local/go/bin:${PATH}" -RUN curl -s -L -o /tmp/go-1.23.0.tar.gz https://go.dev/dl/go1.23.0.linux-${TARGETARCH}.tar.gz -RUN tar -C /usr/local -xzf /tmp/go-1.23.0.tar.gz +# Copy the registry.ddbuild.io credential helper from the upstream docker image so `docker pull` +# against registry.ddbuild.io continues to work. ECR/GCR helpers are intentionally omitted — +# jobs that pull from those registries perform explicit `docker login` beforehand. +COPY --from=registry.ddbuild.io/images/docker:27.3.1 /usr/bin/docker-credential-ci /usr/bin/docker-credential-ci -# Install a number of packages/tools by hand, because the versions in Ubuntu are too old, or just aren't available -# from the normal package manager repositories. -RUN apt-get update && \ - dd-package --bucket binaries-ddbuild-io-prod --package devtools/dd-package-dev --distribution "20.04" +# Minimal docker client config: only the ci helper for registry.ddbuild.io. +RUN mkdir -p /root/.docker && \ + printf '{\n "credHelpers": {\n "registry-staging.ddbuild.io": "ci",\n "registry.ddbuild.io": "ci"\n }\n}\n' > /root/.docker/config.json -COPY .ci/install-pr-commenter.sh / -RUN chmod +x /install-pr-commenter.sh && /install-pr-commenter.sh - -COPY .ci/install-bloaty.sh / -RUN chmod +x /install-bloaty.sh && /install-bloaty.sh +# Install Go, which we need to build AWS-LC in FIPS-compliant mode. +ENV PATH="/usr/local/go/bin:${PATH}" +RUN curl -s -L -o /tmp/go-1.23.0.tar.gz https://go.dev/dl/go1.23.0.linux-${TARGETARCH}.tar.gz && \ + tar -C /usr/local -xzf /tmp/go-1.23.0.tar.gz && \ + rm -f /tmp/go-1.23.0.tar.gz COPY .ci/install-protoc.sh / RUN chmod +x /install-protoc.sh && /install-protoc.sh -COPY .ci/install-awscli.sh / -RUN chmod +x /install-awscli.sh && /install-awscli.sh - -COPY .ci/install-vault.sh / -RUN chmod +x /install-vault.sh && /install-vault.sh - -# Install Rust and common Cargo tooling that we depend on. +# Install Rust and common Cargo tooling that we depend on. Nightly is installed with a minimal profile +# since we only use it to run rustfmt with Edition 2024. RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain ${RUST_VERSION} ENV PATH="/root/.cargo/bin:${PATH}" -RUN rustup toolchain add nightly && \ - rustup component add clippy && \ - rustup component add --toolchain nightly rustfmt +RUN rustup component add clippy && \ + rustup toolchain install nightly --profile minimal --component rustfmt -# Pre-install the relevant Cargo tools we use in the build process. +# Pre-install the relevant Cargo tools we use in the build process. Drop the Cargo registry cache +# immediately after, since the tools themselves live in /root/.cargo/bin and jobs repopulate the +# registry from Cargo.lock on first build. WORKDIR /tmp COPY ./Makefile /tmp COPY ./bin/agent-data-plane/Cargo.toml /tmp/bin/agent-data-plane/Cargo.toml -RUN CI=true make cargo-preinstall +RUN CI=true make cargo-preinstall && \ + rm -rf /root/.cargo/registry /tmp/Makefile /tmp/bin + +# Final sweep: remove anything leftover in /tmp, apt metadata, docs/man pages, and ephemeral caches. +# The nightly toolchain's rust-std (in lib/rustlib/*/lib) is only used when compiling Rust code; +# since we only use nightly to run rustfmt, we can drop it. rustfmt itself still links dynamically +# against libLLVM / librustc_driver in lib/, so only the rustlib subtree is safe to remove. +RUN find /tmp -mindepth 1 -delete && \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb && \ + rm -rf /root/.cache && \ + rm -rf /root/.cargo/registry && \ + rm -rf /root/.rustup/toolchains/nightly-*/lib/rustlib/*/lib && \ + rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/locale/* COPY .ci/images/build/entrypoint.sh / RUN chmod +x /entrypoint.sh diff --git a/.ci/images/smp/Dockerfile b/.ci/images/smp/Dockerfile index 1a67e61ca5..3a7b8dde46 100644 --- a/.ci/images/smp/Dockerfile +++ b/.ci/images/smp/Dockerfile @@ -1,26 +1,54 @@ +# Builder stage: compile bloaty from source. Its build-only deps (cmake, ninja-build, g++) stay here. +FROM registry.ddbuild.io/docker:24.0.4-jammy AS bloaty-builder + +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=Etc/UTC + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl cmake ninja-build g++ bzip2 && \ + rm -rf /var/lib/apt/lists/* + +COPY .ci/install-bloaty.sh /install-bloaty.sh +RUN chmod +x /install-bloaty.sh && /install-bloaty.sh + +# Final stage: runtime-only dependencies. Build toolchain is not present here. FROM registry.ddbuild.io/docker:24.0.4-jammy +ARG RUST_VERSION=1.93.0 + ENV DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC RUN --mount=type=bind,source=.ci/configure-apt-mirror.sh,target=/tmp/configure-apt-mirror.sh \ sh /tmp/configure-apt-mirror.sh -# Install basic utilities and an updated compiler/binutils toolchain, which is necessary for compiling. +# Runtime utilities for the benchmark/binary-size-analysis jobs. python3 + binutils are needed by +# analyze-binary-size.py (objcopy / strip). No C/C++ toolchain here — bloaty is built in the +# dedicated builder stage above and copied in below. RUN apt-get update && \ - apt-get install -y --no-install-recommends curl ca-certificates awscli lsb-release git jq bc bzip2 && \ - apt-get clean + apt-get install -y --no-install-recommends curl ca-certificates lsb-release git jq bc bzip2 python3 binutils && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* RUN apt-get update && \ - dd-package --bucket binaries-ddbuild-io-prod --package devtools/dd-package-dev --distribution "20.04" + dd-package --bucket binaries-ddbuild-io-prod --package devtools/dd-package-dev --distribution "20.04" && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -# Install pr-commenter which we will need to post comments to Github. -COPY .ci/images/smp/install-pr-commenter.sh / +# Install Rust which is required by the Binary Size Analysis job. +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain ${RUST_VERSION} +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install pr-commenter for posting analysis reports back to GitHub. +COPY .ci/install-pr-commenter.sh / RUN chmod +x /install-pr-commenter.sh && /install-pr-commenter.sh -# Build and install bloaty for binary size analysis. -COPY .ci/images/smp/install-bloaty.sh / -RUN chmod +x /install-bloaty.sh && /install-bloaty.sh +# Install AWS CLI for retrieiving credentials and settings for the SMP image registry location. +COPY .ci/install-awscli.sh / +RUN chmod +x /install-awscli.sh && /install-awscli.sh + +# Copy just the compiled bloaty binary from the builder stage. +COPY --from=bloaty-builder /usr/local/bin/bloaty /usr/local/bin/bloaty COPY .ci/images/smp/entrypoint.sh / RUN chmod +x /entrypoint.sh diff --git a/.gitlab/benchmark.yml b/.gitlab/benchmark.yml index 196e9c63b2..642e1c521e 100644 --- a/.gitlab/benchmark.yml +++ b/.gitlab/benchmark.yml @@ -120,7 +120,7 @@ run-benchmarks-adp: needs: - build-adp-baseline-image - build-adp-comparison-image - image: "${SALUKI_BUILD_CI_IMAGE}" + image: "${SALUKI_SMP_CI_IMAGE}" before_script: - *setup-smp-env artifacts: @@ -172,7 +172,7 @@ binary-size-analysis: - if: !reference [.on_mq_branch, rules, if] when: never - if: !reference [.on_development_branch, rules, if] - image: "${SALUKI_BUILD_CI_IMAGE}" + image: "${SALUKI_SMP_CI_IMAGE}" needs: - build-adp-baseline-image - build-adp-comparison-image diff --git a/.gitlab/fuzz.yml b/.gitlab/fuzz.yml index 4e6c724395..65c0cf80b6 100644 --- a/.gitlab/fuzz.yml +++ b/.gitlab/fuzz.yml @@ -15,6 +15,8 @@ fuzz:infra: # Install Python dependencies for fuzz_infra.py - apt-get update && apt-get install -y python3 python3-pip - pip3 install requests toml # parsing of cargo toml files + # Install vault (used by fuzz_infra.py to fetch the fuzzing platform auth token). + - bash .ci/install-vault.sh script: # There's a bug with `cargo-fuzz` not actually respecting .cargo/config.toml where we normally # define the unstable config flag for `tokio`, which we need to properly compile things. diff --git a/.gitlab/internal.yml b/.gitlab/internal.yml index 18ff34a791..e625d1414b 100644 --- a/.gitlab/internal.yml +++ b/.gitlab/internal.yml @@ -1,4 +1,4 @@ -generate-build-ci-image: +.generate-ci-image-definition: stage: internal image: ${DOCKER_BUILD_IMAGE} id_tokens: @@ -11,88 +11,48 @@ generate-build-ci-image: allow_failure: true - if: $CI_PIPELINE_SOURCE == "schedule" && $BUILD_HELPER_IMAGES == "true" allow_failure: true - before_script: - - export RUST_VERSION=$(grep channel rust-toolchain.toml | cut -d '"' -f 2) - - echo RUST_VERSION=${RUST_VERSION} script: + - export IMAGE_REF=${SALUKI_IMAGE_REPO_BASE}/${IMAGE_NAME}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - docker buildx build --platform linux/amd64,linux/arm64 - --tag ${SALUKI_IMAGE_REPO_BASE}/build-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - --tag ${SALUKI_IMAGE_REPO_BASE}/build-ci:latest + --tag ${IMAGE_REF} --label git.repository=${CI_PROJECT_NAME} --label git.branch=${CI_COMMIT_REF_NAME} --label git.commit=${CI_COMMIT_SHA} --label ci.pipeline_id=${CI_PIPELINE_ID} --label ci.job_id=${CI_JOB_ID} - --build-arg CI=true - --build-arg RUST_VERSION=${RUST_VERSION} - --build-arg DD_AGENT_IMAGE=registry.datadoghq.com/agent:latest-jmx - --squash + ${BUILD_ARGS} --push - --metadata-file ./build-ci-metadata - --file .ci/images/build/Dockerfile + --metadata-file ./build.metadata + --file ${DOCKERFILE} . - - ddsign sign ${SALUKI_IMAGE_REPO_BASE}/build-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - --docker-metadata-file ./build-ci-metadata + # Now flatten the image, realizing any space savings from duplicate layers, cleaned up cruft, etc. + - crane flatten --tag ${IMAGE_REF} ${IMAGE_REF} + # Tag this as `latest` so we can immediately take advatange of it. + - crane tag ${IMAGE_REF} latest + # Recalculate the image digest post-flattening and update the build metadata so we can sign the flattened image. + - NEW_DIGEST=$(crane digest ${IMAGE_REF}) + - jq --arg d "$NEW_DIGEST" '.["containerimage.digest"] = $d' build.metadata > build.metadata.new && mv build.metadata.new build.metadata + - ddsign sign ${IMAGE_REF} --docker-metadata-file ./build.metadata + +generate-build-ci-image: + extends: [.generate-ci-image-definition] + before_script: + - export RUST_VERSION=$(grep channel rust-toolchain.toml | cut -d '"' -f 2) + - echo RUST_VERSION=${RUST_VERSION} + - export BUILD_ARGS="--build-arg RUST_VERSION=${RUST_VERSION}" + variables: + IMAGE_NAME: build-ci + DOCKERFILE: .ci/images/build/Dockerfile generate-general-ci-image: - stage: internal - image: ${DOCKER_BUILD_IMAGE} - id_tokens: - DDSIGN_ID_TOKEN: - aud: image-integrity - needs: [] - rules: - - if: $CI_PIPELINE_SOURCE == "web" - when: manual - allow_failure: true - - if: $CI_PIPELINE_SOURCE == "schedule" && $BUILD_HELPER_IMAGES == "true" - allow_failure: true - script: - - docker buildx build - --platform linux/amd64,linux/arm64 - --tag ${SALUKI_IMAGE_REPO_BASE}/general-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - --tag ${SALUKI_IMAGE_REPO_BASE}/general-ci:latest - --label git.repository=${CI_PROJECT_NAME} - --label git.branch=${CI_COMMIT_REF_NAME} - --label git.commit=${CI_COMMIT_SHA} - --label ci.pipeline_id=${CI_PIPELINE_ID} - --label ci.job_id=${CI_JOB_ID} - --squash - --push - --metadata-file ./general-ci-metadata - --file .ci/images/general/Dockerfile - . - - ddsign sign ${SALUKI_IMAGE_REPO_BASE}/general-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - --docker-metadata-file ./general-ci-metadata + extends: [.generate-ci-image-definition] + variables: + IMAGE_NAME: general-ci + DOCKERFILE: .ci/images/general/Dockerfile generate-smp-ci-image: - stage: internal - image: ${DOCKER_BUILD_IMAGE} - id_tokens: - DDSIGN_ID_TOKEN: - aud: image-integrity - needs: [] - rules: - - if: $CI_PIPELINE_SOURCE == "web" - when: manual - allow_failure: true - - if: $CI_PIPELINE_SOURCE == "schedule" && $BUILD_HELPER_IMAGES == "true" - allow_failure: true - script: - - docker buildx build - --platform linux/amd64,linux/arm64 - --tag ${SALUKI_IMAGE_REPO_BASE}/smp-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - --tag ${SALUKI_IMAGE_REPO_BASE}/smp-ci:latest - --label git.repository=${CI_PROJECT_NAME} - --label git.branch=${CI_COMMIT_REF_NAME} - --label git.commit=${CI_COMMIT_SHA} - --label ci.pipeline_id=${CI_PIPELINE_ID} - --label ci.job_id=${CI_JOB_ID} - --squash - --push - --metadata-file ./smp-ci-metadata - --file .ci/images/smp/Dockerfile - . - - ddsign sign ${SALUKI_IMAGE_REPO_BASE}/smp-ci:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - --docker-metadata-file ./smp-ci-metadata + extends: [.generate-ci-image-definition] + variables: + IMAGE_NAME: smp-ci + DOCKERFILE: .ci/images/smp/Dockerfile