diff --git a/modules/l2/_common/mkosi.conf b/modules/l2/_common/mkosi.conf index 027a4076..86e23021 100644 --- a/modules/l2/_common/mkosi.conf +++ b/modules/l2/_common/mkosi.conf @@ -8,9 +8,12 @@ ExtraTrees=modules/l2/_common/mkosi.extra PostInstallationScripts=modules/l2/_common/mkosi.postinst.chroot SyncScripts=modules/l2/_common//mkosi.sync -Packages=fluent-bit +Packages=curl + fluent-bit + logrotate prometheus-node-exporter prometheus-process-exporter + rsyslog usrmerge xfsprogs diff --git a/modules/l2/_common/mkosi.extra/etc/flashbots/l2.yaml b/modules/l2/_common/mkosi.extra/etc/flashbots/l2.yaml index bcc1a81a..f5a56c34 100644 --- a/modules/l2/_common/mkosi.extra/etc/flashbots/l2.yaml +++ b/modules/l2/_common/mkosi.extra/etc/flashbots/l2.yaml @@ -2,7 +2,7 @@ gcp_ops_agent: git_reference: 2.57.0 gomplate: - git_reference: v4.3.0 + git_reference: v5.0.0 vault: git_reference: v1.20.1 diff --git a/modules/l2/_common/mkosi.extra/etc/systemd/system/vault-agent.service b/modules/l2/_common/mkosi.extra/etc/systemd/system/vault-agent.service index 96f27849..198b4830 100644 --- a/modules/l2/_common/mkosi.extra/etc/systemd/system/vault-agent.service +++ b/modules/l2/_common/mkosi.extra/etc/systemd/system/vault-agent.service @@ -18,6 +18,7 @@ RestartSec=5s TimeoutStopSec=30 ExecStartPre=/usr/bin/gomplate \ + --datasource gcp=gcp+meta:/// \ --left-delim "[[" \ --right-delim "]]" \ --input-dir "/etc/vault-agent/gomplate" \ diff --git a/modules/l2/_common/mkosi.extra/usr/bin/ptlb-routes-nanny.sh b/modules/l2/_common/mkosi.extra/usr/bin/ptlb-routes-nanny.sh index cdd30336..35d51925 100755 --- a/modules/l2/_common/mkosi.extra/usr/bin/ptlb-routes-nanny.sh +++ b/modules/l2/_common/mkosi.extra/usr/bin/ptlb-routes-nanny.sh @@ -20,16 +20,22 @@ for line in "$( for idx in $( curl \ + --fail \ --header "metadata-flavor: Google" \ - --max-time 1 \ + --retry 100 \ + --retry-all-errors \ + --retry-delay 1 \ --show-error \ --silent \ http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/forwarded-ips/ ); do ip=$( curl \ + --fail \ --header "metadata-flavor: Google" \ - --max-time 1 \ + --retry 100 \ + --retry-all-errors \ + --retry-delay 1 \ --show-error \ --silent \ http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/forwarded-ips/${idx} diff --git a/modules/l2/_common/mkosi.postinst.chroot b/modules/l2/_common/mkosi.postinst.chroot index 2c0d8e90..3b9d62c5 100755 --- a/modules/l2/_common/mkosi.postinst.chroot +++ b/modules/l2/_common/mkosi.postinst.chroot @@ -4,13 +4,17 @@ set -euxo pipefail # Enable systemd services +systemctl unmask syslog.socket + systemctl add-wants minimal.target \ automount-data.service \ google-cloud-ops-agent-fluent-bit.service \ google-cloud-ops-agent.service \ + logrotate.timer \ prometheus-node-exporter.service \ prometheus-process-exporter.service \ ptlb-routes-nanny.timer \ + rsyslog.service \ vault-agent.service # Remove automatically generated vault cert diff --git a/modules/l2/_gcp/mkosi.conf b/modules/l2/_gcp/mkosi.conf index cc85c9ac..c9069f50 100644 --- a/modules/l2/_gcp/mkosi.conf +++ b/modules/l2/_gcp/mkosi.conf @@ -4,3 +4,5 @@ Profiles=gcp [Content] ExtraTrees=modules/l2/_gcp/mkosi.extra PostInstallationScripts=modules/l2/_gcp/mkosi.postinst.chroot + +Packages=curl diff --git a/modules/l2/_gcp/mkosi.extra/etc/systemd/system/set-hostname.service b/modules/l2/_gcp/mkosi.extra/etc/systemd/system/set-hostname.service index 263867fa..e80fbd72 100644 --- a/modules/l2/_gcp/mkosi.extra/etc/systemd/system/set-hostname.service +++ b/modules/l2/_gcp/mkosi.extra/etc/systemd/system/set-hostname.service @@ -4,11 +4,17 @@ ConditionFirstBoot=yes After=network.target network-setup.service Wants=network-setup.service +StartLimitIntervalSec=0 +StartLimitBurst=0 + [Service] User=root Group=root Type=oneshot -ExecStart=/usr/bin/set-hostname-gcp +ExecStart=/usr/bin/set-hostname-gcp.sh + +Restart=on-failure +RestartSec=5 [Install] WantedBy=default.target diff --git a/modules/l2/_gcp/mkosi.extra/usr/bin/set-hostname-gcp b/modules/l2/_gcp/mkosi.extra/usr/bin/set-hostname-gcp deleted file mode 100755 index 779d2430..00000000 --- a/modules/l2/_gcp/mkosi.extra/usr/bin/set-hostname-gcp +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -hostname=$(curl --header "Metadata-Flavor: Google" --fail --silent --show-error \ - --retry 100 --retry-delay 1 --retry-all-errors \ - http://169.254.169.254/computeMetadata/v1/instance/name) -echo "Setting hostname to '${hostname}'..." -hostname "${hostname}" -echo "127.0.0.1 ${hostname}" >> /etc/hosts diff --git a/modules/l2/_gcp/mkosi.extra/usr/bin/set-hostname-gcp.sh b/modules/l2/_gcp/mkosi.extra/usr/bin/set-hostname-gcp.sh new file mode 100755 index 00000000..faf648b6 --- /dev/null +++ b/modules/l2/_gcp/mkosi.extra/usr/bin/set-hostname-gcp.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -euxo pipefail + +if hostname=$( + curl \ + --connect-timeout 1 \ + --fail \ + --header "Metadata-Flavor: Google" \ + --retry 10 \ + --retry-all-errors \ + --retry-delay 1 \ + --show-error \ + --silent \ + http://169.254.169.254/computeMetadata/v1/instance/name + ); then + + echo "Setting hostname to '${hostname}'..." + + hostname "${hostname}" + echo "127.0.0.1 ${hostname}" >> /etc/hosts + + systemctl restart rsyslog.service || true +else + echo "Failed to get instance name from metadata service" + exit 1 +fi diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.build b/modules/l2/op-rbuilder-bproxy/mkosi.build index 629e0c8c..dcfecd51 100755 --- a/modules/l2/op-rbuilder-bproxy/mkosi.build +++ b/modules/l2/op-rbuilder-bproxy/mkosi.build @@ -2,7 +2,7 @@ set -euxo pipefail -ENV_YAML="$SRCDIR/modules/l2/op-rbuilder/mkosi.extra/etc/flashbots/op-rbuilder.yaml" +ENV_YAML="$SRCDIR/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/flashbots/op-rbuilder.yaml" BPROXY_REF=$(mkosi-chroot yq -r .bproxy.git_reference < "$ENV_YAML") NODE_HEALTHCHECKER_REF=$(mkosi-chroot yq -r .node_healthchecker.git_reference < "$ENV_YAML") diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.conf b/modules/l2/op-rbuilder-bproxy/mkosi.conf index 6ff09886..50ea7154 100644 --- a/modules/l2/op-rbuilder-bproxy/mkosi.conf +++ b/modules/l2/op-rbuilder-bproxy/mkosi.conf @@ -2,9 +2,9 @@ WithNetwork=true [Content] -BuildScripts=modules/l2/op-rbuilder/mkosi.build -ExtraTrees=modules/l2/op-rbuilder/mkosi.extra -PostInstallationScripts=modules/l2/op-rbuilder/mkosi.postinst.chroot +BuildScripts=modules/l2/op-rbuilder-bproxy/mkosi.build +ExtraTrees=modules/l2/op-rbuilder-bproxy/mkosi.extra +PostInstallationScripts=modules/l2/op-rbuilder-bproxy/mkosi.postinst.chroot Packages=libtss2-dev sudo diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/flashbots/op-rbuilder.yaml b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/flashbots/op-rbuilder.yaml index dcfbe317..a81f8d5e 100644 --- a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/flashbots/op-rbuilder.yaml +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/flashbots/op-rbuilder.yaml @@ -1,9 +1,9 @@ bproxy: - git_reference: v0.0.92-hf.3 + git_reference: v0.0.93 node_healthchecker: git_reference: v0.1.11 op_rbuilder: - git_reference: op-rbuilder/v0.4.0 + git_reference: op-rbuilder/v0.3.3 rust: version: 1.94.0 tdx_quote_provider: diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy.service.ctmpl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy.service.ctmpl new file mode 100644 index 00000000..f029b731 --- /dev/null +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy.service.ctmpl @@ -0,0 +1,90 @@ +[Install] +WantedBy=default.target + +[Unit] +Description=L2 builder proxy +After=network.target +Wants=network.target + +[Service] +Type=simple +SyslogIdentifier=bproxy +User=op-rbuilder +Group=optimism + +Restart=always +RestartSec=5 +TimeoutStopSec=60 + +ExecStartPre=+/usr/bin/mkdir -p /etc/bproxy +ExecStartPre=+/usr/bin/chown -R op-rbuilder:optimism /etc/bproxy + +((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data )) +((- $tls_crt := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_crt )) +((- $tls_key := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_tls[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data.tls_key )) + +ExecStart=/usr/bin/bproxy serve \ + --authrpc-backend http://127.0.0.1:18651 \ + --authrpc-backend-timeout 5s \ + --authrpc-client-idle-connection-timeout 15m \ + --authrpc-deduplicate-fcus \ + --authrpc-enabled \ + --authrpc-healthcheck http://127.0.0.1:8080 \ + --authrpc-listen-address 0.0.0.0:8651 \ + --authrpc-max-backend-connections-per-host 1 \ + --authrpc-max-request-size 150 \ + --authrpc-max-response-size 1150 \ + ((- if $service.authrpc_peers )) + ((- range $idx, $url := $service.authrpc_peers )) + --authrpc-peers '(( printf "%s" $url ))' \ + ((- end )) + --authrpc-remove-backend-from-peers \ + ((- end )) + ((- if $tls_crt )) + --authrpc-tls-crt /etc/bproxy/tls.crt \ + ((- end )) + ((- if $tls_key )) + --authrpc-tls-key /etc/bproxy/tls.key \ + ((- end )) + --authrpc-use-priority-queue \ + ((- if $service.feat_flashblocks ))(( if $service.feat_flashblocks | parseBool )) + --flashblocks-backend ws://127.0.0.1:11111 \ + --flashblocks-enabled \ + --flashblocks-healthcheck http://127.0.0.1:8080 \ + --flashblocks-listen-address 0.0.0.0:1111 \ + ((- if $tls_crt )) + --flashblocks-tls-crt /etc/bproxy/tls.crt \ + ((- end )) + ((- if $tls_key )) + --flashblocks-tls-key /etc/bproxy/tls.key \ + ((- end )) + ((- end ))(( end )) + --metrics-listen-address 0.0.0.0:6785 \ + --rpc-backend http://127.0.0.1:18645 \ + --rpc-backend-timeout 5s \ + --rpc-enabled \ + --rpc-healthcheck http://127.0.0.1:8080 \ + --rpc-listen-address 0.0.0.0:8645 \ + --rpc-max-backend-connections-per-host 512 \ + --rpc-max-request-size 150 \ + --rpc-max-response-size 1150 \ + ((- if $service.rpc_peers )) + ((- range $idx, $url := $service.rpc_peers )) + --rpc-peers '(( printf "%s" $url ))' \ + ((- end )) + --rpc-remove-backend-from-peers \ + ((- end )) + ((- if $tls_crt )) + --rpc-tls-crt /etc/bproxy/tls.crt \ + ((- end )) + ((- if $tls_key )) + --rpc-tls-key /etc/bproxy/tls.key \ + ((- end )) + --rpc-use-priority-queue \ + ((- if $service.bproxy_custom_flags ))(( range $idx, $flag := $service.bproxy_custom_flags )) + (( printf "%s" $flag )) \ + ((- end ))(( end )) + +ExecStop=/usr/bin/sh -c "kill -1 $( pgrep node-health ) | true" +ExecStop=/usr/bin/sleep 15 +ExecStop=/usr/bin/sh -c "PID=$( pgrep bproxy ); if [ \"0${PID}\" -gt 0 ]; then kill -2 ${PID}; while kill -0 ${PID} 2>/dev/null; do sleep 1; done; fi" diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy.service.hcl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy.service.hcl index 0a406054..f675bec4 100644 --- a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy.service.hcl +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/bproxy.service.hcl @@ -14,6 +14,8 @@ template { command = ["/bin/sh", "-c", <<-EOT + printf '{"@level":"info","@message":"rendered template","@destination":"/etc/systemd/system/bproxy.service","@content":"%s"}\n' "$( cat /etc/systemd/system/bproxy.service | base64 -w 0 )" + systemctl daemon-reload systemctl add-wants minimal.target bproxy.service systemctl restart bproxy.service diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl index 27c006d8..e83c7dfa 100644 --- a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl @@ -47,7 +47,7 @@ ExecStart=/usr/bin/op-rbuilder node \ ((- if $service.genesis_json )) --chain '/var/opt/optimism/rbuilder/genesis.json' \ ((- else ))(( if $service.network_name )) - --chain '(( $service.network_name ))' \ + --chain '(( $service.network_name | trimSuffix "-mainnet" ))' \ ((- end ))(( end )) --color 'never' \ --datadir '/var/opt/optimism/rbuilder' \ diff --git a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl index 9696e71f..5fa35dee 100644 --- a/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl +++ b/modules/l2/op-rbuilder-bproxy/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl @@ -14,7 +14,7 @@ template { command = ["/bin/sh", "-c", <<-EOT - # op-rbuilder + printf '{"@level":"info","@message":"rendered template","@destination":"/etc/systemd/system/op-rbuilder.service","@content":"%s"}\n' "$( cat /etc/systemd/system/op-rbuilder.service | base64 -w 0 )" systemctl daemon-reload systemctl add-wants minimal.target op-rbuilder.service @@ -23,7 +23,7 @@ template { PID=$( pgrep node-health ); if [ 0${PID} -gt 0 ]; then kill -1 ${PID} || true; fi sleep 5 - PID=$( pgrep rproxy ); if [ 0${PID} -gt 0 ]; then kill -1 ${PID} || true; fi + PID=$( pgrep bproxy ); if [ 0${PID} -gt 0 ]; then kill -1 ${PID} || true; fi systemctl restart op-rbuilder.service systemctl restart node-healthchecker.service diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/flashbots/op-rbuilder.yaml b/modules/l2/op-rbuilder/mkosi.extra/etc/flashbots/op-rbuilder.yaml index 79c62c6b..a76cba35 100644 --- a/modules/l2/op-rbuilder/mkosi.extra/etc/flashbots/op-rbuilder.yaml +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/flashbots/op-rbuilder.yaml @@ -1,7 +1,7 @@ node_healthchecker: git_reference: v0.1.11 op_rbuilder: - git_reference: op-rbuilder/v0.4.0 + git_reference: op-rbuilder/v0.3.3 rproxy: git_reference: v0.0.11 rust: diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl index 41c25948..602eca01 100644 --- a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.ctmpl @@ -47,7 +47,7 @@ ExecStart=/usr/bin/op-rbuilder node \ ((- if $service.genesis_json )) --chain '/var/opt/optimism/rbuilder/genesis.json' \ ((- else ))(( if $service.network_name )) - --chain '(( $service.network_name ))' \ + --chain '(( $service.network_name | trimSuffix "-mainnet" ))' \ ((- end ))(( end )) --color 'never' \ --datadir '/var/opt/optimism/rbuilder' \ diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl index 9696e71f..65755ba8 100644 --- a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/op-rbuilder.service.hcl @@ -14,7 +14,7 @@ template { command = ["/bin/sh", "-c", <<-EOT - # op-rbuilder + printf '{"@level":"info","@message":"rendered template","@destination":"/etc/systemd/system/op-rbuilder.service","@content":"%s"}\n' "$( cat /etc/systemd/system/op-rbuilder.service | base64 -w 0 )" systemctl daemon-reload systemctl add-wants minimal.target op-rbuilder.service diff --git a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.hcl b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.hcl index 0e1a979a..65c7f8c8 100644 --- a/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.hcl +++ b/modules/l2/op-rbuilder/mkosi.extra/etc/vault-agent/gomplate/rproxy.service.hcl @@ -14,6 +14,8 @@ template { command = ["/bin/sh", "-c", <<-EOT + printf '{"@level":"info","@message":"rendered template","@destination":"/etc/systemd/system/rproxy.service","@content":"%s"}\n' "$( cat /etc/systemd/system/rproxy.service | base64 -w 0 )" + systemctl daemon-reload systemctl add-wants minimal.target rproxy.service systemctl restart rproxy.service