diff --git a/.github/workflows/helm-chart.yaml b/.github/workflows/helm-chart.yaml index da529c5303..d443bb53f9 100644 --- a/.github/workflows/helm-chart.yaml +++ b/.github/workflows/helm-chart.yaml @@ -44,6 +44,17 @@ jobs: - name: "Checkout code" uses: actions/checkout@v6 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: pip install -r tools/ci/helm_ci/requirements.txt + + - name: Run validate values + run: python tools/ci/helm_ci/validate_helm.py + - name: "Lint Helm chart" run: helm lint ./helm diff --git a/.gitignore b/.gitignore index 845fa7ff97..c73e5d83c9 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,10 @@ website/versioned_docs website/versioned_sidebars website/versions.json website/pnpm-lock.yaml + +### Python ### +.venv/ +venv/ +env/ +.env +__pycache__/ diff --git a/helm/templates/sts-coordinator.yaml b/helm/templates/sts-coordinator.yaml index c443e230fb..1175431fbb 100644 --- a/helm/templates/sts-coordinator.yaml +++ b/helm/templates/sts-coordinator.yaml @@ -24,7 +24,7 @@ metadata: {{- include "fluss.labels" . | nindent 4 }} spec: serviceName: coordinator-server-hs - replicas: {{ .Values.coordinator.numberOfReplicas }} + replicas: {{ .Values.coordinatorServer.replicas }} selector: matchLabels: {{- include "fluss.selectorLabels" . | nindent 6 }} @@ -66,9 +66,9 @@ spec: {{- end }} ports: - name: internal - containerPort: {{ .Values.listeners.internal.port }} + containerPort: {{ .Values.ports.internal }} - name: client - containerPort: {{ .Values.listeners.client.port }} + containerPort: {{ .Values.ports.client }} command: - "/bin/sh" - "-c" @@ -76,8 +76,8 @@ spec: export FLUSS_SERVER_ID=${POD_NAME##*-} && \ cp /opt/conf/server.yaml $FLUSS_HOME/conf && \ - BIND_LISTENERS="INTERNAL://${POD_IP}:{{ .Values.listeners.internal.port }}, CLIENT://${POD_IP}:{{ .Values.listeners.client.port }}" && \ - ADVERTISED_LISTENERS="CLIENT://${POD_NAME}.coordinator-server-hs.${POD_NAMESPACE}.svc.cluster.local:{{ .Values.listeners.client.port }}" && \ + BIND_LISTENERS="INTERNAL://${POD_IP}:{{ .Values.ports.internal }}, CLIENT://${POD_IP}:{{ .Values.ports.client }}" && \ + ADVERTISED_LISTENERS="CLIENT://${POD_NAME}.coordinator-server-hs.${POD_NAMESPACE}.svc.cluster.local:{{ .Values.ports.client }}" && \ echo "" >> $FLUSS_HOME/conf/server.yaml && \ echo "bind.listeners: ${BIND_LISTENERS}" >> $FLUSS_HOME/conf/server.yaml && \ @@ -85,21 +85,21 @@ spec: bin/coordinator-server.sh start-foreground livenessProbe: - failureThreshold: 100 - timeoutSeconds: 1 - initialDelaySeconds: 10 - periodSeconds: 3 + failureThreshold: {{ .Values.coordinatorServer.livenessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.coordinatorServer.livenessProbe.timeoutSeconds }} + initialDelaySeconds: {{ .Values.coordinatorServer.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.coordinatorServer.livenessProbe.periodSeconds }} tcpSocket: - port: {{ .Values.listeners.client.port }} + port: {{ .Values.ports.client }} readinessProbe: - failureThreshold: 100 - timeoutSeconds: 1 - initialDelaySeconds: 10 - periodSeconds: 3 + failureThreshold: {{ .Values.coordinatorServer.readinessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.coordinatorServer.readinessProbe.timeoutSeconds }} + initialDelaySeconds: {{ .Values.coordinatorServer.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.coordinatorServer.readinessProbe.periodSeconds }} tcpSocket: - port: {{ .Values.listeners.client.port }} + port: {{ .Values.ports.client }} resources: - {{- toYaml .Values.resources.tabletServer | nindent 12 }} + {{- toYaml .Values.coordinatorServer.resources | nindent 12 }} volumeMounts: - name: fluss-conf mountPath: /opt/conf @@ -114,7 +114,7 @@ spec: - name: fluss-conf configMap: name: fluss-conf-file - {{- if not .Values.coordinator.storage.enabled }} + {{- if not .Values.coordinatorServer.persistence.enabled }} - name: data emptyDir: {} {{- end }} @@ -123,7 +123,7 @@ spec: secret: secretName: {{ include "fluss.security.sasl.configName" . }} {{- end }} - {{- if .Values.coordinator.storage.enabled }} + {{- if .Values.coordinatorServer.persistence.enabled }} volumeClaimTemplates: - metadata: name: data @@ -131,6 +131,6 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: {{ .Values.coordinator.storage.size }} - storageClassName: {{ .Values.coordinator.storage.storageClass }} + storage: {{ .Values.coordinatorServer.persistence.size }} + storageClassName: {{ .Values.coordinatorServer.persistence.storageClass }} {{- end}} diff --git a/helm/templates/sts-tablet.yaml b/helm/templates/sts-tablet.yaml index 1ffe1af310..4cb7d88235 100644 --- a/helm/templates/sts-tablet.yaml +++ b/helm/templates/sts-tablet.yaml @@ -24,7 +24,7 @@ metadata: {{- include "fluss.labels" . | nindent 4 }} spec: serviceName: tablet-server-hs - replicas: {{ .Values.tablet.numberOfReplicas }} + replicas: {{ .Values.tabletServer.replicas }} selector: matchLabels: {{- include "fluss.selectorLabels" . | nindent 6 }} @@ -62,9 +62,9 @@ spec: {{- end }} ports: - name: internal - containerPort: {{ .Values.listeners.internal.port }} + containerPort: {{ .Values.ports.internal }} - name: client - containerPort: {{ .Values.listeners.client.port }} + containerPort: {{ .Values.ports.client }} command: - "/bin/sh" - "-c" @@ -72,8 +72,8 @@ spec: export FLUSS_SERVER_ID=${POD_NAME##*-} && \ cp /opt/conf/server.yaml $FLUSS_HOME/conf && \ - BIND_LISTENERS="INTERNAL://${POD_IP}:{{ .Values.listeners.internal.port }}, CLIENT://${POD_IP}:{{ .Values.listeners.client.port }}" && \ - ADVERTISED_LISTENERS="CLIENT://${POD_NAME}.tablet-server-hs.${POD_NAMESPACE}.svc.cluster.local:{{ .Values.listeners.client.port }}" && \ + BIND_LISTENERS="INTERNAL://${POD_IP}:{{ .Values.ports.internal }}, CLIENT://${POD_IP}:{{ .Values.ports.client }}" && \ + ADVERTISED_LISTENERS="CLIENT://${POD_NAME}.tablet-server-hs.${POD_NAMESPACE}.svc.cluster.local:{{ .Values.ports.client }}" && \ echo "" >> $FLUSS_HOME/conf/server.yaml && \ echo "tablet-server.id: ${FLUSS_SERVER_ID}" >> $FLUSS_HOME/conf/server.yaml && \ @@ -82,21 +82,21 @@ spec: bin/tablet-server.sh start-foreground livenessProbe: - failureThreshold: 100 - timeoutSeconds: 1 - initialDelaySeconds: 10 - periodSeconds: 3 + failureThreshold: {{ .Values.tabletServer.livenessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.tabletServer.livenessProbe.timeoutSeconds }} + initialDelaySeconds: {{ .Values.tabletServer.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.tabletServer.livenessProbe.periodSeconds }} tcpSocket: - port: {{ .Values.listeners.client.port }} + port: {{ .Values.ports.client }} readinessProbe: - failureThreshold: 100 - timeoutSeconds: 1 - initialDelaySeconds: 10 - periodSeconds: 3 + failureThreshold: {{ .Values.tabletServer.readinessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.tabletServer.readinessProbe.timeoutSeconds }} + initialDelaySeconds: {{ .Values.tabletServer.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.tabletServer.readinessProbe.periodSeconds }} tcpSocket: - port: {{ .Values.listeners.client.port }} + port: {{ .Values.ports.client }} resources: - {{- toYaml .Values.resources.tabletServer | nindent 12 }} + {{- toYaml .Values.tabletServer.resources | nindent 12 }} volumeMounts: - name: fluss-conf mountPath: /opt/conf @@ -111,7 +111,7 @@ spec: - name: fluss-conf configMap: name: fluss-conf-file - {{- if not .Values.tablet.storage.enabled }} + {{- if not .Values.tabletServer.persistence.enabled }} - name: data emptyDir: {} {{- end }} @@ -120,7 +120,7 @@ spec: secret: secretName: {{ include "fluss.security.sasl.configName" . }} {{- end }} - {{- if .Values.tablet.storage.enabled }} + {{- if .Values.tabletServer.persistence.enabled }} volumeClaimTemplates: - metadata: name: data @@ -128,6 +128,6 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: {{ .Values.tablet.storage.size }} - storageClassName: {{ .Values.tablet.storage.storageClass }} + storage: {{ .Values.tabletServer.persistence.size }} + storageClassName: {{ .Values.tabletServer.persistence.storageClass }} {{- end}} diff --git a/helm/templates/svc-coordinator.yaml b/helm/templates/svc-coordinator.yaml index 64a622ae28..263d0d87ad 100644 --- a/helm/templates/svc-coordinator.yaml +++ b/helm/templates/svc-coordinator.yaml @@ -29,11 +29,11 @@ spec: ports: - name: internal protocol: TCP - port: {{ .Values.listeners.internal.port }} + port: {{ .Values.ports.internal }} targetPort: internal - name: client protocol: TCP - port: {{ .Values.listeners.client.port }} + port: {{ .Values.ports.client }} targetPort: client selector: {{- include "fluss.selectorLabels" . | nindent 4 }} diff --git a/helm/templates/svc-tablet.yaml b/helm/templates/svc-tablet.yaml index 517653e87f..062cdb989e 100644 --- a/helm/templates/svc-tablet.yaml +++ b/helm/templates/svc-tablet.yaml @@ -29,11 +29,11 @@ spec: ports: - name: internal protocol: TCP - port: {{ .Values.listeners.internal.port }} + port: {{ .Values.ports.internal }} targetPort: internal - name: client protocol: TCP - port: {{ .Values.listeners.client.port }} + port: {{ .Values.ports.client }} targetPort: client selector: {{- include "fluss.selectorLabels" . | nindent 4 }} diff --git a/helm/values.schema.json b/helm/values.schema.json new file mode 100644 index 0000000000..33a2a762d7 --- /dev/null +++ b/helm/values.schema.json @@ -0,0 +1,395 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "description": "Default values for fluss. Declare variables to be passed into your templates.", + "type": "object", + "x-docsSectionOrder": [ + "Image", + "Coordinator Server", + "Tablet Server", + "Configuration Overrides", + "Ports", + "Security", + "Service Account" + ], + "additionalProperties": false, + "properties": { + "image": { + "description": "Fluss image configuration", + "type": "object", + "x-docsSection": "Image", + "additionalProperties": false, + "properties": { + "registry": { + "description": "Container registry", + "type": "string", + "default": "" + }, + "repository": { + "description": "Fluss image repository", + "type": "string", + "default": "apache/fluss" + }, + "tag": { + "description": "Fluss image tag", + "type": "string", + "default": "0.10.0-incubating" + }, + "pullPolicy": { + "description": "The fluss image pull policy", + "type": "string", + "enum": ["Always", "Never", "IfNotPresent"], + "default": "IfNotPresent" + }, + "pullSecrets": { + "description": "List of existing Kubernetes secrets containing Base64 encoded credentials to connect to private registries (will get passed to image.pullSecrets).", + "type": "array", + "default": [], + "items": { + "type": "string" + }, + "uniqueItems": true + } + } + }, + "coordinatorServer": { + "description": "Coordinator server configuration", + "type": "object", + "x-docsSection": "Coordinator Server", + "additionalProperties": false, + "properties": { + "replicas": { + "description": "Amount of coordinator server replicas.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "livenessProbe": { + "description": "Liveness probe configuration for coordinator server containers.", + "type": "object", + "properties": { + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": "integer", + "default": 100 + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": "integer", + "default": 1 + }, + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated.", + "type": "integer", + "default": 10 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": "integer", + "default": 3 + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration for coordinator server containers.", + "type": "object", + "properties": { + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": "integer", + "default": 100 + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": "integer", + "default": 1 + }, + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated.", + "type": "integer", + "default": 10 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": "integer", + "default": 3 + } + } + }, + "resources": { + "description": "Resources for coordinator server pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ] + }, + "persistence": { + "description": "Persistence configuration for coordinator servers.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable persistent volumes.", + "type": "boolean", + "default": false + }, + "size": { + "description": "Volume size for the coordinator server.", + "type": "string", + "default": "1Gi" + }, + "storageClass": { + "description": "Storage class of the persistent volume.", + "type": ["string", "null"], + "default": null + } + } + } + } + }, + "tabletServer": { + "type": "object", + "description": "Tablet server configuration", + "x-docsSection": "Tablet Server", + "additionalProperties": false, + "properties": { + "replicas": { + "description": "Amount of tablet server replicas.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "livenessProbe": { + "description": "Liveness probe configuration for tablet server containers.", + "type": "object", + "properties": { + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": "integer", + "default": 100 + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": "integer", + "default": 1 + }, + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated.", + "type": "integer", + "default": 10 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": "integer", + "default": 3 + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration for tablet server containers.", + "type": "object", + "properties": { + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": "integer", + "default": 100 + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": "integer", + "default": 1 + }, + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated.", + "type": "integer", + "default": 10 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": "integer", + "default": 3 + } + } + }, + "resources": { + "description": "Resources for tablet server pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ] + }, + "persistence": { + "description": "Persistence configuration for tablet servers.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable persistent volumes.", + "type": "boolean", + "default": false + }, + "size": { + "description": "Volume size for the tablet server.", + "type": "string", + "default": "1Gi" + }, + "storageClass": { + "description": "Storage class of the persistent volume.", + "type": ["string", "null"], + "default": null + } + } + } + } + }, + "configurationOverrides": { + "description": "Fluss configuration overrides (see detail: https://fluss.apache.org/docs/maintenance/configuration/)", + "type": "object", + "x-docsSection": "Configuration Overrides", + "default": { + "default.bucket.number": 3, + "default.replication.factor": 3, + "zookeeper.path.root": "/fluss", + "zookeeper.address": "zk-zookeeper.{{ .Release.Namespace }}.svc.cluster.local:2181", + "data.dir": "/tmp/fluss/data", + "internal.listener.name": "INTERNAL", + "remote.data.dir": "/tmp/fluss/remote-data" + }, + "additionalProperties": { + "type": ["string", "number", "boolean"] + } + }, + "ports": { + "description": "Fluss ports configuration", + "type": "object", + "x-docsSection": "Ports", + "additionalProperties": false, + "properties": { + "internal": { + "description": "Internal port for Fluss", + "type": "integer", + "default": 9123 + }, + "client": { + "description": "Client port for Fluss", + "type": "integer", + "default": 9124 + } + } + }, + "security": { + "description": "Fluss security configuration", + "type": "object", + "x-docsSection": "Security", + "additionalProperties": false, + "properties": { + "client": { + "type": "object", + "additionalProperties": false, + "properties": { + "sasl": { + "type": "object", + "additionalProperties": false, + "properties": { + "mechanism": { + "description": "SASL mechanism for client authentication", + "type": "string", + "default": "" + }, + "plain": { + "type": "object", + "additionalProperties": false, + "properties": { + "users": { + "description": "List of SASL PLAIN users", + "type": "array", + "default": [], + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "internal": { + "type": "object", + "additionalProperties": false, + "properties": { + "sasl": { + "type": "object", + "additionalProperties": false, + "properties": { + "mechanism": { + "description": "SASL mechanism for internal communication", + "type": "string", + "default": "" + }, + "plain": { + "type": "object", + "additionalProperties": false, + "properties": { + "username": { + "description": "Username for SASL PLAIN authentication", + "type": "string", + "default": "" + }, + "password": { + "description": "Password for SASL PLAIN authentication", + "type": "string", + "default": "" + } + } + } + } + } + } + } + } + }, + "serviceAccount": { + "description": "ServiceAccount configuration", + "type": "object", + "x-docsSection": "Service Account", + "properties": { + "create": { + "description": "Allow to create a service account", + "type": "boolean", + "default": false + }, + "name": { + "description": "Name of the service account", + "type": "string", + "default": "" + }, + "annotations": { + "description": "Additional annotations to apply to the ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/helm/values.schema.schema.json b/helm/values.schema.schema.json new file mode 100644 index 0000000000..64179d5a7a --- /dev/null +++ b/helm/values.schema.schema.json @@ -0,0 +1,104 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "description": "This schema is used to validate `values.schema.json` to ensure each parameter has `default` and `description` set, and that top level properties have a `x-docsSection` set.", + "definitions": { + "leafs": { + "additionalProperties": { + "if": { + "not": { + "properties": { + "description": { + "pattern": "^Labels for the configmap$|^Labels for the secret$|^Annotations for the configmap$|^Annotations for the secret$" + } + } + } + }, + "then": { + "additionalProperties": { + "$ref": "#/definitions/leafs" + } + } + }, + "if": { + "oneOf": [ + { + "properties": { + "type": { + "const": "integer" + } + } + }, + { + "properties": { + "type": { + "const": "number" + } + } + }, + { + "properties": { + "type": { + "const": "string" + } + } + }, + { + "properties": { + "type": { + "const": "boolean" + } + } + }, + { + "properties": { + "type": { + "const": "object" + }, + "properties": false + } + }, + { + "properties": { + "type": { + "const": "array" + }, + "items": false + } + } + ] + }, + "then": { + "required": [ + "description", + "default" + ] + } + } + }, + "required": [ + "x-docsSectionOrder" + ], + "properties": { + "section_order": { + "type": "array", + "items": { + "type": "string" + } + }, + "properties": { + "additionalProperties": { + "allOf": [ + { + "$ref": "#/definitions/leafs" + }, + { + "required": [ + "x-docsSection" + ] + } + ] + } + } + } +} diff --git a/helm/values.yaml b/helm/values.yaml index 521553f671..dfbe12ff9b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -27,36 +27,78 @@ image: pullPolicy: IfNotPresent pullSecrets: [] -# Fluss server configuration options -configurationOverrides: - default.bucket.number: 3 - default.replication.factor: 3 - zookeeper.path.root: /fluss - zookeeper.address: zk-zookeeper.{{ .Release.Namespace }}.svc.cluster.local:2181 - remote.data.dir: /tmp/fluss/remote-data - data.dir: /tmp/fluss/data - internal.listener.name: INTERNAL +coordinatorServer: + replicas: 1 + + livenessProbe: + failureThreshold: 100 + timeoutSeconds: 1 + initialDelaySeconds: 10 + periodSeconds: 3 + + readinessProbe: + failureThreshold: 100 + timeoutSeconds: 1 + initialDelaySeconds: 10 + periodSeconds: 3 -tablet: - numberOfReplicas: 3 - storage: + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + persistence: enabled: false size: 1Gi storageClass: -coordinator: - numberOfReplicas: 1 - storage: +tabletServer: + replicas: 3 + + livenessProbe: + failureThreshold: 100 + timeoutSeconds: 1 + initialDelaySeconds: 10 + periodSeconds: 3 + + readinessProbe: + failureThreshold: 100 + timeoutSeconds: 1 + initialDelaySeconds: 10 + periodSeconds: 3 + + resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + + persistence: enabled: false size: 1Gi storageClass: +# Fluss server configuration options +configurationOverrides: + # See detail https://fluss.apache.org/docs/maintenance/configuration/ + default.bucket.number: 3 + default.replication.factor: 3 + zookeeper.path.root: /fluss + zookeeper.address: zk-zookeeper.{{ .Release.Namespace }}.svc.cluster.local:2181 + data.dir: /tmp/fluss/data + internal.listener.name: INTERNAL + # File systems configuration (see detail https://fluss.apache.org/docs/maintenance/filesystems/overview/) + remote.data.dir: /tmp/fluss/remote-data + # Fluss listener configurations -listeners: - internal: - port: 9123 - client: - port: 9124 +ports: + internal: 9123 + client: 9124 # Fluss security configurations security: @@ -75,26 +117,6 @@ security: username: "" password: "" -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # coordinatorServer: - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - # tabletServer: - # requests: - # cpu: 100m - # memory: 128Mi - # limits: - # cpu: 100m - # memory: 128Mi - serviceAccount: create: false # If not set and create is true, a name is generated using the fullname template diff --git a/tools/ci/helm_ci/requirements.txt b/tools/ci/helm_ci/requirements.txt new file mode 100644 index 0000000000..db8390f8b4 --- /dev/null +++ b/tools/ci/helm_ci/requirements.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +jsonschema==4.26.0 +PyYAML==6.0.3 \ No newline at end of file diff --git a/tools/ci/helm_ci/validate_helm.py b/tools/ci/helm_ci/validate_helm.py new file mode 100644 index 0000000000..6e45a29a5d --- /dev/null +++ b/tools/ci/helm_ci/validate_helm.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "jsonschema==4.26.0", +# "PyYAML==6.0.3", +# ] +# /// + +import logging +import json +import yaml +from jsonschema import Draft7Validator + +from pathlib import Path + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", +) +log = logging.getLogger(__name__) + + +def get_parent_dir(file, steps): + parent_path = Path(file).resolve() + for _ in range(steps): + parent_path = parent_path.parent + return parent_path + + +root_dir = get_parent_dir(__file__, 4) +helm_dir = root_dir / "helm" +schema_meta_file = str(helm_dir / "values.schema.schema.json") +schema_file = str(helm_dir / "values.schema.json") +values_file = str(helm_dir / "values.yaml") + + +def validate_values_with_schema(schema_file: str, value_file: str): + log.info(f"Validating {value_file}") + with open(schema_file) as f: + schema_values = json.load(f) + + if value_file.endswith("json"): + with open(value_file) as f: + values = json.load(f) + else: + with open(value_file) as f: + values = yaml.safe_load(f) + + validator = Draft7Validator(schema_values) + errors = sorted(validator.iter_errors(values), key=lambda e: e.path) + + if errors: + for e in errors: + log.error(e.message) + else: + log.info(f"{value_file} is valid") + + +if __name__ == "__main__": + log.info("Validating Helm chart values") + # validate values.schema.json against values.schema.schema.json + validate_values_with_schema(schema_file=schema_meta_file, value_file=schema_file) + # validate values.yaml against values.schema.json + validate_values_with_schema(schema_file=schema_file, value_file=values_file)