define autoscaling behaviour with rolling update strategy

README.md

@@ -197,11 +197,9 @@ Add new variable to `deploy-project.sh` and specify your redis version
   ...
 ```
 
-### Enable Horizontal pod autoscaling
+### Horizontal pod autoscaling
 
-Add new variables to `deploy-project.sh` to enable pod autoscaling:
-
-- Enable this functionality:
+- You can disable autoscaling by setting `ENABLE_AUTOSCALING=false` in `deploy-project.sh`:
   ```diff
   ...
   function deploy() {
@@ -210,7 +208,7 @@ Add new variables to `deploy-project.sh` to enable pod autoscaling:
           ...
       )
 
-  +   ENABLE_AUTOSCALING=true
+  +   ENABLE_AUTOSCALING=false
   ...
   ```
 - If you need more replicas, then you can adjust those variables (default values are set to 2):

UPGRADE.md

@@ -12,6 +12,7 @@
 
 ## Upgrade from v5.0.0 to v5.1.0
 
+- define autoscaling behavior with rolling update strategy ([#78](https://github.com/shopsys/deployment/pull/78))
 - removed GCloud-specific docker registry secret creation ([#77](https://github.com/shopsys/deployment/pull/77))
 - improved probes and graceful shutdown for storefront ([#76](https://github.com/shopsys/deployment/pull/76))
 - added probes for RabbitMQ with graceful shutdown ([#75](https://github.com/shopsys/deployment/pull/75))

deploy/parts/autoscaling.sh

@@ -1,37 +1,52 @@
 #!/bin/bash -e
 
+patch_rollout() {
+    local file="$1" min_replicas="$2"
+
+    if (( min_replicas >= 4 )); then
+        yq e -i '.spec.strategy.rollingUpdate.maxUnavailable = "25%"' "$file"
+    fi
+
+    if (( min_replicas >= 6 )); then
+        yq e -i '.spec.strategy.rollingUpdate.maxSurge = "25%"' "$file"
+    fi
+}
+
 echo -n "Prepare Autoscaling "
 
 assertVariable "BASE_PATH"
 assertVariable "CONFIGURATION_TARGET_PATH"
+assertVariable "RUNNING_PRODUCTION"
 
-if [ -z ${ENABLE_AUTOSCALING} ]; then
-    ENABLE_AUTOSCALING=false
-fi
-
-if [ ${ENABLE_AUTOSCALING} = true ]; then
-    if [ -z ${MIN_PHP_FPM_REPLICAS} ]; then
-        MIN_PHP_FPM_REPLICAS=2
-    fi
+if [[ "${ENABLE_AUTOSCALING:-true}" == "true" ]]; then
 
-    if [ -z ${MAX_PHP_FPM_REPLICAS} ]; then
-        MAX_PHP_FPM_REPLICAS=3
-    fi
+    MIN_PHP_FPM_REPLICAS="${MIN_PHP_FPM_REPLICAS:-2}"
+    MAX_PHP_FPM_REPLICAS="${MAX_PHP_FPM_REPLICAS:-3}"
+    MIN_STOREFRONT_REPLICAS="${MIN_STOREFRONT_REPLICAS:-2}"
+    MAX_STOREFRONT_REPLICAS="${MAX_STOREFRONT_REPLICAS:-3}"
 
-    if [ -z ${MIN_STOREFRONT_REPLICAS} ]; then
+    if [[ "${RUNNING_PRODUCTION}" -eq 0 || "${DOWNSCALE_RESOURCE:-0}" -eq 1 ]]; then
+        MIN_PHP_FPM_REPLICAS=2
+        MAX_PHP_FPM_REPLICAS=2
         MIN_STOREFRONT_REPLICAS=2
-    fi
-
-    if [ -z ${MAX_STOREFRONT_REPLICAS} ]; then
-        MAX_STOREFRONT_REPLICAS=3
+        MAX_STOREFRONT_REPLICAS=2
     fi
 
     yq e -i ".spec.minReplicas=${MIN_PHP_FPM_REPLICAS}" "${CONFIGURATION_TARGET_PATH}/horizontalPodAutoscaler.yaml"
     yq e -i ".spec.maxReplicas=${MAX_PHP_FPM_REPLICAS}" "${CONFIGURATION_TARGET_PATH}/horizontalPodAutoscaler.yaml"
 
     yq e -i ".spec.minReplicas=${MIN_STOREFRONT_REPLICAS}" "${CONFIGURATION_TARGET_PATH}/horizontalStorefrontAutoscaler.yaml"
     yq e -i ".spec.maxReplicas=${MAX_STOREFRONT_REPLICAS}" "${CONFIGURATION_TARGET_PATH}/horizontalStorefrontAutoscaler.yaml"
+
+    patch_rollout "${CONFIGURATION_TARGET_PATH}/deployments/webserver-php-fpm.yaml" "${MIN_PHP_FPM_REPLICAS}"
+    patch_rollout "${CONFIGURATION_TARGET_PATH}/deployments/storefront.yaml" "${MIN_STOREFRONT_REPLICAS}"
 fi
 
+yq e -i '
+    .resources += [
+      "../../horizontalPodAutoscaler.yaml",
+      "../../horizontalStorefrontAutoscaler.yaml"
+    ]
+' "${CONFIGURATION_TARGET_PATH}/kustomize/webserver/kustomization.yaml"
 
 echo -e "[${GREEN}OK${NO_COLOR}]"

deploy/parts/deploy.sh

@@ -11,7 +11,6 @@ assertVariable "DEPLOY_REGISTER_USER"
 assertVariable "DEPLOY_REGISTER_PASSWORD"
 
 assertVariable "BASIC_AUTH_PATH"
-assertVariable "ENABLE_AUTOSCALING"
 assertVariable "RUNNING_PRODUCTION"
 FIRST_DEPLOY_LOAD_DEMO_DATA=${FIRST_DEPLOY_LOAD_DEMO_DATA:-0}
 
@@ -158,40 +157,12 @@ if [ $DISPLAY_FINAL_CONFIGURATION -eq "1" ]; then
     echo ""
 fi
 
-if [ ${ENABLE_AUTOSCALING} = true ]; then
-    echo -n "    Delete previous Horizontal pod autoscaler for Backend "
-    runCommand "SKIP" "kubectl delete hpa webserver-php-fpm --namespace=${PROJECT_NAME}"
-
-    echo -n "    Delete previous Horizontal pod autoscaler for Storefront "
-    runCommand "SKIP" "kubectl delete hpa storefront --namespace=${PROJECT_NAME}"
-fi
-
 echo -n "    Deploy Webserver and PHP-FPM container with Storefront"
 runCommand "ERROR" "kustomize build --load-restrictor LoadRestrictionsNone  \"${CONFIGURATION_TARGET_PATH}/kustomize/webserver\" | kubectl apply -f -"
 
 echo -n "    Waiting for start new PHP-FPM and Storefront container (In case of fail you need to manually check what is state of application)"
 runCommand "ERROR" "kubectl rollout status --namespace=${PROJECT_NAME} deployment/webserver-php-fpm deployment/storefront --watch"
 
-if [ ${ENABLE_AUTOSCALING} = true ]; then
-    echo -n "    Deploy Horizontal pod autoscaler for Backend "
-
-    if [ ${RUNNING_PRODUCTION} -eq "0" ]; then
-        yq e -i '.spec.minReplicas = 2' "${CONFIGURATION_TARGET_PATH}/horizontalPodAutoscaler.yaml"
-        yq e -i '.spec.maxReplicas = 2' "${CONFIGURATION_TARGET_PATH}/horizontalPodAutoscaler.yaml"
-    fi
-
-    runCommand "ERROR" "kubectl apply -f ${CONFIGURATION_TARGET_PATH}/horizontalPodAutoscaler.yaml"
-
-    echo -n "    Deploy Horizontal pod autoscaler for Storefront "
-
-    if [ ${RUNNING_PRODUCTION} -eq "0" ]; then
-        yq e -i '.spec.minReplicas = 2' "${CONFIGURATION_TARGET_PATH}/horizontalStorefrontAutoscaler.yaml"
-        yq e -i '.spec.maxReplicas = 2' "${CONFIGURATION_TARGET_PATH}/horizontalStorefrontAutoscaler.yaml"
-    fi
-
-    runCommand "ERROR" "kubectl apply -f ${CONFIGURATION_TARGET_PATH}/horizontalStorefrontAutoscaler.yaml"
-fi
-
 RUNNING_WEBSERVER_PHP_FPM_POD=$(kubectl get pods --namespace=${PROJECT_NAME} --field-selector=status.phase=Running -l app=webserver-php-fpm -o=jsonpath='{.items[?(@.status.containerStatuses[0].state.running)].metadata.name}')
 
 echo -n "Disable maintenance page "

kubernetes/deployments/storefront.yaml

@@ -4,6 +4,11 @@ metadata:
     name: storefront
 spec:
     replicas: 1
+    strategy:
+        rollingUpdate:
+            maxSurge: 50%
+            maxUnavailable: 0
+        type: RollingUpdate
     selector:
         matchLabels:
             app: storefront

kubernetes/deployments/webserver-php-fpm.yaml

@@ -9,7 +9,7 @@ spec:
     replicas: 1
     strategy:
         rollingUpdate:
-            maxSurge: 1
+            maxSurge: 50%
             maxUnavailable: 0
         type: RollingUpdate
     selector:

kubernetes/horizontalPodAutoscaler.yaml

@@ -10,6 +10,27 @@ spec:
         name: webserver-php-fpm
     minReplicas: 2
     maxReplicas: 2
+    behavior:
+        scaleUp:
+            stabilizationWindowSeconds: 60
+            selectPolicy: Max
+            policies:
+                - type: Pods
+                  value: 2
+                  periodSeconds: 60
+                - type: Percent
+                  value: 50
+                  periodSeconds: 60
+        scaleDown:
+            stabilizationWindowSeconds: 600
+            selectPolicy: Max
+            policies:
+                - type: Pods
+                  value: 1
+                  periodSeconds: 60
+                - type: Percent
+                  value: 30
+                  periodSeconds: 60
     metrics:
         -   type: ContainerResource
             containerResource:

kubernetes/horizontalStorefrontAutoscaler.yaml

@@ -10,6 +10,27 @@ spec:
         name: storefront
     minReplicas: 2
     maxReplicas: 2
+    behavior:
+        scaleUp:
+            stabilizationWindowSeconds: 60
+            selectPolicy: Max
+            policies:
+                - type: Pods
+                  value: 2
+                  periodSeconds: 60
+                - type: Percent
+                  value: 50
+                  periodSeconds: 60
+        scaleDown:
+            stabilizationWindowSeconds: 600
+            selectPolicy: Max
+            policies:
+                - type: Pods
+                  value: 1
+                  periodSeconds: 60
+                - type: Percent
+                  value: 30
+                  periodSeconds: 60
     metrics:
         -   type: Resource
             resource:

tests/run-tests.sh

@@ -178,8 +178,6 @@ build_outputs() {
     fi
 
     # Copy individual files that aren't built by kustomize
-    cp "${config_path}/horizontalPodAutoscaler.yaml" "${output_dir}/" 2>/dev/null || true
-    cp "${config_path}/horizontalStorefrontAutoscaler.yaml" "${output_dir}/" 2>/dev/null || true
     cp "${config_path}/namespace.yaml" "${output_dir}/" 2>/dev/null || true
 
     echo "$output_dir"

tests/scenarios/basic-production/deploy-project.sh

@@ -5,7 +5,6 @@ source "$(dirname "$0")/../../lib/scenario-base.sh"
 # Scenario-specific configuration
 DOMAINS=(DOMAIN_HOSTNAME_1 DOMAIN_HOSTNAME_2)
 export RUNNING_PRODUCTION=1
-ENABLE_AUTOSCALING=true
 
 declare -A CRON_INSTANCES=(
     ["cron"]='*/5 * * * *'

tests/scenarios/basic-production/expected/webserver.yaml

@@ -581,6 +581,11 @@ spec:
   selector:
     matchLabels:
       app: storefront
+  strategy:
+    rollingUpdate:
+      maxSurge: 50%
+      maxUnavailable: 0
+    type: RollingUpdate
   template:
     metadata:
       annotations:
@@ -674,7 +679,7 @@ spec:
       app: webserver-php-fpm
   strategy:
     rollingUpdate:
-      maxSurge: 1
+      maxSurge: 50%
       maxUnavailable: 0
     type: RollingUpdate
   template:
@@ -924,6 +929,89 @@ spec:
           defaultMode: 420
           secretName: fe-api-keys
 ---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: storefront
+  namespace: myproject-production
+spec:
+  behavior:
+    scaleDown:
+      policies:
+      - periodSeconds: 60
+        type: Pods
+        value: 1
+      - periodSeconds: 60
+        type: Percent
+        value: 30
+      selectPolicy: Max
+      stabilizationWindowSeconds: 600
+    scaleUp:
+      policies:
+      - periodSeconds: 60
+        type: Pods
+        value: 2
+      - periodSeconds: 60
+        type: Percent
+        value: 50
+      selectPolicy: Max
+      stabilizationWindowSeconds: 60
+  maxReplicas: 3
+  metrics:
+  - resource:
+      name: cpu
+      target:
+        averageUtilization: 120
+        type: Utilization
+    type: Resource
+  minReplicas: 2
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: storefront
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: webserver-php-fpm
+  namespace: myproject-production
+spec:
+  behavior:
+    scaleDown:
+      policies:
+      - periodSeconds: 60
+        type: Pods
+        value: 1
+      - periodSeconds: 60
+        type: Percent
+        value: 30
+      selectPolicy: Max
+      stabilizationWindowSeconds: 600
+    scaleUp:
+      policies:
+      - periodSeconds: 60
+        type: Pods
+        value: 2
+      - periodSeconds: 60
+        type: Percent
+        value: 50
+      selectPolicy: Max
+      stabilizationWindowSeconds: 60
+  maxReplicas: 3
+  metrics:
+  - containerResource:
+      container: php-fpm
+      name: cpu
+      target:
+        averageUtilization: 120
+        type: Utilization
+    type: ContainerResource
+  minReplicas: 2
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: webserver-php-fpm
+---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata: