From 768e8fdcb2f095ae5f4ec3faa7e60381e72ee8ac Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 3 Mar 2026 14:04:07 +0530 Subject: [PATCH 01/10] ## Dev Board Ticket https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/2942 ## Changes - Update package `osm-osw-reformatter` version from `0.3.1` to `0.3.2` ## Testing - TDEI Portal --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4f0b448..de75baf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ pydantic==1.10.4 python-ms-core==0.0.23 uvicorn==0.20.0 html_testRunner==1.2.1 -osm-osw-reformatter==0.3.1 +osm-osw-reformatter==0.3.2 numpy==1.26.4 \ No newline at end of file From b841745831a5341af67fa5fb8bffc1003cbb19f4 Mon Sep 17 00:00:00 2001 From: Naresh Kumar D Date: Thu, 5 Mar 2026 16:55:46 +0530 Subject: [PATCH 02/10] stops after first message stops after first message --- requirements.txt | 3 ++- src/config.py | 1 + src/service/osw_formatter_service.py | 30 +++++++++++++++++++++++++++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index de75baf..7337c94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ +--extra-index-url https://test.pypi.org/simple/ fastapi==0.88.0 pydantic==1.10.4 -python-ms-core==0.0.23 +python-ms-core==0.0.24 uvicorn==0.20.0 html_testRunner==1.2.1 osm-osw-reformatter==0.3.2 diff --git a/src/config.py b/src/config.py index 88a4a57..68ee8ff 100644 --- a/src/config.py +++ b/src/config.py @@ -18,6 +18,7 @@ class Settings(BaseSettings): app_name: str = 'python-osw-formatter' event_bus = EventBusSettings() max_concurrent_messages: int = os.environ.get('MAX_CONCURRENT_MESSAGES', 2) + max_receivable_messages: int = os.environ.get('MAX_RECEIVABLE_MESSAGES', -1) def get_root_directory(self) -> str: return os.path.dirname(os.path.abspath(__file__)) diff --git a/src/service/osw_formatter_service.py b/src/service/osw_formatter_service.py index 2e398cc..7483d52 100644 --- a/src/service/osw_formatter_service.py +++ b/src/service/osw_formatter_service.py @@ -36,6 +36,7 @@ def __init__(self): self.logger = self.core.get_logger() self.storage_client = self.core.get_storage_client() self.container_name = self._settings.event_bus.container_name + self._shutdown_triggered = threading.Event() self.listening_thread = threading.Thread(target=self.start_listening) self.listening_thread.start() self.download_dir = self._settings.get_download_directory() @@ -81,9 +82,12 @@ def process(message: QueueMessage) -> None: logger.error(f"Error occurred while processing message, {e}") self.send_status(result=ValidationResult(is_valid=False, validation_message=str(e)), upload_message=message) + finally: + self._stop_server_and_container() self.listening_topic.subscribe( - subscription=self.subscription_name, callback=process + subscription=self.subscription_name, callback=process, + max_receivable_messages=self._settings.max_receivable_messages ) def format(self, received_message: OSWValidationMessage): @@ -300,3 +304,27 @@ def _prepare_upload_file(self, formatter: OSWFormat, generated_files): def stop_listening(self): self.listening_thread.join(timeout=0) return + + def _stop_server_and_container(self, delay_seconds: float = 0.0): + """ + Attempt to gracefully stop the current process (stopping FastAPI/uvicorn and the Docker container). + """ + logger.info('Gracefully stopping FastAPI/uvicorn and Docker container') + if self._shutdown_triggered.is_set(): + logger.info('Server stop already in progress; skipping duplicate trigger.') + return + self._shutdown_triggered.set() + logger.info('Server stop triggered; scheduling shutdown.') + def _terminate(): + if delay_seconds: + time.sleep(delay_seconds) + try: + logger.info('Sending SIGTERM to stop server/container.') + os.kill(os.getpid(), signal.SIGTERM) + except Exception as err: + logger.warning(f'Error occurred while sending SIGTERM: {err}') + finally: + logger.info('Forcing process exit to stop server/container.') + os._exit(0) + + threading.Thread(target=_terminate, daemon=True).start() \ No newline at end of file From 444e11ade4aa14eeb9c0de4631797b960606048c Mon Sep 17 00:00:00 2001 From: sujata-m Date: Thu, 12 Mar 2026 15:33:36 +0530 Subject: [PATCH 03/10] Increased terminate instance time to 2 secs --- .gitignore | 3 ++- src/config.py | 9 +++++++-- src/service/osw_formatter_service.py | 3 ++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 2191abc..6f19d42 100644 --- a/.gitignore +++ b/.gitignore @@ -134,4 +134,5 @@ dmypy.json reports/ test_report.html integration_test_report.html -test_results \ No newline at end of file +test_results +.idea \ No newline at end of file diff --git a/src/config.py b/src/config.py index 68ee8ff..389db87 100644 --- a/src/config.py +++ b/src/config.py @@ -17,8 +17,13 @@ class EventBusSettings: class Settings(BaseSettings): app_name: str = 'python-osw-formatter' event_bus = EventBusSettings() - max_concurrent_messages: int = os.environ.get('MAX_CONCURRENT_MESSAGES', 2) - max_receivable_messages: int = os.environ.get('MAX_RECEIVABLE_MESSAGES', -1) + max_concurrent_messages: int = os.environ.get('MAX_CONCURRENT_MESSAGES', 1) + # Single-run worker should consume one message and then shut down. + max_receivable_messages: int = os.environ.get('MAX_RECEIVABLE_MESSAGES', 1) + # Wait for queue message completion/abandon settlement before terminating process. + message_settle_wait_seconds: float = os.environ.get('MESSAGE_SETTLE_WAIT_SECONDS', 10.0) + # Delay gives queue client time to settle/complete the in-flight message before exit. + shutdown_delay_seconds: float = os.environ.get('SHUTDOWN_DELAY_SECONDS', 2.0) def get_root_directory(self) -> str: return os.path.dirname(os.path.abspath(__file__)) diff --git a/src/service/osw_formatter_service.py b/src/service/osw_formatter_service.py index 7483d52..22c20d5 100644 --- a/src/service/osw_formatter_service.py +++ b/src/service/osw_formatter_service.py @@ -1,6 +1,7 @@ import gc import os import time +import signal import logging import traceback import urllib.parse @@ -83,7 +84,7 @@ def process(message: QueueMessage) -> None: self.send_status(result=ValidationResult(is_valid=False, validation_message=str(e)), upload_message=message) finally: - self._stop_server_and_container() + self._stop_server_and_container(delay_seconds=2) self.listening_topic.subscribe( subscription=self.subscription_name, callback=process, From 89075a815ff0ec81331e784ac77fb135676c2af3 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Wed, 18 Mar 2026 12:08:46 +0530 Subject: [PATCH 04/10] Multi-processing fix --- requirements.txt | 2 +- src/service/osw_formatter_service.py | 6 +++--- .../service/test_osw_formatter_service.py | 18 +++++++++++++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7337c94..6772f69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ --extra-index-url https://test.pypi.org/simple/ fastapi==0.88.0 pydantic==1.10.4 -python-ms-core==0.0.24 +python-ms-core==0.2.5.2 uvicorn==0.20.0 html_testRunner==1.2.1 osm-osw-reformatter==0.3.2 diff --git a/src/service/osw_formatter_service.py b/src/service/osw_formatter_service.py index 22c20d5..564b3cc 100644 --- a/src/service/osw_formatter_service.py +++ b/src/service/osw_formatter_service.py @@ -58,7 +58,7 @@ def process(message: QueueMessage) -> None: messageId=message.messageId, data=queue_message['data'] ) - logger.info(f'Received on demand request: {ondemand_request.data.jobId}') + logger.info(f'Received on demand request: {ondemand_request.data.jobId}, Core: {Core.__version__}') self.process_on_demand_format(request=ondemand_request) except Exception as e: logger.error(f"Error occurred while processing on demand message, {e}") @@ -83,13 +83,13 @@ def process(message: QueueMessage) -> None: logger.error(f"Error occurred while processing message, {e}") self.send_status(result=ValidationResult(is_valid=False, validation_message=str(e)), upload_message=message) - finally: - self._stop_server_and_container(delay_seconds=2) self.listening_topic.subscribe( subscription=self.subscription_name, callback=process, max_receivable_messages=self._settings.max_receivable_messages ) + logger.info('Listener finished processing available messages; stopping server/container.') + self._stop_server_and_container(delay_seconds=self._settings.shutdown_delay_seconds) def format(self, received_message: OSWValidationMessage): tdei_record_id: str = "" diff --git a/tests/unit_tests/service/test_osw_formatter_service.py b/tests/unit_tests/service/test_osw_formatter_service.py index 844b4e3..163750f 100644 --- a/tests/unit_tests/service/test_osw_formatter_service.py +++ b/tests/unit_tests/service/test_osw_formatter_service.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Optional from dataclasses import dataclass, asdict -from unittest.mock import Mock, MagicMock, patch, call +from unittest.mock import ANY, Mock, MagicMock, patch, call from src.service.osw_formatter_service import OSWFomatterService from src.osw_format import OSWFormat from src.models.queue_message_content import ValidationResult @@ -50,6 +50,9 @@ def setUp(self): self.formatter.core.return_value = MagicMock() self.formatter.core.get_topic = MagicMock() self.formatter.core.get_topic.return_value = MagicMock() + self.formatter._settings = MagicMock() + self.formatter._settings.max_receivable_messages = 1 + self.formatter._settings.shutdown_delay_seconds = 2.0 self.formatter.download_dir = DOWNLOAD_PATH @patch.object(OSWFomatterService, 'start_listening') @@ -60,6 +63,19 @@ def test_start_listening(self, mock_start_listening): # Assert mock_start_listening.assert_called_once() + @patch.object(OSWFomatterService, '_stop_server_and_container') + def test_start_listening_stops_container_after_subscribe_returns(self, mock_stop_server_and_container): + self.formatter.start_listening() + + self.formatter.listening_topic.subscribe.assert_called_once_with( + subscription=self.formatter.subscription_name, + callback=ANY, + max_receivable_messages=self.formatter._settings.max_receivable_messages, + ) + mock_stop_server_and_container.assert_called_once_with( + delay_seconds=self.formatter._settings.shutdown_delay_seconds + ) + @patch('src.service.osw_formatter_service.OSWFormat') @patch.object(OSWFormat, 'download_single_file') @patch.object(OSWFomatterService, 'send_status') From c6c5c1237d7e3432f336f7d41dbc6931e8a6bd82 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Thu, 19 Mar 2026 15:59:40 +0530 Subject: [PATCH 05/10] Added pipeline --- .github/workflows/dev_workflow_func_app.yaml | 105 ++++++++++++++++++ .github/workflows/prod_workflow_func_app.yaml | 105 ++++++++++++++++++ .../workflows/stage_workflow_func_app.yaml | 105 ++++++++++++++++++ requirements.txt | 3 +- 4 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/dev_workflow_func_app.yaml create mode 100644 .github/workflows/prod_workflow_func_app.yaml create mode 100644 .github/workflows/stage_workflow_func_app.yaml diff --git a/.github/workflows/dev_workflow_func_app.yaml b/.github/workflows/dev_workflow_func_app.yaml new file mode 100644 index 0000000..b1beb19 --- /dev/null +++ b/.github/workflows/dev_workflow_func_app.yaml @@ -0,0 +1,105 @@ +--- +# Dev Workflow – build image and deploy to Azure Function App (Development environment). +# +# Required secrets (Settings → Environments → Development → Environment secrets): +# - REGISTRY_DOMAIN – Azure Container Registry login server (e.g. myregistry.azurecr.io) +# - REGISTRY_USERNAME – ACR username +# - REGISTRY_PASSWORD – ACR password +# - REGISTRY_REPO – Repository name in ACR for this app +# - TDEI_CORE_AZURE_CREDS – Azure service principal JSON (for az login) +# +# Required variables (Settings → Environments → Development → Environment variables): +# - FUNCTION_APP_NAME – Azure Function App name to deploy to +# - RESOURCE_GROUP – Azure resource group containing the Function App +# +# Optional variables (defaults used if not set): +# - RESTART_APP – Set to 'true' or 'false'; default 'true' +# - APP_SETTINGS_JSON – JSON object of extra app settings to apply; default '{}' +# +######### Dev Workflow ######## +on: + pull_request: + branches: [dev] + types: + - closed + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + Build: + environment: Development + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + steps: + - uses: actions/checkout@v2 + - uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.REGISTRY_DOMAIN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Publish image to Azure Registry + run: | + docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'main' && 'prod' || github.ref_name }}${{ github.ref_name != 'main' && '-latest' || 'latest' }} . + docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags + deploy: + environment: Development + runs-on: ubuntu-latest + needs: [Build] + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{ secrets.TDEI_CORE_AZURE_CREDS }} + + - name: Resolve deploy config from environment + id: deploy_config + run: | + echo "function_app_name=${{ vars.FUNCTION_APP_NAME }}" >> "$GITHUB_OUTPUT" + echo "resource_group=${{ vars.RESOURCE_GROUP }}" >> "$GITHUB_OUTPUT" + echo "aci_image=${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}" >> "$GITHUB_OUTPUT" + echo "restart_app=${{ vars.RESTART_APP || 'true' }}" >> "$GITHUB_OUTPUT" + + - name: Log target environment + shell: bash + run: | + echo "Deploying to:" + echo " Function App: ${{ steps.deploy_config.outputs.function_app_name }}" + echo " Resource Group: ${{ steps.deploy_config.outputs.resource_group }}" + echo " ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }}" + + - name: Update app settings (ACI_IMAGE + extras) + shell: bash + run: | + python - <<'PY' + import json + import os + app_settings = os.environ.get("APP_SETTINGS_JSON", "{}") + data = json.loads(app_settings) if app_settings else {} + data["ACI_IMAGE"] = os.environ["ACI_IMAGE"] + with open("/tmp/appsettings.txt", "w", encoding="utf-8") as handle: + for key, value in data.items(): + handle.write(f"{key}={value}\n") + PY + echo "Updating only provided settings (no clearing of others)." + az functionapp config appsettings set \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ + --settings $(cat /tmp/appsettings.txt | tr '\n' ' ') + env: + ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }} + APP_SETTINGS_JSON: ${{ vars.APP_SETTINGS_JSON || '{}' }} + + - name: Restart function app + if: ${{ steps.deploy_config.outputs.restart_app == 'true' }} + shell: bash + run: | + az functionapp restart \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ No newline at end of file diff --git a/.github/workflows/prod_workflow_func_app.yaml b/.github/workflows/prod_workflow_func_app.yaml new file mode 100644 index 0000000..923c11f --- /dev/null +++ b/.github/workflows/prod_workflow_func_app.yaml @@ -0,0 +1,105 @@ +--- +# Prod Workflow – build image and deploy to Azure Function App (Production environment). +# +# Required secrets (Settings → Environments → Production → Environment secrets): +# - REGISTRY_DOMAIN – Azure Container Registry login server (e.g. myregistry.azurecr.io) +# - REGISTRY_USERNAME – ACR username +# - REGISTRY_PASSWORD – ACR password +# - REGISTRY_REPO – Repository name in ACR for this app +# - TDEI_CORE_AZURE_CREDS – Azure service principal JSON (for az login) +# +# Required variables (Settings → Environments → Production → Environment variables): +# - FUNCTION_APP_NAME – Azure Function App name to deploy to +# - RESOURCE_GROUP – Azure resource group containing the Function App +# +# Optional variables (defaults used if not set): +# - RESTART_APP – Set to 'true' or 'false'; default 'true' +# - APP_SETTINGS_JSON – JSON object of extra app settings to apply; default '{}' +# +######### Prod Workflow ######## +on: + pull_request: + branches: [main] + types: + - closed + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + Build: + environment: Production + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + steps: + - uses: actions/checkout@v2 + - uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.REGISTRY_DOMAIN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Publish image to Azure Registry + run: | + docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'main' && 'prod' || github.ref_name }}${{ github.ref_name != 'main' && '-latest' || 'latest' }} . + docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags + deploy: + environment: Production + runs-on: ubuntu-latest + needs: [Build] + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{ secrets.TDEI_CORE_AZURE_CREDS }} + + - name: Resolve deploy config from environment + id: deploy_config + run: | + echo "function_app_name=${{ vars.FUNCTION_APP_NAME }}" >> "$GITHUB_OUTPUT" + echo "resource_group=${{ vars.RESOURCE_GROUP }}" >> "$GITHUB_OUTPUT" + echo "aci_image=${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}" >> "$GITHUB_OUTPUT" + echo "restart_app=${{ vars.RESTART_APP || 'true' }}" >> "$GITHUB_OUTPUT" + + - name: Log target environment + shell: bash + run: | + echo "Deploying to:" + echo " Function App: ${{ steps.deploy_config.outputs.function_app_name }}" + echo " Resource Group: ${{ steps.deploy_config.outputs.resource_group }}" + echo " ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }}" + + - name: Update app settings (ACI_IMAGE + extras) + shell: bash + run: | + python - <<'PY' + import json + import os + app_settings = os.environ.get("APP_SETTINGS_JSON", "{}") + data = json.loads(app_settings) if app_settings else {} + data["ACI_IMAGE"] = os.environ["ACI_IMAGE"] + with open("/tmp/appsettings.txt", "w", encoding="utf-8") as handle: + for key, value in data.items(): + handle.write(f"{key}={value}\n") + PY + echo "Updating only provided settings (no clearing of others)." + az functionapp config appsettings set \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ + --settings $(cat /tmp/appsettings.txt | tr '\n' ' ') + env: + ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }} + APP_SETTINGS_JSON: ${{ vars.APP_SETTINGS_JSON || '{}' }} + + - name: Restart function app + if: ${{ steps.deploy_config.outputs.restart_app == 'true' }} + shell: bash + run: | + az functionapp restart \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ No newline at end of file diff --git a/.github/workflows/stage_workflow_func_app.yaml b/.github/workflows/stage_workflow_func_app.yaml new file mode 100644 index 0000000..07b0e76 --- /dev/null +++ b/.github/workflows/stage_workflow_func_app.yaml @@ -0,0 +1,105 @@ +--- +# Stage Workflow – build image and deploy to Azure Function App (Stage environment). +# +# Required secrets (Settings → Environments → Stage → Environment secrets): +# - REGISTRY_DOMAIN – Azure Container Registry login server (e.g. myregistry.azurecr.io) +# - REGISTRY_USERNAME – ACR username +# - REGISTRY_PASSWORD – ACR password +# - REGISTRY_REPO – Repository name in ACR for this app +# - TDEI_CORE_AZURE_CREDS – Azure service principal JSON (for az login) +# +# Required variables (Settings → Environments → Stage → Environment variables): +# - FUNCTION_APP_NAME – Azure Function App name to deploy to +# - RESOURCE_GROUP – Azure resource group containing the Function App +# +# Optional variables (defaults used if not set): +# - RESTART_APP – Set to 'true' or 'false'; default 'true' +# - APP_SETTINGS_JSON – JSON object of extra app settings to apply; default '{}' +# +######### Stage Workflow ######## +on: + pull_request: + branches: [stage] + types: + - closed + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + Build: + environment: Stage + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + steps: + - uses: actions/checkout@v2 + - uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.REGISTRY_DOMAIN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Publish image to Azure Registry + run: | + docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'main' && 'prod' || github.ref_name }}${{ github.ref_name != 'main' && '-latest' || 'latest' }} . + docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags + deploy: + environment: Stage + runs-on: ubuntu-latest + needs: [Build] + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{ secrets.TDEI_CORE_AZURE_CREDS }} + + - name: Resolve deploy config from environment + id: deploy_config + run: | + echo "function_app_name=${{ vars.FUNCTION_APP_NAME }}" >> "$GITHUB_OUTPUT" + echo "resource_group=${{ vars.RESOURCE_GROUP }}" >> "$GITHUB_OUTPUT" + echo "aci_image=${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}" >> "$GITHUB_OUTPUT" + echo "restart_app=${{ vars.RESTART_APP || 'true' }}" >> "$GITHUB_OUTPUT" + + - name: Log target environment + shell: bash + run: | + echo "Deploying to:" + echo " Function App: ${{ steps.deploy_config.outputs.function_app_name }}" + echo " Resource Group: ${{ steps.deploy_config.outputs.resource_group }}" + echo " ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }}" + + - name: Update app settings (ACI_IMAGE + extras) + shell: bash + run: | + python - <<'PY' + import json + import os + app_settings = os.environ.get("APP_SETTINGS_JSON", "{}") + data = json.loads(app_settings) if app_settings else {} + data["ACI_IMAGE"] = os.environ["ACI_IMAGE"] + with open("/tmp/appsettings.txt", "w", encoding="utf-8") as handle: + for key, value in data.items(): + handle.write(f"{key}={value}\n") + PY + echo "Updating only provided settings (no clearing of others)." + az functionapp config appsettings set \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ + --settings $(cat /tmp/appsettings.txt | tr '\n' ' ') + env: + ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }} + APP_SETTINGS_JSON: ${{ vars.APP_SETTINGS_JSON || '{}' }} + + - name: Restart function app + if: ${{ steps.deploy_config.outputs.restart_app == 'true' }} + shell: bash + run: | + az functionapp restart \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6772f69..a44ce6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ ---extra-index-url https://test.pypi.org/simple/ fastapi==0.88.0 pydantic==1.10.4 -python-ms-core==0.2.5.2 +python-ms-core==0.0.25 uvicorn==0.20.0 html_testRunner==1.2.1 osm-osw-reformatter==0.3.2 From 7d086b88aa87461f8f332e9bad2422f703e54450 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Thu, 19 Mar 2026 16:51:32 +0530 Subject: [PATCH 06/10] Fixed unit test cases --- tests/unit_tests/service/test_service.py | 44 +++++++++++++++++------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/tests/unit_tests/service/test_service.py b/tests/unit_tests/service/test_service.py index cc8d016..addc118 100644 --- a/tests/unit_tests/service/test_service.py +++ b/tests/unit_tests/service/test_service.py @@ -8,21 +8,36 @@ class TestOSWFormatterService(unittest.TestCase): - @patch('src.service.osw_formatter_service.Settings') - @patch('src.service.osw_formatter_service.Core') - def setUp(self, mock_core, mock_settings): - # Mock Settings - mock_settings.return_value.event_bus.validation_subscription = 'test_subscription' - mock_settings.return_value.event_bus.validation_topic = 'test_request_topic' - mock_settings.return_value.event_bus.formatter_topic = 'test_response_topic' - mock_settings.return_value.max_concurrent_messages = 10 - mock_settings.return_value.get_download_directory.return_value = '/tmp' - mock_settings.return_value.event_bus.container_name = 'test_container' - - # Mock Core + def setUp(self): + self.settings_patcher = patch.object(OSWFomatterService, '_settings') + mock_settings = self.settings_patcher.start() + self.addCleanup(self.settings_patcher.stop) + mock_settings.event_bus = MagicMock() + mock_settings.event_bus.validation_subscription = 'test_subscription' + mock_settings.event_bus.validation_topic = 'test_request_topic' + mock_settings.event_bus.formatter_topic = 'test_response_topic' + mock_settings.event_bus.container_name = 'test_container' + mock_settings.max_concurrent_messages = 10 + mock_settings.max_receivable_messages = 1 + mock_settings.shutdown_delay_seconds = 0 + mock_settings.get_download_directory.return_value = '/tmp' + + self.core_patcher = patch('src.service.osw_formatter_service.Core') + mock_core = self.core_patcher.start() + self.addCleanup(self.core_patcher.stop) + mock_core.__version__ = '0.test' mock_core.return_value.get_topic.return_value = MagicMock() mock_core.return_value.get_storage_client.return_value = MagicMock() + self.thread_patcher = patch('src.service.osw_formatter_service.threading.Thread') + mock_thread = self.thread_patcher.start() + self.addCleanup(self.thread_patcher.stop) + mock_thread.return_value = MagicMock() + + self.stop_server_patcher = patch.object(OSWFomatterService, '_stop_server_and_container') + self.stop_server_patcher.start() + self.addCleanup(self.stop_server_patcher.stop) + # Initialize InclinationService with mocked dependencies self.service = OSWFomatterService() self.service.storage_client = MagicMock() @@ -77,6 +92,7 @@ def test_on_demand_request_success(self, mock_queue_message, mock_logger, mock_r self.service.process_on_demand_format = MagicMock() # Act + self.service.start_listening() callback = self.service.listening_topic.subscribe.call_args[1]['callback'] callback(mock_message) @@ -86,7 +102,8 @@ def test_on_demand_request_success(self, mock_queue_message, mock_logger, mock_r messageId='1234', data={'jobId': '5678'} ) - mock_logger.info.assert_called_with('Received on demand request: 5678') + logged_message = mock_logger.info.call_args[0][0] + self.assertIn('Received on demand request: 5678', logged_message) self.service.process_on_demand_format.assert_called_once_with(request=mock_request.return_value) @@ -105,6 +122,7 @@ def test_on_demand_request_exception(self, mock_logger, mock_response, mock_requ self.service.send_on_demand_response = MagicMock() # Act + self.service.start_listening() callback = self.service.listening_topic.subscribe.call_args[1]['callback'] callback(mock_message) From 277938c8f9177c93d2696285282c1eac1ce53dfd Mon Sep 17 00:00:00 2001 From: sujata-m Date: Thu, 19 Mar 2026 19:06:50 +0530 Subject: [PATCH 07/10] Fixed pipeline --- .github/workflows/unit_tests.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 90ce6b6..901e5d7 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -13,6 +13,7 @@ jobs: env: DATABASE_NAME: test_database + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} steps: - name: Checkout code @@ -20,8 +21,8 @@ jobs: - name: Updating and installing GDAL run: | - sudo apt update - sudo apt install gdal-bin libgdal-dev python3-gdal + sudo apt-get update + sudo apt-get install -y gdal-bin libgdal-dev python3-gdal - name: Set up Python uses: actions/setup-python@v2 @@ -31,8 +32,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install GDAL==3.4.1 - pip install -r requirements.txt + GDAL_VERSION=$(gdal-config --version) + python -m pip install coverage "GDAL==${GDAL_VERSION}" + python -m pip install -r requirements.txt - name: Determine output folder id: set_output_folder @@ -70,11 +72,12 @@ jobs: coverage report --fail-under=85 - name: Upload report to Azure + if: ${{ env.AZURE_STORAGE_CONNECTION_STRING != '' }} uses: LanceMcCarthy/Action-AzureBlobUpload@v2 with: source_folder: 'test_results' destination_folder: '${{ env.output_folder }}' - connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} + connection_string: ${{ env.AZURE_STORAGE_CONNECTION_STRING }} container_name: 'osw-formatter-service' clean_destination_folder: false - delete_if_exists: false \ No newline at end of file + delete_if_exists: false From de51bd70dc1ba2e30b6d262609ee6f18b4dd2dec Mon Sep 17 00:00:00 2001 From: sujata-m Date: Thu, 19 Mar 2026 19:08:50 +0530 Subject: [PATCH 08/10] Fixed pipeline --- .github/workflows/unit_tests.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 901e5d7..796c13f 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -13,7 +13,6 @@ jobs: env: DATABASE_NAME: test_database - AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} steps: - name: Checkout code @@ -21,8 +20,8 @@ jobs: - name: Updating and installing GDAL run: | - sudo apt-get update - sudo apt-get install -y gdal-bin libgdal-dev python3-gdal + sudo apt update + sudo apt install gdal-bin libgdal-dev python3-gdal - name: Set up Python uses: actions/setup-python@v2 @@ -32,9 +31,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - GDAL_VERSION=$(gdal-config --version) - python -m pip install coverage "GDAL==${GDAL_VERSION}" - python -m pip install -r requirements.txt + pip install GDAL==3.4.1 + pip install -r requirements.txt - name: Determine output folder id: set_output_folder @@ -62,7 +60,9 @@ jobs: log_file="test_results/${timestamp}_report.log" echo -e "\nTest Cases Report Report\n" >> $log_file # Run the tests and append output to the log file - python -m coverage run --source=src -m unittest discover -s tests/unit_tests >> $log_file 2>&1 + python -m coverage run --source=src -m unittest discover -s tests/unit_tests -v 2>&1 | tee -a "$log_file" + test_status=${PIPESTATUS[0]} + echo -e "\nCoverage Report\n" >> $log_file coverage report >> $log_file coverage xml @@ -72,12 +72,11 @@ jobs: coverage report --fail-under=85 - name: Upload report to Azure - if: ${{ env.AZURE_STORAGE_CONNECTION_STRING != '' }} uses: LanceMcCarthy/Action-AzureBlobUpload@v2 with: source_folder: 'test_results' destination_folder: '${{ env.output_folder }}' - connection_string: ${{ env.AZURE_STORAGE_CONNECTION_STRING }} + connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} container_name: 'osw-formatter-service' clean_destination_folder: false - delete_if_exists: false + delete_if_exists: false \ No newline at end of file From e9fdc86a3ed0f9973985359a266a194089b638c5 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Thu, 19 Mar 2026 19:13:39 +0530 Subject: [PATCH 09/10] Fixed pipeline --- .github/workflows/unit_tests.yaml | 37 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 796c13f..67486f1 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -42,30 +42,35 @@ jobs: else branch_name=$GITHUB_REF_NAME fi - if [[ $branch_name == "main" ]]; then - echo "output_folder=prod" >> $GITHUB_ENV - elif [[ $branch_name == "stage" ]]; then - echo "output_folder=stage" >> $GITHUB_ENV - elif [[ $branch_name == "dev" ]]; then - echo "output_folder=dev" >> $GITHUB_ENV - else - echo "Unknown branch: $branch_name" - exit 1 - fi + + case "$branch_name" in + main) echo "output_folder=prod" >> "$GITHUB_ENV" ;; + stage) echo "output_folder=stage" >> "$GITHUB_ENV" ;; + dev) echo "output_folder=dev" >> "$GITHUB_ENV" ;; + *) echo "Unknown branch: $branch_name"; exit 1 ;; + esac - name: Run tests with coverage run: | - timestamp=$(date '+%Y-%m-%d_%H-%M-%S') + set -o pipefail + timestamp="$(date '+%Y-%m-%d_%H-%M-%S')" mkdir -p test_results log_file="test_results/${timestamp}_report.log" - echo -e "\nTest Cases Report Report\n" >> $log_file - # Run the tests and append output to the log file + + { + echo + echo "Test Cases Report" + echo + } | tee -a "$log_file" + + # Run unittest in verbose mode; mirror output to console and file python -m coverage run --source=src -m unittest discover -s tests/unit_tests -v 2>&1 | tee -a "$log_file" test_status=${PIPESTATUS[0]} - echo -e "\nCoverage Report\n" >> $log_file - coverage report >> $log_file - coverage xml + echo -e "\nCoverage Report\n" | tee -a "$log_file" + coverage report 2>&1 | tee -a "$log_file" + + exit $test_status - name: Check coverage run: | From afd6db95a1a6c27cf03d49dbaf1773161b5573ca Mon Sep 17 00:00:00 2001 From: sujata-m Date: Thu, 19 Mar 2026 19:16:23 +0530 Subject: [PATCH 10/10] Fixed test cases --- tests/unit_tests/test_osw_format.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit_tests/test_osw_format.py b/tests/unit_tests/test_osw_format.py index ce9daf6..3eced93 100644 --- a/tests/unit_tests/test_osw_format.py +++ b/tests/unit_tests/test_osw_format.py @@ -208,6 +208,8 @@ def test_download_single_file_file_not_found(self): class TesOSWFormatCleanUp(unittest.TestCase): + def setUp(self): + os.makedirs(DOWNLOAD_FILE_PATH, exist_ok=True) def test_clean_up_file_exists(self): filename = 'file1.txt'