Handle GitHub API timeouts while fetching org DSN

src/sentry_config.py

@@ -4,13 +4,13 @@
 import logging
 import os
 from configparser import ConfigParser
-from functools import lru_cache
 
 import requests
 
 LOGGING_LEVEL = os.environ.get("LOGGING_LEVEL", logging.INFO)
 logger = logging.getLogger(__name__)
 logger.setLevel(LOGGING_LEVEL)
+GH_API_TIMEOUT_SECONDS = float(os.environ.get("GH_API_TIMEOUT_SECONDS", "5"))
 
 SENTRY_CONFIG_API_URL = (
     "https://api.github.com/repos/{owner}/.sentry/contents/sentry_config.ini"
@@ -23,26 +23,21 @@
         "Accept": "application/vnd.github+json",
         "Authorization": f"token {token}",
     }
-    try:
-        api_url = SENTRY_CONFIG_API_URL.replace("{owner}", org)
+    api_url = SENTRY_CONFIG_API_URL.replace("{owner}", org)
 
-        # - Get meta about sentry_config.ini file
-        resp = requests.get(api_url, headers=headers)
-        resp.raise_for_status()
-        meta = resp.json()
+    # - Get meta about sentry_config.ini file
+    resp = requests.get(api_url, headers=headers, timeout=GH_API_TIMEOUT_SECONDS)
+    resp.raise_for_status()
+    meta = resp.json()
 
-        if meta["type"] != "file":
-            # XXX: custom error
-            raise Exception(meta["type"])
+    if meta["type"] != "file":
+        # XXX: custom error
+        raise Exception(meta["type"])
 
-        assert meta["encoding"] == "base64", meta["encoding"]
-        file_contents = base64.b64decode(meta["content"]).decode()
+    assert meta["encoding"] == "base64", meta["encoding"]
+    file_contents = base64.b64decode(meta["content"]).decode()
 
-        # - Read ini file and assertions
-        cp = ConfigParser()
-        cp.read_string(file_contents)
-        return cp.get("sentry-github-actions-app", "dsn")
-
-    except Exception as e:
-        logger.exception(e)
-        raise e
+    # - Read ini file and assertions
+    cp = ConfigParser()
+    cp.read_string(file_contents)
+    return cp.get("sentry-github-actions-app", "dsn")

src/web_app_handler.py

@@ -6,6 +6,8 @@
 import os
 from typing import NamedTuple
 
+import requests
+
 from .github_app import GithubAppToken
 from .github_sdk import GithubClient
 from src.sentry_config import fetch_dsn_for_github_org
@@ -37,28 +39,35 @@ def handle_event(self, data, headers):
             installation_id = data["installation"]["id"]
             org = data["repository"]["owner"]["login"]
 
-            # We are executing in Github App mode
-            if self.config.gh_app:
-                with GithubAppToken(**self.config.gh_app._asdict()).get_token(
-                    installation_id
-                ) as token:
+            try:
+                # We are executing in Github App mode
+                if self.config.gh_app:
+                    with GithubAppToken(**self.config.gh_app._asdict()).get_token(
+                        installation_id
+                    ) as token:
+                        # Once the Sentry org has a .sentry repo we can remove the DSN from the deployment
+                        dsn = fetch_dsn_for_github_org(org, token)
+                        client = GithubClient(
+                            token=token,
+                            dsn=dsn,
+                            dry_run=self.dry_run,
+                        )
+                        client.send_trace(data["workflow_job"])
+                else:
                     # Once the Sentry org has a .sentry repo we can remove the DSN from the deployment
-                    dsn = fetch_dsn_for_github_org(org, token)
+                    dsn = fetch_dsn_for_github_org(org, self.config.gh.token)
                     client = GithubClient(
-                        token=token,
+                        token=self.config.gh.token,
                         dsn=dsn,
                         dry_run=self.dry_run,
                     )
                     client.send_trace(data["workflow_job"])
-            else:
-                # Once the Sentry org has a .sentry repo we can remove the DSN from the deployment
-                dsn = fetch_dsn_for_github_org(org, token)
-                client = GithubClient(
-                    token=self.config.gh.token,
-                    dsn=dsn,
-                    dry_run=self.dry_run,
+            except requests.exceptions.Timeout as e:
+                logger.warning(
+                    f"Timed out while contacting GitHub APIs for org '{org}'. Skipping trace forwarding.",
+                    exc_info=e,
                 )
-                client.send_trace(data["workflow_job"])
+                reason = "Timed out while contacting GitHub APIs."
 
         return reason, http_code
 

tests/test_sentry_config_file.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
 from unittest import TestCase
+from unittest.mock import Mock
+from unittest.mock import patch
 
 import responses
 
+from src import sentry_config
 from src.sentry_config import fetch_dsn_for_github_org
 from src.sentry_config import SENTRY_CONFIG_API_URL as api_url
 
@@ -47,6 +50,21 @@ def setUp(self) -> None:
     def test_fetch_parse_sentry_config_file(self) -> None:
         assert fetch_dsn_for_github_org(org, token) == expected_dsn
 
+    def test_fetch_uses_configured_timeout(self) -> None:
+        mocked_response = Mock()
+        mocked_response.json.return_value = sentry_config_file_meta
+        mocked_response.raise_for_status.return_value = None
+        with patch("src.sentry_config.requests.get", return_value=mocked_response) as get:
+            assert fetch_dsn_for_github_org(org, token) == expected_dsn
+            get.assert_called_once_with(
+                self.api_url,
+                headers={
+                    "Accept": "application/vnd.github+json",
+                    "Authorization": f"token {token}",
+                },
+                timeout=sentry_config.GH_API_TIMEOUT_SECONDS,
+            )
+
     def test_fetch_private_repo(self) -> None:
         pass
 

tests/test_web_app_handler.py

@@ -4,6 +4,7 @@
 from unittest import mock
 
 import pytest
+import requests
 
 from src.web_app_handler import WebAppHandler
 
@@ -108,3 +109,20 @@ def test_handle_event_with_secret(monkeypatch, webhook_event):
     )
     assert reason == "OK"
     assert http_code == 200
+
+
+def test_handle_event_tolerates_github_timeout(monkeypatch, webhook_event):
+    monkeypatch.setenv("GH_TOKEN", "fake_pat")
+    monkeypatch.delenv("GH_APP_ID", raising=False)
+    handler = WebAppHandler()
+
+    with mock.patch(
+        "src.web_app_handler.fetch_dsn_for_github_org",
+        side_effect=requests.exceptions.Timeout(),
+    ):
+        reason, http_code = handler.handle_event(
+            data=webhook_event["payload"],
+            headers=webhook_event["headers"],
+        )
+    assert reason == "Timed out while contacting GitHub APIs."
+    assert http_code == 200