diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9cb24494540..c364fb4f161 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1159,6 +1159,28 @@ jobs: # azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" # displayName: "Execution Performance" +- job: CheckExternalUrls + displayName: "Check External Source URLs" + + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - bash: | + set -ev + if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then + # If CI is set to shallow fetch, target branch should be explicitly fetched. + git fetch origin --depth=1 $(System.PullRequest.TargetBranch) + python scripts/ci/validate_external_source_urls.py --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) + else + # Non-PR builds may use shallow checkout where HEAD^ is missing. + echo "Skipping external URL checks for Non-PR builds" + fi + displayName: 'Validate External Source URLs' + - job: CheckLinter displayName: "Check CLI Linter" diff --git a/scripts/ci/validate_external_source_urls.py b/scripts/ci/validate_external_source_urls.py new file mode 100644 index 00000000000..f4b454afef4 --- /dev/null +++ b/scripts/ci/validate_external_source_urls.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Fail CI if forbidden raw GitHub URL is introduced in new diff lines.""" + +import argparse +import fnmatch +import re +import subprocess +import sys + + +FORBIDDEN_EXTERNAL_URL_PATTERN = re.compile( + r"https://raw\.githubusercontent\.com" +) +RECOMMENDED_INTERNAL_URL = "https://azcliprod.blob.core.windows.net/cli" + +# Paths matching these glob patterns are excluded from the check. +# Exclusions cover documentation, test source files, test recordings, and test data. +EXCLUDED_PATH_PATTERNS = [ + "*.md", + "*.rst", + "doc/*", + "docs/*", + "*/doc/*", + "*/docs/*", + "scripts/*", + "*/tests/recordings/*", + "*/tests/*.py", + "*/tests/*.json", + "*/tests/*.yaml", + "*/tests/*.yml", + "*/tests/*/recordings/*", + "*/tests/*/test_*.py", + "*/tests/*/*.json", + "*/tests/*/*.yaml", + "*/tests/*/*.yml", +] + + +def _is_excluded(file_path: str) -> bool: + """Return True if *file_path* matches one of the exclusion glob patterns.""" + for pattern in EXCLUDED_PATH_PATTERNS: + if fnmatch.fnmatch(file_path, pattern): + return True + return False + + +def _run_diff(src: str, tgt: str, cached: bool = False) -> str: + cmd = ["git", "diff", "--unified=0", "--no-color"] + if cached: + cmd.append("--cached") + else: + cmd.append(f"{tgt}...{src}") + + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError(proc.stderr.strip() or "git diff failed") + return proc.stdout + + +def _find_violations(diff_text: str): + violations = [] + current_file = "" + + for line in diff_text.splitlines(): + if line.startswith("+++ b/"): + current_file = line[6:] + continue + + if not line.startswith("+") or line.startswith("+++"): + continue + + added_line = line[1:] + if FORBIDDEN_EXTERNAL_URL_PATTERN.search(added_line) and not _is_excluded(current_file): + violations.append((current_file or "", added_line.strip())) + + return violations + + +def main() -> int: + parser = argparse.ArgumentParser(description="Check diff for forbidden raw github URL usage.") + parser.add_argument("--src", default="HEAD", help="Source ref/commit for git diff.") + parser.add_argument("--tgt", default="HEAD~1", help="Target ref/commit for git diff.") + parser.add_argument("--cached", action="store_true", help="Check staged changes in git index.") + args = parser.parse_args() + + try: + diff_text = _run_diff(src=args.src, tgt=args.tgt, cached=args.cached) + except Exception as ex: # pylint: disable=broad-except + if args.cached: + print(f"Unable to evaluate staged diff: {ex}", file=sys.stderr) + else: + print(f"Unable to evaluate diff between '{args.tgt}' and '{args.src}': {ex}", file=sys.stderr) + return 1 + + violations = _find_violations(diff_text) + if not violations: + print("No forbidden external github URL found in added lines.") + return 0 + + print("Found forbidden external github URL in this change:", file=sys.stderr) + for file_path, content in violations: + print(f" - {file_path}: {content}", file=sys.stderr) + + print( + f"Use '{RECOMMENDED_INTERNAL_URL}' instead of raw GitHub URLs to limit external system access.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/src/azure-cli-core/azure/cli/core/cloud.py b/src/azure-cli-core/azure/cli/core/cloud.py index 7c9808cf3c4..a402bffb027 100644 --- a/src/azure-cli-core/azure/cli/core/cloud.py +++ b/src/azure-cli-core/azure/cli/core/cloud.py @@ -502,7 +502,7 @@ class CloudNameEnum: # pylint: disable=too-few-public-methods active_directory_resource_id='https://management.sovcloud-api.fr/', active_directory_graph_resource_id='https://graph.svc.sovcloud.fr/', microsoft_graph_resource_id='https://graph.svc.sovcloud.fr', - vm_image_alias_doc='https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/arm-compute/quickstart-templates/aliases.json', + vm_image_alias_doc='https://azcliprod.blob.core.windows.net/cli/vm/aliases_master.json', media_resource_id='https://rest.media.sovcloud-api.fr', ossrdbms_resource_id='https://ossrdbms-aad.database.sovcloud-api.fr', portal='https://portal.sovcloud-azure.fr'), diff --git a/src/azure-cli/azure/cli/command_modules/cloud/tests/latest/recordings/test_cloud_scenario.yaml b/src/azure-cli/azure/cli/command_modules/cloud/tests/latest/recordings/test_cloud_scenario.yaml index e3592d909dd..d64c20b1ee9 100644 --- a/src/azure-cli/azure/cli/command_modules/cloud/tests/latest/recordings/test_cloud_scenario.yaml +++ b/src/azure-cli/azure/cli/command_modules/cloud/tests/latest/recordings/test_cloud_scenario.yaml @@ -14,7 +14,7 @@ interactions: uri: https://management.azure.com/metadata/endpoints?api-version=2022-09-01 response: body: - string: '{"portal":"https://portal.azure.com","authentication":{"loginEndpoint":"https://login.microsoftonline.com","audiences":["https://management.core.windows.net/","https://management.azure.com/"],"tenant":"common","identityProvider":"AAD"},"media":"https://rest.media.azure.net","graphAudience":"https://graph.windows.net/","graph":"https://graph.windows.net/","name":"AzureCloud","suffixes":{"azureDataLakeStoreFileSystem":"azuredatalakestore.net","acrLoginServer":"azurecr.io","sqlServerHostname":"database.windows.net","azureDataLakeAnalyticsCatalogAndJob":"azuredatalakeanalytics.net","keyVaultDns":"vault.azure.net","storage":"core.windows.net","azureFrontDoorEndpointSuffix":"azurefd.net","storageSyncEndpointSuffix":"afs.azure.net","mhsmDns":"managedhsm.azure.net","mysqlServerEndpoint":"mysql.database.azure.com","postgresqlServerEndpoint":"postgres.database.azure.com","mariadbServerEndpoint":"mariadb.database.azure.com","synapseAnalytics":"dev.azuresynapse.net","attestationEndpoint":"attest.azure.net"},"batch":"https://batch.core.windows.net/","resourceManager":"https://management.azure.com/","vmImageAliasDoc":"https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/arm-compute/quickstart-templates/aliases.json","activeDirectoryDataLake":"https://datalake.azure.net/","sqlManagement":"https://management.core.windows.net:8443/","microsoftGraphResourceId":"https://graph.microsoft.com/","appInsightsResourceId":"https://api.applicationinsights.io","appInsightsTelemetryChannelResourceId":"https://dc.applicationinsights.azure.com/v2/track","attestationResourceId":"https://attest.azure.net","synapseAnalyticsResourceId":"https://dev.azuresynapse.net","logAnalyticsResourceId":"https://api.loganalytics.io","ossrDbmsResourceId":"https://ossrdbms-aad.database.windows.net"}' + string: '{"portal":"https://portal.azure.com","authentication":{"loginEndpoint":"https://login.microsoftonline.com","audiences":["https://management.core.windows.net/","https://management.azure.com/"],"tenant":"common","identityProvider":"AAD"},"media":"https://rest.media.azure.net","graphAudience":"https://graph.windows.net/","graph":"https://graph.windows.net/","name":"AzureCloud","suffixes":{"azureDataLakeStoreFileSystem":"azuredatalakestore.net","acrLoginServer":"azurecr.io","sqlServerHostname":"database.windows.net","azureDataLakeAnalyticsCatalogAndJob":"azuredatalakeanalytics.net","keyVaultDns":"vault.azure.net","storage":"core.windows.net","azureFrontDoorEndpointSuffix":"azurefd.net","storageSyncEndpointSuffix":"afs.azure.net","mhsmDns":"managedhsm.azure.net","mysqlServerEndpoint":"mysql.database.azure.com","postgresqlServerEndpoint":"postgres.database.azure.com","mariadbServerEndpoint":"mariadb.database.azure.com","synapseAnalytics":"dev.azuresynapse.net","attestationEndpoint":"attest.azure.net"},"batch":"https://batch.core.windows.net/","resourceManager":"https://management.azure.com/","vmImageAliasDoc":"https://azcliprod.blob.core.windows.net/cli/vm/aliases_master.json","activeDirectoryDataLake":"https://datalake.azure.net/","sqlManagement":"https://management.core.windows.net:8443/","microsoftGraphResourceId":"https://graph.microsoft.com/","appInsightsResourceId":"https://api.applicationinsights.io","appInsightsTelemetryChannelResourceId":"https://dc.applicationinsights.azure.com/v2/track","attestationResourceId":"https://attest.azure.net","synapseAnalyticsResourceId":"https://dev.azuresynapse.net","logAnalyticsResourceId":"https://api.loganalytics.io","ossrDbmsResourceId":"https://ossrdbms-aad.database.windows.net"}' headers: cache-control: - no-cache diff --git a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/recordings/test_delete_dependent_resources.yaml b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/recordings/test_delete_dependent_resources.yaml index 94939a0ffa0..9eda3926193 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/recordings/test_delete_dependent_resources.yaml +++ b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/recordings/test_delete_dependent_resources.yaml @@ -54,7 +54,7 @@ interactions: User-Agent: - python-requests/2.25.1 method: GET - uri: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/arm-compute/quickstart-templates/aliases.json + uri: https://azcliprod.blob.core.windows.net/cli/vm/aliases_master.json response: body: string: "{\n \"$schema\": \"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json\",\n